Simplified release process based on feedback

Previously, littlefs had mutable versions. That is, anytime a new commit
landed on master, the bot would update the most recent version to
contain the patch. The idea was that this would make sure users always
had the most recent bug fixes. Immutable snapshots could be accessed
through the git hashes.

However, at this point multiple developers have pointed out that this is
confusing, with mutable versions being non-standard and surprising.

This new release process adopts SemVer in its entirety, with
incrementing patch numbers and immutable versions.

When a new commit lands on master:
1. The major/minor version is taken from lfs.h
2. The most recent patch version is looked up on GitHub and incremented
3. A changelog is built out of the commits to the previous version
4. A new release is created on GitHub

Additionally, any commits that land while CI is still running are
coalesced together. Which means multiple PRs can land in a single
release.

.travis.yml

@@ -134,53 +134,44 @@ jobs:
         - STAGE=deploy
         - NAME=deploy
       script:
-        # Update tag for version defined in lfs.h
+        # Find version defined in lfs.h
         - LFS_VERSION=$(grep -ox '#define LFS_VERSION .*' lfs.h | cut -d ' ' -f3)
         - LFS_VERSION_MAJOR=$((0xffff & ($LFS_VERSION >> 16)))
         - LFS_VERSION_MINOR=$((0xffff & ($LFS_VERSION >>  0)))
-        - LFS_VERSION="v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR"
-        - echo "littlefs version $LFS_VERSION"
+        # Grab latests patch from repo tags, default to 0
+        - LFS_VERSION_PATCH=$(curl -f -u "$GEKY_BOT_RELEASES"
+                https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs
+                | jq 'map(.ref | match(
+                    "refs/tags/v'"$LFS_VERSION_MAJOR"'\\.'"$LFS_VERSION_MINOR"'\\.(.*)$")
+                    .captures[].string | tonumber + 1) | max // 0')
+        # We have our new version
+        - LFS_VERSION="v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.$LFS_VERSION_PATCH"
+        - echo "VERSION $LFS_VERSION"
         - |
-          curl -u $GEKY_BOT_RELEASES -X POST \
-            https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs \
-            -d "{
-              \"ref\": \"refs/tags/$LFS_VERSION\",
-              \"sha\": \"$TRAVIS_COMMIT\"
-            }"
-        - |
-          curl -f -u $GEKY_BOT_RELEASES -X PATCH \
-            https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs/tags/$LFS_VERSION \
-            -d "{
-              \"sha\": \"$TRAVIS_COMMIT\"
-            }"
-        # Create release notes from commits
-        - LFS_PREV_VERSION="v$LFS_VERSION_MAJOR.$(($LFS_VERSION_MINOR-1))"
-        - |
-          if [ $(git tag -l "$LFS_PREV_VERSION") ]
+          # Check that we're the most recent commit
+          CURRENT_COMMIT=$(curl -f -u "$GEKY_BOT_RELEASES" \
+                https://api.github.com/repos/$TRAVIS_REPO_SLUG/commits/master \
+                | jq -re '.sha')
+          if [ "$TRAVIS_COMMIT" == "$CURRENT_COMMIT" ]
           then
-            curl -u $GEKY_BOT_RELEASES -X POST \
+            # Build release notes
+            PREV=$(git tag --sort=-v:refname -l "v*.*.*" | head -1)
+            if [ ! -z "$PREV" ]
+            then
+                echo "PREV $PREV"
+                CHANGES=$'### Changes\n\n'$( \
+                    git log --oneline $PREV.. --grep='^Merge' --invert-grep)
+                printf "CHANGES\n%s\n\n" "$CHANGES"
+            fi
+            # Create the release
+            curl -f -u "$GEKY_BOT_RELEASES" -X POST \
                 https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases \
                 -d "{
                     \"tag_name\": \"$LFS_VERSION\",
-                    \"name\": \"$LFS_VERSION\"
+                    \"target_commitish\": \"$TRAVIS_COMMIT\",
+                    \"name\": \"${LFS_VERSION%.0}\",
+                    \"body\": $(jq -sR '.' <<< "$CHANGES")
                 }"
-            RELEASE=$(
-                curl -f -u $GEKY_BOT_RELEASES \
-                    https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases/tags/$LFS_VERSION
-            )
-            CHANGES=$(
-                git log --oneline $LFS_PREV_VERSION.. --grep='^Merge' --invert-grep
-            )
-            curl -f -u $GEKY_BOT_RELEASES -X PATCH \
-                https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases/$(
-                    jq -r '.id' <<< "$RELEASE"
-                ) \
-                -d "$(
-                    jq -s '{
-                        "body": ((.[0] // "" | sub("(?<=\n)#+ Changes.*"; ""; "mi"))
-                            + "### Changes\n\n" + .[1])
-                    }' <(jq '.body' <<< "$RELEASE") <(jq -sR '.' <<< "$CHANGES")
-                )"
           fi
 
 # Manage statuses

lfs.c

@@ -417,11 +417,14 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_dir_t *dir) {
     // rather than clobbering one of the blocks we just pretend
     // the revision may be valid
     int err = lfs_bd_read(lfs, dir->pair[0], 0, &dir->d.rev, 4);
-    dir->d.rev = lfs_fromle32(dir->d.rev);
-    if (err) {
+    if (err && err != LFS_ERR_CORRUPT) {
         return err;
     }
 
+    if (err != LFS_ERR_CORRUPT) {
+        dir->d.rev = lfs_fromle32(dir->d.rev);
+    }
+
     // set defaults
     dir->d.rev += 1;
     dir->d.size = sizeof(dir->d)+4;
@@ -445,6 +448,9 @@ static int lfs_dir_fetch(lfs_t *lfs,
         int err = lfs_bd_read(lfs, tpair[i], 0, &test, sizeof(test));
         lfs_dir_fromle32(&test);
         if (err) {
+            if (err == LFS_ERR_CORRUPT) {
+                continue;
+            }
             return err;
         }
 
@@ -464,6 +470,9 @@ static int lfs_dir_fetch(lfs_t *lfs,
         err = lfs_bd_crc(lfs, tpair[i], sizeof(test),
                 (0x7fffffff & test.size) - sizeof(test), &crc);
         if (err) {
+            if (err == LFS_ERR_CORRUPT) {
+                continue;
+            }
             return err;
         }
 
@@ -1282,8 +1291,9 @@ static int lfs_ctz_traverse(lfs_t *lfs,
 
 
 /// Top level file operations ///
-int lfs_file_open(lfs_t *lfs, lfs_file_t *file,
-        const char *path, int flags) {
+int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file,
+        const char *path, int flags,
+        const struct lfs_file_config *cfg) {
     // deorphan if we haven't yet, needed at most once after poweron
     if ((flags & 3) != LFS_O_RDONLY && !lfs->deorphaned) {
         int err = lfs_deorphan(lfs);
@@ -1323,6 +1333,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file,
     }
 
     // setup file struct
+    file->cfg = cfg;
     file->pair[0] = cwd.pair[0];
     file->pair[1] = cwd.pair[1];
     file->poff = entry.off;
@@ -1340,7 +1351,10 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file,
     }
 
     // allocate buffer if needed
-    if (lfs->cfg->file_buffer) {
+    file->cache.block = 0xffffffff;
+    if (file->cfg && file->cfg->buffer) {
+        file->cache.buffer = file->cfg->buffer;
+    } else if (lfs->cfg->file_buffer) {
         if (lfs->files) {
             // already in use
             return LFS_ERR_NOMEM;
@@ -1368,6 +1382,11 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file,
     return 0;
 }
 
+int lfs_file_open(lfs_t *lfs, lfs_file_t *file,
+        const char *path, int flags) {
+    return lfs_file_opencfg(lfs, file, path, flags, NULL);
+}
+
 int lfs_file_close(lfs_t *lfs, lfs_file_t *file) {
     int err = lfs_file_sync(lfs, file);
 
@@ -1380,7 +1399,7 @@ int lfs_file_close(lfs_t *lfs, lfs_file_t *file) {
     }
 
     // clean up memory
-    if (!lfs->cfg->file_buffer) {
+    if (!(file->cfg && file->cfg->buffer) && !lfs->cfg->file_buffer) {
         lfs_free(file->cache.buffer);
     }
 
@@ -1997,6 +2016,21 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) {
 
 
 /// Filesystem operations ///
+static void lfs_deinit(lfs_t *lfs) {
+    // free allocated memory
+    if (!lfs->cfg->read_buffer) {
+        lfs_free(lfs->rcache.buffer);
+    }
+
+    if (!lfs->cfg->prog_buffer) {
+        lfs_free(lfs->pcache.buffer);
+    }
+
+    if (!lfs->cfg->lookahead_buffer) {
+        lfs_free(lfs->free.buffer);
+    }
+}
+
 static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) {
     lfs->cfg = cfg;
 
@@ -2006,7 +2040,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) {
     } else {
         lfs->rcache.buffer = lfs_malloc(lfs->cfg->read_size);
         if (!lfs->rcache.buffer) {
-            return LFS_ERR_NOMEM;
+            goto cleanup;
         }
     }
 
@@ -2016,7 +2050,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) {
     } else {
         lfs->pcache.buffer = lfs_malloc(lfs->cfg->prog_size);
         if (!lfs->pcache.buffer) {
-            return LFS_ERR_NOMEM;
+            goto cleanup;
         }
     }
 
@@ -2032,7 +2066,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) {
     } else {
         lfs->free.buffer = lfs_malloc(lfs->cfg->lookahead/8);
         if (!lfs->free.buffer) {
-            return LFS_ERR_NOMEM;
+            goto cleanup;
         }
     }
 
@@ -2052,23 +2086,10 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) {
     lfs->deorphaned = false;
 
     return 0;
-}
 
-static int lfs_deinit(lfs_t *lfs) {
-    // free allocated memory
-    if (!lfs->cfg->read_buffer) {
-        lfs_free(lfs->rcache.buffer);
-    }
-
-    if (!lfs->cfg->prog_buffer) {
-        lfs_free(lfs->pcache.buffer);
-    }
-
-    if (!lfs->cfg->lookahead_buffer) {
-        lfs_free(lfs->free.buffer);
-    }
-
-    return 0;
+cleanup:
+    lfs_deinit(lfs);
+    return LFS_ERR_NOMEM;
 }
 
 int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) {
@@ -2088,19 +2109,19 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) {
     lfs_dir_t superdir;
     err = lfs_dir_alloc(lfs, &superdir);
     if (err) {
-        return err;
+        goto cleanup;
     }
 
     // write root directory
     lfs_dir_t root;
     err = lfs_dir_alloc(lfs, &root);
     if (err) {
-        return err;
+        goto cleanup;
     }
 
     err = lfs_dir_commit(lfs, &root, NULL, 0);
     if (err) {
-        return err;
+        goto cleanup;
     }
 
     lfs->root[0] = root.pair[0];
@@ -2131,24 +2152,28 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) {
                  &superblock.d, sizeof(superblock.d)}
             }, 1);
         if (err && err != LFS_ERR_CORRUPT) {
-            return err;
+            goto cleanup;
         }
 
         valid = valid || !err;
     }
 
     if (!valid) {
-        return LFS_ERR_CORRUPT;
+        err = LFS_ERR_CORRUPT;
+        goto cleanup;
     }
 
     // sanity check that fetch works
     err = lfs_dir_fetch(lfs, &superdir, (const lfs_block_t[2]){0, 1});
     if (err) {
-        return err;
+        goto cleanup;
     }
 
     lfs_alloc_ack(lfs);
-    return lfs_deinit(lfs);
+
+cleanup:
+    lfs_deinit(lfs);
+    return err;
 }
 
 int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) {
@@ -2168,7 +2193,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) {
     lfs_superblock_t superblock;
     err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1});
     if (err && err != LFS_ERR_CORRUPT) {
-        return err;
+        goto cleanup;
     }
 
     if (!err) {
@@ -2176,7 +2201,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) {
                 &superblock.d, sizeof(superblock.d));
         lfs_superblock_fromle32(&superblock.d);
         if (err) {
-            return err;
+            goto cleanup;
         }
 
         lfs->root[0] = superblock.d.root[0];
@@ -2185,7 +2210,8 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) {
 
     if (err || memcmp(superblock.d.magic, "littlefs", 8) != 0) {
         LFS_ERROR("Invalid superblock at %d %d", 0, 1);
-        return LFS_ERR_CORRUPT;
+        err = LFS_ERR_CORRUPT;
+        goto cleanup;
     }
 
     uint16_t major_version = (0xffff & (superblock.d.version >> 16));
@@ -2193,14 +2219,21 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) {
     if ((major_version != LFS_DISK_VERSION_MAJOR ||
          minor_version > LFS_DISK_VERSION_MINOR)) {
         LFS_ERROR("Invalid version %d.%d", major_version, minor_version);
-        return LFS_ERR_INVAL;
+        err = LFS_ERR_INVAL;
+        goto cleanup;
     }
 
     return 0;
+
+cleanup:
+
+    lfs_deinit(lfs);
+    return err;
 }
 
 int lfs_unmount(lfs_t *lfs) {
-    return lfs_deinit(lfs);
+    lfs_deinit(lfs);
+    return 0;
 }
 
 

lfs.h

@@ -21,7 +21,7 @@ extern "C"
 // Software library version
 // Major (top-nibble), incremented on backwards incompatible changes
 // Minor (bottom-nibble), incremented on feature additions
-#define LFS_VERSION 0x00010004
+#define LFS_VERSION 0x00010005
 #define LFS_VERSION_MAJOR (0xffff & (LFS_VERSION >> 16))
 #define LFS_VERSION_MINOR (0xffff & (LFS_VERSION >>  0))
 
@@ -167,6 +167,12 @@ struct lfs_config {
     void *file_buffer;
 };
 
+// Optional configuration provided during lfs_file_opencfg
+struct lfs_file_config {
+    // Optional, statically allocated buffer for files. Must be program sized.
+    // If NULL, malloc will be used by default.
+    void *buffer;
+};
 
 // File info structure
 struct lfs_info {
@@ -214,6 +220,7 @@ typedef struct lfs_file {
     lfs_block_t head;
     lfs_size_t size;
 
+    const struct lfs_file_config *cfg;
     uint32_t flags;
     lfs_off_t pos;
     lfs_block_t block;
@@ -281,7 +288,8 @@ typedef struct lfs {
 // Format a block device with the littlefs
 //
 // Requires a littlefs object and config struct. This clobbers the littlefs
-// object, and does not leave the filesystem mounted.
+// object, and does not leave the filesystem mounted. The config struct must
+// be zeroed for defaults and backwards compatibility.
 //
 // Returns a negative error code on failure.
 int lfs_format(lfs_t *lfs, const struct lfs_config *config);
@@ -290,7 +298,8 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *config);
 //
 // Requires a littlefs object and config struct. Multiple filesystems
 // may be mounted simultaneously with multiple littlefs objects. Both
-// lfs and config must be allocated while mounted.
+// lfs and config must be allocated while mounted. The config struct must
+// be zeroed for defaults and backwards compatibility.
 //
 // Returns a negative error code on failure.
 int lfs_mount(lfs_t *lfs, const struct lfs_config *config);
@@ -328,14 +337,27 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info);
 
 // Open a file
 //
-// The mode that the file is opened in is determined
-// by the flags, which are values from the enum lfs_open_flags
-// that are bitwise-ored together.
+// The mode that the file is opened in is determined by the flags, which
+// are values from the enum lfs_open_flags that are bitwise-ored together.
 //
 // Returns a negative error code on failure.
 int lfs_file_open(lfs_t *lfs, lfs_file_t *file,
         const char *path, int flags);
 
+// Open a file with extra configuration
+//
+// The mode that the file is opened in is determined by the flags, which
+// are values from the enum lfs_open_flags that are bitwise-ored together.
+//
+// The config struct provides additional config options per file as described
+// above. The config struct must be allocated while the file is open, and the
+// config struct must be zeroed for defaults and backwards compatibility.
+//
+// Returns a negative error code on failure.
+int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file,
+        const char *path, int flags,
+        const struct lfs_file_config *config);
+
 // Close a file
 //
 // Any pending writes are written out to storage as though