Changed release script to generate drafts

Interesting open-source projects that I've ran into around embedded
storage. May be interesting to others in the embedded space.

Added mklfs, SPIFFS, and Dhara.

Also a thanks to jolivepetrus for posting the mklfs tool he put
together.

.travis.yml

@@ -35,7 +35,7 @@ script:
     if [ "$TRAVIS_TEST_RESULT" -eq 0 ]
     then
         CURR=$(tail -n1 sizes | awk '{print $1}')
-        PREV=$(curl -u "$GEKY_BOT_STATUSES" https://api.github.com/repos/$TRAVIS_REPO_SLUG/status/master \
+        PREV=$(curl -u $GEKY_BOT_STATUSES https://api.github.com/repos/$TRAVIS_REPO_SLUG/status/master \
             | jq -re "select(.sha != \"$TRAVIS_COMMIT\")
                 | .statuses[] | select(.context == \"$STAGE/$NAME\").description
                 | capture(\"code size is (?<size>[0-9]+)\").size" \
@@ -100,7 +100,6 @@ jobs:
       env:
         - STAGE=test
         - NAME=littlefs-fuse
-      if: branch !~ -prefix$
       install:
         - sudo apt-get install libfuse-dev
         - git clone --depth 1 https://github.com/geky/littlefs-fuse
@@ -129,88 +128,70 @@ jobs:
         - ls
         - make -B test_dirs test_files QUIET=1
 
-    # Automatically create releases
+      # Automatically update releases
     - stage: deploy
       env:
         - STAGE=deploy
         - NAME=deploy
       script:
+        # Find version defined in lfs.h
+        - LFS_VERSION=$(grep -ox '#define LFS_VERSION .*' lfs.h | cut -d ' ' -f3)
+        - LFS_VERSION_MAJOR=$((0xffff & ($LFS_VERSION >> 16)))
+        - LFS_VERSION_MINOR=$((0xffff & ($LFS_VERSION >>  0)))
+        # Grab latests patch from repo tags, default to 0, needs finagling to get past github's pagination api
+        - PREV_URL=https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs/tags/v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.
+        - PREV_URL=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL" -I
+                | sed -n '/^Link/{s/.*<\(.*\)>; rel="last"/\1/;p;q0};$q1'
+                || echo $PREV_URL)
+        - LFS_VERSION_PATCH=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL"
+                | jq 'map(.ref | match("\\bv.*\\..*\\.(.*)$";"g")
+                    .captures[].string | tonumber) | max + 1'
+                || echo 0)
+        # We have our new version
+        - LFS_VERSION="v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.$LFS_VERSION_PATCH"
+        - echo "VERSION $LFS_VERSION"
         - |
-          bash << 'SCRIPT'
-          set -ev
-          # Find version defined in lfs.h
-          LFS_VERSION=$(grep -ox '#define LFS_VERSION .*' lfs.h | cut -d ' ' -f3)
-          LFS_VERSION_MAJOR=$((0xffff & ($LFS_VERSION >> 16)))
-          LFS_VERSION_MINOR=$((0xffff & ($LFS_VERSION >>  0)))
-          # Grab latests patch from repo tags, default to 0, needs finagling
-          # to get past github's pagination api
-          PREV_URL=https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs/tags/v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.
-          PREV_URL=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL" -I \
-              | sed -n '/^Link/{s/.*<\(.*\)>; rel="last"/\1/;p;q0};$q1' \
-              || echo $PREV_URL)
-          LFS_VERSION_PATCH=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL" \
-              | jq 'map(.ref | match("\\bv.*\\..*\\.(.*)$";"g")
-                  .captures[].string | tonumber) | max + 1' \
-              || echo 0)
-          # We have our new version
-          LFS_VERSION="v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.$LFS_VERSION_PATCH"
-          echo "VERSION $LFS_VERSION"
           # Check that we're the most recent commit
           CURRENT_COMMIT=$(curl -f -u "$GEKY_BOT_RELEASES" \
-              https://api.github.com/repos/$TRAVIS_REPO_SLUG/commits/master \
-              | jq -re '.sha')
-          [ "$TRAVIS_COMMIT" == "$CURRENT_COMMIT" ] || exit 0
-          # Create major branch
-          git branch v$LFS_VERSION_MAJOR HEAD
-          # Create major prefix branch
-          git config user.name "geky bot"
-          git config user.email "bot@geky.net"
-          git fetch https://github.com/$TRAVIS_REPO_SLUG.git \
-              --depth=50 v$LFS_VERSION_MAJOR-prefix || true
-          ./scripts/prefix.py lfs$LFS_VERSION_MAJOR
-          git branch v$LFS_VERSION_MAJOR-prefix $( \
-              git commit-tree $(git write-tree) \
-                  $(git rev-parse --verify -q FETCH_HEAD | sed -e 's/^/-p /') \
-                  -p HEAD \
-                  -m "Generated v$LFS_VERSION_MAJOR prefixes")
-          git reset --hard
-          # Update major version branches (vN and vN-prefix)
-          git push https://$GEKY_BOT_RELEASES@github.com/$TRAVIS_REPO_SLUG.git \
-              v$LFS_VERSION_MAJOR \
-              v$LFS_VERSION_MAJOR-prefix
-          # Create patch version tag (vN.N.N)
-          curl -f -u "$GEKY_BOT_RELEASES" -X POST \
-              https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs \
-              -d "{
-                  \"ref\": \"refs/tags/$LFS_VERSION\",
-                  \"sha\": \"$TRAVIS_COMMIT\"
-              }"
-          # Create minor release?
-          [[ "$LFS_VERSION" == *.0 ]] || exit 0
-          # Build release notes
-          PREV=$(git tag --sort=-v:refname -l "v*.0" | head -1)
-          if [ ! -z "$PREV" ]
+                https://api.github.com/repos/$TRAVIS_REPO_SLUG/commits/master \
+                | jq -re '.sha')
+          if [ "$TRAVIS_COMMIT" == "$CURRENT_COMMIT" ]
           then
-              echo "PREV $PREV"
-              CHANGES=$'### Changes\n\n'$( \
-                  git log --oneline $PREV.. --grep='^Merge' --invert-grep)
-              printf "CHANGES\n%s\n\n" "$CHANGES"
+            # Create a simple tag
+            curl -f -u "$GEKY_BOT_RELEASES" -X POST \
+                https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs \
+                -d "{
+                    \"ref\": \"refs/tags/$LFS_VERSION\",
+                    \"sha\": \"$TRAVIS_COMMIT\"
+                }"
+            # Minor release?
+            if [[ "$LFS_VERSION" == *.0 ]]
+            then
+              # Build release notes
+              PREV=$(git tag --sort=-v:refname -l "v*.0" | head -1)
+              if [ ! -z "$PREV" ]
+              then
+                  echo "PREV $PREV"
+                  CHANGES=$'### Changes\n\n'$( \
+                      git log --oneline $PREV.. --grep='^Merge' --invert-grep)
+                  printf "CHANGES\n%s\n\n" "$CHANGES"
+              fi
+              # Create the release
+              curl -f -u "$GEKY_BOT_RELEASES" -X POST \
+                  https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases \
+                  -d "{
+                      \"tag_name\": \"$LFS_VERSION\",
+                      \"name\": \"${LFS_VERSION%.0}\",
+                      \"draft\": true,
+                      \"body\": $(jq -sR '.' <<< "$CHANGES")
+                  }"
+            fi
           fi
-          # Create the release
-          curl -f -u "$GEKY_BOT_RELEASES" -X POST \
-              https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases \
-              -d "{
-                  \"tag_name\": \"$LFS_VERSION\",
-                  \"name\": \"${LFS_VERSION%.0}\",
-                  \"draft\": true,
-                  \"body\": $(jq -sR '.' <<< "$CHANGES")
-              }" #"
-          SCRIPT
 
 # Manage statuses
 before_install:
   - |
-    curl -u "$GEKY_BOT_STATUSES" -X POST \
+    curl -u $GEKY_BOT_STATUSES -X POST \
         https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
         -d "{
             \"context\": \"$STAGE/$NAME\",
@@ -221,7 +202,7 @@ before_install:
 
 after_failure:
   - |
-    curl -u "$GEKY_BOT_STATUSES" -X POST \
+    curl -u $GEKY_BOT_STATUSES -X POST \
         https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
         -d "{
             \"context\": \"$STAGE/$NAME\",
@@ -232,7 +213,7 @@ after_failure:
 
 after_success:
   - |
-    curl -u "$GEKY_BOT_STATUSES" -X POST \
+    curl -u $GEKY_BOT_STATUSES -X POST \
         https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
         -d "{
             \"context\": \"$STAGE/$NAME\",

README.md

@@ -111,9 +111,9 @@ filesystem until sync or close is called on the file.
 
 ## Other notes
 
-All littlefs calls have the potential to return a negative error code. The 
-errors can be either one of those found in the `enum lfs_error` in 
-[lfs.h](lfs.h), or an error returned by the user's block device operations.
+All littlefs have the potential to return a negative error code. The errors
+can be either one of those found in the `enum lfs_error` in [lfs.h](lfs.h),
+or an error returned by the user's block device operations.
 
 In the configuration struct, the `prog` and `erase` function provided by the
 user may return a `LFS_ERR_CORRUPT` error if the implementation already can

lfs.c

@@ -888,7 +888,7 @@ nextname:
         }
 
         // check that entry has not been moved
-        if (!lfs->moving && entry->d.type & 0x80) {
+        if (entry->d.type & 0x80) {
             int moved = lfs_moved(lfs, &entry->d.u);
             if (moved < 0 || moved) {
                 return (moved < 0) ? moved : LFS_ERR_NOENT;
@@ -1644,11 +1644,6 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file,
         file->pos = file->size;
     }
 
-    if (file->pos + size > LFS_FILE_MAX) {
-        // larger than file limit?
-        return LFS_ERR_FBIG;
-    }
-
     if (!(file->flags & LFS_F_WRITING) && file->pos > file->size) {
         // fill with zeros
         lfs_off_t pos = file->pos;
@@ -1735,24 +1730,24 @@ lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file,
         return err;
     }
 
-    // find new pos
-    lfs_soff_t npos = file->pos;
-    if (whence == LFS_SEEK_SET) {
-        npos = off;
-    } else if (whence == LFS_SEEK_CUR) {
-        npos = file->pos + off;
-    } else if (whence == LFS_SEEK_END) {
-        npos = file->size + off;
-    }
-
-    if (npos < 0 || npos > LFS_FILE_MAX) {
-        // file position out of range
-        return LFS_ERR_INVAL;
-    }
-
     // update pos
-    file->pos = npos;
-    return npos;
+    if (whence == LFS_SEEK_SET) {
+        file->pos = off;
+    } else if (whence == LFS_SEEK_CUR) {
+        if (off < 0 && (lfs_off_t)-off > file->pos) {
+            return LFS_ERR_INVAL;
+        }
+
+        file->pos = file->pos + off;
+    } else if (whence == LFS_SEEK_END) {
+        if (off < 0 && (lfs_off_t)-off > file->size) {
+            return LFS_ERR_INVAL;
+        }
+
+        file->pos = file->size + off;
+    }
+
+    return file->pos;
 }
 
 int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) {
@@ -1927,14 +1922,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) {
     // find old entry
     lfs_dir_t oldcwd;
     lfs_entry_t oldentry;
-    int err = lfs_dir_find(lfs, &oldcwd, &oldentry, &(const char *){oldpath});
-    if (err) {
-        return err;
-    }
-
-    // mark as moving
-    oldentry.d.type |= 0x80;
-    err = lfs_dir_update(lfs, &oldcwd, &oldentry, NULL);
+    int err = lfs_dir_find(lfs, &oldcwd, &oldentry, &oldpath);
     if (err) {
         return err;
     }
@@ -1947,9 +1935,11 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) {
         return err;
     }
 
-    // must have same type
     bool prevexists = (err != LFS_ERR_NOENT);
-    if (prevexists && preventry.d.type != (0x7f & oldentry.d.type)) {
+    bool samepair = (lfs_paircmp(oldcwd.pair, newcwd.pair) == 0);
+
+    // must have same type
+    if (prevexists && preventry.d.type != oldentry.d.type) {
         return LFS_ERR_ISDIR;
     }
 
@@ -1966,6 +1956,18 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) {
         }
     }
 
+    // mark as moving
+    oldentry.d.type |= 0x80;
+    err = lfs_dir_update(lfs, &oldcwd, &oldentry, NULL);
+    if (err) {
+        return err;
+    }
+
+    // update pair if newcwd == oldcwd
+    if (samepair) {
+        newcwd = oldcwd;
+    }
+
     // move to new location
     lfs_entry_t newentry = preventry;
     newentry.d = oldentry.d;
@@ -1984,13 +1986,10 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) {
         }
     }
 
-    // fetch old pair again in case dir block changed
-    lfs->moving = true;
-    err = lfs_dir_find(lfs, &oldcwd, &oldentry, &oldpath);
-    if (err) {
-        return err;
+    // update pair if newcwd == oldcwd
+    if (samepair) {
+        oldcwd = newcwd;
     }
-    lfs->moving = false;
 
     // remove old entry
     err = lfs_dir_remove(lfs, &oldcwd, &oldentry);
@@ -2088,7 +2087,6 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) {
     lfs->files = NULL;
     lfs->dirs = NULL;
     lfs->deorphaned = false;
-    lfs->moving = false;
 
     return 0;
 

lfs.h

@@ -21,7 +21,7 @@ extern "C"
 // Software library version
 // Major (top-nibble), incremented on backwards incompatible changes
 // Minor (bottom-nibble), incremented on feature additions
-#define LFS_VERSION 0x00010007
+#define LFS_VERSION 0x00010006
 #define LFS_VERSION_MAJOR (0xffff & (LFS_VERSION >> 16))
 #define LFS_VERSION_MINOR (0xffff & (LFS_VERSION >>  0))
 
@@ -49,11 +49,6 @@ typedef uint32_t lfs_block_t;
 #define LFS_NAME_MAX 255
 #endif
 
-// Max file size in bytes
-#ifndef LFS_FILE_MAX
-#define LFS_FILE_MAX 2147483647
-#endif
-
 // Possible error codes, these are negative to allow
 // valid positive return values
 enum lfs_error {
@@ -66,7 +61,6 @@ enum lfs_error {
     LFS_ERR_ISDIR    = -21,  // Entry is a dir
     LFS_ERR_NOTEMPTY = -39,  // Dir is not empty
     LFS_ERR_BADF     = -9,   // Bad file number
-    LFS_ERR_FBIG     = -27,  // File too large
     LFS_ERR_INVAL    = -22,  // Invalid parameter
     LFS_ERR_NOSPC    = -28,  // No space left on device
     LFS_ERR_NOMEM    = -12,  // No more memory available
@@ -286,7 +280,6 @@ typedef struct lfs {
 
     lfs_free_t free;
     bool deorphaned;
-    bool moving;
 } lfs_t;
 
 

scripts/prefix.py

@@ -1,61 +0,0 @@
-#!/usr/bin/env python
-
-# This script replaces prefixes of files, and symbols in that file.
-# Useful for creating different versions of the codebase that don't
-# conflict at compile time.
-#
-# example:
-# $ ./scripts/prefix.py lfs2
-
-import os
-import os.path
-import re
-import glob
-import itertools
-import tempfile
-import shutil
-import subprocess
-
-DEFAULT_PREFIX = "lfs"
-
-def subn(from_prefix, to_prefix, name):
-    name, count1 = re.subn('\\b'+from_prefix, to_prefix, name)
-    name, count2 = re.subn('\\b'+from_prefix.upper(), to_prefix.upper(), name)
-    name, count3 = re.subn('\\B-D'+from_prefix.upper(),
-            '-D'+to_prefix.upper(), name)
-    return name, count1+count2+count3
-
-def main(from_prefix, to_prefix=None, files=None):
-    if not to_prefix:
-        from_prefix, to_prefix = DEFAULT_PREFIX, from_prefix
-
-    if not files:
-        files = subprocess.check_output([
-                'git', 'ls-tree', '-r', '--name-only', 'HEAD']).split()
-
-    for oldname in files:
-        # Rename any matching file names
-        newname, namecount = subn(from_prefix, to_prefix, oldname)
-        if namecount:
-            subprocess.check_call(['git', 'mv', oldname, newname])
-
-        # Rename any prefixes in file
-        count = 0
-        with open(newname+'~', 'w') as tempf:
-            with open(newname) as newf:
-                for line in newf:
-                    line, n = subn(from_prefix, to_prefix, line)
-                    count += n
-                    tempf.write(line)
-        shutil.copystat(newname, newname+'~')
-        os.rename(newname+'~', newname)
-        subprocess.check_call(['git', 'add', newname])
-
-        # Summary
-        print '%s: %d replacements' % (
-                '%s -> %s' % (oldname, newname) if namecount else oldname,
-                count)
-
-if __name__ == "__main__":
-    import sys
-    sys.exit(main(*sys.argv[1:]))

tests/test_dirs.sh

@@ -326,42 +326,13 @@ tests/test.py << TEST
     lfs_unmount(&lfs) => 0;
 TEST
 
-echo "--- Multi-block rename ---"
-tests/test.py << TEST
-    lfs_mount(&lfs, &cfg) => 0;
-    for (int i = 0; i < $LARGESIZE; i++) {
-        sprintf((char*)buffer, "cactus/test%d", i);
-        sprintf((char*)wbuffer, "cactus/tedd%d", i);
-        lfs_rename(&lfs, (char*)buffer, (char*)wbuffer) => 0;
-    }
-    lfs_unmount(&lfs) => 0;
-TEST
-tests/test.py << TEST
-    lfs_mount(&lfs, &cfg) => 0;
-    lfs_dir_open(&lfs, &dir[0], "cactus") => 0;
-    lfs_dir_read(&lfs, &dir[0], &info) => 1;
-    strcmp(info.name, ".") => 0;
-    info.type => LFS_TYPE_DIR;
-    lfs_dir_read(&lfs, &dir[0], &info) => 1;
-    strcmp(info.name, "..") => 0;
-    info.type => LFS_TYPE_DIR;
-    for (int i = 0; i < $LARGESIZE; i++) {
-        sprintf((char*)buffer, "tedd%d", i);
-        lfs_dir_read(&lfs, &dir[0], &info) => 1;
-        strcmp(info.name, (char*)buffer) => 0;
-        info.type => LFS_TYPE_DIR;
-    }
-    lfs_dir_read(&lfs, &dir[0], &info) => 0;
-    lfs_unmount(&lfs) => 0;
-TEST
-
 echo "--- Multi-block remove ---"
 tests/test.py << TEST
     lfs_mount(&lfs, &cfg) => 0;
     lfs_remove(&lfs, "cactus") => LFS_ERR_NOTEMPTY;
 
     for (int i = 0; i < $LARGESIZE; i++) {
-        sprintf((char*)buffer, "cactus/tedd%d", i);
+        sprintf((char*)buffer, "cactus/test%d", i);
         lfs_remove(&lfs, (char*)buffer) => 0;
     }
 
@@ -420,43 +391,13 @@ tests/test.py << TEST
     lfs_unmount(&lfs) => 0;
 TEST
 
-echo "--- Multi-block rename with files ---"
-tests/test.py << TEST
-    lfs_mount(&lfs, &cfg) => 0;
-    for (int i = 0; i < $LARGESIZE; i++) {
-        sprintf((char*)buffer, "prickly-pear/test%d", i);
-        sprintf((char*)wbuffer, "prickly-pear/tedd%d", i);
-        lfs_rename(&lfs, (char*)buffer, (char*)wbuffer) => 0;
-    }
-    lfs_unmount(&lfs) => 0;
-TEST
-tests/test.py << TEST
-    lfs_mount(&lfs, &cfg) => 0;
-    lfs_dir_open(&lfs, &dir[0], "prickly-pear") => 0;
-    lfs_dir_read(&lfs, &dir[0], &info) => 1;
-    strcmp(info.name, ".") => 0;
-    info.type => LFS_TYPE_DIR;
-    lfs_dir_read(&lfs, &dir[0], &info) => 1;
-    strcmp(info.name, "..") => 0;
-    info.type => LFS_TYPE_DIR;
-    for (int i = 0; i < $LARGESIZE; i++) {
-        sprintf((char*)buffer, "tedd%d", i);
-        lfs_dir_read(&lfs, &dir[0], &info) => 1;
-        strcmp(info.name, (char*)buffer) => 0;
-        info.type => LFS_TYPE_REG;
-        info.size => 6;
-    }
-    lfs_dir_read(&lfs, &dir[0], &info) => 0;
-    lfs_unmount(&lfs) => 0;
-TEST
-
 echo "--- Multi-block remove with files ---"
 tests/test.py << TEST
     lfs_mount(&lfs, &cfg) => 0;
     lfs_remove(&lfs, "prickly-pear") => LFS_ERR_NOTEMPTY;
 
     for (int i = 0; i < $LARGESIZE; i++) {
-        sprintf((char*)buffer, "prickly-pear/tedd%d", i);
+        sprintf((char*)buffer, "prickly-pear/test%d", i);
         lfs_remove(&lfs, (char*)buffer) => 0;
     }