Fixed 32-bit/64-bit Ubuntu multilib issue in Travis

Kind of a two-fold issue. One, the programming to the middle of inline
files was causing the cache to get updated to a half programmed state.
While fine, as all programs do occur in order in a block, this is less
efficient when writing to inline files as it would cause the inline file
to need to be reread even if it fits in the cache.

Two, the rereading of the inline file was broken and passed the file's
tag all the way to where a user would expect an error. This was easy to
fix but adds to the reasons we should have test coverage information.

Found by ebinans

.travis.yml

@@ -36,7 +36,7 @@ script:
     if [ "$TRAVIS_TEST_RESULT" -eq 0 ]
     then
         CURR=$(tail -n1 sizes | awk '{print $1}')
-        PREV=$(curl -u $GEKY_BOT_STATUSES https://api.github.com/repos/$TRAVIS_REPO_SLUG/status/master \
+        PREV=$(curl -u "$GEKY_BOT_STATUSES" https://api.github.com/repos/$TRAVIS_REPO_SLUG/status/master \
             | jq -re "select(.sha != \"$TRAVIS_COMMIT\")
                 | .statuses[] | select(.context == \"$STAGE/$NAME\").description
                 | capture(\"code size is (?<size>[0-9]+)\").size" \
@@ -66,7 +66,10 @@ jobs:
         - CC="arm-linux-gnueabi-gcc --static -mthumb"
         - EXEC="qemu-arm"
       install:
-        - sudo apt-get install gcc-arm-linux-gnueabi qemu-user
+        - sudo apt-get install
+              gcc-arm-linux-gnueabi
+              libc6-dev-armel-cross
+              qemu-user
         - arm-linux-gnueabi-gcc --version
         - qemu-arm -version
 
@@ -78,7 +81,10 @@ jobs:
         - CC="powerpc-linux-gnu-gcc --static"
         - EXEC="qemu-ppc"
       install:
-        - sudo apt-get install gcc-powerpc-linux-gnu qemu-user
+        - sudo apt-get install
+              gcc-powerpc-linux-gnu
+              libc6-dev-powerpc-cross
+              qemu-user
         - powerpc-linux-gnu-gcc --version
         - qemu-ppc -version
 
@@ -90,9 +96,10 @@ jobs:
         - CC="mips-linux-gnu-gcc --static"
         - EXEC="qemu-mips"
       install:
-        - sudo add-apt-repository -y "deb http://archive.ubuntu.com/ubuntu/ xenial main universe"
+        - sudo apt-get install
-        - sudo apt-get -qq update
+              gcc-mips-linux-gnu
-        - sudo apt-get install gcc-mips-linux-gnu qemu-user
+              libc6-dev-mips-cross
+              qemu-user
         - mips-linux-gnu-gcc --version
         - qemu-mips -version
 
@@ -101,6 +108,7 @@ jobs:
       env:
         - STAGE=test
         - NAME=littlefs-fuse
+      if: branch !~ -prefix$
       install:
         - sudo apt-get install libfuse-dev
         - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v2-alpha
@@ -113,7 +121,7 @@ jobs:
 
         - mkdir mount
         - sudo chmod a+rw /dev/loop0
-        - dd if=/dev/zero bs=512 count=2048 of=disk
+        - dd if=/dev/zero bs=512 count=4096 of=disk
         - losetup /dev/loop0 disk
       script:
         # self-host test
@@ -126,73 +134,144 @@ jobs:
         - mkdir mount/littlefs
         - cp -r $(git ls-tree --name-only HEAD) mount/littlefs
         - cd mount/littlefs
+        - stat .
         - ls -flh
         - make -B test_dirs test_files QUIET=1
 
-      # Automatically update releases
+    # self-host with littlefs-fuse for fuzz test
+    - stage: test
+      env:
+        - STAGE=test
+        - NAME=littlefs-migration
+      if: branch !~ -prefix$
+      install:
+        - sudo apt-get install libfuse-dev
+        - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v2-alpha v2
+        - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v1 v1
+        - fusermount -V
+        - gcc --version
+      before_script:
+        # setup disk for littlefs-fuse
+        - rm -rf v2/littlefs/*
+        - cp -r $(git ls-tree --name-only HEAD) v2/littlefs
+
+        - mkdir mount
+        - sudo chmod a+rw /dev/loop0
+        - dd if=/dev/zero bs=512 count=4096 of=disk
+        - losetup /dev/loop0 disk
+      script:
+        # compile v1 and v2
+        - make -C v1
+        - make -C v2
+
+        # run self-host test with v1
+        - v1/lfs --format /dev/loop0
+        - v1/lfs /dev/loop0 mount
+
+        - ls mount
+        - mkdir mount/littlefs
+        - cp -r $(git ls-tree --name-only HEAD) mount/littlefs
+        - cd mount/littlefs
+        - stat .
+        - ls -flh
+        - make -B test_dirs test_files QUIET=1
+
+        # attempt to migrate
+        - cd ../..
+        - fusermount -u mount
+
+        - v2/lfs --migrate /dev/loop0
+        - v2/lfs /dev/loop0 mount
+
+        # run self-host test with v2 right where we left off
+        - ls mount
+        - cd mount/littlefs
+        - stat .
+        - ls -flh
+        - make -B test_dirs test_files QUIET=1
+
+    # Automatically create releases
     - stage: deploy
       env:
         - STAGE=deploy
         - NAME=deploy
       script:
-        # Find version defined in lfs.h
-        - LFS_VERSION=$(grep -ox '#define LFS_VERSION .*' lfs.h | cut -d ' ' -f3)
-        - LFS_VERSION_MAJOR=$((0xffff & ($LFS_VERSION >> 16)))
-        - LFS_VERSION_MINOR=$((0xffff & ($LFS_VERSION >>  0)))
-        # Grab latests patch from repo tags, default to 0, needs finagling to get past github's pagination api
-        - PREV_URL=https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs/tags/v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.
-        - PREV_URL=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL" -I
-                | sed -n '/^Link/{s/.*<\(.*\)>; rel="last"/\1/;p;q0};$q1'
-                || echo $PREV_URL)
-        - LFS_VERSION_PATCH=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL"
-                | jq 'map(.ref | match("\\bv.*\\..*\\.(.*)$";"g")
-                    .captures[].string | tonumber) | max + 1'
-                || echo 0)
-        # We have our new version
-        - LFS_VERSION="v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.$LFS_VERSION_PATCH"
-        - echo "VERSION $LFS_VERSION"
         - |
+          bash << 'SCRIPT'
+          set -ev
+          # Find version defined in lfs.h
+          LFS_VERSION=$(grep -ox '#define LFS_VERSION .*' lfs.h | cut -d ' ' -f3)
+          LFS_VERSION_MAJOR=$((0xffff & ($LFS_VERSION >> 16)))
+          LFS_VERSION_MINOR=$((0xffff & ($LFS_VERSION >>  0)))
+          # Grab latests patch from repo tags, default to 0, needs finagling
+          # to get past github's pagination api
+          PREV_URL=https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs/tags/v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.
+          PREV_URL=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL" -I \
+              | sed -n '/^Link/{s/.*<\(.*\)>; rel="last"/\1/;p;q0};$q1' \
+              || echo $PREV_URL)
+          LFS_VERSION_PATCH=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL" \
+              | jq 'map(.ref | match("\\bv.*\\..*\\.(.*)$";"g")
+                  .captures[].string | tonumber) | max + 1' \
+              || echo 0)
+          # We have our new version
+          LFS_VERSION="v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.$LFS_VERSION_PATCH"
+          echo "VERSION $LFS_VERSION"
           # Check that we're the most recent commit
           CURRENT_COMMIT=$(curl -f -u "$GEKY_BOT_RELEASES" \
-                https://api.github.com/repos/$TRAVIS_REPO_SLUG/commits/master \
+              https://api.github.com/repos/$TRAVIS_REPO_SLUG/commits/master \
-                | jq -re '.sha')
+              | jq -re '.sha')
-          if [ "$TRAVIS_COMMIT" == "$CURRENT_COMMIT" ]
+          [ "$TRAVIS_COMMIT" == "$CURRENT_COMMIT" ] || exit 0
+          # Create major branch
+          git branch v$LFS_VERSION_MAJOR HEAD
+          # Create major prefix branch
+          git config user.name "geky bot"
+          git config user.email "bot@geky.net"
+          git fetch https://github.com/$TRAVIS_REPO_SLUG.git \
+              --depth=50 v$LFS_VERSION_MAJOR-prefix || true
+          ./scripts/prefix.py lfs$LFS_VERSION_MAJOR
+          git branch v$LFS_VERSION_MAJOR-prefix $( \
+              git commit-tree $(git write-tree) \
+                  $(git rev-parse --verify -q FETCH_HEAD | sed -e 's/^/-p /') \
+                  -p HEAD \
+                  -m "Generated v$LFS_VERSION_MAJOR prefixes")
+          git reset --hard
+          # Update major version branches (vN and vN-prefix)
+          git push https://$GEKY_BOT_RELEASES@github.com/$TRAVIS_REPO_SLUG.git \
+              v$LFS_VERSION_MAJOR \
+              v$LFS_VERSION_MAJOR-prefix
+          # Create patch version tag (vN.N.N)
+          curl -f -u "$GEKY_BOT_RELEASES" -X POST \
+              https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs \
+              -d "{
+                  \"ref\": \"refs/tags/$LFS_VERSION\",
+                  \"sha\": \"$TRAVIS_COMMIT\"
+              }"
+          # Create minor release?
+          [[ "$LFS_VERSION" == *.0 ]] || exit 0
+          # Build release notes
+          PREV=$(git tag --sort=-v:refname -l "v*.0" | head -1)
+          if [ ! -z "$PREV" ]
           then
-            # Create a simple tag
+              echo "PREV $PREV"
-            curl -f -u "$GEKY_BOT_RELEASES" -X POST \
+              CHANGES=$'### Changes\n\n'$( \
-                https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs \
+                  git log --oneline $PREV.. --grep='^Merge' --invert-grep)
-                -d "{
+              printf "CHANGES\n%s\n\n" "$CHANGES"
-                    \"ref\": \"refs/tags/$LFS_VERSION\",
-                    \"sha\": \"$TRAVIS_COMMIT\"
-                }"
-            # Minor release?
-            if [[ "$LFS_VERSION" == *.0 ]]
-            then
-              # Build release notes
-              PREV=$(git tag --sort=-v:refname -l "v*.0" | head -1)
-              if [ ! -z "$PREV" ]
-              then
-                  echo "PREV $PREV"
-                  CHANGES=$'### Changes\n\n'$( \
-                      git log --oneline $PREV.. --grep='^Merge' --invert-grep)
-                  printf "CHANGES\n%s\n\n" "$CHANGES"
-              fi
-              # Create the release
-              curl -f -u "$GEKY_BOT_RELEASES" -X POST \
-                  https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases \
-                  -d "{
-                      \"tag_name\": \"$LFS_VERSION\",
-                      \"name\": \"${LFS_VERSION%.0}\",
-                      \"draft\": true,
-                      \"body\": $(jq -sR '.' <<< "$CHANGES")
-                  }"
-            fi
           fi
+          # Create the release
+          curl -f -u "$GEKY_BOT_RELEASES" -X POST \
+              https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases \
+              -d "{
+                  \"tag_name\": \"$LFS_VERSION\",
+                  \"name\": \"${LFS_VERSION%.0}\",
+                  \"draft\": true,
+                  \"body\": $(jq -sR '.' <<< "$CHANGES")
+              }" #"
+          SCRIPT
 
 # Manage statuses
 before_install:
   - |
-    curl -u $GEKY_BOT_STATUSES -X POST \
+    curl -u "$GEKY_BOT_STATUSES" -X POST \
         https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
         -d "{
             \"context\": \"$STAGE/$NAME\",
@@ -203,7 +282,7 @@ before_install:
 
 after_failure:
   - |
-    curl -u $GEKY_BOT_STATUSES -X POST \
+    curl -u "$GEKY_BOT_STATUSES" -X POST \
         https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
         -d "{
             \"context\": \"$STAGE/$NAME\",
@@ -214,7 +293,7 @@ after_failure:
 
 after_success:
   - |
-    curl -u $GEKY_BOT_STATUSES -X POST \
+    curl -u "$GEKY_BOT_STATUSES" -X POST \
         https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
         -d "{
             \"context\": \"$STAGE/$NAME\",

Makefile

@@ -26,7 +26,9 @@ override CFLAGS += -m$(WORD)
 endif
 override CFLAGS += -I.
 override CFLAGS += -std=c99 -Wall -pedantic
-override CFLAGS += -Wshadow -Wunused-parameter -Wjump-misses-init -Wsign-compare
+override CFLAGS += -Wextra -Wshadow -Wjump-misses-init
+# Remove missing-field-initializers because of GCC bug
+override CFLAGS += -Wno-missing-field-initializers
 
 
 all: $(TARGET)

README.md

@@ -1,6 +1,6 @@
-## The little filesystem
+## littlefs
 
-A little fail-safe filesystem designed for embedded systems.
+A little fail-safe filesystem designed for microcontrollers.
 
 ```
    | | |     .---._____
@@ -11,17 +11,19 @@ A little fail-safe filesystem designed for embedded systems.
    | | |
 ```
 
-**Bounded RAM/ROM** - The littlefs is designed to work with a limited amount
+**Power-loss resilience** - littlefs is designed to handle random power
-of memory. Recursion is avoided and dynamic memory is limited to configurable
+failures. All file operations have strong copy-on-write guarantees and if
-buffers that can be provided statically.
+power is lost the filesystem will fall back to the last known good state.
 
-**Power-loss resilient** - The littlefs is designed for systems that may have
+**Dynamic wear leveling** - littlefs is designed with flash in mind, and
-random power failures. The littlefs has strong copy-on-write guarantees and
+provides wear leveling over dynamic blocks. Additionally, littlefs can
-storage on disk is always kept in a valid state.
+detect bad blocks and work around them.
 
-**Wear leveling** - Since the most common form of embedded storage is erodible
+**Bounded RAM/ROM** - littlefs is designed to work with a small amount of
-flash memories, littlefs provides a form of dynamic wear leveling for systems
+memory. RAM usage is strictly bounded, which means RAM consumption does not
-that can not fit a full flash translation layer.
+change as the filesystem grows. The filesystem contains no unbounded
+recursion and dynamic memory is limited to configurable buffers that can be
+provided statically.
 
 ## Example
 
@@ -91,11 +93,11 @@ int main(void) {
 Detailed documentation (or at least as much detail as is currently available)
 can be found in the comments in [lfs.h](lfs.h).
 
-As you may have noticed, littlefs takes in a configuration structure that
+littlefs takes in a configuration structure that defines how the filesystem
-defines how the filesystem operates. The configuration struct provides the
+operates. The configuration struct provides the filesystem with the block
-filesystem with the block device operations and dimensions, tweakable
+device operations and dimensions, tweakable parameters that tradeoff memory
-parameters that tradeoff memory usage for performance, and optional
+usage for performance, and optional static buffers if the user wants to avoid
-static buffers if the user wants to avoid dynamic memory.
+dynamic memory.
 
 The state of the littlefs is stored in the `lfs_t` type which is left up
 to the user to allocate, allowing multiple filesystems to be in use
@@ -107,14 +109,14 @@ directory functions, with the deviation that the allocation of filesystem
 structures must be provided by the user.
 
 All POSIX operations, such as remove and rename, are atomic, even in event
-of power-loss. Additionally, no file updates are actually committed to the
+of power-loss. Additionally, no file updates are not actually committed to
-filesystem until sync or close is called on the file.
+the filesystem until sync or close is called on the file.
 
 ## Other notes
 
-All littlefs have the potential to return a negative error code. The errors
+All littlefs calls have the potential to return a negative error code. The
-can be either one of those found in the `enum lfs_error` in [lfs.h](lfs.h),
+errors can be either one of those found in the `enum lfs_error` in
-or an error returned by the user's block device operations.
+[lfs.h](lfs.h), or an error returned by the user's block device operations.
 
 In the configuration struct, the `prog` and `erase` function provided by the
 user may return a `LFS_ERR_CORRUPT` error if the implementation already can
@@ -128,14 +130,60 @@ from memory, otherwise data integrity can not be guaranteed. If the `write`
 function does not perform caching, and therefore each `read` or `write` call
 hits the memory, the `sync` function can simply return 0.
 
-## Reference material
+## Design
 
-[DESIGN.md](DESIGN.md) - DESIGN.md contains a fully detailed dive into how
+At a high level, littlefs is a block based filesystem that uses small logs to
-littlefs actually works. I would encourage you to read it since the
+store metadata and larger copy-on-write (COW) structures to store file data.
-solutions and tradeoffs at work here are quite interesting.
 
-[SPEC.md](SPEC.md) - SPEC.md contains the on-disk specification of littlefs
+In littlefs, these ingredients form a sort of two-layered cake, with the small
-with all the nitty-gritty details. Can be useful for developing tooling.
+logs (called metadata pairs) providing fast updates to metadata anywhere on
+storage, while the COW structures store file data compactly and without any
+wear amplification cost.
+
+Both of these data structures are built out of blocks, which are fed by a
+common block allocator. By limiting the number of erases allowed on a block
+per allocation, the allocator provides dynamic wear leveling over the entire
+filesystem.
+
+```
+                    root
+                   .--------.--------.
+                   | A'| B'|         |
+                   |   |   |->       |
+                   |   |   |         |
+                   '--------'--------'
+                .----'   '--------------.
+       A       v                 B       v
+      .--------.--------.       .--------.--------.
+      | C'| D'|         |       | E'|new|         |
+      |   |   |->       |       |   | E'|->       |
+      |   |   |         |       |   |   |         |
+      '--------'--------'       '--------'--------'
+      .-'   '--.                  |   '------------------.
+     v          v              .-'                        v
+.--------.  .--------.        v                       .--------.
+|   C    |  |   D    |   .--------.       write       | new E  |
+|        |  |        |   |   E    |        ==>        |        |
+|        |  |        |   |        |                   |        |
+'--------'  '--------'   |        |                   '--------'
+                         '--------'                   .-'    |
+                         .-'    '-.    .-------------|------'
+                        v          v  v              v
+                   .--------.  .--------.       .--------.
+                   |   F    |  |   G    |       | new F  |
+                   |        |  |        |       |        |
+                   |        |  |        |       |        |
+                   '--------'  '--------'       '--------'
+```
+
+More details on how littlefs works can be found in [DESIGN.md](DESIGN.md) and
+[SPEC.md](SPEC.md).
+
+- [DESIGN.md](DESIGN.md) - A fully detailed dive into how littlefs works.
+  I would suggest reading it as the tradeoffs at work are quite interesting.
+
+- [SPEC.md](SPEC.md) - The on-disk specification of littlefs with all the
+  nitty-gritty details. May be useful for tooling development.
 
 ## Testing
 
@@ -149,9 +197,9 @@ make test
 
 ## License
 
-The littlefs is provided under the [BSD-3-Clause](https://spdx.org/licenses/BSD-3-Clause.html)
+The littlefs is provided under the [BSD-3-Clause] license. See
-license. See [LICENSE.md](LICENSE.md) for more information. Contributions to
+[LICENSE.md](LICENSE.md) for more information. Contributions to this project
-this project are accepted under the same license.
+are accepted under the same license.
 
 Individual files contain the following tag instead of the full license text.
 
@@ -162,32 +210,39 @@ License Identifiers that are here available: http://spdx.org/licenses/
 
 ## Related projects
 
-[Mbed OS](https://github.com/ARMmbed/mbed-os/tree/master/features/filesystem/littlefs) -
+- [littlefs-fuse] - A [FUSE] wrapper for littlefs. The project allows you to
-The easiest way to get started with littlefs is to jump into [Mbed](https://os.mbed.com/),
+  mount littlefs directly on a Linux machine. Can be useful for debugging
-which already has block device drivers for most forms of embedded storage. The
+  littlefs if you have an SD card handy.
-littlefs is available in Mbed OS as the [LittleFileSystem](https://os.mbed.com/docs/latest/reference/littlefilesystem.html)
-class.
 
-[littlefs-fuse](https://github.com/geky/littlefs-fuse) - A [FUSE](https://github.com/libfuse/libfuse)
+- [littlefs-js] - A javascript wrapper for littlefs. I'm not sure why you would
-wrapper for littlefs. The project allows you to mount littlefs directly on a
+  want this, but it is handy for demos.  You can see it in action
-Linux machine. Can be useful for debugging littlefs if you have an SD card
+  [here][littlefs-js-demo].
-handy.
 
-[littlefs-js](https://github.com/geky/littlefs-js) - A javascript wrapper for
+- [mklfs] - A command line tool built by the [Lua RTOS] guys for making
-littlefs. I'm not sure why you would want this, but it is handy for demos.
+  littlefs images from a host PC. Supports Windows, Mac OS, and Linux.
-You can see it in action [here](http://littlefs.geky.net/demo.html).
 
-[mklfs](https://github.com/whitecatboard/Lua-RTOS-ESP32/tree/master/components/mklfs/src) -
+- [Mbed OS] - The easiest way to get started with littlefs is to jump into Mbed
-A command line tool built by the [Lua RTOS](https://github.com/whitecatboard/Lua-RTOS-ESP32)
+  which already has block device drivers for most forms of embedded storage.
-guys for making littlefs images from a host PC. Supports Windows, Mac OS,
+  littlefs is available in Mbed OS as the [LittleFileSystem] class.
-and Linux.
 
-[SPIFFS](https://github.com/pellepl/spiffs) - Another excellent embedded
+- [SPIFFS] - Another excellent embedded filesystem for NOR flash. As a more
-filesystem for NOR flash. As a more traditional logging filesystem with full
+  traditional logging filesystem with full static wear-leveling, SPIFFS will
-static wear-leveling, SPIFFS will likely outperform littlefs on small
+  likely outperform littlefs on small memories such as the internal flash on
-memories such as the internal flash on microcontrollers.
+  microcontrollers.
 
-[Dhara](https://github.com/dlbeer/dhara) - An interesting NAND flash
+- [Dhara] - An interesting NAND flash translation layer designed for small
-translation layer designed for small MCUs. It offers static wear-leveling and
+  MCUs. It offers static wear-leveling and power-resilience with only a fixed
-power-resilience with only a fixed O(|address|) pointer structure stored on
+  _O(|address|)_ pointer structure stored on each block and in RAM.
-each block and in RAM.
+
+
+[BSD-3-Clause]: https://spdx.org/licenses/BSD-3-Clause.html
+[littlefs-fuse]: https://github.com/geky/littlefs-fuse
+[FUSE]: https://github.com/libfuse/libfuse
+[littlefs-js]: https://github.com/geky/littlefs-js
+[littlefs-js-demo]:http://littlefs.geky.net/demo.html
+[mklfs]: https://github.com/whitecatboard/Lua-RTOS-ESP32/tree/master/components/mklfs/src
+[Lua RTOS]: https://github.com/whitecatboard/Lua-RTOS-ESP32
+[Mbed OS]: https://github.com/armmbed/mbed-os
+[LittleFileSystem]: https://os.mbed.com/docs/mbed-os/v5.12/apis/littlefilesystem.html
+[SPIFFS]: https://github.com/pellepl/spiffs
+[Dhara]: https://github.com/dlbeer/dhara

lfs.h

@@ -215,8 +215,9 @@ struct lfs_config {
     // By default lfs_malloc is used to allocate this buffer.
     void *prog_buffer;
 
-    // Optional statically allocated program buffer. Must be lookahead_size.
+    // Optional statically allocated lookahead buffer. Must be lookahead_size
-    // By default lfs_malloc is used to allocate this buffer.
+    // and aligned to a 64-bit boundary. By default lfs_malloc is used to
+    // allocate this buffer.
     void *lookahead_buffer;
 
     // Optional upper limit on length of file names in bytes. No downside for
@@ -380,6 +381,10 @@ typedef struct lfs {
     lfs_size_t name_max;
     lfs_size_t file_max;
     lfs_size_t attr_max;
+
+#ifdef LFS_MIGRATE
+    struct lfs1 *lfs1;
+#endif
 } lfs_t;
 
 
@@ -573,7 +578,8 @@ int lfs_dir_close(lfs_t *lfs, lfs_dir_t *dir);
 // Read an entry in the directory
 //
 // Fills out the info structure, based on the specified file or directory.
-// Returns a negative error code on failure.
+// Returns a positive value on success, 0 at the end of directory,
+// or a negative error code on failure.
 int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info);
 
 // Change the position of the directory
@@ -617,6 +623,21 @@ lfs_ssize_t lfs_fs_size(lfs_t *lfs);
 // Returns a negative error code on failure.
 int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data);
 
+#ifdef LFS_MIGRATE
+// Attempts to migrate a previous version of littlefs
+//
+// Behaves similarly to the lfs_format function. Attempts to mount
+// the previous version of littlefs and update the filesystem so it can be
+// mounted with the current version of littlefs.
+//
+// Requires a littlefs object and config struct. This clobbers the littlefs
+// object, and does not leave the filesystem mounted. The config struct must
+// be zeroed for defaults and backwards compatibility.
+//
+// Returns a negative error code on failure.
+int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg);
+#endif
+
 
 #ifdef __cplusplus
 } /* extern "C" */

lfs_util.h

@@ -143,14 +143,14 @@ static inline int lfs_scmp(uint32_t a, uint32_t b) {
 // Convert between 32-bit little-endian and native order
 static inline uint32_t lfs_fromle32(uint32_t a) {
 #if !defined(LFS_NO_INTRINSICS) && ( \
-    (defined(  BYTE_ORDER  ) &&   BYTE_ORDER   ==   ORDER_LITTLE_ENDIAN  ) || \
+    (defined(  BYTE_ORDER  ) && defined(  ORDER_LITTLE_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_LITTLE_ENDIAN  ) || \
-    (defined(__BYTE_ORDER  ) && __BYTE_ORDER   == __ORDER_LITTLE_ENDIAN  ) || \
+    (defined(__BYTE_ORDER  ) && defined(__ORDER_LITTLE_ENDIAN  ) && __BYTE_ORDER   == __ORDER_LITTLE_ENDIAN  ) || \
-    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
+    (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
     return a;
 #elif !defined(LFS_NO_INTRINSICS) && ( \
-    (defined(  BYTE_ORDER  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
+    (defined(  BYTE_ORDER  ) && defined(  ORDER_BIG_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
-    (defined(__BYTE_ORDER  ) && __BYTE_ORDER   == __ORDER_BIG_ENDIAN  ) || \
+    (defined(__BYTE_ORDER  ) && defined(__ORDER_BIG_ENDIAN  ) && __BYTE_ORDER   == __ORDER_BIG_ENDIAN  ) || \
-    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
+    (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
     return __builtin_bswap32(a);
 #else
     return (((uint8_t*)&a)[0] <<  0) |
@@ -167,14 +167,14 @@ static inline uint32_t lfs_tole32(uint32_t a) {
 // Convert between 32-bit big-endian and native order
 static inline uint32_t lfs_frombe32(uint32_t a) {
 #if !defined(LFS_NO_INTRINSICS) && ( \
-    (defined(  BYTE_ORDER  ) &&   BYTE_ORDER   ==   ORDER_LITTLE_ENDIAN  ) || \
+    (defined(  BYTE_ORDER  ) && defined(  ORDER_LITTLE_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_LITTLE_ENDIAN  ) || \
-    (defined(__BYTE_ORDER  ) && __BYTE_ORDER   == __ORDER_LITTLE_ENDIAN  ) || \
+    (defined(__BYTE_ORDER  ) && defined(__ORDER_LITTLE_ENDIAN  ) && __BYTE_ORDER   == __ORDER_LITTLE_ENDIAN  ) || \
-    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
+    (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
     return __builtin_bswap32(a);
 #elif !defined(LFS_NO_INTRINSICS) && ( \
-    (defined(  BYTE_ORDER  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
+    (defined(  BYTE_ORDER  ) && defined(  ORDER_BIG_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
-    (defined(__BYTE_ORDER  ) && __BYTE_ORDER   == __ORDER_BIG_ENDIAN  ) || \
+    (defined(__BYTE_ORDER  ) && defined(__ORDER_BIG_ENDIAN  ) && __BYTE_ORDER   == __ORDER_BIG_ENDIAN  ) || \
-    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
+    (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
     return a;
 #else
     return (((uint8_t*)&a)[0] << 24) |
@@ -192,6 +192,7 @@ static inline uint32_t lfs_tobe32(uint32_t a) {
 uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size);
 
 // Allocate memory, only used if buffers are not provided to littlefs
+// Note, memory must be 64-bit aligned
 static inline void *lfs_malloc(size_t size) {
 #ifndef LFS_NO_MALLOC
     return malloc(size);

scripts/prefix.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python2
+
+# This script replaces prefixes of files, and symbols in that file.
+# Useful for creating different versions of the codebase that don't
+# conflict at compile time.
+#
+# example:
+# $ ./scripts/prefix.py lfs2
+
+import os
+import os.path
+import re
+import glob
+import itertools
+import tempfile
+import shutil
+import subprocess
+
+DEFAULT_PREFIX = "lfs"
+
+def subn(from_prefix, to_prefix, name):
+    name, count1 = re.subn('\\b'+from_prefix, to_prefix, name)
+    name, count2 = re.subn('\\b'+from_prefix.upper(), to_prefix.upper(), name)
+    name, count3 = re.subn('\\B-D'+from_prefix.upper(),
+            '-D'+to_prefix.upper(), name)
+    return name, count1+count2+count3
+
+def main(from_prefix, to_prefix=None, files=None):
+    if not to_prefix:
+        from_prefix, to_prefix = DEFAULT_PREFIX, from_prefix
+
+    if not files:
+        files = subprocess.check_output([
+                'git', 'ls-tree', '-r', '--name-only', 'HEAD']).split()
+
+    for oldname in files:
+        # Rename any matching file names
+        newname, namecount = subn(from_prefix, to_prefix, oldname)
+        if namecount:
+            subprocess.check_call(['git', 'mv', oldname, newname])
+
+        # Rename any prefixes in file
+        count = 0
+        with open(newname+'~', 'w') as tempf:
+            with open(newname) as newf:
+                for line in newf:
+                    line, n = subn(from_prefix, to_prefix, line)
+                    count += n
+                    tempf.write(line)
+        shutil.copystat(newname, newname+'~')
+        os.rename(newname+'~', newname)
+        subprocess.check_call(['git', 'add', newname])
+
+        # Summary
+        print '%s: %d replacements' % (
+                '%s -> %s' % (oldname, newname) if namecount else oldname,
+                count)
+
+if __name__ == "__main__":
+    import sys
+    sys.exit(main(*sys.argv[1:]))

tests/corrupt.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 import struct
 import sys

tests/stats.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 import struct
 import sys

tests/test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 import re
 import sys

tests/test_paths.sh

@@ -128,6 +128,14 @@ tests/test.py << TEST
     lfs_mkdir(&lfs, "/") => LFS_ERR_EXIST;
     lfs_file_open(&lfs, &file[0], "/", LFS_O_WRONLY | LFS_O_CREAT)
         => LFS_ERR_ISDIR;
+
+    // more corner cases
+    lfs_remove(&lfs, "") => LFS_ERR_INVAL;
+    lfs_remove(&lfs, ".") => LFS_ERR_INVAL;
+    lfs_remove(&lfs, "..") => LFS_ERR_INVAL;
+    lfs_remove(&lfs, "/") => LFS_ERR_INVAL;
+    lfs_remove(&lfs, "//") => LFS_ERR_INVAL;
+    lfs_remove(&lfs, "./") => LFS_ERR_INVAL;
     lfs_unmount(&lfs) => 0;
 TEST
 
@@ -165,5 +173,29 @@ tests/test.py << TEST
     lfs_unmount(&lfs) => 0;
 TEST
 
+echo "--- Really big path test ---"
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    memset(buffer, 'w', LFS_NAME_MAX);
+    buffer[LFS_NAME_MAX+1] = '\0';
+    lfs_mkdir(&lfs, (char*)buffer) => 0;
+    lfs_remove(&lfs, (char*)buffer) => 0;
+    lfs_file_open(&lfs, &file[0], (char*)buffer,
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_remove(&lfs, (char*)buffer) => 0;
+
+    memcpy(buffer, "coffee/", strlen("coffee/"));
+    memset(buffer+strlen("coffee/"), 'w', LFS_NAME_MAX);
+    buffer[strlen("coffee/")+LFS_NAME_MAX+1] = '\0';
+    lfs_mkdir(&lfs, (char*)buffer) => 0;
+    lfs_remove(&lfs, (char*)buffer) => 0;
+    lfs_file_open(&lfs, &file[0], (char*)buffer,
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_remove(&lfs, (char*)buffer) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+
 echo "--- Results ---"
 tests/stats.py

tests/test_seek.sh

@@ -357,5 +357,73 @@ tests/test.py << TEST
     lfs_unmount(&lfs) => 0;
 TEST
 
+echo "--- Inline write and seek ---"
+for SIZE in $SMALLSIZE $MEDIUMSIZE $LARGESIZE
+do
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "hello/tinykitty$SIZE",
+            LFS_O_RDWR | LFS_O_CREAT) => 0;
+    int j = 0;
+    int k = 0;
+
+    memcpy(buffer, "abcdefghijklmnopqrstuvwxyz", 26);
+    for (unsigned i = 0; i < $SIZE; i++) {
+        lfs_file_write(&lfs, &file[0], &buffer[j++ % 26], 1) => 1;
+        lfs_file_tell(&lfs, &file[0]) => i+1;
+        lfs_file_size(&lfs, &file[0]) => i+1;
+    }
+
+    lfs_file_seek(&lfs, &file[0], 0, LFS_SEEK_SET) => 0;
+    lfs_file_tell(&lfs, &file[0]) => 0;
+    lfs_file_size(&lfs, &file[0]) => $SIZE;
+    for (unsigned i = 0; i < $SIZE; i++) {
+        uint8_t c;
+        lfs_file_read(&lfs, &file[0], &c, 1) => 1;
+        c => buffer[k++ % 26];
+    }
+
+    lfs_file_sync(&lfs, &file[0]) => 0;
+    lfs_file_tell(&lfs, &file[0]) => $SIZE;
+    lfs_file_size(&lfs, &file[0]) => $SIZE;
+
+    lfs_file_seek(&lfs, &file[0], 0, LFS_SEEK_SET) => 0;
+    for (unsigned i = 0; i < $SIZE; i++) {
+        lfs_file_write(&lfs, &file[0], &buffer[j++ % 26], 1) => 1;
+        lfs_file_tell(&lfs, &file[0]) => i+1;
+        lfs_file_size(&lfs, &file[0]) => $SIZE;
+        lfs_file_sync(&lfs, &file[0]) => 0;
+        lfs_file_tell(&lfs, &file[0]) => i+1;
+        lfs_file_size(&lfs, &file[0]) => $SIZE;
+        if (i < $SIZE-2) {
+            uint8_t c[3];
+            lfs_file_seek(&lfs, &file[0], -1, LFS_SEEK_CUR) => i;
+            lfs_file_read(&lfs, &file[0], &c, 3) => 3;
+            lfs_file_tell(&lfs, &file[0]) => i+3;
+            lfs_file_size(&lfs, &file[0]) => $SIZE;
+            lfs_file_seek(&lfs, &file[0], i+1, LFS_SEEK_SET) => i+1;
+            lfs_file_tell(&lfs, &file[0]) => i+1;
+            lfs_file_size(&lfs, &file[0]) => $SIZE;
+        }
+    }
+
+    lfs_file_seek(&lfs, &file[0], 0, LFS_SEEK_SET) => 0;
+    lfs_file_tell(&lfs, &file[0]) => 0;
+    lfs_file_size(&lfs, &file[0]) => $SIZE;
+    for (unsigned i = 0; i < $SIZE; i++) {
+        uint8_t c;
+        lfs_file_read(&lfs, &file[0], &c, 1) => 1;
+        c => buffer[k++ % 26];
+    }
+
+    lfs_file_sync(&lfs, &file[0]) => 0;
+    lfs_file_tell(&lfs, &file[0]) => $SIZE;
+    lfs_file_size(&lfs, &file[0]) => $SIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+done
+
 echo "--- Results ---"
 tests/stats.py

tests/test_truncate.sh

@@ -11,6 +11,150 @@ tests/test.py << TEST
     lfs_format(&lfs, &cfg) => 0;
 TEST
 
+echo "--- Simple truncate ---"
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldynoop",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+
+    strcpy((char*)buffer, "hair");
+    size = strlen((char*)buffer);
+    for (lfs_off_t j = 0; j < $LARGESIZE; j += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldynoop", LFS_O_RDWR) => 0;
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_truncate(&lfs, &file[0], $MEDIUMSIZE) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldynoop", LFS_O_RDONLY) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    size = strlen("hair");
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_read(&lfs, &file[0], buffer, size) => size;
+        memcmp(buffer, "hair", size) => 0;
+    }
+    lfs_file_read(&lfs, &file[0], buffer, size) => 0;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+
+echo "--- Truncate and read ---"
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldyread",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+
+    strcpy((char*)buffer, "hair");
+    size = strlen((char*)buffer);
+    for (lfs_off_t j = 0; j < $LARGESIZE; j += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldyread", LFS_O_RDWR) => 0;
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_truncate(&lfs, &file[0], $MEDIUMSIZE) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    size = strlen("hair");
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_read(&lfs, &file[0], buffer, size) => size;
+        memcmp(buffer, "hair", size) => 0;
+    }
+    lfs_file_read(&lfs, &file[0], buffer, size) => 0;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldyread", LFS_O_RDONLY) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    size = strlen("hair");
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_read(&lfs, &file[0], buffer, size) => size;
+        memcmp(buffer, "hair", size) => 0;
+    }
+    lfs_file_read(&lfs, &file[0], buffer, size) => 0;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+
+echo "--- Truncate and write ---"
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldywrite",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+
+    strcpy((char*)buffer, "hair");
+    size = strlen((char*)buffer);
+    for (lfs_off_t j = 0; j < $LARGESIZE; j += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldywrite", LFS_O_RDWR) => 0;
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_truncate(&lfs, &file[0], $MEDIUMSIZE) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    strcpy((char*)buffer, "bald");
+    size = strlen((char*)buffer);
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldywrite", LFS_O_RDONLY) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    size = strlen("bald");
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_read(&lfs, &file[0], buffer, size) => size;
+        memcmp(buffer, "bald", size) => 0;
+    }
+    lfs_file_read(&lfs, &file[0], buffer, size) => 0;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+
+# More aggressive general truncation tests
 truncate_test() {
 STARTSIZES="$1"
 STARTSEEKS="$2"