Merge pull request #85 from ARMmbed/v2-alpha

v2: Metadata logging, custom attributes, inline files, and a major version bump

.travis.yml

@@ -36,7 +36,7 @@ script:
     if [ "$TRAVIS_TEST_RESULT" -eq 0 ]
     then
         CURR=$(tail -n1 sizes | awk '{print $1}')
-        PREV=$(curl -u $GEKY_BOT_STATUSES https://api.github.com/repos/$TRAVIS_REPO_SLUG/status/master \
+        PREV=$(curl -u "$GEKY_BOT_STATUSES" https://api.github.com/repos/$TRAVIS_REPO_SLUG/status/master \
             | jq -re "select(.sha != \"$TRAVIS_COMMIT\")
                 | .statuses[] | select(.context == \"$STAGE/$NAME\").description
                 | capture(\"code size is (?<size>[0-9]+)\").size" \
@@ -101,6 +101,7 @@ jobs:
       env:
         - STAGE=test
         - NAME=littlefs-fuse
+      if: branch !~ -prefix$
       install:
         - sudo apt-get install libfuse-dev
         - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v2-alpha
@@ -113,7 +114,7 @@ jobs:
 
         - mkdir mount
         - sudo chmod a+rw /dev/loop0
-        - dd if=/dev/zero bs=512 count=2048 of=disk
+        - dd if=/dev/zero bs=512 count=4096 of=disk
         - losetup /dev/loop0 disk
       script:
         # self-host test
@@ -126,73 +127,143 @@ jobs:
         - mkdir mount/littlefs
         - cp -r $(git ls-tree --name-only HEAD) mount/littlefs
         - cd mount/littlefs
+        - stat .
         - ls -flh
         - make -B test_dirs test_files QUIET=1
 
-      # Automatically update releases
+    # self-host with littlefs-fuse for fuzz test
+    - stage: test
+      env:
+        - STAGE=test
+        - NAME=littlefs-migration
+      install:
+        - sudo apt-get install libfuse-dev
+        - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v2-alpha v2
+        - git clone --depth 1 https://github.com/geky/littlefs-fuse v1
+        - fusermount -V
+        - gcc --version
+      before_script:
+        # setup disk for littlefs-fuse
+        - rm -rf v2/littlefs/*
+        - cp -r $(git ls-tree --name-only HEAD) v2/littlefs
+
+        - mkdir mount
+        - sudo chmod a+rw /dev/loop0
+        - dd if=/dev/zero bs=512 count=4096 of=disk
+        - losetup /dev/loop0 disk
+      script:
+        # compile v1 and v2
+        - make -C v1
+        - make -C v2
+
+        # run self-host test with v1
+        - v1/lfs --format /dev/loop0
+        - v1/lfs /dev/loop0 mount
+
+        - ls mount
+        - mkdir mount/littlefs
+        - cp -r $(git ls-tree --name-only HEAD) mount/littlefs
+        - cd mount/littlefs
+        - stat .
+        - ls -flh
+        - make -B test_dirs test_files QUIET=1
+
+        # attempt to migrate
+        - cd ../..
+        - fusermount -u mount
+
+        - v2/lfs --migrate /dev/loop0
+        - v2/lfs /dev/loop0 mount
+
+        # run self-host test with v2 right where we left off
+        - ls mount
+        - cd mount/littlefs
+        - stat .
+        - ls -flh
+        - make -B test_dirs test_files QUIET=1
+
+    # Automatically create releases
     - stage: deploy
       env:
         - STAGE=deploy
         - NAME=deploy
       script:
-        # Find version defined in lfs.h
-        - LFS_VERSION=$(grep -ox '#define LFS_VERSION .*' lfs.h | cut -d ' ' -f3)
-        - LFS_VERSION_MAJOR=$((0xffff & ($LFS_VERSION >> 16)))
-        - LFS_VERSION_MINOR=$((0xffff & ($LFS_VERSION >>  0)))
-        # Grab latests patch from repo tags, default to 0, needs finagling to get past github's pagination api
-        - PREV_URL=https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs/tags/v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.
-        - PREV_URL=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL" -I
-                | sed -n '/^Link/{s/.*<\(.*\)>; rel="last"/\1/;p;q0};$q1'
-                || echo $PREV_URL)
-        - LFS_VERSION_PATCH=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL"
-                | jq 'map(.ref | match("\\bv.*\\..*\\.(.*)$";"g")
-                    .captures[].string | tonumber) | max + 1'
-                || echo 0)
-        # We have our new version
-        - LFS_VERSION="v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.$LFS_VERSION_PATCH"
-        - echo "VERSION $LFS_VERSION"
         - |
+          bash << 'SCRIPT'
+          set -ev
+          # Find version defined in lfs.h
+          LFS_VERSION=$(grep -ox '#define LFS_VERSION .*' lfs.h | cut -d ' ' -f3)
+          LFS_VERSION_MAJOR=$((0xffff & ($LFS_VERSION >> 16)))
+          LFS_VERSION_MINOR=$((0xffff & ($LFS_VERSION >>  0)))
+          # Grab latests patch from repo tags, default to 0, needs finagling
+          # to get past github's pagination api
+          PREV_URL=https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs/tags/v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.
+          PREV_URL=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL" -I \
+              | sed -n '/^Link/{s/.*<\(.*\)>; rel="last"/\1/;p;q0};$q1' \
+              || echo $PREV_URL)
+          LFS_VERSION_PATCH=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL" \
+              | jq 'map(.ref | match("\\bv.*\\..*\\.(.*)$";"g")
+                  .captures[].string | tonumber) | max + 1' \
+              || echo 0)
+          # We have our new version
+          LFS_VERSION="v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.$LFS_VERSION_PATCH"
+          echo "VERSION $LFS_VERSION"
           # Check that we're the most recent commit
           CURRENT_COMMIT=$(curl -f -u "$GEKY_BOT_RELEASES" \
-                https://api.github.com/repos/$TRAVIS_REPO_SLUG/commits/master \
+              https://api.github.com/repos/$TRAVIS_REPO_SLUG/commits/master \
-                | jq -re '.sha')
+              | jq -re '.sha')
-          if [ "$TRAVIS_COMMIT" == "$CURRENT_COMMIT" ]
+          [ "$TRAVIS_COMMIT" == "$CURRENT_COMMIT" ] || exit 0
+          # Create major branch
+          git branch v$LFS_VERSION_MAJOR HEAD
+          # Create major prefix branch
+          git config user.name "geky bot"
+          git config user.email "bot@geky.net"
+          git fetch https://github.com/$TRAVIS_REPO_SLUG.git \
+              --depth=50 v$LFS_VERSION_MAJOR-prefix || true
+          ./scripts/prefix.py lfs$LFS_VERSION_MAJOR
+          git branch v$LFS_VERSION_MAJOR-prefix $( \
+              git commit-tree $(git write-tree) \
+                  $(git rev-parse --verify -q FETCH_HEAD | sed -e 's/^/-p /') \
+                  -p HEAD \
+                  -m "Generated v$LFS_VERSION_MAJOR prefixes")
+          git reset --hard
+          # Update major version branches (vN and vN-prefix)
+          git push https://$GEKY_BOT_RELEASES@github.com/$TRAVIS_REPO_SLUG.git \
+              v$LFS_VERSION_MAJOR \
+              v$LFS_VERSION_MAJOR-prefix
+          # Create patch version tag (vN.N.N)
+          curl -f -u "$GEKY_BOT_RELEASES" -X POST \
+              https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs \
+              -d "{
+                  \"ref\": \"refs/tags/$LFS_VERSION\",
+                  \"sha\": \"$TRAVIS_COMMIT\"
+              }"
+          # Create minor release?
+          [[ "$LFS_VERSION" == *.0 ]] || exit 0
+          # Build release notes
+          PREV=$(git tag --sort=-v:refname -l "v*.0" | head -1)
+          if [ ! -z "$PREV" ]
           then
-            # Create a simple tag
+              echo "PREV $PREV"
-            curl -f -u "$GEKY_BOT_RELEASES" -X POST \
+              CHANGES=$'### Changes\n\n'$( \
-                https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs \
+                  git log --oneline $PREV.. --grep='^Merge' --invert-grep)
-                -d "{
+              printf "CHANGES\n%s\n\n" "$CHANGES"
-                    \"ref\": \"refs/tags/$LFS_VERSION\",
-                    \"sha\": \"$TRAVIS_COMMIT\"
-                }"
-            # Minor release?
-            if [[ "$LFS_VERSION" == *.0 ]]
-            then
-              # Build release notes
-              PREV=$(git tag --sort=-v:refname -l "v*.0" | head -1)
-              if [ ! -z "$PREV" ]
-              then
-                  echo "PREV $PREV"
-                  CHANGES=$'### Changes\n\n'$( \
-                      git log --oneline $PREV.. --grep='^Merge' --invert-grep)
-                  printf "CHANGES\n%s\n\n" "$CHANGES"
-              fi
-              # Create the release
-              curl -f -u "$GEKY_BOT_RELEASES" -X POST \
-                  https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases \
-                  -d "{
-                      \"tag_name\": \"$LFS_VERSION\",
-                      \"name\": \"${LFS_VERSION%.0}\",
-                      \"draft\": true,
-                      \"body\": $(jq -sR '.' <<< "$CHANGES")
-                  }"
-            fi
           fi
+          # Create the release
+          curl -f -u "$GEKY_BOT_RELEASES" -X POST \
+              https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases \
+              -d "{
+                  \"tag_name\": \"$LFS_VERSION\",
+                  \"name\": \"${LFS_VERSION%.0}\",
+                  \"draft\": true,
+                  \"body\": $(jq -sR '.' <<< "$CHANGES")
+              }" #"
+          SCRIPT
 
 # Manage statuses
 before_install:
   - |
-    curl -u $GEKY_BOT_STATUSES -X POST \
+    curl -u "$GEKY_BOT_STATUSES" -X POST \
         https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
         -d "{
             \"context\": \"$STAGE/$NAME\",
@@ -203,7 +274,7 @@ before_install:
 
 after_failure:
   - |
-    curl -u $GEKY_BOT_STATUSES -X POST \
+    curl -u "$GEKY_BOT_STATUSES" -X POST \
         https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
         -d "{
             \"context\": \"$STAGE/$NAME\",
@@ -214,7 +285,7 @@ after_failure:
 
 after_success:
   - |
-    curl -u $GEKY_BOT_STATUSES -X POST \
+    curl -u "$GEKY_BOT_STATUSES" -X POST \
         https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
         -d "{
             \"context\": \"$STAGE/$NAME\",

Makefile

@@ -26,7 +26,9 @@ override CFLAGS += -m$(WORD)
 endif
 override CFLAGS += -I.
 override CFLAGS += -std=c99 -Wall -pedantic
-override CFLAGS += -Wshadow -Wunused-parameter -Wjump-misses-init -Wsign-compare
+override CFLAGS += -Wextra -Wshadow -Wjump-misses-init
+# Remove missing-field-initializers because of GCC bug
+override CFLAGS += -Wno-missing-field-initializers
 
 
 all: $(TARGET)

README.md

@@ -1,6 +1,6 @@
-## The little filesystem
+## littlefs
 
-A little fail-safe filesystem designed for embedded systems.
+A little fail-safe filesystem designed for microcontrollers.
 
 ```
    | | |     .---._____
@@ -11,17 +11,19 @@ A little fail-safe filesystem designed for embedded systems.
    | | |
 ```
 
-**Bounded RAM/ROM** - The littlefs is designed to work with a limited amount
+**Power-loss resilience** - littlefs is designed to handle random power
-of memory. Recursion is avoided and dynamic memory is limited to configurable
+failures. All file operations have strong copy-on-write guarantees and if
-buffers that can be provided statically.
+power is lost the filesystem will fall back to the last known good state.
 
-**Power-loss resilient** - The littlefs is designed for systems that may have
+**Dynamic wear leveling** - littlefs is designed with flash in mind, and
-random power failures. The littlefs has strong copy-on-write guarantees and
+provides wear leveling over dynamic blocks. Additionally, littlefs can
-storage on disk is always kept in a valid state.
+detect bad blocks and work around them.
 
-**Wear leveling** - Since the most common form of embedded storage is erodible
+**Bounded RAM/ROM** - littlefs is designed to work with a small amount of
-flash memories, littlefs provides a form of dynamic wear leveling for systems
+memory. RAM usage is strictly bounded, which means RAM consumption does not
-that can not fit a full flash translation layer.
+change as the filesystem grows. The filesystem contains no unbounded
+recursion and dynamic memory is limited to configurable buffers that can be
+provided statically.
 
 ## Example
 
@@ -91,11 +93,11 @@ int main(void) {
 Detailed documentation (or at least as much detail as is currently available)
 can be found in the comments in [lfs.h](lfs.h).
 
-As you may have noticed, littlefs takes in a configuration structure that
+littlefs takes in a configuration structure that defines how the filesystem
-defines how the filesystem operates. The configuration struct provides the
+operates. The configuration struct provides the filesystem with the block
-filesystem with the block device operations and dimensions, tweakable
+device operations and dimensions, tweakable parameters that tradeoff memory
-parameters that tradeoff memory usage for performance, and optional
+usage for performance, and optional static buffers if the user wants to avoid
-static buffers if the user wants to avoid dynamic memory.
+dynamic memory.
 
 The state of the littlefs is stored in the `lfs_t` type which is left up
 to the user to allocate, allowing multiple filesystems to be in use
@@ -107,14 +109,14 @@ directory functions, with the deviation that the allocation of filesystem
 structures must be provided by the user.
 
 All POSIX operations, such as remove and rename, are atomic, even in event
-of power-loss. Additionally, no file updates are actually committed to the
+of power-loss. Additionally, no file updates are not actually committed to
-filesystem until sync or close is called on the file.
+the filesystem until sync or close is called on the file.
 
 ## Other notes
 
-All littlefs have the potential to return a negative error code. The errors
+All littlefs calls have the potential to return a negative error code. The
-can be either one of those found in the `enum lfs_error` in [lfs.h](lfs.h),
+errors can be either one of those found in the `enum lfs_error` in
-or an error returned by the user's block device operations.
+[lfs.h](lfs.h), or an error returned by the user's block device operations.
 
 In the configuration struct, the `prog` and `erase` function provided by the
 user may return a `LFS_ERR_CORRUPT` error if the implementation already can
@@ -128,14 +130,60 @@ from memory, otherwise data integrity can not be guaranteed. If the `write`
 function does not perform caching, and therefore each `read` or `write` call
 hits the memory, the `sync` function can simply return 0.
 
-## Reference material
+## Design
 
-[DESIGN.md](DESIGN.md) - DESIGN.md contains a fully detailed dive into how
+At a high level, littlefs is a block based filesystem that uses small logs to
-littlefs actually works. I would encourage you to read it since the
+store metadata and larger copy-on-write (COW) structures to store file data.
-solutions and tradeoffs at work here are quite interesting.
 
-[SPEC.md](SPEC.md) - SPEC.md contains the on-disk specification of littlefs
+In littlefs, these ingredients form a sort of two-layered cake, with the small
-with all the nitty-gritty details. Can be useful for developing tooling.
+logs (called metadata pairs) providing fast updates to metadata anywhere on
+storage, while the COW structures store file data compactly and without any
+wear amplification cost.
+
+Both of these data structures are built out of blocks, which are fed by a
+common block allocator. By limiting the number of erases allowed on a block
+per allocation, the allocator provides dynamic wear leveling over the entire
+filesystem.
+
+```
+                    root
+                   .--------.--------.
+                   | A'| B'|         |
+                   |   |   |->       |
+                   |   |   |         |
+                   '--------'--------'
+                .----'   '--------------.
+       A       v                 B       v
+      .--------.--------.       .--------.--------.
+      | C'| D'|         |       | E'|new|         |
+      |   |   |->       |       |   | E'|->       |
+      |   |   |         |       |   |   |         |
+      '--------'--------'       '--------'--------'
+      .-'   '--.                  |   '------------------.
+     v          v              .-'                        v
+.--------.  .--------.        v                       .--------.
+|   C    |  |   D    |   .--------.       write       | new E  |
+|        |  |        |   |   E    |        ==>        |        |
+|        |  |        |   |        |                   |        |
+'--------'  '--------'   |        |                   '--------'
+                         '--------'                   .-'    |
+                         .-'    '-.    .-------------|------'
+                        v          v  v              v
+                   .--------.  .--------.       .--------.
+                   |   F    |  |   G    |       | new F  |
+                   |        |  |        |       |        |
+                   |        |  |        |       |        |
+                   '--------'  '--------'       '--------'
+```
+
+More details on how littlefs works can be found in [DESIGN.md](DESIGN.md) and
+[SPEC.md](SPEC.md).
+
+- [DESIGN.md](DESIGN.md) - A fully detailed dive into how littlefs works.
+  I would suggest reading it as the tradeoffs at work are quite interesting.
+
+- [SPEC.md](SPEC.md) - The on-disk specification of littlefs with all the
+  nitty-gritty details. May be useful for tooling development.
 
 ## Testing
 
@@ -149,9 +197,9 @@ make test
 
 ## License
 
-The littlefs is provided under the [BSD-3-Clause](https://spdx.org/licenses/BSD-3-Clause.html)
+The littlefs is provided under the [BSD-3-Clause] license. See
-license. See [LICENSE.md](LICENSE.md) for more information. Contributions to
+[LICENSE.md](LICENSE.md) for more information. Contributions to this project
-this project are accepted under the same license.
+are accepted under the same license.
 
 Individual files contain the following tag instead of the full license text.
 
@@ -162,32 +210,39 @@ License Identifiers that are here available: http://spdx.org/licenses/
 
 ## Related projects
 
-[Mbed OS](https://github.com/ARMmbed/mbed-os/tree/master/features/filesystem/littlefs) -
+- [littlefs-fuse] - A [FUSE] wrapper for littlefs. The project allows you to
-The easiest way to get started with littlefs is to jump into [Mbed](https://os.mbed.com/),
+  mount littlefs directly on a Linux machine. Can be useful for debugging
-which already has block device drivers for most forms of embedded storage. The
+  littlefs if you have an SD card handy.
-littlefs is available in Mbed OS as the [LittleFileSystem](https://os.mbed.com/docs/latest/reference/littlefilesystem.html)
-class.
 
-[littlefs-fuse](https://github.com/geky/littlefs-fuse) - A [FUSE](https://github.com/libfuse/libfuse)
+- [littlefs-js] - A javascript wrapper for littlefs. I'm not sure why you would
-wrapper for littlefs. The project allows you to mount littlefs directly on a
+  want this, but it is handy for demos.  You can see it in action
-Linux machine. Can be useful for debugging littlefs if you have an SD card
+  [here][littlefs-js-demo].
-handy.
 
-[littlefs-js](https://github.com/geky/littlefs-js) - A javascript wrapper for
+- [mklfs] - A command line tool built by the [Lua RTOS] guys for making
-littlefs. I'm not sure why you would want this, but it is handy for demos.
+  littlefs images from a host PC. Supports Windows, Mac OS, and Linux.
-You can see it in action [here](http://littlefs.geky.net/demo.html).
 
-[mklfs](https://github.com/whitecatboard/Lua-RTOS-ESP32/tree/master/components/mklfs/src) -
+- [Mbed OS] - The easiest way to get started with littlefs is to jump into Mbed
-A command line tool built by the [Lua RTOS](https://github.com/whitecatboard/Lua-RTOS-ESP32)
+  which already has block device drivers for most forms of embedded storage.
-guys for making littlefs images from a host PC. Supports Windows, Mac OS,
+  littlefs is available in Mbed OS as the [LittleFileSystem] class.
-and Linux.
 
-[SPIFFS](https://github.com/pellepl/spiffs) - Another excellent embedded
+- [SPIFFS] - Another excellent embedded filesystem for NOR flash. As a more
-filesystem for NOR flash. As a more traditional logging filesystem with full
+  traditional logging filesystem with full static wear-leveling, SPIFFS will
-static wear-leveling, SPIFFS will likely outperform littlefs on small
+  likely outperform littlefs on small memories such as the internal flash on
-memories such as the internal flash on microcontrollers.
+  microcontrollers.
 
-[Dhara](https://github.com/dlbeer/dhara) - An interesting NAND flash
+- [Dhara] - An interesting NAND flash translation layer designed for small
-translation layer designed for small MCUs. It offers static wear-leveling and
+  MCUs. It offers static wear-leveling and power-resilience with only a fixed
-power-resilience with only a fixed O(|address|) pointer structure stored on
+  _O(|address|)_ pointer structure stored on each block and in RAM.
-each block and in RAM.
+
+
+[BSD-3-Clause]: https://spdx.org/licenses/BSD-3-Clause.html
+[littlefs-fuse]: https://github.com/geky/littlefs-fuse
+[FUSE]: https://github.com/libfuse/libfuse
+[littlefs-js]: https://github.com/geky/littlefs-js
+[littlefs-js-demo]:http://littlefs.geky.net/demo.html
+[mklfs]: https://github.com/whitecatboard/Lua-RTOS-ESP32/tree/master/components/mklfs/src
+[Lua RTOS]: https://github.com/whitecatboard/Lua-RTOS-ESP32
+[Mbed OS]: https://github.com/armmbed/mbed-os
+[LittleFileSystem]: https://os.mbed.com/docs/mbed-os/v5.12/apis/littlefilesystem.html
+[SPIFFS]: https://github.com/pellepl/spiffs
+[Dhara]: https://github.com/dlbeer/dhara

lfs.c

@@ -439,6 +439,10 @@ static int lfs_fs_relocate(lfs_t *lfs,
         const lfs_block_t oldpair[2], lfs_block_t newpair[2]);
 static int lfs_fs_forceconsistency(lfs_t *lfs);
 static int lfs_deinit(lfs_t *lfs);
+#ifdef LFS_MIGRATE
+static int lfs1_traverse(lfs_t *lfs,
+        int (*cb)(void*, lfs_block_t), void *data);
+#endif
 
 /// Block allocator ///
 static int lfs_alloc_lookahead(void *p, lfs_block_t block) {
@@ -765,7 +769,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
     lfs_stag_t besttag = -1;
 
     // find the block with the most recent revision
-    uint32_t revs[2];
+    uint32_t revs[2] = {0, 0};
     int r = 0;
     for (int i = 0; i < 2; i++) {
         int err = lfs_bd_read(lfs,
@@ -776,7 +780,8 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
             return err;
         }
 
-        if (lfs_scmp(revs[i], revs[(i+1)%2]) > 0 || err == LFS_ERR_CORRUPT) {
+        if (err != LFS_ERR_CORRUPT &&
+                lfs_scmp(revs[i], revs[(i+1)%2]) > 0) {
             r = i;
         }
     }
@@ -822,7 +827,8 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
             // next commit not yet programmed or we're not in valid range
             if (!lfs_tag_isvalid(tag) ||
                     off + lfs_tag_dsize(tag) > lfs->cfg->block_size) {
-                dir->erased = (lfs_tag_type1(ptag) == LFS_TYPE_CRC);
+                dir->erased = (lfs_tag_type1(ptag) == LFS_TYPE_CRC &&
+                        dir->off % lfs->cfg->prog_size == 0);
                 break;
             }
 
@@ -849,7 +855,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
                 }
 
                 // reset the next bit if we need to
-                ptag ^= (lfs_tag_chunk(tag) & 1) << 31;
+                ptag ^= (lfs_tag_chunk(tag) & 1U) << 31;
 
                 // toss our crc into the filesystem seed for
                 // pseudorandom numbers
@@ -1316,15 +1322,14 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir) {
     int err = lfs_bd_read(lfs,
             NULL, &lfs->rcache, sizeof(dir->rev),
             dir->pair[0], 0, &dir->rev, sizeof(dir->rev));
-    if (err) {
-        return err;
-    }
-
     dir->rev = lfs_fromle32(dir->rev);
     if (err && err != LFS_ERR_CORRUPT) {
         return err;
     }
 
+    // make sure we don't immediately evict
+    dir->rev += dir->rev & 1;
+
     // set defaults
     dir->off = sizeof(dir->rev);
     dir->etag = 0xffffffff;
@@ -1415,7 +1420,8 @@ static int lfs_dir_compact(lfs_t *lfs,
     bool relocated = false;
     bool exhausted = false;
 
-    while (true) {
+    // should we split?
+    while (end - begin > 1) {
         // find size
         lfs_size_t size = 0;
         int err = lfs_dir_traverse(lfs,
@@ -1430,9 +1436,11 @@ static int lfs_dir_compact(lfs_t *lfs,
 
         // space is complicated, we need room for tail, crc, gstate,
         // cleanup delete, and we cap at half a block to give room
-        // for metadata updates
+        // for metadata updates.
-        if (size <= lfs_min(lfs->cfg->block_size - 36,
+        if (end - begin < 0xff &&
-                lfs_alignup(lfs->cfg->block_size/2, lfs->cfg->prog_size))) {
+                size <= lfs_min(lfs->cfg->block_size - 36,
+                    lfs_alignup(lfs->cfg->block_size/2,
+                        lfs->cfg->prog_size))) {
             break;
         }
 
@@ -1457,7 +1465,8 @@ static int lfs_dir_compact(lfs_t *lfs,
 
     // increment revision count
     dir->rev += 1;
-    if (lfs->cfg->block_cycles && dir->rev % lfs->cfg->block_cycles == 0) {
+    if (lfs->cfg->block_cycles &&
+            (dir->rev % (lfs->cfg->block_cycles+1) == 0)) {
         if (lfs_pair_cmp(dir->pair, (const lfs_block_t[2]){0, 1}) == 0) {
             // oh no! we're writing too much to the superblock,
             // should we expand?
@@ -1491,7 +1500,7 @@ static int lfs_dir_compact(lfs_t *lfs,
 
     // begin loop to commit compaction to blocks until a compact sticks
     while (true) {
-        if (true) {
+        {
             // There's nothing special about our global delta, so feed it into
             // our local global delta
             int err = lfs_dir_getgstate(lfs, dir, &lfs->gdelta);
@@ -1590,13 +1599,12 @@ static int lfs_dir_compact(lfs_t *lfs,
             dir->count = end - begin;
             dir->off = commit.off;
             dir->etag = commit.ptag;
-            dir->erased = true;
+            dir->erased = (dir->off % lfs->cfg->prog_size == 0);
             // note we able to have already handled move here
             if (lfs_gstate_hasmovehere(&lfs->gpending, dir->pair)) {
                 lfs_gstate_xormove(&lfs->gpending,
                     &lfs->gpending, 0x3ff, NULL);
             }
-
         }
         break;
 
@@ -1705,7 +1713,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir,
         }
     }
 
-    if (dir->erased) {
+    if (dir->erased || dir->count >= 0xff) {
         // try to commit
         struct lfs_commit commit = {
             .block = dir->pair[0],
@@ -1887,7 +1895,7 @@ int lfs_mkdir(lfs_t *lfs, const char *path) {
     // now insert into our parent block
     lfs_pair_tole32(dir.pair);
     err = lfs_dir_commit(lfs, &cwd, LFS_MKATTRS(
-            {LFS_MKTAG(LFS_TYPE_CREATE, id, 0)},
+            {LFS_MKTAG(LFS_TYPE_CREATE, id, 0), NULL},
             {LFS_MKTAG(LFS_TYPE_DIR, id, nlen), path},
             {LFS_MKTAG(LFS_TYPE_DIRSTRUCT, id, 8), dir.pair},
             {!cwd.split
@@ -2116,7 +2124,7 @@ static int lfs_ctz_extend(lfs_t *lfs,
         }
         LFS_ASSERT(nblock >= 2 && nblock <= lfs->cfg->block_count);
 
-        if (true) {
+        {
             err = lfs_bd_erase(lfs, nblock);
             if (err) {
                 if (err == LFS_ERR_CORRUPT) {
@@ -2294,9 +2302,9 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file,
 
         // get next slot and create entry to remember name
         err = lfs_dir_commit(lfs, &file->m, LFS_MKATTRS(
-                {LFS_MKTAG(LFS_TYPE_CREATE, file->id, 0)},
+                {LFS_MKTAG(LFS_TYPE_CREATE, file->id, 0), NULL},
                 {LFS_MKTAG(LFS_TYPE_REG, file->id, nlen), path},
-                {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, file->id, 0)}));
+                {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, file->id, 0), NULL}));
         if (err) {
             err = LFS_ERR_NAMETOOLONG;
             goto cleanup;
@@ -2375,7 +2383,8 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file,
         if (file->ctz.size > 0) {
             lfs_stag_t res = lfs_dir_get(lfs, &file->m,
                     LFS_MKTAG(0x700, 0x3ff, 0),
-                    LFS_MKTAG(LFS_TYPE_STRUCT, file->id, file->cache.size),
+                    LFS_MKTAG(LFS_TYPE_STRUCT, file->id,
+                        lfs_min(file->cache.size, 0x3fe)),
                     file->cache.buffer);
             if (res < 0) {
                 err = res;
@@ -2570,6 +2579,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) {
     while (true) {
         int err = lfs_file_flush(lfs, file);
         if (err) {
+            file->flags |= LFS_F_ERRED;
             return err;
         }
 
@@ -2605,6 +2615,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) {
                 if (err == LFS_ERR_NOSPC && (file->flags & LFS_F_INLINE)) {
                     goto relocate;
                 }
+                file->flags |= LFS_F_ERRED;
                 return err;
             }
 
@@ -2618,6 +2629,7 @@ relocate:
         file->off = file->pos;
         err = lfs_file_relocate(lfs, file);
         if (err) {
+            file->flags |= LFS_F_ERRED;
             return err;
         }
     }
@@ -2943,8 +2955,8 @@ int lfs_remove(lfs_t *lfs, const char *path) {
 
     lfs_mdir_t cwd;
     lfs_stag_t tag = lfs_dir_find(lfs, &cwd, &path, NULL);
-    if (tag < 0) {
+    if (tag < 0 || lfs_tag_id(tag) == 0x3ff) {
-        return tag;
+        return (tag < 0) ? tag : LFS_ERR_INVAL;
     }
 
     lfs_mdir_t dir;
@@ -2973,7 +2985,7 @@ int lfs_remove(lfs_t *lfs, const char *path) {
 
     // delete the entry
     err = lfs_dir_commit(lfs, &cwd, LFS_MKATTRS(
-            {LFS_MKTAG(LFS_TYPE_DELETE, lfs_tag_id(tag), 0)}));
+            {LFS_MKTAG(LFS_TYPE_DELETE, lfs_tag_id(tag), 0), NULL}));
     if (err) {
         return err;
     }
@@ -3006,16 +3018,17 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) {
     // find old entry
     lfs_mdir_t oldcwd;
     lfs_stag_t oldtag = lfs_dir_find(lfs, &oldcwd, &oldpath, NULL);
-    if (oldtag < 0) {
+    if (oldtag < 0 || lfs_tag_id(oldtag) == 0x3ff) {
-        return oldtag;
+        return (oldtag < 0) ? oldtag : LFS_ERR_INVAL;
     }
 
     // find new entry
     lfs_mdir_t newcwd;
     uint16_t newid;
     lfs_stag_t prevtag = lfs_dir_find(lfs, &newcwd, &newpath, &newid);
-    if (prevtag < 0 && !(prevtag == LFS_ERR_NOENT && newid != 0x3ff)) {
+    if ((prevtag < 0 || lfs_tag_id(prevtag) == 0x3ff) &&
-        return err;
+            !(prevtag == LFS_ERR_NOENT && newid != 0x3ff)) {
+        return (prevtag < 0) ? prevtag : LFS_ERR_INVAL;
     }
 
     lfs_mdir_t prevdir;
@@ -3067,8 +3080,8 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) {
     err = lfs_dir_commit(lfs, &newcwd, LFS_MKATTRS(
             {prevtag != LFS_ERR_NOENT
                 ? LFS_MKTAG(LFS_TYPE_DELETE, newid, 0)
-                : LFS_MKTAG(LFS_FROM_NOOP, 0, 0)},
+                : LFS_MKTAG(LFS_FROM_NOOP, 0, 0), NULL},
-            {LFS_MKTAG(LFS_TYPE_CREATE, newid, 0)},
+            {LFS_MKTAG(LFS_TYPE_CREATE, newid, 0), NULL},
             {LFS_MKTAG(lfs_tag_type3(oldtag), newid, strlen(newpath)),
                 newpath},
             {LFS_MKTAG(LFS_FROM_MOVE, newid, lfs_tag_id(oldtag)), &oldcwd}));
@@ -3186,6 +3199,9 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) {
     LFS_ASSERT(4*lfs_npw2(0xffffffff / (lfs->cfg->block_size-2*4))
             <= lfs->cfg->block_size);
 
+    // we don't support some corner cases
+    LFS_ASSERT(lfs->cfg->block_cycles < 0xffffffff);
+
     // setup read cache
     if (lfs->cfg->read_buffer) {
         lfs->rcache.buffer = lfs->cfg->read_buffer;
@@ -3213,8 +3229,9 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) {
     lfs_cache_zero(lfs, &lfs->pcache);
 
     // setup lookahead, must be multiple of 64-bits
-    LFS_ASSERT(lfs->cfg->lookahead_size % 8 == 0);
     LFS_ASSERT(lfs->cfg->lookahead_size > 0);
+    LFS_ASSERT(lfs->cfg->lookahead_size % 8 == 0 &&
+            (uintptr_t)lfs->cfg->lookahead_buffer % 8 == 0);
     if (lfs->cfg->lookahead_buffer) {
         lfs->free.buffer = lfs->cfg->lookahead_buffer;
     } else {
@@ -3252,6 +3269,9 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) {
     lfs->gstate = (struct lfs_gstate){0};
     lfs->gpending = (struct lfs_gstate){0};
     lfs->gdelta = (struct lfs_gstate){0};
+#ifdef LFS_MIGRATE
+    lfs->lfs1 = NULL;
+#endif
 
     return 0;
 
@@ -3279,7 +3299,7 @@ static int lfs_deinit(lfs_t *lfs) {
 
 int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) {
     int err = 0;
-    if (true) {
+    {
         err = lfs_init(lfs, cfg);
         if (err) {
             return err;
@@ -3312,7 +3332,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) {
 
         lfs_superblock_tole32(&superblock);
         err = lfs_dir_commit(lfs, &root, LFS_MKATTRS( 
-                {LFS_MKTAG(LFS_TYPE_CREATE, 0, 0)},
+                {LFS_MKTAG(LFS_TYPE_CREATE, 0, 0), NULL},
                 {LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), "littlefs"},
                 {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)),
                     &superblock}));
@@ -3462,6 +3482,20 @@ int lfs_fs_traverse(lfs_t *lfs,
         int (*cb)(void *data, lfs_block_t block), void *data) {
     // iterate over metadata pairs
     lfs_mdir_t dir = {.tail = {0, 1}};
+
+#ifdef LFS_MIGRATE
+    // also consider v1 blocks during migration
+    if (lfs->lfs1) {
+        int err = lfs1_traverse(lfs, cb, data);
+        if (err) {
+            return err;
+        }
+
+        dir.tail[0] = lfs->root[0];
+        dir.tail[1] = lfs->root[1];
+    }
+#endif
+
     while (!lfs_pair_isnull(dir.tail)) {
         for (int i = 0; i < 2; i++) {
             int err = cb(data, dir.tail[i]);
@@ -3792,3 +3826,634 @@ lfs_ssize_t lfs_fs_size(lfs_t *lfs) {
 
   return size;
 }
+
+#ifdef LFS_MIGRATE
+////// Migration from littelfs v1 below this //////
+
+/// Version info ///
+
+// Software library version
+// Major (top-nibble), incremented on backwards incompatible changes
+// Minor (bottom-nibble), incremented on feature additions
+#define LFS1_VERSION 0x00010007
+#define LFS1_VERSION_MAJOR (0xffff & (LFS1_VERSION >> 16))
+#define LFS1_VERSION_MINOR (0xffff & (LFS1_VERSION >>  0))
+
+// Version of On-disk data structures
+// Major (top-nibble), incremented on backwards incompatible changes
+// Minor (bottom-nibble), incremented on feature additions
+#define LFS1_DISK_VERSION 0x00010001
+#define LFS1_DISK_VERSION_MAJOR (0xffff & (LFS1_DISK_VERSION >> 16))
+#define LFS1_DISK_VERSION_MINOR (0xffff & (LFS1_DISK_VERSION >>  0))
+
+
+/// v1 Definitions ///
+
+// File types
+enum lfs1_type {
+    LFS1_TYPE_REG        = 0x11,
+    LFS1_TYPE_DIR        = 0x22,
+    LFS1_TYPE_SUPERBLOCK = 0x2e,
+};
+
+typedef struct lfs1 {
+    lfs_block_t root[2];
+} lfs1_t;
+
+typedef struct lfs1_entry {
+    lfs_off_t off;
+
+    struct lfs1_disk_entry {
+        uint8_t type;
+        uint8_t elen;
+        uint8_t alen;
+        uint8_t nlen;
+        union {
+            struct {
+                lfs_block_t head;
+                lfs_size_t size;
+            } file;
+            lfs_block_t dir[2];
+        } u;
+    } d;
+} lfs1_entry_t;
+
+typedef struct lfs1_dir {
+    struct lfs1_dir *next;
+    lfs_block_t pair[2];
+    lfs_off_t off;
+
+    lfs_block_t head[2];
+    lfs_off_t pos;
+
+    struct lfs1_disk_dir {
+        uint32_t rev;
+        lfs_size_t size;
+        lfs_block_t tail[2];
+    } d;
+} lfs1_dir_t;
+
+typedef struct lfs1_superblock {
+    lfs_off_t off;
+
+    struct lfs1_disk_superblock {
+        uint8_t type;
+        uint8_t elen;
+        uint8_t alen;
+        uint8_t nlen;
+        lfs_block_t root[2];
+        uint32_t block_size;
+        uint32_t block_count;
+        uint32_t version;
+        char magic[8];
+    } d;
+} lfs1_superblock_t;
+
+
+/// Low-level wrappers v1->v2 ///
+void lfs1_crc(uint32_t *crc, const void *buffer, size_t size) {
+    *crc = lfs_crc(*crc, buffer, size);
+}
+
+static int lfs1_bd_read(lfs_t *lfs, lfs_block_t block,
+        lfs_off_t off, void *buffer, lfs_size_t size) {
+    // if we ever do more than writes to alternating pairs,
+    // this may need to consider pcache
+    return lfs_bd_read(lfs, &lfs->pcache, &lfs->rcache, size,
+            block, off, buffer, size);
+}
+
+static int lfs1_bd_crc(lfs_t *lfs, lfs_block_t block,
+        lfs_off_t off, lfs_size_t size, uint32_t *crc) {
+    for (lfs_off_t i = 0; i < size; i++) {
+        uint8_t c;
+        int err = lfs1_bd_read(lfs, block, off+i, &c, 1);
+        if (err) {
+            return err;
+        }
+
+        lfs1_crc(crc, &c, 1);
+    }
+
+    return 0;
+}
+
+
+/// Endian swapping functions ///
+static void lfs1_dir_fromle32(struct lfs1_disk_dir *d) {
+    d->rev     = lfs_fromle32(d->rev);
+    d->size    = lfs_fromle32(d->size);
+    d->tail[0] = lfs_fromle32(d->tail[0]);
+    d->tail[1] = lfs_fromle32(d->tail[1]);
+}
+
+static void lfs1_dir_tole32(struct lfs1_disk_dir *d) {
+    d->rev     = lfs_tole32(d->rev);
+    d->size    = lfs_tole32(d->size);
+    d->tail[0] = lfs_tole32(d->tail[0]);
+    d->tail[1] = lfs_tole32(d->tail[1]);
+}
+
+static void lfs1_entry_fromle32(struct lfs1_disk_entry *d) {
+    d->u.dir[0] = lfs_fromle32(d->u.dir[0]);
+    d->u.dir[1] = lfs_fromle32(d->u.dir[1]);
+}
+
+static void lfs1_entry_tole32(struct lfs1_disk_entry *d) {
+    d->u.dir[0] = lfs_tole32(d->u.dir[0]);
+    d->u.dir[1] = lfs_tole32(d->u.dir[1]);
+}
+
+static void lfs1_superblock_fromle32(struct lfs1_disk_superblock *d) {
+    d->root[0]     = lfs_fromle32(d->root[0]);
+    d->root[1]     = lfs_fromle32(d->root[1]);
+    d->block_size  = lfs_fromle32(d->block_size);
+    d->block_count = lfs_fromle32(d->block_count);
+    d->version     = lfs_fromle32(d->version);
+}
+
+
+///// Metadata pair and directory operations ///
+static inline lfs_size_t lfs1_entry_size(const lfs1_entry_t *entry) {
+    return 4 + entry->d.elen + entry->d.alen + entry->d.nlen;
+}
+
+static int lfs1_dir_fetch(lfs_t *lfs,
+        lfs1_dir_t *dir, const lfs_block_t pair[2]) {
+    // copy out pair, otherwise may be aliasing dir
+    const lfs_block_t tpair[2] = {pair[0], pair[1]};
+    bool valid = false;
+
+    // check both blocks for the most recent revision
+    for (int i = 0; i < 2; i++) {
+        struct lfs1_disk_dir test;
+        int err = lfs1_bd_read(lfs, tpair[i], 0, &test, sizeof(test));
+        lfs1_dir_fromle32(&test);
+        if (err) {
+            if (err == LFS_ERR_CORRUPT) {
+                continue;
+            }
+            return err;
+        }
+
+        if (valid && lfs_scmp(test.rev, dir->d.rev) < 0) {
+            continue;
+        }
+
+        if ((0x7fffffff & test.size) < sizeof(test)+4 ||
+            (0x7fffffff & test.size) > lfs->cfg->block_size) {
+            continue;
+        }
+
+        uint32_t crc = 0xffffffff;
+        lfs1_dir_tole32(&test);
+        lfs1_crc(&crc, &test, sizeof(test));
+        lfs1_dir_fromle32(&test);
+        err = lfs1_bd_crc(lfs, tpair[i], sizeof(test),
+                (0x7fffffff & test.size) - sizeof(test), &crc);
+        if (err) {
+            if (err == LFS_ERR_CORRUPT) {
+                continue;
+            }
+            return err;
+        }
+
+        if (crc != 0) {
+            continue;
+        }
+
+        valid = true;
+
+        // setup dir in case it's valid
+        dir->pair[0] = tpair[(i+0) % 2];
+        dir->pair[1] = tpair[(i+1) % 2];
+        dir->off = sizeof(dir->d);
+        dir->d = test;
+    }
+
+    if (!valid) {
+        LFS_ERROR("Corrupted dir pair at %" PRIu32 " %" PRIu32 ,
+                tpair[0], tpair[1]);
+        return LFS_ERR_CORRUPT;
+    }
+
+    return 0;
+}
+
+static int lfs1_dir_next(lfs_t *lfs, lfs1_dir_t *dir, lfs1_entry_t *entry) {
+    while (dir->off + sizeof(entry->d) > (0x7fffffff & dir->d.size)-4) {
+        if (!(0x80000000 & dir->d.size)) {
+            entry->off = dir->off;
+            return LFS_ERR_NOENT;
+        }
+
+        int err = lfs1_dir_fetch(lfs, dir, dir->d.tail);
+        if (err) {
+            return err;
+        }
+
+        dir->off = sizeof(dir->d);
+        dir->pos += sizeof(dir->d) + 4;
+    }
+
+    int err = lfs1_bd_read(lfs, dir->pair[0], dir->off,
+            &entry->d, sizeof(entry->d));
+    lfs1_entry_fromle32(&entry->d);
+    if (err) {
+        return err;
+    }
+
+    entry->off = dir->off;
+    dir->off += lfs1_entry_size(entry);
+    dir->pos += lfs1_entry_size(entry);
+    return 0;
+}
+
+/// littlefs v1 specific operations ///
+int lfs1_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) {
+    if (lfs_pair_isnull(lfs->lfs1->root)) {
+        return 0;
+    }
+
+    // iterate over metadata pairs
+    lfs1_dir_t dir;
+    lfs1_entry_t entry;
+    lfs_block_t cwd[2] = {0, 1};
+
+    while (true) {
+        for (int i = 0; i < 2; i++) {
+            int err = cb(data, cwd[i]);
+            if (err) {
+                return err;
+            }
+        }
+
+        int err = lfs1_dir_fetch(lfs, &dir, cwd);
+        if (err) {
+            return err;
+        }
+
+        // iterate over contents
+        while (dir.off + sizeof(entry.d) <= (0x7fffffff & dir.d.size)-4) {
+            err = lfs1_bd_read(lfs, dir.pair[0], dir.off,
+                    &entry.d, sizeof(entry.d));
+            lfs1_entry_fromle32(&entry.d);
+            if (err) {
+                return err;
+            }
+
+            dir.off += lfs1_entry_size(&entry);
+            if ((0x70 & entry.d.type) == (0x70 & LFS1_TYPE_REG)) {
+                err = lfs_ctz_traverse(lfs, NULL, &lfs->rcache,
+                        entry.d.u.file.head, entry.d.u.file.size, cb, data);
+                if (err) {
+                    return err;
+                }
+            }
+        }
+
+        // we also need to check if we contain a threaded v2 directory
+        lfs_mdir_t dir2 = {.split=true, .tail={cwd[0], cwd[1]}};
+        while (dir2.split) {
+            err = lfs_dir_fetch(lfs, &dir2, dir2.tail);
+            if (err) {
+                break;
+            }
+
+            for (int i = 0; i < 2; i++) {
+                err = cb(data, dir2.pair[i]);
+                if (err) {
+                    return err;
+                }
+            }
+        }
+
+        cwd[0] = dir.d.tail[0];
+        cwd[1] = dir.d.tail[1];
+
+        if (lfs_pair_isnull(cwd)) {
+            break;
+        }
+    }
+
+    return 0;
+}
+
+static int lfs1_moved(lfs_t *lfs, const void *e) {
+    if (lfs_pair_isnull(lfs->lfs1->root)) {
+        return 0;
+    }
+
+    // skip superblock
+    lfs1_dir_t cwd;
+    int err = lfs1_dir_fetch(lfs, &cwd, (const lfs_block_t[2]){0, 1});
+    if (err) {
+        return err;
+    }
+
+    // iterate over all directory directory entries
+    lfs1_entry_t entry;
+    while (!lfs_pair_isnull(cwd.d.tail)) {
+        err = lfs1_dir_fetch(lfs, &cwd, cwd.d.tail);
+        if (err) {
+            return err;
+        }
+
+        while (true) {
+            err = lfs1_dir_next(lfs, &cwd, &entry);
+            if (err && err != LFS_ERR_NOENT) {
+                return err;
+            }
+
+            if (err == LFS_ERR_NOENT) {
+                break;
+            }
+
+            if (!(0x80 & entry.d.type) &&
+                 memcmp(&entry.d.u, e, sizeof(entry.d.u)) == 0) {
+                return true;
+            }
+        }
+    }
+
+    return false;
+}
+
+/// Filesystem operations ///
+static int lfs1_mount(lfs_t *lfs, struct lfs1 *lfs1,
+        const struct lfs_config *cfg) {
+    int err = 0;
+    {
+        err = lfs_init(lfs, cfg);
+        if (err) {
+            return err;
+        }
+
+        lfs->lfs1 = lfs1;
+        lfs->lfs1->root[0] = 0xffffffff;
+        lfs->lfs1->root[1] = 0xffffffff;
+
+        // setup free lookahead
+        lfs->free.off = 0;
+        lfs->free.size = 0;
+        lfs->free.i = 0;
+        lfs_alloc_ack(lfs);
+
+        // load superblock
+        lfs1_dir_t dir;
+        lfs1_superblock_t superblock;
+        err = lfs1_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1});
+        if (err && err != LFS_ERR_CORRUPT) {
+            goto cleanup;
+        }
+
+        if (!err) {
+            err = lfs1_bd_read(lfs, dir.pair[0], sizeof(dir.d),
+                    &superblock.d, sizeof(superblock.d));
+            lfs1_superblock_fromle32(&superblock.d);
+            if (err) {
+                goto cleanup;
+            }
+
+            lfs->lfs1->root[0] = superblock.d.root[0];
+            lfs->lfs1->root[1] = superblock.d.root[1];
+        }
+
+        if (err || memcmp(superblock.d.magic, "littlefs", 8) != 0) {
+            LFS_ERROR("Invalid superblock at %d %d", 0, 1);
+            err = LFS_ERR_CORRUPT;
+            goto cleanup;
+        }
+
+        uint16_t major_version = (0xffff & (superblock.d.version >> 16));
+        uint16_t minor_version = (0xffff & (superblock.d.version >>  0));
+        if ((major_version != LFS1_DISK_VERSION_MAJOR ||
+             minor_version > LFS1_DISK_VERSION_MINOR)) {
+            LFS_ERROR("Invalid version %d.%d", major_version, minor_version);
+            err = LFS_ERR_INVAL;
+            goto cleanup;
+        }
+
+        return 0;
+    }
+
+cleanup:
+    lfs_deinit(lfs);
+    return err;
+}
+
+static int lfs1_unmount(lfs_t *lfs) {
+    return lfs_deinit(lfs);
+}
+
+/// v1 migration ///
+int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg) {
+    struct lfs1 lfs1;
+    int err = lfs1_mount(lfs, &lfs1, cfg);
+    if (err) {
+        return err;
+    }
+
+    {
+        // iterate through each directory, copying over entries
+        // into new directory
+        lfs1_dir_t dir1;
+        lfs_mdir_t dir2;
+        dir1.d.tail[0] = lfs->lfs1->root[0];
+        dir1.d.tail[1] = lfs->lfs1->root[1];
+        while (!lfs_pair_isnull(dir1.d.tail)) {
+            // iterate old dir
+            err = lfs1_dir_fetch(lfs, &dir1, dir1.d.tail);
+            if (err) {
+                goto cleanup;
+            }
+
+            // create new dir and bind as temporary pretend root
+            err = lfs_dir_alloc(lfs, &dir2);
+            if (err) {
+                goto cleanup;
+            }
+
+            dir2.rev = dir1.d.rev;
+            dir1.head[0] = dir1.pair[0];
+            dir1.head[1] = dir1.pair[1];
+            lfs->root[0] = dir2.pair[0];
+            lfs->root[1] = dir2.pair[1];
+
+            err = lfs_dir_commit(lfs, &dir2, NULL, 0);
+            if (err) {
+                goto cleanup;
+            }
+
+            while (true) {
+                lfs1_entry_t entry1;
+                err = lfs1_dir_next(lfs, &dir1, &entry1);
+                if (err && err != LFS_ERR_NOENT) {
+                    goto cleanup;
+                }
+
+                if (err == LFS_ERR_NOENT) {
+                    break;
+                }
+
+                // check that entry has not been moved
+                if (entry1.d.type & 0x80) {
+                    int moved = lfs1_moved(lfs, &entry1.d.u);
+                    if (moved < 0) {
+                        err = moved;
+                        goto cleanup;
+                    }
+
+                    if (moved) {
+                        continue;
+                    }
+
+                    entry1.d.type &= ~0x80;
+                }
+                
+                // also fetch name
+                char name[LFS_NAME_MAX+1];
+                memset(name, 0, sizeof(name));
+                err = lfs1_bd_read(lfs, dir1.pair[0],
+                        entry1.off + 4+entry1.d.elen+entry1.d.alen,
+                        name, entry1.d.nlen);
+                if (err) {
+                    goto cleanup;
+                }
+
+                bool isdir = (entry1.d.type == LFS1_TYPE_DIR);
+
+                // create entry in new dir
+                err = lfs_dir_fetch(lfs, &dir2, lfs->root);
+                if (err) {
+                    goto cleanup;
+                }
+
+                uint16_t id;
+                err = lfs_dir_find(lfs, &dir2, &(const char*){name}, &id);
+                if (!(err == LFS_ERR_NOENT && id != 0x3ff)) {
+                    err = (err < 0) ? err : LFS_ERR_EXIST;
+                    goto cleanup;
+                }
+
+                lfs1_entry_tole32(&entry1.d);
+                err = lfs_dir_commit(lfs, &dir2, LFS_MKATTRS(
+                        {LFS_MKTAG(LFS_TYPE_CREATE, id, 0), NULL},
+                        {LFS_MKTAG(
+                            isdir ? LFS_TYPE_DIR : LFS_TYPE_REG,
+                            id, entry1.d.nlen), name},
+                        {LFS_MKTAG(
+                            isdir ? LFS_TYPE_DIRSTRUCT : LFS_TYPE_CTZSTRUCT,
+                            id, sizeof(&entry1.d.u)), &entry1.d.u}));
+                lfs1_entry_fromle32(&entry1.d);
+                if (err) {
+                    goto cleanup;
+                }
+            }
+
+            if (!lfs_pair_isnull(dir1.d.tail)) {
+                // find last block and update tail to thread into fs
+                err = lfs_dir_fetch(lfs, &dir2, lfs->root);
+                if (err) {
+                    goto cleanup;
+                }
+
+                while (dir2.split) {
+                    err = lfs_dir_fetch(lfs, &dir2, dir2.tail);
+                    if (err) {
+                        goto cleanup;
+                    }
+                }
+
+                lfs_pair_tole32(dir2.pair);
+                err = lfs_dir_commit(lfs, &dir2, LFS_MKATTRS(
+                        {LFS_MKTAG(LFS_TYPE_SOFTTAIL, 0x3ff, 0),
+                            dir1.d.tail}));
+                lfs_pair_fromle32(dir2.pair);
+                if (err) {
+                    goto cleanup;
+                }
+            }
+
+            // Copy over first block to thread into fs. Unfortunately
+            // if this fails there is not much we can do.
+            LFS_DEBUG("Migrating %"PRIu32" %"PRIu32" -> %"PRIu32" %"PRIu32,
+                    lfs->root[0], lfs->root[1], dir1.head[0], dir1.head[1]);
+
+            err = lfs_bd_erase(lfs, dir1.head[1]);
+            if (err) {
+                goto cleanup;
+            }
+
+            err = lfs_dir_fetch(lfs, &dir2, lfs->root);
+            if (err) {
+                goto cleanup;
+            }
+
+            for (lfs_off_t i = 0; i < dir2.off; i++) {
+                uint8_t dat;
+                err = lfs_bd_read(lfs,
+                        NULL, &lfs->rcache, dir2.off,
+                        dir2.pair[0], i, &dat, 1);
+                if (err) {
+                    goto cleanup;
+                }
+
+                err = lfs_bd_prog(lfs,
+                        &lfs->pcache, &lfs->rcache, true,
+                        dir1.head[1], i, &dat, 1);
+                if (err) {
+                    goto cleanup;
+                }
+            }
+        }
+
+        // Create new superblock. This marks a successful migration!
+        err = lfs1_dir_fetch(lfs, &dir1, (const lfs_block_t[2]){0, 1});
+        if (err) {
+            goto cleanup;
+        }
+
+        dir2.pair[0] = dir1.pair[0];
+        dir2.pair[1] = dir1.pair[1];
+        dir2.rev = dir1.d.rev;
+        dir2.off = sizeof(dir2.rev);
+        dir2.etag = 0xffffffff;
+        dir2.count = 0;
+        dir2.tail[0] = lfs->lfs1->root[0];
+        dir2.tail[1] = lfs->lfs1->root[1];
+        dir2.erased = false;
+        dir2.split = true;
+
+        lfs_superblock_t superblock = {
+            .version     = LFS_DISK_VERSION,
+            .block_size  = lfs->cfg->block_size,
+            .block_count = lfs->cfg->block_count,
+            .name_max    = lfs->name_max,
+            .file_max    = lfs->file_max,
+            .attr_max    = lfs->attr_max,
+        };
+
+        lfs_superblock_tole32(&superblock);
+        err = lfs_dir_commit(lfs, &dir2, LFS_MKATTRS(
+                {LFS_MKTAG(LFS_TYPE_CREATE, 0, 0), NULL},
+                {LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), "littlefs"},
+                {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)),
+                    &superblock}));
+        if (err) {
+            goto cleanup;
+        }
+
+        // sanity check that fetch works
+        err = lfs_dir_fetch(lfs, &dir2, (const lfs_block_t[2]){0, 1});
+        if (err) {
+            goto cleanup;
+        }
+    }
+
+cleanup:
+    lfs1_unmount(lfs);
+    return err;
+}
+
+#endif

lfs.h

@@ -215,8 +215,9 @@ struct lfs_config {
     // By default lfs_malloc is used to allocate this buffer.
     void *prog_buffer;
 
-    // Optional statically allocated program buffer. Must be lookahead_size.
+    // Optional statically allocated lookahead buffer. Must be lookahead_size
-    // By default lfs_malloc is used to allocate this buffer.
+    // and aligned to a 64-bit boundary. By default lfs_malloc is used to
+    // allocate this buffer.
     void *lookahead_buffer;
 
     // Optional upper limit on length of file names in bytes. No downside for
@@ -380,6 +381,10 @@ typedef struct lfs {
     lfs_size_t name_max;
     lfs_size_t file_max;
     lfs_size_t attr_max;
+
+#ifdef LFS_MIGRATE
+    struct lfs1 *lfs1;
+#endif
 } lfs_t;
 
 
@@ -617,6 +622,21 @@ lfs_ssize_t lfs_fs_size(lfs_t *lfs);
 // Returns a negative error code on failure.
 int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data);
 
+#ifdef LFS_MIGRATE
+// Attempts to migrate a previous version of littlefs
+//
+// Behaves similarly to the lfs_format function. Attempts to mount
+// the previous version of littlefs and update the filesystem so it can be
+// mounted with the current version of littlefs.
+//
+// Requires a littlefs object and config struct. This clobbers the littlefs
+// object, and does not leave the filesystem mounted. The config struct must
+// be zeroed for defaults and backwards compatibility.
+//
+// Returns a negative error code on failure.
+int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg);
+#endif
+
 
 #ifdef __cplusplus
 } /* extern "C" */

lfs_util.h

@@ -192,6 +192,7 @@ static inline uint32_t lfs_tobe32(uint32_t a) {
 uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size);
 
 // Allocate memory, only used if buffers are not provided to littlefs
+// Note, memory must be 64-bit aligned
 static inline void *lfs_malloc(size_t size) {
 #ifndef LFS_NO_MALLOC
     return malloc(size);

scripts/prefix.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+
+# This script replaces prefixes of files, and symbols in that file.
+# Useful for creating different versions of the codebase that don't
+# conflict at compile time.
+#
+# example:
+# $ ./scripts/prefix.py lfs2
+
+import os
+import os.path
+import re
+import glob
+import itertools
+import tempfile
+import shutil
+import subprocess
+
+DEFAULT_PREFIX = "lfs"
+
+def subn(from_prefix, to_prefix, name):
+    name, count1 = re.subn('\\b'+from_prefix, to_prefix, name)
+    name, count2 = re.subn('\\b'+from_prefix.upper(), to_prefix.upper(), name)
+    name, count3 = re.subn('\\B-D'+from_prefix.upper(),
+            '-D'+to_prefix.upper(), name)
+    return name, count1+count2+count3
+
+def main(from_prefix, to_prefix=None, files=None):
+    if not to_prefix:
+        from_prefix, to_prefix = DEFAULT_PREFIX, from_prefix
+
+    if not files:
+        files = subprocess.check_output([
+                'git', 'ls-tree', '-r', '--name-only', 'HEAD']).split()
+
+    for oldname in files:
+        # Rename any matching file names
+        newname, namecount = subn(from_prefix, to_prefix, oldname)
+        if namecount:
+            subprocess.check_call(['git', 'mv', oldname, newname])
+
+        # Rename any prefixes in file
+        count = 0
+        with open(newname+'~', 'w') as tempf:
+            with open(newname) as newf:
+                for line in newf:
+                    line, n = subn(from_prefix, to_prefix, line)
+                    count += n
+                    tempf.write(line)
+        shutil.copystat(newname, newname+'~')
+        os.rename(newname+'~', newname)
+        subprocess.check_call(['git', 'add', newname])
+
+        # Summary
+        print '%s: %d replacements' % (
+                '%s -> %s' % (oldname, newname) if namecount else oldname,
+                count)
+
+if __name__ == "__main__":
+    import sys
+    sys.exit(main(*sys.argv[1:]))

tests/test_paths.sh

@@ -128,6 +128,14 @@ tests/test.py << TEST
     lfs_mkdir(&lfs, "/") => LFS_ERR_EXIST;
     lfs_file_open(&lfs, &file[0], "/", LFS_O_WRONLY | LFS_O_CREAT)
         => LFS_ERR_ISDIR;
+
+    // more corner cases
+    lfs_remove(&lfs, "") => LFS_ERR_INVAL;
+    lfs_remove(&lfs, ".") => LFS_ERR_INVAL;
+    lfs_remove(&lfs, "..") => LFS_ERR_INVAL;
+    lfs_remove(&lfs, "/") => LFS_ERR_INVAL;
+    lfs_remove(&lfs, "//") => LFS_ERR_INVAL;
+    lfs_remove(&lfs, "./") => LFS_ERR_INVAL;
     lfs_unmount(&lfs) => 0;
 TEST
 
@@ -165,5 +173,29 @@ tests/test.py << TEST
     lfs_unmount(&lfs) => 0;
 TEST
 
+echo "--- Really big path test ---"
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    memset(buffer, 'w', LFS_NAME_MAX);
+    buffer[LFS_NAME_MAX+1] = '\0';
+    lfs_mkdir(&lfs, (char*)buffer) => 0;
+    lfs_remove(&lfs, (char*)buffer) => 0;
+    lfs_file_open(&lfs, &file[0], (char*)buffer,
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_remove(&lfs, (char*)buffer) => 0;
+
+    memcpy(buffer, "coffee/", strlen("coffee/"));
+    memset(buffer+strlen("coffee/"), 'w', LFS_NAME_MAX);
+    buffer[strlen("coffee/")+LFS_NAME_MAX+1] = '\0';
+    lfs_mkdir(&lfs, (char*)buffer) => 0;
+    lfs_remove(&lfs, (char*)buffer) => 0;
+    lfs_file_open(&lfs, &file[0], (char*)buffer,
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_remove(&lfs, (char*)buffer) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+
 echo "--- Results ---"
 tests/stats.py