Fixed issue where inlined files were not cleaned up

A relatively late change in v2 was to switch from implicit replacement
of inline files to explicit replacement, which requires explicitly
deleting the inline struct tag before creating a CTZ skip-list tag.

Unfortunately I never added the actual logic to delete the tag. This
went unnoticed as the later CTZ skip-list tag always overrides the
search for the struct tag during a directory fetch.

To fix we need to add an explicit delete. Also there was some clean up
necessary for actually removing delete tags during compaction as well
as some refactoring around outlining.

Found by Johnxjj

.travis.yml

@@ -36,7 +36,7 @@ script:
     if [ "$TRAVIS_TEST_RESULT" -eq 0 ]
     then
         CURR=$(tail -n1 sizes | awk '{print $1}')
-        PREV=$(curl -u $GEKY_BOT_STATUSES https://api.github.com/repos/$TRAVIS_REPO_SLUG/status/master \
+        PREV=$(curl -u "$GEKY_BOT_STATUSES" https://api.github.com/repos/$TRAVIS_REPO_SLUG/status/master \
             | jq -re "select(.sha != \"$TRAVIS_COMMIT\")
                 | .statuses[] | select(.context == \"$STAGE/$NAME\").description
                 | capture(\"code size is (?<size>[0-9]+)\").size" \
@@ -66,7 +66,10 @@ jobs:
         - CC="arm-linux-gnueabi-gcc --static -mthumb"
         - EXEC="qemu-arm"
       install:
-        - sudo apt-get install gcc-arm-linux-gnueabi qemu-user
+        - sudo apt-get install
+              gcc-arm-linux-gnueabi
+              libc6-dev-armel-cross
+              qemu-user
         - arm-linux-gnueabi-gcc --version
         - qemu-arm -version
 
@@ -78,7 +81,10 @@ jobs:
         - CC="powerpc-linux-gnu-gcc --static"
         - EXEC="qemu-ppc"
       install:
-        - sudo apt-get install gcc-powerpc-linux-gnu qemu-user
+        - sudo apt-get install
+              gcc-powerpc-linux-gnu
+              libc6-dev-powerpc-cross
+              qemu-user
         - powerpc-linux-gnu-gcc --version
         - qemu-ppc -version
 
@@ -90,9 +96,10 @@ jobs:
         - CC="mips-linux-gnu-gcc --static"
         - EXEC="qemu-mips"
       install:
-        - sudo add-apt-repository -y "deb http://archive.ubuntu.com/ubuntu/ xenial main universe"
-        - sudo apt-get -qq update
-        - sudo apt-get install gcc-mips-linux-gnu qemu-user
+        - sudo apt-get install
+              gcc-mips-linux-gnu
+              libc6-dev-mips-cross
+              qemu-user
         - mips-linux-gnu-gcc --version
         - qemu-mips -version
 
@@ -101,6 +108,7 @@ jobs:
       env:
         - STAGE=test
         - NAME=littlefs-fuse
+      if: branch !~ -prefix$
       install:
         - sudo apt-get install libfuse-dev
         - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v2-alpha
@@ -113,7 +121,7 @@ jobs:
 
         - mkdir mount
         - sudo chmod a+rw /dev/loop0
-        - dd if=/dev/zero bs=512 count=2048 of=disk
+        - dd if=/dev/zero bs=512 count=4096 of=disk
         - losetup /dev/loop0 disk
       script:
         # self-host test
@@ -126,73 +134,144 @@ jobs:
         - mkdir mount/littlefs
         - cp -r $(git ls-tree --name-only HEAD) mount/littlefs
         - cd mount/littlefs
+        - stat .
         - ls -flh
         - make -B test_dirs test_files QUIET=1
 
-      # Automatically update releases
+    # self-host with littlefs-fuse for fuzz test
+    - stage: test
+      env:
+        - STAGE=test
+        - NAME=littlefs-migration
+      if: branch !~ -prefix$
+      install:
+        - sudo apt-get install libfuse-dev
+        - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v2-alpha v2
+        - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v1 v1
+        - fusermount -V
+        - gcc --version
+      before_script:
+        # setup disk for littlefs-fuse
+        - rm -rf v2/littlefs/*
+        - cp -r $(git ls-tree --name-only HEAD) v2/littlefs
+
+        - mkdir mount
+        - sudo chmod a+rw /dev/loop0
+        - dd if=/dev/zero bs=512 count=4096 of=disk
+        - losetup /dev/loop0 disk
+      script:
+        # compile v1 and v2
+        - make -C v1
+        - make -C v2
+
+        # run self-host test with v1
+        - v1/lfs --format /dev/loop0
+        - v1/lfs /dev/loop0 mount
+
+        - ls mount
+        - mkdir mount/littlefs
+        - cp -r $(git ls-tree --name-only HEAD) mount/littlefs
+        - cd mount/littlefs
+        - stat .
+        - ls -flh
+        - make -B test_dirs test_files QUIET=1
+
+        # attempt to migrate
+        - cd ../..
+        - fusermount -u mount
+
+        - v2/lfs --migrate /dev/loop0
+        - v2/lfs /dev/loop0 mount
+
+        # run self-host test with v2 right where we left off
+        - ls mount
+        - cd mount/littlefs
+        - stat .
+        - ls -flh
+        - make -B test_dirs test_files QUIET=1
+
+    # Automatically create releases
     - stage: deploy
       env:
         - STAGE=deploy
         - NAME=deploy
       script:
-        # Find version defined in lfs.h
-        - LFS_VERSION=$(grep -ox '#define LFS_VERSION .*' lfs.h | cut -d ' ' -f3)
-        - LFS_VERSION_MAJOR=$((0xffff & ($LFS_VERSION >> 16)))
-        - LFS_VERSION_MINOR=$((0xffff & ($LFS_VERSION >>  0)))
-        # Grab latests patch from repo tags, default to 0, needs finagling to get past github's pagination api
-        - PREV_URL=https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs/tags/v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.
-        - PREV_URL=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL" -I
-                | sed -n '/^Link/{s/.*<\(.*\)>; rel="last"/\1/;p;q0};$q1'
-                || echo $PREV_URL)
-        - LFS_VERSION_PATCH=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL"
-                | jq 'map(.ref | match("\\bv.*\\..*\\.(.*)$";"g")
-                    .captures[].string | tonumber) | max + 1'
-                || echo 0)
-        # We have our new version
-        - LFS_VERSION="v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.$LFS_VERSION_PATCH"
-        - echo "VERSION $LFS_VERSION"
         - |
+          bash << 'SCRIPT'
+          set -ev
+          # Find version defined in lfs.h
+          LFS_VERSION=$(grep -ox '#define LFS_VERSION .*' lfs.h | cut -d ' ' -f3)
+          LFS_VERSION_MAJOR=$((0xffff & ($LFS_VERSION >> 16)))
+          LFS_VERSION_MINOR=$((0xffff & ($LFS_VERSION >>  0)))
+          # Grab latests patch from repo tags, default to 0, needs finagling
+          # to get past github's pagination api
+          PREV_URL=https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs/tags/v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.
+          PREV_URL=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL" -I \
+              | sed -n '/^Link/{s/.*<\(.*\)>; rel="last"/\1/;p;q0};$q1' \
+              || echo $PREV_URL)
+          LFS_VERSION_PATCH=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL" \
+              | jq 'map(.ref | match("\\bv.*\\..*\\.(.*)$";"g")
+                  .captures[].string | tonumber) | max + 1' \
+              || echo 0)
+          # We have our new version
+          LFS_VERSION="v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.$LFS_VERSION_PATCH"
+          echo "VERSION $LFS_VERSION"
           # Check that we're the most recent commit
           CURRENT_COMMIT=$(curl -f -u "$GEKY_BOT_RELEASES" \
-                https://api.github.com/repos/$TRAVIS_REPO_SLUG/commits/master \
-                | jq -re '.sha')
-          if [ "$TRAVIS_COMMIT" == "$CURRENT_COMMIT" ]
+              https://api.github.com/repos/$TRAVIS_REPO_SLUG/commits/master \
+              | jq -re '.sha')
+          [ "$TRAVIS_COMMIT" == "$CURRENT_COMMIT" ] || exit 0
+          # Create major branch
+          git branch v$LFS_VERSION_MAJOR HEAD
+          # Create major prefix branch
+          git config user.name "geky bot"
+          git config user.email "bot@geky.net"
+          git fetch https://github.com/$TRAVIS_REPO_SLUG.git \
+              --depth=50 v$LFS_VERSION_MAJOR-prefix || true
+          ./scripts/prefix.py lfs$LFS_VERSION_MAJOR
+          git branch v$LFS_VERSION_MAJOR-prefix $( \
+              git commit-tree $(git write-tree) \
+                  $(git rev-parse --verify -q FETCH_HEAD | sed -e 's/^/-p /') \
+                  -p HEAD \
+                  -m "Generated v$LFS_VERSION_MAJOR prefixes")
+          git reset --hard
+          # Update major version branches (vN and vN-prefix)
+          git push https://$GEKY_BOT_RELEASES@github.com/$TRAVIS_REPO_SLUG.git \
+              v$LFS_VERSION_MAJOR \
+              v$LFS_VERSION_MAJOR-prefix
+          # Create patch version tag (vN.N.N)
+          curl -f -u "$GEKY_BOT_RELEASES" -X POST \
+              https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs \
+              -d "{
+                  \"ref\": \"refs/tags/$LFS_VERSION\",
+                  \"sha\": \"$TRAVIS_COMMIT\"
+              }"
+          # Create minor release?
+          [[ "$LFS_VERSION" == *.0 ]] || exit 0
+          # Build release notes
+          PREV=$(git tag --sort=-v:refname -l "v*.0" | head -1)
+          if [ ! -z "$PREV" ]
           then
-            # Create a simple tag
-            curl -f -u "$GEKY_BOT_RELEASES" -X POST \
-                https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs \
-                -d "{
-                    \"ref\": \"refs/tags/$LFS_VERSION\",
-                    \"sha\": \"$TRAVIS_COMMIT\"
-                }"
-            # Minor release?
-            if [[ "$LFS_VERSION" == *.0 ]]
-            then
-              # Build release notes
-              PREV=$(git tag --sort=-v:refname -l "v*.0" | head -1)
-              if [ ! -z "$PREV" ]
-              then
-                  echo "PREV $PREV"
-                  CHANGES=$'### Changes\n\n'$( \
-                      git log --oneline $PREV.. --grep='^Merge' --invert-grep)
-                  printf "CHANGES\n%s\n\n" "$CHANGES"
-              fi
-              # Create the release
-              curl -f -u "$GEKY_BOT_RELEASES" -X POST \
-                  https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases \
-                  -d "{
-                      \"tag_name\": \"$LFS_VERSION\",
-                      \"name\": \"${LFS_VERSION%.0}\",
-                      \"draft\": true,
-                      \"body\": $(jq -sR '.' <<< "$CHANGES")
-                  }"
-            fi
+              echo "PREV $PREV"
+              CHANGES=$'### Changes\n\n'$( \
+                  git log --oneline $PREV.. --grep='^Merge' --invert-grep)
+              printf "CHANGES\n%s\n\n" "$CHANGES"
           fi
+          # Create the release
+          curl -f -u "$GEKY_BOT_RELEASES" -X POST \
+              https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases \
+              -d "{
+                  \"tag_name\": \"$LFS_VERSION\",
+                  \"name\": \"${LFS_VERSION%.0}\",
+                  \"draft\": true,
+                  \"body\": $(jq -sR '.' <<< "$CHANGES")
+              }" #"
+          SCRIPT
 
 # Manage statuses
 before_install:
   - |
-    curl -u $GEKY_BOT_STATUSES -X POST \
+    curl -u "$GEKY_BOT_STATUSES" -X POST \
         https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
         -d "{
             \"context\": \"$STAGE/$NAME\",
@@ -203,7 +282,7 @@ before_install:
 
 after_failure:
   - |
-    curl -u $GEKY_BOT_STATUSES -X POST \
+    curl -u "$GEKY_BOT_STATUSES" -X POST \
         https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
         -d "{
             \"context\": \"$STAGE/$NAME\",
@@ -214,7 +293,7 @@ after_failure:
 
 after_success:
   - |
-    curl -u $GEKY_BOT_STATUSES -X POST \
+    curl -u "$GEKY_BOT_STATUSES" -X POST \
         https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
         -d "{
             \"context\": \"$STAGE/$NAME\",

Makefile

@@ -26,7 +26,9 @@ override CFLAGS += -m$(WORD)
 endif
 override CFLAGS += -I.
 override CFLAGS += -std=c99 -Wall -pedantic
-override CFLAGS += -Wshadow -Wunused-parameter -Wjump-misses-init -Wsign-compare
+override CFLAGS += -Wextra -Wshadow -Wjump-misses-init
+# Remove missing-field-initializers because of GCC bug
+override CFLAGS += -Wno-missing-field-initializers
 
 
 all: $(TARGET)

README.md

@@ -1,6 +1,6 @@
-## The little filesystem
+## littlefs
 
-A little fail-safe filesystem designed for embedded systems.
+A little fail-safe filesystem designed for microcontrollers.
 
 ```
    | | |     .---._____
@@ -11,17 +11,19 @@ A little fail-safe filesystem designed for embedded systems.
    | | |
 ```
 
-**Bounded RAM/ROM** - The littlefs is designed to work with a limited amount
-of memory. Recursion is avoided and dynamic memory is limited to configurable
-buffers that can be provided statically.
+**Power-loss resilience** - littlefs is designed to handle random power
+failures. All file operations have strong copy-on-write guarantees and if
+power is lost the filesystem will fall back to the last known good state.
 
-**Power-loss resilient** - The littlefs is designed for systems that may have
-random power failures. The littlefs has strong copy-on-write guarantees and
-storage on disk is always kept in a valid state.
+**Dynamic wear leveling** - littlefs is designed with flash in mind, and
+provides wear leveling over dynamic blocks. Additionally, littlefs can
+detect bad blocks and work around them.
 
-**Wear leveling** - Since the most common form of embedded storage is erodible
-flash memories, littlefs provides a form of dynamic wear leveling for systems
-that can not fit a full flash translation layer.
+**Bounded RAM/ROM** - littlefs is designed to work with a small amount of
+memory. RAM usage is strictly bounded, which means RAM consumption does not
+change as the filesystem grows. The filesystem contains no unbounded
+recursion and dynamic memory is limited to configurable buffers that can be
+provided statically.
 
 ## Example
 
@@ -91,11 +93,11 @@ int main(void) {
 Detailed documentation (or at least as much detail as is currently available)
 can be found in the comments in [lfs.h](lfs.h).
 
-As you may have noticed, littlefs takes in a configuration structure that
-defines how the filesystem operates. The configuration struct provides the
-filesystem with the block device operations and dimensions, tweakable
-parameters that tradeoff memory usage for performance, and optional
-static buffers if the user wants to avoid dynamic memory.
+littlefs takes in a configuration structure that defines how the filesystem
+operates. The configuration struct provides the filesystem with the block
+device operations and dimensions, tweakable parameters that tradeoff memory
+usage for performance, and optional static buffers if the user wants to avoid
+dynamic memory.
 
 The state of the littlefs is stored in the `lfs_t` type which is left up
 to the user to allocate, allowing multiple filesystems to be in use
@@ -107,14 +109,14 @@ directory functions, with the deviation that the allocation of filesystem
 structures must be provided by the user.
 
 All POSIX operations, such as remove and rename, are atomic, even in event
-of power-loss. Additionally, no file updates are actually committed to the
-filesystem until sync or close is called on the file.
+of power-loss. Additionally, no file updates are not actually committed to
+the filesystem until sync or close is called on the file.
 
 ## Other notes
 
-All littlefs have the potential to return a negative error code. The errors
-can be either one of those found in the `enum lfs_error` in [lfs.h](lfs.h),
-or an error returned by the user's block device operations.
+All littlefs calls have the potential to return a negative error code. The
+errors can be either one of those found in the `enum lfs_error` in
+[lfs.h](lfs.h), or an error returned by the user's block device operations.
 
 In the configuration struct, the `prog` and `erase` function provided by the
 user may return a `LFS_ERR_CORRUPT` error if the implementation already can
@@ -128,14 +130,60 @@ from memory, otherwise data integrity can not be guaranteed. If the `write`
 function does not perform caching, and therefore each `read` or `write` call
 hits the memory, the `sync` function can simply return 0.
 
-## Reference material
+## Design
 
-[DESIGN.md](DESIGN.md) - DESIGN.md contains a fully detailed dive into how
-littlefs actually works. I would encourage you to read it since the
-solutions and tradeoffs at work here are quite interesting.
+At a high level, littlefs is a block based filesystem that uses small logs to
+store metadata and larger copy-on-write (COW) structures to store file data.
 
-[SPEC.md](SPEC.md) - SPEC.md contains the on-disk specification of littlefs
-with all the nitty-gritty details. Can be useful for developing tooling.
+In littlefs, these ingredients form a sort of two-layered cake, with the small
+logs (called metadata pairs) providing fast updates to metadata anywhere on
+storage, while the COW structures store file data compactly and without any
+wear amplification cost.
+
+Both of these data structures are built out of blocks, which are fed by a
+common block allocator. By limiting the number of erases allowed on a block
+per allocation, the allocator provides dynamic wear leveling over the entire
+filesystem.
+
+```
+                    root
+                   .--------.--------.
+                   | A'| B'|         |
+                   |   |   |->       |
+                   |   |   |         |
+                   '--------'--------'
+                .----'   '--------------.
+       A       v                 B       v
+      .--------.--------.       .--------.--------.
+      | C'| D'|         |       | E'|new|         |
+      |   |   |->       |       |   | E'|->       |
+      |   |   |         |       |   |   |         |
+      '--------'--------'       '--------'--------'
+      .-'   '--.                  |   '------------------.
+     v          v              .-'                        v
+.--------.  .--------.        v                       .--------.
+|   C    |  |   D    |   .--------.       write       | new E  |
+|        |  |        |   |   E    |        ==>        |        |
+|        |  |        |   |        |                   |        |
+'--------'  '--------'   |        |                   '--------'
+                         '--------'                   .-'    |
+                         .-'    '-.    .-------------|------'
+                        v          v  v              v
+                   .--------.  .--------.       .--------.
+                   |   F    |  |   G    |       | new F  |
+                   |        |  |        |       |        |
+                   |        |  |        |       |        |
+                   '--------'  '--------'       '--------'
+```
+
+More details on how littlefs works can be found in [DESIGN.md](DESIGN.md) and
+[SPEC.md](SPEC.md).
+
+- [DESIGN.md](DESIGN.md) - A fully detailed dive into how littlefs works.
+  I would suggest reading it as the tradeoffs at work are quite interesting.
+
+- [SPEC.md](SPEC.md) - The on-disk specification of littlefs with all the
+  nitty-gritty details. May be useful for tooling development.
 
 ## Testing
 
@@ -149,9 +197,9 @@ make test
 
 ## License
 
-The littlefs is provided under the [BSD-3-Clause](https://spdx.org/licenses/BSD-3-Clause.html)
-license. See [LICENSE.md](LICENSE.md) for more information. Contributions to
-this project are accepted under the same license.
+The littlefs is provided under the [BSD-3-Clause] license. See
+[LICENSE.md](LICENSE.md) for more information. Contributions to this project
+are accepted under the same license.
 
 Individual files contain the following tag instead of the full license text.
 
@@ -162,32 +210,39 @@ License Identifiers that are here available: http://spdx.org/licenses/
 
 ## Related projects
 
-[Mbed OS](https://github.com/ARMmbed/mbed-os/tree/master/features/filesystem/littlefs) -
-The easiest way to get started with littlefs is to jump into [Mbed](https://os.mbed.com/),
-which already has block device drivers for most forms of embedded storage. The
-littlefs is available in Mbed OS as the [LittleFileSystem](https://os.mbed.com/docs/latest/reference/littlefilesystem.html)
-class.
+- [littlefs-fuse] - A [FUSE] wrapper for littlefs. The project allows you to
+  mount littlefs directly on a Linux machine. Can be useful for debugging
+  littlefs if you have an SD card handy.
 
-[littlefs-fuse](https://github.com/geky/littlefs-fuse) - A [FUSE](https://github.com/libfuse/libfuse)
-wrapper for littlefs. The project allows you to mount littlefs directly on a
-Linux machine. Can be useful for debugging littlefs if you have an SD card
-handy.
+- [littlefs-js] - A javascript wrapper for littlefs. I'm not sure why you would
+  want this, but it is handy for demos.  You can see it in action
+  [here][littlefs-js-demo].
 
-[littlefs-js](https://github.com/geky/littlefs-js) - A javascript wrapper for
-littlefs. I'm not sure why you would want this, but it is handy for demos.
-You can see it in action [here](http://littlefs.geky.net/demo.html).
+- [mklfs] - A command line tool built by the [Lua RTOS] guys for making
+  littlefs images from a host PC. Supports Windows, Mac OS, and Linux.
 
-[mklfs](https://github.com/whitecatboard/Lua-RTOS-ESP32/tree/master/components/mklfs/src) -
-A command line tool built by the [Lua RTOS](https://github.com/whitecatboard/Lua-RTOS-ESP32)
-guys for making littlefs images from a host PC. Supports Windows, Mac OS,
-and Linux.
+- [Mbed OS] - The easiest way to get started with littlefs is to jump into Mbed
+  which already has block device drivers for most forms of embedded storage.
+  littlefs is available in Mbed OS as the [LittleFileSystem] class.
 
-[SPIFFS](https://github.com/pellepl/spiffs) - Another excellent embedded
-filesystem for NOR flash. As a more traditional logging filesystem with full
-static wear-leveling, SPIFFS will likely outperform littlefs on small
-memories such as the internal flash on microcontrollers.
+- [SPIFFS] - Another excellent embedded filesystem for NOR flash. As a more
+  traditional logging filesystem with full static wear-leveling, SPIFFS will
+  likely outperform littlefs on small memories such as the internal flash on
+  microcontrollers.
 
-[Dhara](https://github.com/dlbeer/dhara) - An interesting NAND flash
-translation layer designed for small MCUs. It offers static wear-leveling and
-power-resilience with only a fixed O(|address|) pointer structure stored on
-each block and in RAM.
+- [Dhara] - An interesting NAND flash translation layer designed for small
+  MCUs. It offers static wear-leveling and power-resilience with only a fixed
+  _O(|address|)_ pointer structure stored on each block and in RAM.
+
+
+[BSD-3-Clause]: https://spdx.org/licenses/BSD-3-Clause.html
+[littlefs-fuse]: https://github.com/geky/littlefs-fuse
+[FUSE]: https://github.com/libfuse/libfuse
+[littlefs-js]: https://github.com/geky/littlefs-js
+[littlefs-js-demo]:http://littlefs.geky.net/demo.html
+[mklfs]: https://github.com/whitecatboard/Lua-RTOS-ESP32/tree/master/components/mklfs/src
+[Lua RTOS]: https://github.com/whitecatboard/Lua-RTOS-ESP32
+[Mbed OS]: https://github.com/armmbed/mbed-os
+[LittleFileSystem]: https://os.mbed.com/docs/mbed-os/v5.12/apis/littlefilesystem.html
+[SPIFFS]: https://github.com/pellepl/spiffs
+[Dhara]: https://github.com/dlbeer/dhara

lfs.h

@@ -122,13 +122,13 @@ enum lfs_type {
 // File open flags
 enum lfs_open_flags {
     // open flags
-    LFS_O_RDONLY = 1,         // Open a file as read only
-    LFS_O_WRONLY = 2,         // Open a file as write only
-    LFS_O_RDWR   = 3,         // Open a file as read and write
-    LFS_O_CREAT  = 0x0100,    // Create a file if it does not exist
-    LFS_O_EXCL   = 0x0200,    // Fail if a file already exists
-    LFS_O_TRUNC  = 0x0400,    // Truncate the existing file to zero size
-    LFS_O_APPEND = 0x0800,    // Move to end of file on every write
+    LFS_O_RDONLY  = 1,        // Open a file as read only
+    LFS_O_WRONLY  = 2,        // Open a file as write only
+    LFS_O_RDWR    = 3,        // Open a file as read and write
+    LFS_O_CREAT   = 0x0100,   // Create a file if it does not exist
+    LFS_O_EXCL    = 0x0200,   // Fail if a file already exists
+    LFS_O_TRUNC   = 0x0400,   // Truncate the existing file to zero size
+    LFS_O_APPEND  = 0x0800,   // Move to end of file on every write
 
     // internally used flags
     LFS_F_DIRTY   = 0x010000, // File does not match storage
@@ -136,6 +136,7 @@ enum lfs_open_flags {
     LFS_F_READING = 0x040000, // File has been read since last flush
     LFS_F_ERRED   = 0x080000, // An error occured during write
     LFS_F_INLINE  = 0x100000, // Currently inlined in directory entry
+    LFS_F_OUTLINE = 0x200000, // Need to delete inlined directory entry
 };
 
 // File seek flags
@@ -215,8 +216,9 @@ struct lfs_config {
     // By default lfs_malloc is used to allocate this buffer.
     void *prog_buffer;
 
-    // Optional statically allocated program buffer. Must be lookahead_size.
-    // By default lfs_malloc is used to allocate this buffer.
+    // Optional statically allocated lookahead buffer. Must be lookahead_size
+    // and aligned to a 64-bit boundary. By default lfs_malloc is used to
+    // allocate this buffer.
     void *lookahead_buffer;
 
     // Optional upper limit on length of file names in bytes. No downside for
@@ -380,6 +382,10 @@ typedef struct lfs {
     lfs_size_t name_max;
     lfs_size_t file_max;
     lfs_size_t attr_max;
+
+#ifdef LFS_MIGRATE
+    struct lfs1 *lfs1;
+#endif
 } lfs_t;
 
 
@@ -573,7 +579,8 @@ int lfs_dir_close(lfs_t *lfs, lfs_dir_t *dir);
 // Read an entry in the directory
 //
 // Fills out the info structure, based on the specified file or directory.
-// Returns a negative error code on failure.
+// Returns a positive value on success, 0 at the end of directory,
+// or a negative error code on failure.
 int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info);
 
 // Change the position of the directory
@@ -617,6 +624,21 @@ lfs_ssize_t lfs_fs_size(lfs_t *lfs);
 // Returns a negative error code on failure.
 int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data);
 
+#ifdef LFS_MIGRATE
+// Attempts to migrate a previous version of littlefs
+//
+// Behaves similarly to the lfs_format function. Attempts to mount
+// the previous version of littlefs and update the filesystem so it can be
+// mounted with the current version of littlefs.
+//
+// Requires a littlefs object and config struct. This clobbers the littlefs
+// object, and does not leave the filesystem mounted. The config struct must
+// be zeroed for defaults and backwards compatibility.
+//
+// Returns a negative error code on failure.
+int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg);
+#endif
+
 
 #ifdef __cplusplus
 } /* extern "C" */

lfs_util.h

@@ -143,14 +143,14 @@ static inline int lfs_scmp(uint32_t a, uint32_t b) {
 // Convert between 32-bit little-endian and native order
 static inline uint32_t lfs_fromle32(uint32_t a) {
 #if !defined(LFS_NO_INTRINSICS) && ( \
-    (defined(  BYTE_ORDER  ) &&   BYTE_ORDER   ==   ORDER_LITTLE_ENDIAN  ) || \
-    (defined(__BYTE_ORDER  ) && __BYTE_ORDER   == __ORDER_LITTLE_ENDIAN  ) || \
-    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
+    (defined(  BYTE_ORDER  ) && defined(  ORDER_LITTLE_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_LITTLE_ENDIAN  ) || \
+    (defined(__BYTE_ORDER  ) && defined(__ORDER_LITTLE_ENDIAN  ) && __BYTE_ORDER   == __ORDER_LITTLE_ENDIAN  ) || \
+    (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
     return a;
 #elif !defined(LFS_NO_INTRINSICS) && ( \
-    (defined(  BYTE_ORDER  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
-    (defined(__BYTE_ORDER  ) && __BYTE_ORDER   == __ORDER_BIG_ENDIAN  ) || \
-    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
+    (defined(  BYTE_ORDER  ) && defined(  ORDER_BIG_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
+    (defined(__BYTE_ORDER  ) && defined(__ORDER_BIG_ENDIAN  ) && __BYTE_ORDER   == __ORDER_BIG_ENDIAN  ) || \
+    (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
     return __builtin_bswap32(a);
 #else
     return (((uint8_t*)&a)[0] <<  0) |
@@ -167,14 +167,14 @@ static inline uint32_t lfs_tole32(uint32_t a) {
 // Convert between 32-bit big-endian and native order
 static inline uint32_t lfs_frombe32(uint32_t a) {
 #if !defined(LFS_NO_INTRINSICS) && ( \
-    (defined(  BYTE_ORDER  ) &&   BYTE_ORDER   ==   ORDER_LITTLE_ENDIAN  ) || \
-    (defined(__BYTE_ORDER  ) && __BYTE_ORDER   == __ORDER_LITTLE_ENDIAN  ) || \
-    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
+    (defined(  BYTE_ORDER  ) && defined(  ORDER_LITTLE_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_LITTLE_ENDIAN  ) || \
+    (defined(__BYTE_ORDER  ) && defined(__ORDER_LITTLE_ENDIAN  ) && __BYTE_ORDER   == __ORDER_LITTLE_ENDIAN  ) || \
+    (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
     return __builtin_bswap32(a);
 #elif !defined(LFS_NO_INTRINSICS) && ( \
-    (defined(  BYTE_ORDER  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
-    (defined(__BYTE_ORDER  ) && __BYTE_ORDER   == __ORDER_BIG_ENDIAN  ) || \
-    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
+    (defined(  BYTE_ORDER  ) && defined(  ORDER_BIG_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
+    (defined(__BYTE_ORDER  ) && defined(__ORDER_BIG_ENDIAN  ) && __BYTE_ORDER   == __ORDER_BIG_ENDIAN  ) || \
+    (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
     return a;
 #else
     return (((uint8_t*)&a)[0] << 24) |
@@ -192,6 +192,7 @@ static inline uint32_t lfs_tobe32(uint32_t a) {
 uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size);
 
 // Allocate memory, only used if buffers are not provided to littlefs
+// Note, memory must be 64-bit aligned
 static inline void *lfs_malloc(size_t size) {
 #ifndef LFS_NO_MALLOC
     return malloc(size);

scripts/prefix.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 # This script replaces prefixes of files, and symbols in that file.
 # Useful for creating different versions of the codebase that don't

tests/corrupt.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 import struct
 import sys

tests/stats.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 import struct
 import sys

tests/test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 import re
 import sys

tests/test_paths.sh

@@ -128,6 +128,14 @@ tests/test.py << TEST
     lfs_mkdir(&lfs, "/") => LFS_ERR_EXIST;
     lfs_file_open(&lfs, &file[0], "/", LFS_O_WRONLY | LFS_O_CREAT)
         => LFS_ERR_ISDIR;
+
+    // more corner cases
+    lfs_remove(&lfs, "") => LFS_ERR_INVAL;
+    lfs_remove(&lfs, ".") => LFS_ERR_INVAL;
+    lfs_remove(&lfs, "..") => LFS_ERR_INVAL;
+    lfs_remove(&lfs, "/") => LFS_ERR_INVAL;
+    lfs_remove(&lfs, "//") => LFS_ERR_INVAL;
+    lfs_remove(&lfs, "./") => LFS_ERR_INVAL;
     lfs_unmount(&lfs) => 0;
 TEST
 
@@ -165,5 +173,29 @@ tests/test.py << TEST
     lfs_unmount(&lfs) => 0;
 TEST
 
+echo "--- Really big path test ---"
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    memset(buffer, 'w', LFS_NAME_MAX);
+    buffer[LFS_NAME_MAX+1] = '\0';
+    lfs_mkdir(&lfs, (char*)buffer) => 0;
+    lfs_remove(&lfs, (char*)buffer) => 0;
+    lfs_file_open(&lfs, &file[0], (char*)buffer,
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_remove(&lfs, (char*)buffer) => 0;
+
+    memcpy(buffer, "coffee/", strlen("coffee/"));
+    memset(buffer+strlen("coffee/"), 'w', LFS_NAME_MAX);
+    buffer[strlen("coffee/")+LFS_NAME_MAX+1] = '\0';
+    lfs_mkdir(&lfs, (char*)buffer) => 0;
+    lfs_remove(&lfs, (char*)buffer) => 0;
+    lfs_file_open(&lfs, &file[0], (char*)buffer,
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_remove(&lfs, (char*)buffer) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+
 echo "--- Results ---"
 tests/stats.py

tests/test_seek.sh

@@ -357,5 +357,73 @@ tests/test.py << TEST
     lfs_unmount(&lfs) => 0;
 TEST
 
+echo "--- Inline write and seek ---"
+for SIZE in $SMALLSIZE $MEDIUMSIZE $LARGESIZE
+do
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "hello/tinykitty$SIZE",
+            LFS_O_RDWR | LFS_O_CREAT) => 0;
+    int j = 0;
+    int k = 0;
+
+    memcpy(buffer, "abcdefghijklmnopqrstuvwxyz", 26);
+    for (unsigned i = 0; i < $SIZE; i++) {
+        lfs_file_write(&lfs, &file[0], &buffer[j++ % 26], 1) => 1;
+        lfs_file_tell(&lfs, &file[0]) => i+1;
+        lfs_file_size(&lfs, &file[0]) => i+1;
+    }
+
+    lfs_file_seek(&lfs, &file[0], 0, LFS_SEEK_SET) => 0;
+    lfs_file_tell(&lfs, &file[0]) => 0;
+    lfs_file_size(&lfs, &file[0]) => $SIZE;
+    for (unsigned i = 0; i < $SIZE; i++) {
+        uint8_t c;
+        lfs_file_read(&lfs, &file[0], &c, 1) => 1;
+        c => buffer[k++ % 26];
+    }
+
+    lfs_file_sync(&lfs, &file[0]) => 0;
+    lfs_file_tell(&lfs, &file[0]) => $SIZE;
+    lfs_file_size(&lfs, &file[0]) => $SIZE;
+
+    lfs_file_seek(&lfs, &file[0], 0, LFS_SEEK_SET) => 0;
+    for (unsigned i = 0; i < $SIZE; i++) {
+        lfs_file_write(&lfs, &file[0], &buffer[j++ % 26], 1) => 1;
+        lfs_file_tell(&lfs, &file[0]) => i+1;
+        lfs_file_size(&lfs, &file[0]) => $SIZE;
+        lfs_file_sync(&lfs, &file[0]) => 0;
+        lfs_file_tell(&lfs, &file[0]) => i+1;
+        lfs_file_size(&lfs, &file[0]) => $SIZE;
+        if (i < $SIZE-2) {
+            uint8_t c[3];
+            lfs_file_seek(&lfs, &file[0], -1, LFS_SEEK_CUR) => i;
+            lfs_file_read(&lfs, &file[0], &c, 3) => 3;
+            lfs_file_tell(&lfs, &file[0]) => i+3;
+            lfs_file_size(&lfs, &file[0]) => $SIZE;
+            lfs_file_seek(&lfs, &file[0], i+1, LFS_SEEK_SET) => i+1;
+            lfs_file_tell(&lfs, &file[0]) => i+1;
+            lfs_file_size(&lfs, &file[0]) => $SIZE;
+        }
+    }
+
+    lfs_file_seek(&lfs, &file[0], 0, LFS_SEEK_SET) => 0;
+    lfs_file_tell(&lfs, &file[0]) => 0;
+    lfs_file_size(&lfs, &file[0]) => $SIZE;
+    for (unsigned i = 0; i < $SIZE; i++) {
+        uint8_t c;
+        lfs_file_read(&lfs, &file[0], &c, 1) => 1;
+        c => buffer[k++ % 26];
+    }
+
+    lfs_file_sync(&lfs, &file[0]) => 0;
+    lfs_file_tell(&lfs, &file[0]) => $SIZE;
+    lfs_file_size(&lfs, &file[0]) => $SIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+done
+
 echo "--- Results ---"
 tests/stats.py

tests/test_truncate.sh

@@ -11,6 +11,150 @@ tests/test.py << TEST
     lfs_format(&lfs, &cfg) => 0;
 TEST
 
+echo "--- Simple truncate ---"
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldynoop",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+
+    strcpy((char*)buffer, "hair");
+    size = strlen((char*)buffer);
+    for (lfs_off_t j = 0; j < $LARGESIZE; j += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldynoop", LFS_O_RDWR) => 0;
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_truncate(&lfs, &file[0], $MEDIUMSIZE) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldynoop", LFS_O_RDONLY) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    size = strlen("hair");
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_read(&lfs, &file[0], buffer, size) => size;
+        memcmp(buffer, "hair", size) => 0;
+    }
+    lfs_file_read(&lfs, &file[0], buffer, size) => 0;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+
+echo "--- Truncate and read ---"
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldyread",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+
+    strcpy((char*)buffer, "hair");
+    size = strlen((char*)buffer);
+    for (lfs_off_t j = 0; j < $LARGESIZE; j += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldyread", LFS_O_RDWR) => 0;
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_truncate(&lfs, &file[0], $MEDIUMSIZE) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    size = strlen("hair");
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_read(&lfs, &file[0], buffer, size) => size;
+        memcmp(buffer, "hair", size) => 0;
+    }
+    lfs_file_read(&lfs, &file[0], buffer, size) => 0;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldyread", LFS_O_RDONLY) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    size = strlen("hair");
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_read(&lfs, &file[0], buffer, size) => size;
+        memcmp(buffer, "hair", size) => 0;
+    }
+    lfs_file_read(&lfs, &file[0], buffer, size) => 0;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+
+echo "--- Truncate and write ---"
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldywrite",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+
+    strcpy((char*)buffer, "hair");
+    size = strlen((char*)buffer);
+    for (lfs_off_t j = 0; j < $LARGESIZE; j += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldywrite", LFS_O_RDWR) => 0;
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_truncate(&lfs, &file[0], $MEDIUMSIZE) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    strcpy((char*)buffer, "bald");
+    size = strlen((char*)buffer);
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldywrite", LFS_O_RDONLY) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    size = strlen("bald");
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_read(&lfs, &file[0], buffer, size) => size;
+        memcmp(buffer, "bald", size) => 0;
+    }
+    lfs_file_read(&lfs, &file[0], buffer, size) => 0;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+
+# More aggressive general truncation tests
 truncate_test() {
 STARTSIZES="$1"
 STARTSEEKS="$2"