WIP making good progress, found a solution for the cycle issue

Found solution for cycle issue for tail+branch solution, though it does
require creating an extra metadata-pair on relocate.

Need to get passing the advanced relocation tests, this is currently
failing. Also need to figure out why I added the assert on tail end, it
may be related.

.travis.yml

@@ -1,49 +1,70 @@
-# Environment variables
+# environment variables
 env:
   global:
     - CFLAGS=-Werror
+    - MAKEFLAGS=-j
 
-# Common test script
-script:
+# cache installation dirs
+cache:
+  pip: true
+  directories:
+    - $HOME/.cache/apt
+
+# common installation
+_: &install-common
+  # need toml, also pip3 isn't installed by default?
+  - sudo apt-get install python3 python3-pip
+  - sudo pip3 install toml
+  # setup a ram-backed disk to speed up reentrant tests
+  - mkdir disks
+  - sudo mount -t tmpfs -o size=100m tmpfs disks
+  - export TFLAGS="$TFLAGS --disk=disks/disk"
+
+# test cases
+_: &test-example
   # make sure example can at least compile
-  - sed -n '/``` c/,/```/{/```/d; p;}' README.md > test.c &&
+  - sed -n '/``` c/,/```/{/```/d; p}' README.md > test.c &&
     make all CFLAGS+="
         -Duser_provided_block_device_read=NULL
         -Duser_provided_block_device_prog=NULL
         -Duser_provided_block_device_erase=NULL
         -Duser_provided_block_device_sync=NULL
         -include stdio.h"
+# default tests
+_: &test-default
+  # normal+reentrant tests
+  - make test TFLAGS+="-nrk"
+# common real-life geometries
+_: &test-nor
+  # NOR flash: read/prog = 1 block = 4KiB
+  - make test TFLAGS+="-nrk -DLFS_READ_SIZE=1 -DLFS_BLOCK_SIZE=4096"
+_: &test-emmc
+  # eMMC: read/prog = 512 block = 512
+  - make test TFLAGS+="-nrk -DLFS_READ_SIZE=512 -DLFS_BLOCK_SIZE=512"
+_: &test-nand
+  # NAND flash: read/prog = 4KiB block = 32KiB
+  - make test TFLAGS+="-nrk -DLFS_READ_SIZE=4096 -DLFS_BLOCK_SIZE=\(32*1024\)"
+# other extreme geometries that are useful for testing various corner cases
+_: &test-no-intrinsics
+  - make test TFLAGS+="-nrk -DLFS_NO_INTRINSICS"
+_: &test-no-inline
+  - make test TFLAGS+="-nrk -DLFS_INLINE_MAX=0"
+_: &test-byte-writes
+  - make test TFLAGS+="-nrk -DLFS_READ_SIZE=1 -DLFS_CACHE_SIZE=1"
+_: &test-block-cycles
+  - make test TFLAGS+="-nrk -DLFS_BLOCK_CYCLES=1"
+_: &test-odd-block-count
+  - make test TFLAGS+="-nrk -DLFS_BLOCK_COUNT=1023 -DLFS_LOOKAHEAD_SIZE=256"
+_: &test-odd-block-size
+  - make test TFLAGS+="-nrk -DLFS_READ_SIZE=11 -DLFS_BLOCK_SIZE=704"
 
-  # run tests
-  - make test QUIET=1
-
-  # run tests with a few different configurations
-  - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=1      -DLFS_CACHE_SIZE=4"
-  - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=512    -DLFS_CACHE_SIZE=512 -DLFS_BLOCK_CYCLES=16"
-  - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=8      -DLFS_CACHE_SIZE=16  -DLFS_BLOCK_CYCLES=2"
-  - make test QUIET=1 CFLAGS+="-DLFS_BLOCK_COUNT=1023 -DLFS_LOOKAHEAD_SIZE=256"
-
-  - make clean test QUIET=1 CFLAGS+="-DLFS_INLINE_MAX=0"
-  - make clean test QUIET=1 CFLAGS+="-DLFS_EMUBD_ERASE_VALUE=0xff"
-  - make clean test QUIET=1 CFLAGS+="-DLFS_NO_INTRINSICS"
-
-  # additional configurations that don't support all tests (this should be
-  # fixed but at the moment it is what it is)
-  - make test_files QUIET=1
-        CFLAGS+="-DLFS_READ_SIZE=1 -DLFS_BLOCK_SIZE=4096"
-  - make test_files QUIET=1
-        CFLAGS+="-DLFS_READ_SIZE=\(2*1024\) -DLFS_BLOCK_SIZE=\(64*1024\)"
-  - make test_files QUIET=1
-        CFLAGS+="-DLFS_READ_SIZE=\(8*1024\) -DLFS_BLOCK_SIZE=\(64*1024\)"
-  - make test_files QUIET=1
-        CFLAGS+="-DLFS_READ_SIZE=11 -DLFS_BLOCK_SIZE=704"
-
+# report size 
+_: &report-size
   # compile and find the code size with the smallest configuration
-  - make clean size
-        OBJ="$(ls lfs*.o | tr '\n' ' ')"
+  - make -j1 clean size
+        OBJ="$(ls lfs*.c | sed 's/\.c/\.o/' | tr '\n' ' ')"
         CFLAGS+="-DLFS_NO_ASSERT -DLFS_NO_DEBUG -DLFS_NO_WARN -DLFS_NO_ERROR"
         | tee sizes
-
   # update status if we succeeded, compare with master if possible
   - |
     if [ "$TRAVIS_TEST_RESULT" -eq 0 ]
@@ -51,7 +72,7 @@ script:
         CURR=$(tail -n1 sizes | awk '{print $1}')
         PREV=$(curl -u "$GEKY_BOT_STATUSES" https://api.github.com/repos/$TRAVIS_REPO_SLUG/status/master \
             | jq -re "select(.sha != \"$TRAVIS_COMMIT\")
-                | .statuses[] | select(.context == \"$STAGE/$NAME\").description
+                | .statuses[] | select(.context == \"${TRAVIS_BUILD_STAGE_NAME,,}/$NAME\").description
                 | capture(\"code size is (?<size>[0-9]+)\").size" \
             || echo 0)
   
@@ -62,257 +83,347 @@ script:
         fi
     fi
 
-# CI matrix
+# stage control
+stages:
+  - name: test
+  - name: deploy
+    if: branch = master AND type = push
+
+# job control
 jobs:
-  include:
-    # native testing
-    - stage: test
-      env:
-        - STAGE=test
-        - NAME=littlefs-x86
+  # native testing
+  - &x86
+    stage: test
+    env:
+      - NAME=littlefs-x86
+    install: *install-common
+    script: [*test-example, *report-size]
+  - {<<: *x86, script: [*test-default,          *report-size]}
+  - {<<: *x86, script: [*test-nor,              *report-size]}
+  - {<<: *x86, script: [*test-emmc,             *report-size]}
+  - {<<: *x86, script: [*test-nand,             *report-size]}
+  - {<<: *x86, script: [*test-no-intrinsics,    *report-size]}
+  - {<<: *x86, script: [*test-no-inline,        *report-size]}
+  - {<<: *x86, script: [*test-byte-writes,      *report-size]}
+  - {<<: *x86, script: [*test-block-cycles,     *report-size]}
+  - {<<: *x86, script: [*test-odd-block-count,  *report-size]}
+  - {<<: *x86, script: [*test-odd-block-size,   *report-size]}
 
-    # cross-compile with ARM (thumb mode)
-    - stage: test
-      env:
-        - STAGE=test
-        - NAME=littlefs-arm
-        - CC="arm-linux-gnueabi-gcc --static -mthumb"
-        - EXEC="qemu-arm"
-      install:
-        - sudo apt-get install
-              gcc-arm-linux-gnueabi
-              libc6-dev-armel-cross
-              qemu-user
-        - arm-linux-gnueabi-gcc --version
-        - qemu-arm -version
+  # cross-compile with ARM (thumb mode)
+  - &arm
+    stage: test
+    env:
+      - NAME=littlefs-arm
+      - CC="arm-linux-gnueabi-gcc --static -mthumb"
+      - TFLAGS="$TFLAGS --exec=qemu-arm"
+    install:
+      - *install-common
+      - sudo apt-get install
+            gcc-arm-linux-gnueabi
+            libc6-dev-armel-cross
+            qemu-user
+      - arm-linux-gnueabi-gcc --version
+      - qemu-arm -version
+    script: [*test-example, *report-size]
+  - {<<: *arm, script: [*test-default,          *report-size]}
+  - {<<: *arm, script: [*test-nor,              *report-size]}
+  - {<<: *arm, script: [*test-emmc,             *report-size]}
+  - {<<: *arm, script: [*test-nand,             *report-size]}
+  - {<<: *arm, script: [*test-no-intrinsics,    *report-size]}
+  - {<<: *arm, script: [*test-no-inline,        *report-size]}
+  # it just takes way to long to run byte-level writes in qemu,
+  # note this is still tested in the native tests
+  #- {<<: *arm, script: [*test-byte-writes,      *report-size]}
+  - {<<: *arm, script: [*test-block-cycles,     *report-size]}
+  - {<<: *arm, script: [*test-odd-block-count,  *report-size]}
+  - {<<: *arm, script: [*test-odd-block-size,   *report-size]}
 
-    # cross-compile with PowerPC
-    - stage: test
-      env:
-        - STAGE=test
-        - NAME=littlefs-powerpc
-        - CC="powerpc-linux-gnu-gcc --static"
-        - EXEC="qemu-ppc"
-      install:
-        - sudo apt-get install
-              gcc-powerpc-linux-gnu
-              libc6-dev-powerpc-cross
-              qemu-user
-        - powerpc-linux-gnu-gcc --version
-        - qemu-ppc -version
+  # cross-compile with MIPS
+  - &mips
+    stage: test
+    env:
+      - NAME=littlefs-mips
+      - CC="mips-linux-gnu-gcc --static"
+      - TFLAGS="$TFLAGS --exec=qemu-mips"
+    install:
+      - *install-common
+      - sudo apt-get install
+            gcc-mips-linux-gnu
+            libc6-dev-mips-cross
+            qemu-user
+      - mips-linux-gnu-gcc --version
+      - qemu-mips -version
+    script: [*test-example, *report-size]
+  - {<<: *mips, script: [*test-default,          *report-size]}
+  - {<<: *mips, script: [*test-nor,              *report-size]}
+  - {<<: *mips, script: [*test-emmc,             *report-size]}
+  - {<<: *mips, script: [*test-nand,             *report-size]}
+  - {<<: *mips, script: [*test-no-intrinsics,    *report-size]}
+  - {<<: *mips, script: [*test-no-inline,        *report-size]}
+  # it just takes way to long to run byte-level writes in qemu,
+  # note this is still tested in the native tests
+  #- {<<: *mips, script: [*test-byte-writes,      *report-size]}
+  - {<<: *mips, script: [*test-block-cycles,     *report-size]}
+  - {<<: *mips, script: [*test-odd-block-count,  *report-size]}
+  - {<<: *mips, script: [*test-odd-block-size,   *report-size]}
 
-    # cross-compile with MIPS
-    - stage: test
-      env:
-        - STAGE=test
-        - NAME=littlefs-mips
-        - CC="mips-linux-gnu-gcc --static"
-        - EXEC="qemu-mips"
-      install:
-        - sudo apt-get install
-              gcc-mips-linux-gnu
-              libc6-dev-mips-cross
-              qemu-user
-        - mips-linux-gnu-gcc --version
-        - qemu-mips -version
+  # cross-compile with PowerPC
+  - &powerpc
+    stage: test
+    env:
+      - NAME=littlefs-powerpc
+      - CC="powerpc-linux-gnu-gcc --static"
+      - TFLAGS="$TFLAGS --exec=qemu-ppc"
+    install:
+      - *install-common
+      - sudo apt-get install
+            gcc-powerpc-linux-gnu
+            libc6-dev-powerpc-cross
+            qemu-user
+      - powerpc-linux-gnu-gcc --version
+      - qemu-ppc -version
+    script: [*test-example, *report-size]
+  - {<<: *powerpc, script: [*test-default,          *report-size]}
+  - {<<: *powerpc, script: [*test-nor,              *report-size]}
+  - {<<: *powerpc, script: [*test-emmc,             *report-size]}
+  - {<<: *powerpc, script: [*test-nand,             *report-size]}
+  - {<<: *powerpc, script: [*test-no-intrinsics,    *report-size]}
+  - {<<: *powerpc, script: [*test-no-inline,        *report-size]}
+  # it just takes way to long to run byte-level writes in qemu,
+  # note this is still tested in the native tests
+  #- {<<: *powerpc, script: [*test-byte-writes,      *report-size]}
+  - {<<: *powerpc, script: [*test-block-cycles,     *report-size]}
+  - {<<: *powerpc, script: [*test-odd-block-count,  *report-size]}
+  - {<<: *powerpc, script: [*test-odd-block-size,   *report-size]}
 
-    # self-host with littlefs-fuse for fuzz test
-    - stage: test
-      env:
-        - STAGE=test
-        - NAME=littlefs-fuse
-      if: branch !~ -prefix$
-      install:
-        - sudo apt-get install libfuse-dev
-        - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v2
-        - fusermount -V
-        - gcc --version
-      before_script:
-        # setup disk for littlefs-fuse
-        - rm -rf littlefs-fuse/littlefs/*
-        - cp -r $(git ls-tree --name-only HEAD) littlefs-fuse/littlefs
+  # test under valgrind, checking for memory errors
+  - &valgrind
+    stage: test
+    env:
+      - NAME=littlefs-valgrind
+    install:
+      - *install-common
+      - sudo apt-get install valgrind
+      - valgrind --version
+    script:
+      - make test TFLAGS+="-k --valgrind"
 
-        - mkdir mount
-        - sudo chmod a+rw /dev/loop0
-        - dd if=/dev/zero bs=512 count=4096 of=disk
-        - losetup /dev/loop0 disk
-      script:
-        # self-host test
-        - make -C littlefs-fuse
+  # self-host with littlefs-fuse for fuzz test
+  - stage: test
+    env:
+      - NAME=littlefs-fuse
+    if: branch !~ -prefix$
+    install:
+      - *install-common
+      - sudo apt-get install libfuse-dev
+      - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v2
+      - fusermount -V
+      - gcc --version
 
-        - littlefs-fuse/lfs --format /dev/loop0
-        - littlefs-fuse/lfs /dev/loop0 mount
+      # setup disk for littlefs-fuse
+      - rm -rf littlefs-fuse/littlefs/*
+      - cp -r $(git ls-tree --name-only HEAD) littlefs-fuse/littlefs
 
-        - ls mount
-        - mkdir mount/littlefs
-        - cp -r $(git ls-tree --name-only HEAD) mount/littlefs
-        - cd mount/littlefs
-        - stat .
-        - ls -flh
-        - make -B test_dirs test_files QUIET=1
+      - mkdir mount
+      - sudo chmod a+rw /dev/loop0
+      - dd if=/dev/zero bs=512 count=128K of=disk
+      - losetup /dev/loop0 disk
+    script:
+      # self-host test
+      - make -C littlefs-fuse
 
-    # self-host with littlefs-fuse for fuzz test
-    - stage: test
-      env:
-        - STAGE=test
-        - NAME=littlefs-migration
-      if: branch !~ -prefix$
-      install:
-        - sudo apt-get install libfuse-dev
-        - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v2 v2
-        - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v1 v1
-        - fusermount -V
-        - gcc --version
-      before_script:
-        # setup disk for littlefs-fuse
-        - rm -rf v2/littlefs/*
-        - cp -r $(git ls-tree --name-only HEAD) v2/littlefs
+      - littlefs-fuse/lfs --format /dev/loop0
+      - littlefs-fuse/lfs /dev/loop0 mount
 
-        - mkdir mount
-        - sudo chmod a+rw /dev/loop0
-        - dd if=/dev/zero bs=512 count=4096 of=disk
-        - losetup /dev/loop0 disk
-      script:
-        # compile v1 and v2
-        - make -C v1
-        - make -C v2
+      - ls mount
+      - mkdir mount/littlefs
+      - cp -r $(git ls-tree --name-only HEAD) mount/littlefs
+      - cd mount/littlefs
+      - stat .
+      - ls -flh
+      - make -B test
 
-        # run self-host test with v1
-        - v1/lfs --format /dev/loop0
-        - v1/lfs /dev/loop0 mount
+  # test migration using littlefs-fuse
+  - stage: test
+    env:
+      - NAME=littlefs-migration
+    if: branch !~ -prefix$
+    install:
+      - *install-common
+      - sudo apt-get install libfuse-dev
+      - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v2 v2
+      - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v1 v1
+      - fusermount -V
+      - gcc --version
 
-        - ls mount
-        - mkdir mount/littlefs
-        - cp -r $(git ls-tree --name-only HEAD) mount/littlefs
-        - cd mount/littlefs
-        - stat .
-        - ls -flh
-        - make -B test_dirs test_files QUIET=1
+      # setup disk for littlefs-fuse
+      - rm -rf v2/littlefs/*
+      - cp -r $(git ls-tree --name-only HEAD) v2/littlefs
 
-        # attempt to migrate
-        - cd ../..
-        - fusermount -u mount
+      - mkdir mount
+      - sudo chmod a+rw /dev/loop0
+      - dd if=/dev/zero bs=512 count=128K of=disk
+      - losetup /dev/loop0 disk
+    script:
+      # compile v1 and v2
+      - make -C v1
+      - make -C v2
 
-        - v2/lfs --migrate /dev/loop0
-        - v2/lfs /dev/loop0 mount
+      # run self-host test with v1
+      - v1/lfs --format /dev/loop0
+      - v1/lfs /dev/loop0 mount
 
-        # run self-host test with v2 right where we left off
-        - ls mount
-        - cd mount/littlefs
-        - stat .
-        - ls -flh
-        - make -B test_dirs test_files QUIET=1
+      - ls mount
+      - mkdir mount/littlefs
+      - cp -r $(git ls-tree --name-only HEAD) mount/littlefs
+      - cd mount/littlefs
+      - stat .
+      - ls -flh
+      - make -B test
 
-    # Automatically create releases
-    - stage: deploy
-      env:
-        - STAGE=deploy
-        - NAME=deploy
-      script:
-        - |
-          bash << 'SCRIPT'
-          set -ev
-          # Find version defined in lfs.h
-          LFS_VERSION=$(grep -ox '#define LFS_VERSION .*' lfs.h | cut -d ' ' -f3)
-          LFS_VERSION_MAJOR=$((0xffff & ($LFS_VERSION >> 16)))
-          LFS_VERSION_MINOR=$((0xffff & ($LFS_VERSION >>  0)))
-          # Grab latests patch from repo tags, default to 0, needs finagling
-          # to get past github's pagination api
-          PREV_URL=https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs/tags/v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.
-          PREV_URL=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL" -I \
-              | sed -n '/^Link/{s/.*<\(.*\)>; rel="last"/\1/;p;q0};$q1' \
-              || echo $PREV_URL)
-          LFS_VERSION_PATCH=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL" \
-              | jq 'map(.ref | match("\\bv.*\\..*\\.(.*)$";"g")
-                  .captures[].string | tonumber) | max + 1' \
-              || echo 0)
-          # We have our new version
-          LFS_VERSION="v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.$LFS_VERSION_PATCH"
-          echo "VERSION $LFS_VERSION"
-          # Check that we're the most recent commit
-          CURRENT_COMMIT=$(curl -f -u "$GEKY_BOT_RELEASES" \
-              https://api.github.com/repos/$TRAVIS_REPO_SLUG/commits/master \
-              | jq -re '.sha')
-          [ "$TRAVIS_COMMIT" == "$CURRENT_COMMIT" ] || exit 0
-          # Create major branch
-          git branch v$LFS_VERSION_MAJOR HEAD
-          # Create major prefix branch
-          git config user.name "geky bot"
-          git config user.email "bot@geky.net"
-          git fetch https://github.com/$TRAVIS_REPO_SLUG.git \
-              --depth=50 v$LFS_VERSION_MAJOR-prefix || true
-          ./scripts/prefix.py lfs$LFS_VERSION_MAJOR
-          git branch v$LFS_VERSION_MAJOR-prefix $( \
-              git commit-tree $(git write-tree) \
-                  $(git rev-parse --verify -q FETCH_HEAD | sed -e 's/^/-p /') \
-                  -p HEAD \
-                  -m "Generated v$LFS_VERSION_MAJOR prefixes")
-          git reset --hard
-          # Update major version branches (vN and vN-prefix)
-          git push --atomic https://$GEKY_BOT_RELEASES@github.com/$TRAVIS_REPO_SLUG.git \
-              v$LFS_VERSION_MAJOR \
-              v$LFS_VERSION_MAJOR-prefix
-          # Build release notes
-          PREV=$(git tag --sort=-v:refname -l "v*" | head -1)
-          if [ ! -z "$PREV" ]
-          then
-              echo "PREV $PREV"
-              CHANGES=$(git log --oneline $PREV.. --grep='^Merge' --invert-grep)
-              printf "CHANGES\n%s\n\n" "$CHANGES"
-          fi
-          case ${GEKY_BOT_DRAFT:-minor} in
-              true)  DRAFT=true ;;
-              minor) DRAFT=$(jq -R 'endswith(".0")' <<< "$LFS_VERSION") ;;
-              false) DRAFT=false ;;
-          esac
-          # Create the release and patch version tag (vN.N.N)
-          curl -f -u "$GEKY_BOT_RELEASES" -X POST \
-              https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases \
-              -d "{
-                  \"tag_name\": \"$LFS_VERSION\",
-                  \"name\": \"${LFS_VERSION%.0}\",
-                  \"target_commitish\": \"$TRAVIS_COMMIT\",
-                  \"draft\": $DRAFT,
-                  \"body\": $(jq -sR '.' <<< "$CHANGES")
-              }" #"
-          SCRIPT
+      # attempt to migrate
+      - cd ../..
+      - fusermount -u mount
 
-# Manage statuses
+      - v2/lfs --migrate /dev/loop0
+      - v2/lfs /dev/loop0 mount
+
+      # run self-host test with v2 right where we left off
+      - ls mount
+      - cd mount/littlefs
+      - stat .
+      - ls -flh
+      - make -B test
+
+  # automatically create releases
+  - stage: deploy
+    env:
+      - NAME=deploy
+    script:
+      - |
+        bash << 'SCRIPT'
+        set -ev
+        # Find version defined in lfs.h
+        LFS_VERSION=$(grep -ox '#define LFS_VERSION .*' lfs.h | cut -d ' ' -f3)
+        LFS_VERSION_MAJOR=$((0xffff & ($LFS_VERSION >> 16)))
+        LFS_VERSION_MINOR=$((0xffff & ($LFS_VERSION >>  0)))
+        # Grab latests patch from repo tags, default to 0, needs finagling
+        # to get past github's pagination api
+        PREV_URL=https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs/tags/v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.
+        PREV_URL=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL" -I \
+            | sed -n '/^Link/{s/.*<\(.*\)>; rel="last"/\1/;p;q0};$q1' \
+            || echo $PREV_URL)
+        LFS_VERSION_PATCH=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL" \
+            | jq 'map(.ref | match("\\bv.*\\..*\\.(.*)$";"g")
+                .captures[].string | tonumber) | max + 1' \
+            || echo 0)
+        # We have our new version
+        LFS_VERSION="v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.$LFS_VERSION_PATCH"
+        echo "VERSION $LFS_VERSION"
+        # Check that we're the most recent commit
+        CURRENT_COMMIT=$(curl -f -u "$GEKY_BOT_RELEASES" \
+            https://api.github.com/repos/$TRAVIS_REPO_SLUG/commits/master \
+            | jq -re '.sha')
+        [ "$TRAVIS_COMMIT" == "$CURRENT_COMMIT" ] || exit 0
+        # Create major branch
+        git branch v$LFS_VERSION_MAJOR HEAD
+        # Create major prefix branch
+        git config user.name "geky bot"
+        git config user.email "bot@geky.net"
+        git fetch https://github.com/$TRAVIS_REPO_SLUG.git \
+            --depth=50 v$LFS_VERSION_MAJOR-prefix || true
+        ./scripts/prefix.py lfs$LFS_VERSION_MAJOR
+        git branch v$LFS_VERSION_MAJOR-prefix $( \
+            git commit-tree $(git write-tree) \
+                $(git rev-parse --verify -q FETCH_HEAD | sed -e 's/^/-p /') \
+                -p HEAD \
+                -m "Generated v$LFS_VERSION_MAJOR prefixes")
+        git reset --hard
+        # Update major version branches (vN and vN-prefix)
+        git push --atomic https://$GEKY_BOT_RELEASES@github.com/$TRAVIS_REPO_SLUG.git \
+            v$LFS_VERSION_MAJOR \
+            v$LFS_VERSION_MAJOR-prefix
+        # Build release notes
+        PREV=$(git tag --sort=-v:refname -l "v*" | head -1)
+        if [ ! -z "$PREV" ]
+        then
+            echo "PREV $PREV"
+            CHANGES=$(git log --oneline $PREV.. --grep='^Merge' --invert-grep)
+            printf "CHANGES\n%s\n\n" "$CHANGES"
+        fi
+        case ${GEKY_BOT_DRAFT:-minor} in
+            true)  DRAFT=true ;;
+            minor) DRAFT=$(jq -R 'endswith(".0")' <<< "$LFS_VERSION") ;;
+            false) DRAFT=false ;;
+        esac
+        # Create the release and patch version tag (vN.N.N)
+        curl -f -u "$GEKY_BOT_RELEASES" -X POST \
+            https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases \
+            -d "{
+                \"tag_name\": \"$LFS_VERSION\",
+                \"name\": \"${LFS_VERSION%.0}\",
+                \"target_commitish\": \"$TRAVIS_COMMIT\",
+                \"draft\": $DRAFT,
+                \"body\": $(jq -sR '.' <<< "$CHANGES")
+            }" #"
+        SCRIPT
+
+# manage statuses
 before_install:
   - |
-    curl -u "$GEKY_BOT_STATUSES" -X POST \
-        https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
-        -d "{
-            \"context\": \"$STAGE/$NAME\",
-            \"state\": \"pending\",
-            \"description\": \"${STATUS:-In progress}\",
-            \"target_url\": \"https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$TRAVIS_JOB_ID\"
-        }"
+    # don't clobber other (not us) failures
+    if ! curl https://api.github.com/repos/$TRAVIS_REPO_SLUG/status/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
+        | jq -e ".statuses[] | select(
+            .context == \"${TRAVIS_BUILD_STAGE_NAME,,}/$NAME\" and
+            .state == \"failure\" and
+            (.target_url | endswith(\"$TRAVIS_JOB_NUMBER\") | not))"
+    then
+        curl -u "$GEKY_BOT_STATUSES" -X POST \
+            https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
+            -d "{
+                \"context\": \"${TRAVIS_BUILD_STAGE_NAME,,}/$NAME\",
+                \"state\": \"pending\",
+                \"description\": \"${STATUS:-In progress}\",
+                \"target_url\": \"$TRAVIS_JOB_WEB_URL#$TRAVIS_JOB_NUMBER\"
+            }"
+    fi
 
 after_failure:
   - |
-    curl -u "$GEKY_BOT_STATUSES" -X POST \
-        https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
-        -d "{
-            \"context\": \"$STAGE/$NAME\",
-            \"state\": \"failure\",
-            \"description\": \"${STATUS:-Failed}\",
-            \"target_url\": \"https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$TRAVIS_JOB_ID\"
-        }"
+    # don't clobber other (not us) failures
+    if ! curl https://api.github.com/repos/$TRAVIS_REPO_SLUG/status/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
+        | jq -e ".statuses[] | select(
+            .context == \"${TRAVIS_BUILD_STAGE_NAME,,}/$NAME\" and
+            .state == \"failure\" and
+            (.target_url | endswith(\"$TRAVIS_JOB_NUMBER\") | not))"
+    then
+        curl -u "$GEKY_BOT_STATUSES" -X POST \
+            https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
+            -d "{
+                \"context\": \"${TRAVIS_BUILD_STAGE_NAME,,}/$NAME\",
+                \"state\": \"failure\",
+                \"description\": \"${STATUS:-Failed}\",
+                \"target_url\": \"$TRAVIS_JOB_WEB_URL#$TRAVIS_JOB_NUMBER\"
+            }"
+    fi
 
 after_success:
   - |
-    curl -u "$GEKY_BOT_STATUSES" -X POST \
-        https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
-        -d "{
-            \"context\": \"$STAGE/$NAME\",
-            \"state\": \"success\",
-            \"description\": \"${STATUS:-Passed}\",
-            \"target_url\": \"https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$TRAVIS_JOB_ID\"
-        }"
-
-# Job control
-stages:
-    - name: test
-    - name: deploy
-      if: branch = master AND type = push
+    # don't clobber other (not us) failures
+    # only update if we were last job to mark in progress,
+    # this isn't perfect but is probably good enough
+    if ! curl https://api.github.com/repos/$TRAVIS_REPO_SLUG/status/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
+        | jq -e ".statuses[] | select(
+            .context == \"${TRAVIS_BUILD_STAGE_NAME,,}/$NAME\" and
+            (.state == \"failure\" or .state == \"pending\") and
+            (.target_url | endswith(\"$TRAVIS_JOB_NUMBER\") | not))"
+    then
+        curl -u "$GEKY_BOT_STATUSES" -X POST \
+            https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
+            -d "{
+                \"context\": \"${TRAVIS_BUILD_STAGE_NAME,,}/$NAME\",
+                \"state\": \"success\",
+                \"description\": \"${STATUS:-Passed}\",
+                \"target_url\": \"$TRAVIS_JOB_WEB_URL#$TRAVIS_JOB_NUMBER\"
+            }"
+    fi

lfs.h

@@ -111,12 +111,14 @@ enum lfs_type {
     LFS_TYPE_INLINESTRUCT   = 0x201,
     LFS_TYPE_SOFTTAIL       = 0x600,
     LFS_TYPE_HARDTAIL       = 0x601,
+    LFS_TYPE_BRANCH         = 0x681,
     LFS_TYPE_MOVESTATE      = 0x7ff,
 
     // internal chip sources
     LFS_FROM_NOOP           = 0x000,
     LFS_FROM_MOVE           = 0x101,
-    LFS_FROM_USERATTRS      = 0x102,
+    LFS_FROM_DROP           = 0x102,
+    LFS_FROM_USERATTRS      = 0x103,
 };
 
 // File open flags
@@ -310,8 +312,11 @@ typedef struct lfs_mdir {
     uint32_t etag;
     uint16_t count;
     bool erased;
+    bool first; // TODO come on
     bool split;
+    bool mustrelocate; // TODO not great either
     lfs_block_t tail[2];
+    lfs_block_t branch[2];
 } lfs_mdir_t;
 
 // littlefs directory type
@@ -366,6 +371,9 @@ typedef struct lfs {
     lfs_cache_t pcache;
 
     lfs_block_t root[2];
+    lfs_block_t relocate_tail[2];
+    lfs_block_t relocate_end[2];
+    bool relocate_do_hack; // TODO fixme
     struct lfs_mlist {
         struct lfs_mlist *next;
         uint16_t id;

scripts/explode_asserts.py

@@ -166,8 +166,8 @@ def mkassert(type, comp, lh, rh, size=None):
         'type': type.lower(), 'TYPE': type.upper(),
         'comp': comp.lower(), 'COMP': comp.upper(),
         'prefix': PREFIX.lower(), 'PREFIX': PREFIX.upper(),
-        'lh': lh.strip(),
-        'rh': rh.strip(),
+        'lh': lh.strip(' '),
+        'rh': rh.strip(' '),
         'size': size,
     }
     if size:

scripts/readmdir.py

@@ -18,9 +18,10 @@ TAG_TYPES = {
     'ctzstruct':    (0x7ff, 0x202),
     'inlinestruct': (0x7ff, 0x201),
     'userattr':     (0x700, 0x300),
-    'tail':         (0x700, 0x600),
+    'tail':         (0x700, 0x600), # TODO rename these?
     'softtail':     (0x7ff, 0x600),
     'hardtail':     (0x7ff, 0x601),
+    'branch':       (0x7ff, 0x681),
     'gstate':       (0x700, 0x700),
     'movestate':    (0x7ff, 0x7ff),
     'crc':          (0x700, 0x500),
@@ -103,7 +104,7 @@ class Tag:
 
     def mkmask(self):
         return Tag(
-            0x700 if self.isunique else 0x7ff,
+            0x780 if self.is_('tail') else 0x700 if self.isunique else 0x7ff, # TODO best way?
             0x3ff if self.isattr else 0,
             0)
 
@@ -233,8 +234,8 @@ class MetadataPair:
 
     def __lt__(self, other):
         # corrupt blocks don't count
-        if not self and other:
-            return True
+        if not self or not other:
+            return bool(other)
 
         # use sequence arithmetic to avoid overflow
         return not ((other.rev - self.rev) & 0x80000000)
@@ -318,6 +319,14 @@ def main(args):
 
     # find most recent pair
     mdir = MetadataPair(blocks)
+    print("mdir {%s} rev %d%s%s" % (
+        ', '.join('%#x' % b
+            for b in [args.block1, args.block2]
+            if b is not None),
+        mdir.rev,
+        ' (was %s)' % ', '.join('%d' % m.rev for m in mdir.pair[1:])
+        if len(mdir.pair) > 1 else '',
+        ' (corrupted!)' if not mdir else ''))
     if args.all:
         mdir.dump_all(truncate=not args.no_truncate)
     elif args.log:

scripts/readtree.py

@@ -5,6 +5,7 @@ import sys
 import json
 import io
 import itertools as it
+import collections as c
 from readmdir import Tag, MetadataPair
 
 def popc(x):
@@ -13,31 +14,29 @@ def popc(x):
 def ctz(x):
     return len(bin(x)) - len(bin(x).rstrip('0'))
 
-def dumpentries(args, mdir, f):
+def dumpentries(args, mdir, mdirs, f):
     for k, id_ in enumerate(mdir.ids):
         name = mdir[Tag('name', id_, 0)]
         struct_ = mdir[Tag('struct', id_, 0)]
 
-        f.write("id %d %s %s" % (
+        desc = "id %d %s %s" % (
             id_, name.typerepr(),
-            json.dumps(name.data.decode('utf8'))))
+            json.dumps(name.data.decode('utf8')))
         if struct_.is_('dirstruct'):
-            f.write(" dir {%#x, %#x}" % struct.unpack(
-                '<II', struct_.data[:8].ljust(8, b'\xff')))
+            pair = struct.unpack('<II', struct_.data[:8].ljust(8, b'\xff'))
+            desc += " dir {%#x, %#x}%s" % (
+                pair[0], pair[1],
+                '?' if frozenset(pair) not in mdirs else '')
         if struct_.is_('ctzstruct'):
-            f.write(" ctz {%#x} size %d" % struct.unpack(
-                '<II', struct_.data[:8].ljust(8, b'\xff')))
+            desc += " ctz {%#x} size %d" % struct.unpack(
+                '<II', struct_.data[:8].ljust(8, b'\xff'))
         if struct_.is_('inlinestruct'):
-            f.write(" inline size %d" % struct_.size)
-        f.write("\n")
+            desc += " inline size %d" % struct_.size
 
-        if args.data and struct_.is_('inlinestruct'):
-            for i in range(0, len(struct_.data), 16):
-                f.write("  %08x: %-47s  %-16s\n" % (
-                    i, ' '.join('%02x' % c for c in struct_.data[i:i+16]),
-                    ''.join(c if c >= ' ' and c <= '~' else '.'
-                        for c in map(chr, struct_.data[i:i+16]))))
-        elif args.data and struct_.is_('ctzstruct'):
+        data = None
+        if struct_.is_('inlinestruct'):
+            data = struct_.data
+        elif struct_.is_('ctzstruct'):
             block, size = struct.unpack(
                 '<II', struct_.data[:8].ljust(8, b'\xff'))
             data = []
@@ -51,43 +50,62 @@ def dumpentries(args, mdir, f):
                     data.append(dat[4*(ctz(i)+1) if i != 0 else 0:])
                     block, = struct.unpack('<I', dat[:4].ljust(4, b'\xff'))
                     i -= 1
-
             data = bytes(it.islice(
                 it.chain.from_iterable(reversed(data)), size))
-            for i in range(0, min(len(data), 256)
-                    if not args.no_truncate else len(data), 16):
+
+        f.write("%-45s%s\n" % (desc,
+            "%-23s  %-8s" % (
+                ' '.join('%02x' % c for c in data[:8]),
+                ''.join(c if c >= ' ' and c <= '~' else '.'
+                    for c in map(chr, data[:8])))
+            if not args.no_truncate and len(desc) < 45
+                and data is not None else ""))
+
+        if name.is_('superblock') and struct_.is_('inlinestruct'):
+            f.write(
+                "  block_size %d\n"
+                "  block_count %d\n"
+                "  name_max %d\n"
+                "  file_max %d\n"
+                "  attr_max %d\n" % struct.unpack(
+                    '<IIIII', struct_.data[4:4+20].ljust(20, b'\xff')))
+
+        for tag in mdir.tags:
+            if tag.id==id_ and tag.is_('userattr'):
+                desc = "%s size %d" % (tag.typerepr(), tag.size)
+                f.write("  %-43s%s\n" % (desc,
+                    "%-23s  %-8s" % (
+                        ' '.join('%02x' % c for c in tag.data[:8]),
+                        ''.join(c if c >= ' ' and c <= '~' else '.'
+                            for c in map(chr, tag.data[:8])))
+                    if not args.no_truncate and len(desc) < 43 else ""))
+
+                if args.no_truncate:
+                    for i in range(0, len(tag.data), 16):
+                        f.write("    %08x: %-47s  %-16s\n" % (
+                            i, ' '.join('%02x' % c for c in tag.data[i:i+16]),
+                            ''.join(c if c >= ' ' and c <= '~' else '.'
+                                for c in map(chr, tag.data[i:i+16]))))
+
+        if args.no_truncate and data is not None:
+            for i in range(0, len(data), 16):
                 f.write("  %08x: %-47s  %-16s\n" % (
                     i, ' '.join('%02x' % c for c in data[i:i+16]),
                     ''.join(c if c >= ' ' and c <= '~' else '.'
                         for c in map(chr, data[i:i+16]))))
 
-        for tag in mdir.tags:
-            if tag.id==id_ and tag.is_('userattr'):
-                f.write("id %d %s size %d\n" % (
-                    id_, tag.typerepr(), tag.size))
-
-                if args.data:
-                    for i in range(0, len(tag.data), 16):
-                        f.write("  %-47s  %-16s\n" % (
-                            ' '.join('%02x' % c for c in tag.data[i:i+16]),
-                            ''.join(c if c >= ' ' and c <= '~' else '.'
-                                for c in map(chr, tag.data[i:i+16]))))
-
 def main(args):
+    superblock = None
+    gstate = b'\0\0\0\0\0\0\0\0\0\0\0\0'
+    mdirs = c.OrderedDict()
+    corrupted = []
+    cycle = False
     with open(args.disk, 'rb') as f:
-        dirs = []
-        superblock = None
-        gstate = b''
-        mdirs = []
-        cycle = False
         tail = (args.block1, args.block2)
-        hard = False
-        while True:
-            for m in it.chain((m for d in dirs for m in d), mdirs):
-                if set(m.blocks) == set(tail):
-                    # cycle detected
-                    cycle = m.blocks
-            if cycle:
+        while tail:
+            if frozenset(tail) in mdirs:
+                # cycle detected
+                cycle = tail
                 break
 
             # load mdir
@@ -110,6 +128,13 @@ def main(args):
             except KeyError:
                 mdir.tail = None
 
+            try:
+                mdir.branch = mdir[Tag('branch', 0, 0)]
+                if mdir.branch.size != 8 or mdir.branch.data == 8*b'\xff':
+                    mdir.branch = None
+            except KeyError:
+                mdir.branch = None
+
             # have superblock?
             try:
                 nsuperblock = mdir[
@@ -126,106 +151,135 @@ def main(args):
             except KeyError:
                 pass
 
-            # add to directories
-            mdirs.append(mdir)
-            if mdir.tail is None or not mdir.tail.is_('hardtail'):
-                dirs.append(mdirs)
-                mdirs = []
+            # corrupted?
+            if not mdir:
+                corrupted.append(mdir)
 
-            if mdir.tail is None:
-                break
+            # add to metadata-pairs
+            mdirs[frozenset(mdir.blocks)] = mdir
+            tail = (struct.unpack('<II', mdir.tail.data)
+                if mdir.tail else None)
 
-            tail = struct.unpack('<II', mdir.tail.data)
-            hard = mdir.tail.is_('hardtail')
-
-    # find paths
-    dirtable = {}
-    for dir in dirs:
-        dirtable[frozenset(dir[0].blocks)] = dir
-
-    pending = [("/", dirs[0])]
+    # derive paths and build directories
+    dirs = {}
+    rogue = {}
+    pending = [('/', (args.block1, args.block2))]
     while pending:
-        path, dir = pending.pop(0)
-        for mdir in dir:
+        path, branch = pending.pop(0)
+        dir = []
+        while branch and frozenset(branch) in mdirs:
+            mdir = mdirs[frozenset(branch)]
+            dir.append(mdir)
+
             for tag in mdir.tags:
                 if tag.is_('dir'):
                     try:
-                        npath = tag.data.decode('utf8')
+                        npath = path + '/' + tag.data.decode('utf8')
+                        npath = npath.replace('//', '/')
                         dirstruct = mdir[Tag('dirstruct', tag.id, 0)]
-                        nblocks = struct.unpack('<II', dirstruct.data)
-                        nmdir = dirtable[frozenset(nblocks)]
-                        pending.append(((path + '/' + npath), nmdir))
+                        npair = struct.unpack('<II', dirstruct.data)
+                        pending.append((npath, npair))
                     except KeyError:
                         pass
 
-        dir[0].path = path.replace('//', '/')
+            branch = (struct.unpack('<II', mdir.branch.data)
+                if mdir.branch else None)
 
-    # dump tree
-    if not args.superblock and not args.gstate and not args.mdirs:
-        args.superblock = True
-        args.gstate = True
-        args.mdirs = True
+        if not dir:
+            rogue[path] = branch
+        else:
+            dirs[path] = dir
 
-    if args.superblock and superblock:
-        print("superblock %s v%d.%d" % (
-            json.dumps(superblock[0].data.decode('utf8')),
-            struct.unpack('<H', superblock[1].data[2:2+2])[0],
-            struct.unpack('<H', superblock[1].data[0:0+2])[0]))
-        print(
-            "  block_size %d\n"
-            "  block_count %d\n"
-            "  name_max %d\n"
-            "  file_max %d\n"
-            "  attr_max %d" % struct.unpack(
-                '<IIIII', superblock[1].data[4:4+20].ljust(20, b'\xff')))
+    # also find orphans
+    not_orphans = {frozenset(mdir.blocks)
+        for dir in dirs.values()
+        for mdir in dir}
+    orphans = []
+    for pair, mdir in mdirs.items():
+        if pair not in not_orphans:
+            if len(orphans) > 0 and (pair == frozenset(
+                    struct.unpack('<II', orphans[-1][-1].tail.data))):
+                orphans[-1].append(mdir)
+            else:
+                orphans.append([mdir])
 
-    if args.gstate and gstate:
-        print("gstate 0x%s" % ''.join('%02x' % c for c in gstate))
-        tag = Tag(struct.unpack('<I', gstate[0:4].ljust(4, b'\xff'))[0])
-        blocks = struct.unpack('<II', gstate[4:4+8].ljust(8, b'\xff'))
-        if tag.size:
-            print("  orphans %d" % tag.size)
-        if tag.type:
-            print("  move dir {%#x, %#x} id %d" % (
-                blocks[0], blocks[1], tag.id))
+    # print littlefs + version info
+    version = ('?', '?')
+    if superblock:
+        version = tuple(reversed(
+            struct.unpack('<HH', superblock[1].data[0:4].ljust(4, b'\xff'))))
+    print("%-47s%s" % ("littlefs v%s.%s" % version,
+        "data (truncated, if it fits)"
+        if not any([args.no_truncate, args.tags, args.log, args.all]) else ""))
 
-    if args.mdirs:
-        for i, dir in enumerate(dirs):
-            print("dir %s" % (json.dumps(dir[0].path)
-                if hasattr(dir[0], 'path') else '(orphan)'))
+    # print gstate
+    badgstate = None
+    print("gstate 0x%s" % ''.join('%02x' % c for c in gstate))
+    tag = Tag(struct.unpack('<I', gstate[0:4].ljust(4, b'\xff'))[0])
+    blocks = struct.unpack('<II', gstate[4:4+8].ljust(8, b'\xff'))
+    if tag.size or not tag.isvalid:
+        print("  orphans >=%d" % max(tag.size, 1))
+    if tag.type:
+        if frozenset(blocks) not in mdirs:
+            badgstate = gstate
+        print("  move dir {%#x, %#x}%s id %d" % (
+            blocks[0], blocks[1],
+            '?' if frozenset(blocks) not in mdirs else '',
+            tag.id))
 
-            for j, mdir in enumerate(dir):
-                print("mdir {%#x, %#x} rev %d%s" % (
-                    mdir.blocks[0], mdir.blocks[1], mdir.rev,
-                    ' (corrupted)' if not mdir else ''))
+    # print dir info
+    for path, dir in it.chain(
+            sorted(dirs.items()),
+            zip(it.repeat(None), orphans)):
+        print("dir %s" % json.dumps(path) if path else "orphaned")
 
-                f = io.StringIO()
-                if args.tags:
-                    mdir.dump_tags(f, truncate=not args.no_truncate)
-                elif args.log:
-                    mdir.dump_log(f, truncate=not args.no_truncate)
-                elif args.all:
-                    mdir.dump_all(f, truncate=not args.no_truncate)
-                else:
-                    dumpentries(args, mdir, f)
+        for j, mdir in enumerate(dir):
+            print("mdir {%#x, %#x} rev %d (was %d)%s%s" % (
+                mdir.blocks[0], mdir.blocks[1], mdir.rev, mdir.pair[1].rev,
+                ' (corrupted!)' if not mdir else '',
+                ' -> {%#x, %#x}' % struct.unpack('<II', mdir.tail.data)
+                if mdir.tail else ''))
 
-                lines = list(filter(None, f.getvalue().split('\n')))
-                for k, line in enumerate(lines):
-                    print("%s %s" % (
-                        ' ' if j == len(dir)-1 else
-                        'v' if k == len(lines)-1 else
-                        '|',
-                        line))
+            f = io.StringIO()
+            if args.tags:
+                mdir.dump_tags(f, truncate=not args.no_truncate)
+            elif args.log:
+                mdir.dump_log(f, truncate=not args.no_truncate)
+            elif args.all:
+                mdir.dump_all(f, truncate=not args.no_truncate)
+            else:
+                dumpentries(args, mdir, mdirs, f)
+
+            lines = list(filter(None, f.getvalue().split('\n')))
+            for k, line in enumerate(lines):
+                print("%s %s" % (
+                    ' ' if j == len(dir)-1 else
+                    'v' if k == len(lines)-1 else
+                    '|' if path else '.',
+                    line))
+
+    errcode = 0
+    for mdir in corrupted:
+        errcode = errcode or 1
+        print("*** corrupted mdir {%#x, %#x}! ***" % (
+            mdir.blocks[0], mdir.blocks[1]))
+
+    for path, pair in rogue.items():
+        errcode = errcode or 2
+        print("*** couldn't find dir %s {%#x, %#x}! ***" % (
+            json.dumps(path), pair[0], pair[1]))
+
+    if badgstate:
+        errcode = errcode or 3
+        print("*** bad gstate 0x%s! ***" %
+            ''.join('%02x' % c for c in gstate))
 
     if cycle:
-        print("*** cycle detected! -> {%#x, %#x} ***" % (cycle[0], cycle[1]))
+        errcode = errcode or 4
+        print("*** cycle detected {%#x, %#x}! ***" % (
+            cycle[0], cycle[1]))
 
-    if cycle:
-        return 2
-    elif not all(mdir for dir in dirs for mdir in dir):
-        return 1
-    else:
-        return 0;
+    return errcode
 
 if __name__ == "__main__":
     import argparse
@@ -242,20 +296,12 @@ if __name__ == "__main__":
     parser.add_argument('block2', nargs='?', default=1,
         type=lambda x: int(x, 0),
         help="Optional second block address for finding the root.")
-    parser.add_argument('-s', '--superblock', action='store_true',
-        help="Show contents of the superblock.")
-    parser.add_argument('-g', '--gstate', action='store_true',
-        help="Show contents of global-state.")
-    parser.add_argument('-m', '--mdirs', action='store_true',
-        help="Show contents of metadata-pairs/directories.")
     parser.add_argument('-t', '--tags', action='store_true',
         help="Show metadata tags instead of reconstructing entries.")
     parser.add_argument('-l', '--log', action='store_true',
         help="Show tags in log.")
     parser.add_argument('-a', '--all', action='store_true',
         help="Show all tags in log, included tags in corrupted commits.")
-    parser.add_argument('-d', '--data', action='store_true',
-        help="Also show the raw contents of files/attrs/tags.")
     parser.add_argument('-T', '--no-truncate', action='store_true',
-        help="Don't truncate large amounts of data.")
+        help="Show the full contents of files/attrs/tags.")
     sys.exit(main(parser.parse_args()))

scripts/test.py

@@ -184,7 +184,8 @@ class TestCase:
         elif self.if_ is not None:
             if_ = self.if_
             while True:
-                for k, v in self.defines.items():
+                for k, v in sorted(self.defines.items(),
+                        key=lambda x: len(x[0]), reverse=True):
                     if k in if_:
                         if_ = if_.replace(k, '(%s)' % v)
                         break
@@ -199,22 +200,25 @@ class TestCase:
             return True
 
     def test(self, exec=[], persist=False, cycles=None,
-            gdb=False, failure=None, **args):
+            gdb=False, failure=None, disk=None, **args):
         # build command
         cmd = exec + ['./%s.test' % self.suite.path,
             repr(self.caseno), repr(self.permno)]
 
         # persist disk or keep in RAM for speed?
         if persist:
+            if not disk:
+                disk = self.suite.path + '.disk'
             if persist != 'noerase':
                 try:
-                    os.remove(self.suite.path + '.disk')
+                    with open(disk, 'w') as f:
+                        f.truncate(0)
                     if args.get('verbose', False):
-                        print('rm', self.suite.path + '.disk')
+                        print('truncate --size=0', disk)
                 except FileNotFoundError:
                     pass
 
-            cmd.append(self.suite.path + '.disk')
+            cmd.append(disk)
 
         # simulate power-loss after n cycles?
         if cycles:
@@ -227,7 +231,7 @@ class TestCase:
                 ncmd.extend(['-ex', 'r'])
                 if failure.assert_:
                     ncmd.extend(['-ex', 'up 2'])
-            elif gdb == 'start':
+            elif gdb == 'start' or isinstance(gdb, int):
                 ncmd.extend([
                     '-ex', 'b %s:%d' % (self.suite.path, self.code_lineno),
                     '-ex', 'r'])
@@ -295,11 +299,17 @@ class ValgrindTestCase(TestCase):
         return not self.leaky and super().shouldtest(**args)
 
     def test(self, exec=[], **args):
-        exec = exec + [
+        verbose = args.get('verbose', False)
+        uninit = (self.defines.get('LFS_ERASE_VALUE', None) == -1)
+        exec = [
             'valgrind',
             '--leak-check=full',
+            ] + (['--undef-value-errors=no'] if uninit else []) + [
+            ] + (['--track-origins=yes'] if not uninit else []) + [
             '--error-exitcode=4',
-            '-q']
+            '--error-limit=no',
+            ] + (['--num-callers=1'] if not verbose else []) + [
+            '-q'] + exec
         return super().test(exec=exec, **args)
 
 class ReentrantTestCase(TestCase):
@@ -310,7 +320,7 @@ class ReentrantTestCase(TestCase):
     def shouldtest(self, **args):
         return self.reentrant and super().shouldtest(**args)
 
-    def test(self, exec=[], persist=False, gdb=False, failure=None, **args):
+    def test(self, persist=False, gdb=False, failure=None, **args):
         for cycles in it.count(1):
             # clear disk first?
             if cycles == 1 and persist != 'noerase':
@@ -319,7 +329,9 @@ class ReentrantTestCase(TestCase):
                 persist = 'noerase'
 
             # exact cycle we should drop into debugger?
-            if gdb and failure and failure.cycleno == cycles:
+            if gdb and failure and (
+                    failure.cycleno == cycles or
+                    (isinstance(gdb, int) and gdb == cycles)):
                 return super().test(gdb=gdb, persist=persist, cycles=cycles,
                     failure=failure, **args)
 
@@ -376,10 +388,11 @@ class TestSuite:
             # code lineno?
             if 'code' in case:
                 case['code_lineno'] = code_linenos.pop()
-            # give our case's config a copy of our "global" config
-            for k, v in config.items():
-                if k not in case:
-                    case[k] = v
+            # merge conditions if necessary
+            if 'if' in config and 'if' in case:
+                case['if'] = '(%s) && (%s)' % (config['if'], case['if'])
+            elif 'if' in config:
+                case['if'] = config['if']
             # initialize test case
             self.cases.append(TestCase(case, filter=filter,
                 suite=self, caseno=i+1, lineno=lineno, **args))
@@ -702,8 +715,6 @@ def main(**args):
                         stdout = perm.result.stdout[:-1]
                     else:
                         stdout = perm.result.stdout
-                    if (not args.get('verbose', False) and len(stdout) > 5):
-                        sys.stdout.write('...\n')
                     for line in stdout[-5:]:
                         sys.stdout.write(line)
                 if perm.result.assert_:
@@ -751,7 +762,8 @@ if __name__ == "__main__":
         help="Store disk image in a file.")
     parser.add_argument('-b', '--build', action='store_true',
         help="Only build the tests, do not execute.")
-    parser.add_argument('-g', '--gdb', choices=['init', 'start', 'assert'],
+    parser.add_argument('-g', '--gdb', metavar='{init,start,assert},CYCLE',
+        type=lambda n: n if n in {'init', 'start', 'assert'} else int(n, 0),
         nargs='?', const='assert',
         help="Drop into gdb on test failure.")
     parser.add_argument('--no-internal', action='store_true',
@@ -764,4 +776,6 @@ if __name__ == "__main__":
         help="Run non-leaky tests under valgrind to check for memory leaks.")
     parser.add_argument('-e', '--exec', default=[], type=lambda e: e.split(' '),
         help="Run tests with another executable prefixed on the command line.")
+    parser.add_argument('-d', '--disk',
+        help="Specify a file to use for persistent/reentrant tests.")
     sys.exit(main(**vars(parser.parse_args())))

tests/test_alloc.toml

@@ -1,9 +1,10 @@
 # allocator tests
-# note for these to work there are many constraints on the device geometry
+# note for these to work there are a number constraints on the device geometry
+if = 'LFS_BLOCK_CYCLES == -1'
 
 [[case]] # parallel allocation test
 define.FILES = 3
-define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-4)) / FILES)'
+define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-6)) / FILES)'
 code = '''
     const char *names[FILES] = {"bacon", "eggs", "pancakes"};
     lfs_file_t files[FILES];
@@ -46,7 +47,7 @@ code = '''
 
 [[case]] # serial allocation test
 define.FILES = 3
-define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-4)) / FILES)'
+define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-6)) / FILES)'
 code = '''
     const char *names[FILES] = {"bacon", "eggs", "pancakes"};
 
@@ -85,7 +86,7 @@ code = '''
 
 [[case]] # parallel allocation reuse test
 define.FILES = 3
-define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-4)) / FILES)'
+define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-6)) / FILES)'
 define.CYCLES = [1, 10]
 code = '''
     const char *names[FILES] = {"bacon", "eggs", "pancakes"};
@@ -140,7 +141,7 @@ code = '''
 
 [[case]] # serial allocation reuse test
 define.FILES = 3
-define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-4)) / FILES)'
+define.SIZE = '(((LFS_BLOCK_SIZE-8)*(LFS_BLOCK_COUNT-6)) / FILES)'
 define.CYCLES = [1, 10]
 code = '''
     const char *names[FILES] = {"bacon", "eggs", "pancakes"};

tests/test_badblocks.toml

@@ -1,3 +1,6 @@
+# bad blocks with block cycles should be tested in test_relocations
+if = 'LFS_BLOCK_CYCLES == -1'
+
 [[case]] # single bad blocks
 define.LFS_BLOCK_COUNT = 256 # small bd so test runs faster
 define.LFS_ERASE_CYCLES = 0xffffffff

tests/test_dirs.toml

@@ -155,7 +155,7 @@ code = '''
 '''
 
 [[case]] # reentrant many directory creation/rename/removal
-define.N = [5, 10] # TODO changed from 20, should we be able to do more?
+define.N = [5, 11]
 reentrant = true
 code = '''
     err = lfs_mount(&lfs, &cfg);
@@ -246,6 +246,8 @@ code = '''
                 LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
         lfs_file_close(&lfs, &file) => 0;
     }
+    // TODO rm me
+    lfs_mkdir(&lfs, "a") => 0;
     lfs_unmount(&lfs) => 0;
 
     lfs_mount(&lfs, &cfg) => 0;
@@ -256,6 +258,9 @@ code = '''
     lfs_dir_read(&lfs, &dir, &info) => 1;
     assert(info.type == LFS_TYPE_DIR);
     assert(strcmp(info.name, "..") == 0);
+    lfs_dir_read(&lfs, &dir, &info) => 1;
+    assert(info.type == LFS_TYPE_DIR);
+    assert(strcmp(info.name, "a") == 0);
     for (int i = 0; i < N; i++) {
         sprintf(path, "file%03d", i);
         lfs_dir_read(&lfs, &dir, &info) => 1;

tests/test_entries.toml

@@ -3,7 +3,7 @@
 # still pass with other inline sizes but wouldn't be testing anything.
 
 define.LFS_CACHE_SIZE = 512
-if = 'LFS_CACHE_SIZE == 512'
+if = 'LFS_CACHE_SIZE % LFS_PROG_SIZE == 0 && LFS_CACHE_SIZE == 512'
 
 [[case]] # entry grow test
 code = '''

tests/test_evil.toml

@@ -0,0 +1,285 @@
+# Tests for recovering from conditions which shouldn't normally
+# happen during normal operation of littlefs
+
+# invalid pointer tests (outside of block_count)
+
+[[case]] # invalid tail-pointer test
+define.TAIL_TYPE = ['LFS_TYPE_HARDTAIL', 'LFS_TYPE_SOFTTAIL']
+define.INVALSET = [0x3, 0x1, 0x2]
+in = "lfs.c"
+code = '''
+    // create littlefs
+    lfs_format(&lfs, &cfg) => 0;
+
+    // change tail-pointer to invalid pointers
+    lfs_init(&lfs, &cfg) => 0;
+    lfs_mdir_t mdir;
+    lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0;
+    lfs_dir_commit(&lfs, &mdir, LFS_MKATTRS(
+            {LFS_MKTAG(LFS_TYPE_HARDTAIL, 0x3ff, 8),
+                (lfs_block_t[2]){
+                    (INVALSET & 0x1) ? 0xcccccccc : 0,
+                    (INVALSET & 0x2) ? 0xcccccccc : 0}})) => 0;
+    lfs_deinit(&lfs) => 0;
+
+    // test that mount fails gracefully
+    lfs_mount(&lfs, &cfg) => LFS_ERR_CORRUPT;
+'''
+
+[[case]] # invalid dir pointer test
+define.INVALSET = [0x3, 0x1, 0x2]
+in = "lfs.c"
+code = '''
+    // create littlefs
+    lfs_format(&lfs, &cfg) => 0;
+    // make a dir
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_mkdir(&lfs, "dir_here") => 0;
+    lfs_unmount(&lfs) => 0;
+
+    // change the dir pointer to be invalid
+    lfs_init(&lfs, &cfg) => 0;
+    lfs_mdir_t mdir;
+    lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0;
+    // make sure id 1 == our directory
+    lfs_dir_get(&lfs, &mdir,
+            LFS_MKTAG(0x700, 0x3ff, 0),
+            LFS_MKTAG(LFS_TYPE_NAME, 1, strlen("dir_here")), buffer)
+                => LFS_MKTAG(LFS_TYPE_DIR, 1, strlen("dir_here"));
+    assert(memcmp((char*)buffer, "dir_here", strlen("dir_here")) == 0);
+    // change dir pointer
+    lfs_dir_commit(&lfs, &mdir, LFS_MKATTRS(
+            {LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 1, 8),
+                (lfs_block_t[2]){
+                    (INVALSET & 0x1) ? 0xcccccccc : 0,
+                    (INVALSET & 0x2) ? 0xcccccccc : 0}})) => 0;
+    lfs_deinit(&lfs) => 0;
+
+    // test that accessing our bad dir fails, note there's a number
+    // of ways to access the dir, some can fail, but some don't
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_stat(&lfs, "dir_here", &info) => 0;
+    assert(strcmp(info.name, "dir_here") == 0);
+    assert(info.type == LFS_TYPE_DIR);
+
+    lfs_dir_open(&lfs, &dir, "dir_here") => LFS_ERR_CORRUPT;
+    lfs_stat(&lfs, "dir_here/file_here", &info) => LFS_ERR_CORRUPT;
+    lfs_dir_open(&lfs, &dir, "dir_here/dir_here") => LFS_ERR_CORRUPT;
+    lfs_file_open(&lfs, &file, "dir_here/file_here",
+            LFS_O_RDONLY) => LFS_ERR_CORRUPT;
+    lfs_file_open(&lfs, &file, "dir_here/file_here",
+            LFS_O_WRONLY | LFS_O_CREAT) => LFS_ERR_CORRUPT;
+    lfs_unmount(&lfs) => 0;
+'''
+
+[[case]] # invalid file pointer test
+in = "lfs.c"
+define.SIZE = [10, 1000, 100000] # faked file size
+code = '''
+    // create littlefs
+    lfs_format(&lfs, &cfg) => 0;
+    // make a file
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file, "file_here",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    lfs_file_close(&lfs, &file) => 0;
+    lfs_unmount(&lfs) => 0;
+
+    // change the file pointer to be invalid
+    lfs_init(&lfs, &cfg) => 0;
+    lfs_mdir_t mdir;
+    lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0;
+    // make sure id 1 == our file
+    lfs_dir_get(&lfs, &mdir,
+            LFS_MKTAG(0x700, 0x3ff, 0),
+            LFS_MKTAG(LFS_TYPE_NAME, 1, strlen("file_here")), buffer)
+                => LFS_MKTAG(LFS_TYPE_REG, 1, strlen("file_here"));
+    assert(memcmp((char*)buffer, "file_here", strlen("file_here")) == 0);
+    // change file pointer
+    lfs_dir_commit(&lfs, &mdir, LFS_MKATTRS(
+            {LFS_MKTAG(LFS_TYPE_CTZSTRUCT, 1, sizeof(struct lfs_ctz)),
+                &(struct lfs_ctz){0xcccccccc, lfs_tole32(SIZE)}})) => 0;
+    lfs_deinit(&lfs) => 0;
+
+    // test that accessing our bad file fails, note there's a number
+    // of ways to access the dir, some can fail, but some don't
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_stat(&lfs, "file_here", &info) => 0;
+    assert(strcmp(info.name, "file_here") == 0);
+    assert(info.type == LFS_TYPE_REG);
+    assert(info.size == SIZE);
+
+    lfs_file_open(&lfs, &file, "file_here", LFS_O_RDONLY) => 0;
+    lfs_file_read(&lfs, &file, buffer, SIZE) => LFS_ERR_CORRUPT;
+    lfs_file_close(&lfs, &file) => 0;
+
+    // any allocs that traverse CTZ must unfortunately must fail
+    if (SIZE > 2*LFS_BLOCK_SIZE) {
+        lfs_mkdir(&lfs, "dir_here") => LFS_ERR_CORRUPT;
+    }
+    lfs_unmount(&lfs) => 0;
+'''
+
+[[case]] # invalid pointer in CTZ skip-list test
+define.SIZE = ['2*LFS_BLOCK_SIZE', '3*LFS_BLOCK_SIZE', '4*LFS_BLOCK_SIZE']
+in = "lfs.c"
+code = '''
+    // create littlefs
+    lfs_format(&lfs, &cfg) => 0;
+    // make a file
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file, "file_here",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    for (int i = 0; i < SIZE; i++) {
+        char c = 'c';
+        lfs_file_write(&lfs, &file, &c, 1) => 1;
+    }
+    lfs_file_close(&lfs, &file) => 0;
+    lfs_unmount(&lfs) => 0;
+    // change pointer in CTZ skip-list to be invalid
+    lfs_init(&lfs, &cfg) => 0;
+    lfs_mdir_t mdir;
+    lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0;
+    // make sure id 1 == our file and get our CTZ structure
+    lfs_dir_get(&lfs, &mdir,
+            LFS_MKTAG(0x700, 0x3ff, 0),
+            LFS_MKTAG(LFS_TYPE_NAME, 1, strlen("file_here")), buffer)
+                => LFS_MKTAG(LFS_TYPE_REG, 1, strlen("file_here"));
+    assert(memcmp((char*)buffer, "file_here", strlen("file_here")) == 0);
+    struct lfs_ctz ctz;
+    lfs_dir_get(&lfs, &mdir,
+            LFS_MKTAG(0x700, 0x3ff, 0),
+            LFS_MKTAG(LFS_TYPE_STRUCT, 1, sizeof(struct lfs_ctz)), &ctz)
+                => LFS_MKTAG(LFS_TYPE_CTZSTRUCT, 1, sizeof(struct lfs_ctz));
+    // rewrite block to contain bad pointer
+    uint8_t bbuffer[LFS_BLOCK_SIZE];
+    cfg.read(&cfg, ctz.head, 0, bbuffer, LFS_BLOCK_SIZE) => 0;
+    uint32_t bad = lfs_tole32(0xcccccccc);
+    memcpy(&bbuffer[0], &bad, sizeof(bad));
+    memcpy(&bbuffer[4], &bad, sizeof(bad));
+    cfg.erase(&cfg, ctz.head) => 0;
+    cfg.prog(&cfg, ctz.head, 0, bbuffer, LFS_BLOCK_SIZE) => 0;
+    lfs_deinit(&lfs) => 0;
+
+    // test that accessing our bad file fails, note there's a number
+    // of ways to access the dir, some can fail, but some don't
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_stat(&lfs, "file_here", &info) => 0;
+    assert(strcmp(info.name, "file_here") == 0);
+    assert(info.type == LFS_TYPE_REG);
+    assert(info.size == SIZE);
+
+    lfs_file_open(&lfs, &file, "file_here", LFS_O_RDONLY) => 0;
+    lfs_file_read(&lfs, &file, buffer, SIZE) => LFS_ERR_CORRUPT;
+    lfs_file_close(&lfs, &file) => 0;
+
+    // any allocs that traverse CTZ must unfortunately must fail
+    if (SIZE > 2*LFS_BLOCK_SIZE) {
+        lfs_mkdir(&lfs, "dir_here") => LFS_ERR_CORRUPT;
+    }
+    lfs_unmount(&lfs) => 0;
+'''
+
+
+[[case]] # invalid gstate pointer
+define.INVALSET = [0x3, 0x1, 0x2]
+in = "lfs.c"
+code = '''
+    // create littlefs
+    lfs_format(&lfs, &cfg) => 0;
+
+    // create an invalid gstate
+    lfs_init(&lfs, &cfg) => 0;
+    lfs_mdir_t mdir;
+    lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0;
+    lfs_fs_prepmove(&lfs, 1, (lfs_block_t [2]){
+            (INVALSET & 0x1) ? 0xcccccccc : 0,
+            (INVALSET & 0x2) ? 0xcccccccc : 0});
+    lfs_dir_commit(&lfs, &mdir, NULL, 0) => 0;
+    lfs_deinit(&lfs) => 0;
+
+    // test that mount fails gracefully
+    // mount may not fail, but our first alloc should fail when
+    // we try to fix the gstate
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_mkdir(&lfs, "should_fail") => LFS_ERR_CORRUPT;
+    lfs_unmount(&lfs) => 0;
+'''
+
+# cycle detection/recovery tests
+
+[[case]] # metadata-pair threaded-list loop test
+in = "lfs.c"
+code = '''
+    // create littlefs
+    lfs_format(&lfs, &cfg) => 0;
+
+    // change tail-pointer to point to ourself
+    lfs_init(&lfs, &cfg) => 0;
+    lfs_mdir_t mdir;
+    lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0;
+    lfs_dir_commit(&lfs, &mdir, LFS_MKATTRS(
+            {LFS_MKTAG(LFS_TYPE_HARDTAIL, 0x3ff, 8),
+                (lfs_block_t[2]){0, 1}})) => 0;
+    lfs_deinit(&lfs) => 0;
+
+    // test that mount fails gracefully
+    lfs_mount(&lfs, &cfg) => LFS_ERR_CORRUPT;
+'''
+
+[[case]] # metadata-pair threaded-list 2-length loop test
+in = "lfs.c"
+code = '''
+    // create littlefs with child dir
+    lfs_format(&lfs, &cfg) => 0;
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_mkdir(&lfs, "child") => 0;
+    lfs_unmount(&lfs) => 0;
+
+    // find child
+    lfs_init(&lfs, &cfg) => 0;
+    lfs_mdir_t mdir;
+    lfs_block_t pair[2];
+    lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0;
+    lfs_dir_get(&lfs, &mdir,
+            LFS_MKTAG(0x7ff, 0x3ff, 0),
+            LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 1, sizeof(pair)), pair)
+                => LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 1, sizeof(pair));
+    // change tail-pointer to point to root
+    lfs_dir_fetch(&lfs, &mdir, pair) => 0;
+    lfs_dir_commit(&lfs, &mdir, LFS_MKATTRS(
+            {LFS_MKTAG(LFS_TYPE_HARDTAIL, 0x3ff, 8),
+                (lfs_block_t[2]){0, 1}})) => 0;
+    lfs_deinit(&lfs) => 0;
+
+    // test that mount fails gracefully
+    lfs_mount(&lfs, &cfg) => LFS_ERR_CORRUPT;
+'''
+
+[[case]] # metadata-pair threaded-list 1-length child loop test
+in = "lfs.c"
+code = '''
+    // create littlefs with child dir
+    lfs_format(&lfs, &cfg) => 0;
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_mkdir(&lfs, "child") => 0;
+    lfs_unmount(&lfs) => 0;
+
+    // find child
+    lfs_init(&lfs, &cfg) => 0;
+    lfs_mdir_t mdir;
+    lfs_block_t pair[2];
+    lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0;
+    lfs_dir_get(&lfs, &mdir,
+            LFS_MKTAG(0x7ff, 0x3ff, 0),
+            LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 1, sizeof(pair)), pair)
+                => LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 1, sizeof(pair));
+    // change tail-pointer to point to ourself
+    lfs_dir_fetch(&lfs, &mdir, pair) => 0;
+    lfs_dir_commit(&lfs, &mdir, LFS_MKATTRS(
+            {LFS_MKTAG(LFS_TYPE_HARDTAIL, 0x3ff, 8), pair})) => 0;
+    lfs_deinit(&lfs) => 0;
+
+    // test that mount fails gracefully
+    lfs_mount(&lfs, &cfg) => LFS_ERR_CORRUPT;
+'''

tests/test_exhaustion.toml

@@ -33,6 +33,9 @@ code = '''
                 lfs_ssize_t res = lfs_file_write(&lfs, &file, &c, 1);
                 assert(res == 1 || res == LFS_ERR_NOSPC);
                 if (res == LFS_ERR_NOSPC) {
+                    err = lfs_file_close(&lfs, &file);
+                    assert(err == 0 || err == LFS_ERR_NOSPC);
+                    lfs_unmount(&lfs) => 0;
                     goto exhausted;
                 }
             }
@@ -40,6 +43,7 @@ code = '''
             err = lfs_file_close(&lfs, &file);
             assert(err == 0 || err == LFS_ERR_NOSPC);
             if (err == LFS_ERR_NOSPC) {
+                lfs_unmount(&lfs) => 0;
                 goto exhausted;
             }
         }
@@ -111,6 +115,9 @@ code = '''
                 lfs_ssize_t res = lfs_file_write(&lfs, &file, &c, 1);
                 assert(res == 1 || res == LFS_ERR_NOSPC);
                 if (res == LFS_ERR_NOSPC) {
+                    err = lfs_file_close(&lfs, &file);
+                    assert(err == 0 || err == LFS_ERR_NOSPC);
+                    lfs_unmount(&lfs) => 0;
                     goto exhausted;
                 }
             }
@@ -118,6 +125,7 @@ code = '''
             err = lfs_file_close(&lfs, &file);
             assert(err == 0 || err == LFS_ERR_NOSPC);
             if (err == LFS_ERR_NOSPC) {
+                lfs_unmount(&lfs) => 0;
                 goto exhausted;
             }
         }
@@ -198,6 +206,9 @@ code = '''
                     lfs_ssize_t res = lfs_file_write(&lfs, &file, &c, 1);
                     assert(res == 1 || res == LFS_ERR_NOSPC);
                     if (res == LFS_ERR_NOSPC) {
+                        err = lfs_file_close(&lfs, &file);
+                        assert(err == 0 || err == LFS_ERR_NOSPC);
+                        lfs_unmount(&lfs) => 0;
                         goto exhausted;
                     }
                 }
@@ -205,6 +216,7 @@ code = '''
                 err = lfs_file_close(&lfs, &file);
                 assert(err == 0 || err == LFS_ERR_NOSPC);
                 if (err == LFS_ERR_NOSPC) {
+                    lfs_unmount(&lfs) => 0;
                     goto exhausted;
                 }
             }
@@ -283,6 +295,9 @@ code = '''
                     lfs_ssize_t res = lfs_file_write(&lfs, &file, &c, 1);
                     assert(res == 1 || res == LFS_ERR_NOSPC);
                     if (res == LFS_ERR_NOSPC) {
+                        err = lfs_file_close(&lfs, &file);
+                        assert(err == 0 || err == LFS_ERR_NOSPC);
+                        lfs_unmount(&lfs) => 0;
                         goto exhausted;
                     }
                 }
@@ -290,6 +305,7 @@ code = '''
                 err = lfs_file_close(&lfs, &file);
                 assert(err == 0 || err == LFS_ERR_NOSPC);
                 if (err == LFS_ERR_NOSPC) {
+                    lfs_unmount(&lfs) => 0;
                     goto exhausted;
                 }
             }
@@ -338,9 +354,9 @@ exhausted:
 define.LFS_ERASE_CYCLES = 0xffffffff
 define.LFS_BLOCK_COUNT = 256 # small bd so test runs faster
 define.LFS_BLOCK_CYCLES = [5, 4, 3, 2, 1]
-#define.LFS_BLOCK_CYCLES = [4, 2]
 define.CYCLES = 100
 define.FILES = 10
+if = 'LFS_BLOCK_CYCLES < CYCLES/10'
 code = '''
     lfs_format(&lfs, &cfg) => 0;
     lfs_mount(&lfs, &cfg) => 0;
@@ -364,6 +380,9 @@ code = '''
                 lfs_ssize_t res = lfs_file_write(&lfs, &file, &c, 1);
                 assert(res == 1 || res == LFS_ERR_NOSPC);
                 if (res == LFS_ERR_NOSPC) {
+                    err = lfs_file_close(&lfs, &file);
+                    assert(err == 0 || err == LFS_ERR_NOSPC);
+                    lfs_unmount(&lfs) => 0;
                     goto exhausted;
                 }
             }
@@ -371,6 +390,7 @@ code = '''
             err = lfs_file_close(&lfs, &file);
             assert(err == 0 || err == LFS_ERR_NOSPC);
             if (err == LFS_ERR_NOSPC) {
+                lfs_unmount(&lfs) => 0;
                 goto exhausted;
             }
         }

tests/test_move.toml

@@ -148,6 +148,7 @@ code = '''
 
 [[case]] # move file corrupt source and dest
 in = "lfs.c"
+if = 'LFS_PROG_SIZE <= 0x3fe' # only works with one crc per commit
 code = '''
     lfs_format(&lfs, &cfg) => 0;
     lfs_mount(&lfs, &cfg) => 0;
@@ -239,6 +240,7 @@ code = '''
 
 [[case]] # move file after corrupt
 in = "lfs.c"
+if = 'LFS_PROG_SIZE <= 0x3fe' # only works with one crc per commit
 code = '''
     lfs_format(&lfs, &cfg) => 0;
     lfs_mount(&lfs, &cfg) => 0;
@@ -593,6 +595,7 @@ code = '''
 
 [[case]] # move dir corrupt source and dest
 in = "lfs.c"
+if = 'LFS_PROG_SIZE <= 0x3fe' # only works with one crc per commit
 code = '''
     lfs_format(&lfs, &cfg) => 0;
     lfs_mount(&lfs, &cfg) => 0;
@@ -692,6 +695,7 @@ code = '''
 
 [[case]] # move dir after corrupt
 in = "lfs.c"
+if = 'LFS_PROG_SIZE <= 0x3fe' # only works with one crc per commit
 code = '''
     lfs_format(&lfs, &cfg) => 0;
     lfs_mount(&lfs, &cfg) => 0;

tests/test_orphans.toml

@@ -1,5 +1,6 @@
 [[case]] # orphan test
 in = "lfs.c"
+if = 'LFS_PROG_SIZE <= 0x3fe' # only works with one crc per commit
 code = '''
     lfs_format(&lfs, &cfg) => 0;
     lfs_mount(&lfs, &cfg) => 0;

tests/test_paths.toml

@@ -247,14 +247,14 @@ code = '''
     lfs_mkdir(&lfs, "coffee/coldcoffee") => 0;
 
     memset(path, 'w', LFS_NAME_MAX+1);
-    path[LFS_NAME_MAX+2] = '\0';
+    path[LFS_NAME_MAX+1] = '\0';
     lfs_mkdir(&lfs, path) => LFS_ERR_NAMETOOLONG;
     lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT)
             => LFS_ERR_NAMETOOLONG;
 
     memcpy(path, "coffee/", strlen("coffee/"));
     memset(path+strlen("coffee/"), 'w', LFS_NAME_MAX+1);
-    path[strlen("coffee/")+LFS_NAME_MAX+2] = '\0';
+    path[strlen("coffee/")+LFS_NAME_MAX+1] = '\0';
     lfs_mkdir(&lfs, path) => LFS_ERR_NAMETOOLONG;
     lfs_file_open(&lfs, &file, path, LFS_O_WRONLY | LFS_O_CREAT)
             => LFS_ERR_NAMETOOLONG;
@@ -270,7 +270,6 @@ code = '''
     lfs_mkdir(&lfs, "coffee/warmcoffee") => 0;
     lfs_mkdir(&lfs, "coffee/coldcoffee") => 0;
 
-    lfs_mount(&lfs, &cfg) => 0;
     memset(path, 'w', LFS_NAME_MAX);
     path[LFS_NAME_MAX] = '\0';
     lfs_mkdir(&lfs, path) => 0;

tests/test_relocations.toml

@@ -148,7 +148,7 @@ code = '''
          # almost every tree operation needs a relocation
 reentrant = true
 # TODO fix this case, caused by non-DAG trees
-if = '!(DEPTH == 3 && LFS_CACHE_SIZE != 64)'
+#if = '!(DEPTH == 3 && LFS_CACHE_SIZE != 64)'
 define = [
     {FILES=6,  DEPTH=1, CYCLES=20, LFS_BLOCK_CYCLES=1},
     {FILES=26, DEPTH=1, CYCLES=20, LFS_BLOCK_CYCLES=1},
@@ -210,7 +210,7 @@ code = '''
 [[case]] # reentrant testing for relocations, but now with random renames!
 reentrant = true
 # TODO fix this case, caused by non-DAG trees
-if = '!(DEPTH == 3 && LFS_CACHE_SIZE != 64)'
+#if = '!(DEPTH == 3 && LFS_CACHE_SIZE != 64)'
 define = [
     {FILES=6,  DEPTH=1, CYCLES=20, LFS_BLOCK_CYCLES=1},
     {FILES=26, DEPTH=1, CYCLES=20, LFS_BLOCK_CYCLES=1},

tests/test_superblocks.toml

@@ -27,41 +27,55 @@ code = '''
 '''
 
 [[case]] # expanding superblock
-define.BLOCK_CYCLES = [32, 33, 1]
+define.LFS_BLOCK_CYCLES = [32, 33, 1]
 define.N = [10, 100, 1000]
 code = '''
     lfs_format(&lfs, &cfg) => 0;
     lfs_mount(&lfs, &cfg) => 0;
     for (int i = 0; i < N; i++) {
-        lfs_mkdir(&lfs, "dummy") => 0;
+        lfs_file_open(&lfs, &file, "dummy",
+                LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
+        lfs_file_close(&lfs, &file) => 0;
         lfs_stat(&lfs, "dummy", &info) => 0;
         assert(strcmp(info.name, "dummy") == 0);
+        assert(info.type == LFS_TYPE_REG);
         lfs_remove(&lfs, "dummy") => 0;
     }
     lfs_unmount(&lfs) => 0;
 
     // one last check after power-cycle
     lfs_mount(&lfs, &cfg) => 0;
-    lfs_mkdir(&lfs, "dummy") => 0;
+    lfs_file_open(&lfs, &file, "dummy",
+            LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
+    lfs_file_close(&lfs, &file) => 0;
     lfs_stat(&lfs, "dummy", &info) => 0;
     assert(strcmp(info.name, "dummy") == 0);
+    assert(info.type == LFS_TYPE_REG);
     lfs_unmount(&lfs) => 0;
 '''
 
 [[case]] # expanding superblock with power cycle
-define.BLOCK_CYCLES = [32, 33, 1]
+define.LFS_BLOCK_CYCLES = [32, 33, 1]
 define.N = [10, 100, 1000]
 code = '''
     lfs_format(&lfs, &cfg) => 0;
     for (int i = 0; i < N; i++) {
         lfs_mount(&lfs, &cfg) => 0;
         // remove lingering dummy?
-        err = lfs_remove(&lfs, "dummy");
+        err = lfs_stat(&lfs, "dummy", &info);
         assert(err == 0 || (err == LFS_ERR_NOENT && i == 0));
+        if (!err) {
+            assert(strcmp(info.name, "dummy") == 0);
+            assert(info.type == LFS_TYPE_REG);
+            lfs_remove(&lfs, "dummy") => 0;
+        }
 
-        lfs_mkdir(&lfs, "dummy") => 0;
+        lfs_file_open(&lfs, &file, "dummy",
+                LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
+        lfs_file_close(&lfs, &file) => 0;
         lfs_stat(&lfs, "dummy", &info) => 0;
         assert(strcmp(info.name, "dummy") == 0);
+        assert(info.type == LFS_TYPE_REG);
         lfs_unmount(&lfs) => 0;
     }
 
@@ -69,11 +83,12 @@ code = '''
     lfs_mount(&lfs, &cfg) => 0;
     lfs_stat(&lfs, "dummy", &info) => 0;
     assert(strcmp(info.name, "dummy") == 0);
+    assert(info.type == LFS_TYPE_REG);
     lfs_unmount(&lfs) => 0;
 '''
 
 [[case]] # reentrant expanding superblock
-define.BLOCK_CYCLES = [2, 1]
+define.LFS_BLOCK_CYCLES = [2, 1]
 define.N = 24
 reentrant = true
 code = '''
@@ -85,12 +100,20 @@ code = '''
 
     for (int i = 0; i < N; i++) {
         // remove lingering dummy?
-        err = lfs_remove(&lfs, "dummy");
+        err = lfs_stat(&lfs, "dummy", &info);
         assert(err == 0 || (err == LFS_ERR_NOENT && i == 0));
+        if (!err) {
+            assert(strcmp(info.name, "dummy") == 0);
+            assert(info.type == LFS_TYPE_REG);
+            lfs_remove(&lfs, "dummy") => 0;
+        }
 
-        lfs_mkdir(&lfs, "dummy") => 0;
+        lfs_file_open(&lfs, &file, "dummy",
+                LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
+        lfs_file_close(&lfs, &file) => 0;
         lfs_stat(&lfs, "dummy", &info) => 0;
         assert(strcmp(info.name, "dummy") == 0);
+        assert(info.type == LFS_TYPE_REG);
     }
 
     lfs_unmount(&lfs) => 0;
@@ -99,5 +122,6 @@ code = '''
     lfs_mount(&lfs, &cfg) => 0;
     lfs_stat(&lfs, "dummy", &info) => 0;
     assert(strcmp(info.name, "dummy") == 0);
+    assert(info.type == LFS_TYPE_REG);
     lfs_unmount(&lfs) => 0;
 '''

tests/test_truncate.toml

@@ -100,7 +100,7 @@ code = '''
     lfs_file_open(&lfs, &file, "sequence",
             LFS_O_RDWR | LFS_O_CREAT | LFS_O_TRUNC) => 0;
 
-    size = lfs.cfg->cache_size;
+    size = lfs_min(lfs.cfg->cache_size, sizeof(buffer)/2);
     lfs_size_t qsize = size / 4;
     uint8_t *wb = buffer;
     uint8_t *rb = buffer + size;