Fixed lookahead overflow and removed unbounded lookahead pointers

As pointed out by davidefer, the lookahead pointer modular arithmetic
does not work around integer overflow when the pointer size is not a
multiple of the block count.

To avoid overflow problems, the easy solution is to stop trying to
work around integer overflows and keep the lookahead offset inside the
block device. To make this work, the ack was modified into a resetable
counter that is decremented every block allocation.

As a plus, quite a bit of the allocation logic ended up simplified.

.travis.yml

@@ -1,107 +1,224 @@
+# Environment variables
 env:
+  global:
     - CFLAGS=-Werror
 
+# Common test script
 script:
-    # make sure example can at least compile
-    - sed -n '/``` c/,/```/{/```/d; p;}' README.md > test.c &&
-      make all size CFLAGS+="
+  # make sure example can at least compile
+  - sed -n '/``` c/,/```/{/```/d; p;}' README.md > test.c &&
+    make all CFLAGS+="
         -Duser_provided_block_device_read=NULL
         -Duser_provided_block_device_prog=NULL
         -Duser_provided_block_device_erase=NULL
         -Duser_provided_block_device_sync=NULL
         -include stdio.h"
 
-    # run tests
-    - make test QUIET=1
+  # run tests
+  - make test QUIET=1
 
-    # run tests with a few different configurations
-    - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=1      -DLFS_PROG_SIZE=1"
-    - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=512    -DLFS_PROG_SIZE=512"
-    - make test QUIET=1 CFLAGS+="-DLFS_BLOCK_COUNT=1023 -DLFS_LOOKAHEAD=2048"
+  # run tests with a few different configurations
+  - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=1      -DLFS_PROG_SIZE=1"
+  - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=512    -DLFS_PROG_SIZE=512"
+  - make test QUIET=1 CFLAGS+="-DLFS_BLOCK_COUNT=1023 -DLFS_LOOKAHEAD=2048"
+
+  - make clean test QUIET=1 CFLAGS+="-DLFS_NO_INTRINSICS"
+
+  # compile and find the code size with the smallest configuration
+  - make clean size
+        OBJ="$(ls lfs*.o | tr '\n' ' ')"
+        CFLAGS+="-DLFS_NO{ASSERT,DEBUG,WARN,ERROR}"
+        | tee sizes
+
+  # update status if we succeeded, compare with master if possible
+  - |
+    if [ "$TRAVIS_TEST_RESULT" -eq 0 ]
+    then
+        CURR=$(tail -n1 sizes | awk '{print $1}')
+        PREV=$(curl -u $GEKY_BOT_STATUSES https://api.github.com/repos/$TRAVIS_REPO_SLUG/status/master \
+            | jq -re "select(.sha != \"$TRAVIS_COMMIT\")
+                | .statuses[] | select(.context == \"$STAGE/$NAME\").description
+                | capture(\"code size is (?<size>[0-9]+)\").size" \
+            || echo 0)
+
+        STATUS="Passed, code size is ${CURR}B"
+        if [ "$PREV" -ne 0 ]
+        then
+            STATUS="$STATUS ($(python -c "print '%+.2f' % (100*($CURR-$PREV)/$PREV.0)")%)"
+        fi
+    fi
+
+# CI matrix
+jobs:
+  include:
+    # native testing
+    - stage: test
+      env:
+        - STAGE=test
+        - NAME=littlefs-x86
+
+    # cross-compile with ARM (thumb mode)
+    - stage: test
+      env:
+        - STAGE=test
+        - NAME=littlefs-arm
+        - CC="arm-linux-gnueabi-gcc --static -mthumb"
+        - EXEC="qemu-arm"
+      install:
+        - sudo apt-get install gcc-arm-linux-gnueabi qemu-user
+        - arm-linux-gnueabi-gcc --version
+        - qemu-arm -version
+
+    # cross-compile with PowerPC
+    - stage: test
+      env:
+        - STAGE=test
+        - NAME=littlefs-powerpc
+        - CC="powerpc-linux-gnu-gcc --static"
+        - EXEC="qemu-ppc"
+      install:
+        - sudo apt-get install gcc-powerpc-linux-gnu qemu-user
+        - powerpc-linux-gnu-gcc --version
+        - qemu-ppc -version
+
+    # cross-compile with MIPS
+    - stage: test
+      env:
+        - STAGE=test
+        - NAME=littlefs-mips
+        - CC="mips-linux-gnu-gcc --static"
+        - EXEC="qemu-mips"
+      install:
+        - sudo add-apt-repository -y "deb http://archive.ubuntu.com/ubuntu/ xenial main universe"
+        - sudo apt-get -qq update
+        - sudo apt-get install gcc-mips-linux-gnu qemu-user
+        - mips-linux-gnu-gcc --version
+        - qemu-mips -version
 
     # self-host with littlefs-fuse for fuzz test
-    - make -C littlefs-fuse
+    - stage: test
+      env:
+        - STAGE=test
+        - NAME=littlefs-fuse
+      install:
+        - sudo apt-get install libfuse-dev
+        - git clone --depth 1 https://github.com/geky/littlefs-fuse
+        - fusermount -V
+        - gcc --version
+      before_script:
+        # setup disk for littlefs-fuse
+        - rm -rf littlefs-fuse/littlefs/*
+        - cp -r $(git ls-tree --name-only HEAD) littlefs-fuse/littlefs
 
-    - littlefs-fuse/lfs --format /dev/loop0
-    - littlefs-fuse/lfs /dev/loop0 mount
+        - mkdir mount
+        - sudo chmod a+rw /dev/loop0
+        - dd if=/dev/zero bs=512 count=2048 of=disk
+        - losetup /dev/loop0 disk
+      script:
+        # self-host test
+        - make -C littlefs-fuse
 
-    - ls mount
-    - mkdir mount/littlefs
-    - cp -r $(git ls-tree --name-only HEAD) mount/littlefs
-    - cd mount/littlefs
-    - ls
-    - make -B test_dirs test_files QUIET=1
+        - littlefs-fuse/lfs --format /dev/loop0
+        - littlefs-fuse/lfs /dev/loop0 mount
 
-before_install:
-    - fusermount -V
-    - gcc --version
+        - ls mount
+        - mkdir mount/littlefs
+        - cp -r $(git ls-tree --name-only HEAD) mount/littlefs
+        - cd mount/littlefs
+        - ls
+        - make -B test_dirs test_files QUIET=1
 
-install:
-    - sudo apt-get install libfuse-dev
-    - git clone --depth 1 https://github.com/geky/littlefs-fuse
-
-before_script:
-    - rm -rf littlefs-fuse/littlefs/*
-    - cp -r $(git ls-tree --name-only HEAD) littlefs-fuse/littlefs
-
-    - mkdir mount
-    - sudo chmod a+rw /dev/loop0
-    - dd if=/dev/zero bs=512 count=2048 of=disk
-    - losetup /dev/loop0 disk
-
-deploy:
-    # Let before_deploy take over
-    provider: script
-    script: 'true'
-    on:
-        branch: master
-
-before_deploy:
-    - cd $TRAVIS_BUILD_DIR
-    # Update tag for version defined in lfs.h
-    - LFS_VERSION=$(grep -ox '#define LFS_VERSION .*' lfs.h | cut -d ' ' -f3)
-    - LFS_VERSION_MAJOR=$((0xffff & ($LFS_VERSION >> 16)))
-    - LFS_VERSION_MINOR=$((0xffff & ($LFS_VERSION >>  0)))
-    - LFS_VERSION="v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR"
-    - echo "littlefs version $LFS_VERSION"
-    - |
-      curl -u $GEKY_BOT -X POST \
-        https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs \
-        -d "{
-          \"ref\": \"refs/tags/$LFS_VERSION\",
-          \"sha\": \"$TRAVIS_COMMIT\"
-        }"
-    - |
-      curl -f -u $GEKY_BOT -X PATCH \
-        https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs/tags/$LFS_VERSION \
-        -d "{
-          \"sha\": \"$TRAVIS_COMMIT\"
-        }"
-    # Create release notes from commits
-    - LFS_PREV_VERSION="v$LFS_VERSION_MAJOR.$(($LFS_VERSION_MINOR-1))"
-    - |
-      if [ $(git tag -l "$LFS_PREV_VERSION") ]
-      then
-        curl -u $GEKY_BOT -X POST \
-            https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases \
+      # Automatically update releases
+    - stage: deploy
+      env:
+        - STAGE=deploy
+        - NAME=deploy
+      script:
+        # Update tag for version defined in lfs.h
+        - LFS_VERSION=$(grep -ox '#define LFS_VERSION .*' lfs.h | cut -d ' ' -f3)
+        - LFS_VERSION_MAJOR=$((0xffff & ($LFS_VERSION >> 16)))
+        - LFS_VERSION_MINOR=$((0xffff & ($LFS_VERSION >>  0)))
+        - LFS_VERSION="v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR"
+        - echo "littlefs version $LFS_VERSION"
+        - |
+          curl -u $GEKY_BOT_RELEASES -X POST \
+            https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs \
             -d "{
-                \"tag_name\": \"$LFS_VERSION\",
-                \"name\": \"$LFS_VERSION\"
+              \"ref\": \"refs/tags/$LFS_VERSION\",
+              \"sha\": \"$TRAVIS_COMMIT\"
             }"
-        RELEASE=$(
-            curl -f https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases/tags/$LFS_VERSION
-        )
-        CHANGES=$(
-            git log --oneline $LFS_PREV_VERSION.. --grep='^Merge' --invert-grep
-        )
-        curl -f -u $GEKY_BOT -X PATCH \
-            https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases/$(
-                jq -r '.id' <<< "$RELEASE"
-            ) \
-            -d "$(
-                jq -s '{
-                    "body": ((.[0] // "" | sub("(?<=\n)#+ Changes.*"; ""; "mi"))
-                        + "### Changes\n\n" + .[1])
-                }' <(jq '.body' <<< "$RELEASE") <(jq -sR '.' <<< "$CHANGES")
-            )"
-      fi
+        - |
+          curl -f -u $GEKY_BOT_RELEASES -X PATCH \
+            https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs/tags/$LFS_VERSION \
+            -d "{
+              \"sha\": \"$TRAVIS_COMMIT\"
+            }"
+        # Create release notes from commits
+        - LFS_PREV_VERSION="v$LFS_VERSION_MAJOR.$(($LFS_VERSION_MINOR-1))"
+        - |
+          if [ $(git tag -l "$LFS_PREV_VERSION") ]
+          then
+            curl -u $GEKY_BOT_RELEASES -X POST \
+                https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases \
+                -d "{
+                    \"tag_name\": \"$LFS_VERSION\",
+                    \"name\": \"$LFS_VERSION\"
+                }"
+            RELEASE=$(
+                curl -f -u $GEKY_BOT_RELEASES \
+                    https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases/tags/$LFS_VERSION
+            )
+            CHANGES=$(
+                git log --oneline $LFS_PREV_VERSION.. --grep='^Merge' --invert-grep
+            )
+            curl -f -u $GEKY_BOT_RELEASES -X PATCH \
+                https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases/$(
+                    jq -r '.id' <<< "$RELEASE"
+                ) \
+                -d "$(
+                    jq -s '{
+                        "body": ((.[0] // "" | sub("(?<=\n)#+ Changes.*"; ""; "mi"))
+                            + "### Changes\n\n" + .[1])
+                    }' <(jq '.body' <<< "$RELEASE") <(jq -sR '.' <<< "$CHANGES")
+                )"
+          fi
+
+# Manage statuses
+before_install:
+  - |
+    curl -u $GEKY_BOT_STATUSES -X POST \
+        https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
+        -d "{
+            \"context\": \"$STAGE/$NAME\",
+            \"state\": \"pending\",
+            \"description\": \"${STATUS:-In progress}\",
+            \"target_url\": \"https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$TRAVIS_JOB_ID\"
+        }"
+
+after_failure:
+  - |
+    curl -u $GEKY_BOT_STATUSES -X POST \
+        https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
+        -d "{
+            \"context\": \"$STAGE/$NAME\",
+            \"state\": \"failure\",
+            \"description\": \"${STATUS:-Failed}\",
+            \"target_url\": \"https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$TRAVIS_JOB_ID\"
+        }"
+
+after_success:
+  - |
+    curl -u $GEKY_BOT_STATUSES -X POST \
+        https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \
+        -d "{
+            \"context\": \"$STAGE/$NAME\",
+            \"state\": \"success\",
+            \"description\": \"${STATUS:-Passed}\",
+            \"target_url\": \"https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$TRAVIS_JOB_ID\"
+        }"
+
+# Job control
+stages:
+    - name: test
+    - name: deploy
+      if: branch = master AND type = push

Makefile

@@ -1,8 +1,8 @@
 TARGET = lfs
 
-CC = gcc
-AR = ar
-SIZE = size
+CC ?= gcc
+AR ?= ar
+SIZE ?= size
 
 SRC += $(wildcard *.c emubd/*.c)
 OBJ := $(SRC:.c=.o)
@@ -33,8 +33,8 @@ size: $(OBJ)
 	$(SIZE) -t $^
 
 .SUFFIXES:
-test: test_format test_dirs test_files test_seek test_truncate test_parallel \
-	test_alloc test_paths test_orphan test_move test_corrupt
+test: test_format test_dirs test_files test_seek test_truncate \
+	test_interspersed test_alloc test_paths test_orphan test_move test_corrupt
 test_%: tests/test_%.sh
 ifdef QUIET
 	@./$< | sed -n '/^[-=]/p'

README.md

@@ -115,10 +115,17 @@ All littlefs have the potential to return a negative error code. The errors
 can be either one of those found in the `enum lfs_error` in [lfs.h](lfs.h),
 or an error returned by the user's block device operations.
 
-It should also be noted that the current implementation of littlefs doesn't
-really do anything to ensure that the data written to disk is machine portable.
-This is fine as long as all of the involved machines share endianness
-(little-endian) and don't have strange padding requirements.
+In the configuration struct, the `prog` and `erase` function provided by the
+user may return a `LFS_ERR_CORRUPT` error if the implementation already can
+detect corrupt blocks. However, the wear leveling does not depend on the return
+code of these functions, instead all data is read back and checked for
+integrity.
+
+If your storage caches writes, make sure that the provided `sync` function
+flushes all the data to memory and ensures that the next read fetches the data
+from memory, otherwise data integrity can not be guaranteed. If the `write`
+function does not perform caching, and therefore each `read` or `write` call
+hits the memory, the `sync` function can simply return 0.
 
 ## Reference material
 

lfs.c

@@ -18,17 +18,13 @@
 #include "lfs.h"
 #include "lfs_util.h"
 
-#include <string.h>
-#include <stdlib.h>
-#include <assert.h>
-
 
 /// Caching block device operations ///
 static int lfs_cache_read(lfs_t *lfs, lfs_cache_t *rcache,
         const lfs_cache_t *pcache, lfs_block_t block,
         lfs_off_t off, void *buffer, lfs_size_t size) {
     uint8_t *data = buffer;
-    assert(block < lfs->cfg->block_count);
+    LFS_ASSERT(block < lfs->cfg->block_count);
 
     while (size > 0) {
         if (pcache && block == pcache->block && off >= pcache->off &&
@@ -153,7 +149,7 @@ static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache,
         lfs_cache_t *rcache, lfs_block_t block,
         lfs_off_t off, const void *buffer, lfs_size_t size) {
     const uint8_t *data = buffer;
-    assert(block < lfs->cfg->block_count);
+    LFS_ASSERT(block < lfs->cfg->block_count);
 
     while (size > 0) {
         if (block == pcache->block && off >= pcache->off &&
@@ -180,7 +176,7 @@ static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache,
 
         // pcache must have been flushed, either by programming and
         // entire block or manually flushing the pcache
-        assert(pcache->block == 0xffffffff);
+        LFS_ASSERT(pcache->block == 0xffffffff);
 
         if (off % lfs->cfg->prog_size == 0 &&
                 size >= lfs->cfg->prog_size) {
@@ -274,8 +270,7 @@ int lfs_deorphan(lfs_t *lfs);
 static int lfs_alloc_lookahead(void *p, lfs_block_t block) {
     lfs_t *lfs = p;
 
-    lfs_block_t off = (((lfs_soff_t)(block - lfs->free.begin)
-                % (lfs_soff_t)(lfs->cfg->block_count))
+    lfs_block_t off = ((block - lfs->free.off)
             + lfs->cfg->block_count) % lfs->cfg->block_count;
 
     if (off < lfs->free.size) {
@@ -287,27 +282,39 @@ static int lfs_alloc_lookahead(void *p, lfs_block_t block) {
 
 static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) {
     while (true) {
-        while (lfs->free.off != lfs->free.size) {
-            lfs_block_t off = lfs->free.off;
-            lfs->free.off += 1;
+        while (lfs->free.i != lfs->free.size) {
+            lfs_block_t off = lfs->free.i;
+            lfs->free.i += 1;
+            lfs->free.ack -= 1;
 
             if (!(lfs->free.buffer[off / 32] & (1U << (off % 32)))) {
                 // found a free block
-                *block = (lfs->free.begin + off) % lfs->cfg->block_count;
+                *block = (lfs->free.off + off) % lfs->cfg->block_count;
+
+                // eagerly find next off so an alloc ack can
+                // discredit old lookahead blocks
+                while (lfs->free.i != lfs->free.size &&
+                        (lfs->free.buffer[lfs->free.i / 32]
+                            & (1U << (lfs->free.i % 32)))) {
+                    lfs->free.i += 1;
+                    lfs->free.ack -= 1;
+                }
+
                 return 0;
             }
         }
 
         // check if we have looked at all blocks since last ack
-        if (lfs->free.off == lfs->free.ack - lfs->free.begin) {
-            LFS_WARN("No more free space %d", lfs->free.off + lfs->free.begin);
+        //if (lfs->free.i == lfs->free.ack - lfs->free.off) {
+        if (lfs->free.ack == 0) {
+            LFS_WARN("No more free space %d", lfs->free.i + lfs->free.off);
             return LFS_ERR_NOSPC;
         }
 
-        lfs->free.begin += lfs->free.size;
-        lfs->free.size = lfs_min(lfs->cfg->lookahead,
-                lfs->free.ack - lfs->free.begin);
-        lfs->free.off = 0;
+        lfs->free.off = (lfs->free.off + lfs->free.size)
+                % lfs->cfg->block_count;
+        lfs->free.size = lfs_min(lfs->cfg->lookahead, lfs->free.ack);
+        lfs->free.i = 0;
 
         // find mask of free blocks from tree
         memset(lfs->free.buffer, 0, lfs->cfg->lookahead/8);
@@ -319,7 +326,49 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) {
 }
 
 static void lfs_alloc_ack(lfs_t *lfs) {
-    lfs->free.ack = lfs->free.off-1 + lfs->free.begin + lfs->cfg->block_count;
+    lfs->free.ack = lfs->cfg->block_count;
+}
+
+
+/// Endian swapping functions ///
+static void lfs_dir_fromle32(struct lfs_disk_dir *d) {
+    d->rev     = lfs_fromle32(d->rev);
+    d->size    = lfs_fromle32(d->size);
+    d->tail[0] = lfs_fromle32(d->tail[0]);
+    d->tail[1] = lfs_fromle32(d->tail[1]);
+}
+
+static void lfs_dir_tole32(struct lfs_disk_dir *d) {
+    d->rev     = lfs_tole32(d->rev);
+    d->size    = lfs_tole32(d->size);
+    d->tail[0] = lfs_tole32(d->tail[0]);
+    d->tail[1] = lfs_tole32(d->tail[1]);
+}
+
+static void lfs_entry_fromle32(struct lfs_disk_entry *d) {
+    d->u.dir[0] = lfs_fromle32(d->u.dir[0]);
+    d->u.dir[1] = lfs_fromle32(d->u.dir[1]);
+}
+
+static void lfs_entry_tole32(struct lfs_disk_entry *d) {
+    d->u.dir[0] = lfs_tole32(d->u.dir[0]);
+    d->u.dir[1] = lfs_tole32(d->u.dir[1]);
+}
+
+static void lfs_superblock_fromle32(struct lfs_disk_superblock *d) {
+    d->root[0]     = lfs_fromle32(d->root[0]);
+    d->root[1]     = lfs_fromle32(d->root[1]);
+    d->block_size  = lfs_fromle32(d->block_size);
+    d->block_count = lfs_fromle32(d->block_count);
+    d->version     = lfs_fromle32(d->version);
+}
+
+static void lfs_superblock_tole32(struct lfs_disk_superblock *d) {
+    d->root[0]     = lfs_tole32(d->root[0]);
+    d->root[1]     = lfs_tole32(d->root[1]);
+    d->block_size  = lfs_tole32(d->block_size);
+    d->block_count = lfs_tole32(d->block_count);
+    d->version     = lfs_tole32(d->version);
 }
 
 
@@ -364,6 +413,7 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_dir_t *dir) {
     // rather than clobbering one of the blocks we just pretend
     // the revision may be valid
     int err = lfs_bd_read(lfs, dir->pair[0], 0, &dir->d.rev, 4);
+    dir->d.rev = lfs_fromle32(dir->d.rev);
     if (err) {
         return err;
     }
@@ -389,6 +439,7 @@ static int lfs_dir_fetch(lfs_t *lfs,
     for (int i = 0; i < 2; i++) {
         struct lfs_disk_dir test;
         int err = lfs_bd_read(lfs, tpair[i], 0, &test, sizeof(test));
+        lfs_dir_fromle32(&test);
         if (err) {
             return err;
         }
@@ -403,7 +454,9 @@ static int lfs_dir_fetch(lfs_t *lfs,
         }
 
         uint32_t crc = 0xffffffff;
+        lfs_dir_tole32(&test);
         lfs_crc(&crc, &test, sizeof(test));
+        lfs_dir_fromle32(&test);
         err = lfs_bd_crc(lfs, tpair[i], sizeof(test),
                 (0x7fffffff & test.size) - sizeof(test), &crc);
         if (err) {
@@ -463,8 +516,10 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir,
             }
 
             uint32_t crc = 0xffffffff;
+            lfs_dir_tole32(&dir->d);
             lfs_crc(&crc, &dir->d, sizeof(dir->d));
             err = lfs_bd_prog(lfs, dir->pair[0], 0, &dir->d, sizeof(dir->d));
+            lfs_dir_fromle32(&dir->d);
             if (err) {
                 if (err == LFS_ERR_CORRUPT) {
                     goto relocate;
@@ -511,7 +566,9 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir,
                 }
             }
 
+            crc = lfs_tole32(crc);
             err = lfs_bd_prog(lfs, dir->pair[0], newoff, &crc, 4);
+            crc = lfs_fromle32(crc);
             if (err) {
                 if (err == LFS_ERR_CORRUPT) {
                     goto relocate;
@@ -584,11 +641,14 @@ relocate:
 }
 
 static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir,
-        const lfs_entry_t *entry, const void *data) {
-    return lfs_dir_commit(lfs, dir, (struct lfs_region[]){
+        lfs_entry_t *entry, const void *data) {
+    lfs_entry_tole32(&entry->d);
+    int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){
             {entry->off, sizeof(entry->d), &entry->d, sizeof(entry->d)},
             {entry->off+sizeof(entry->d), entry->d.nlen, data, entry->d.nlen}
         }, data ? 2 : 1);
+    lfs_entry_fromle32(&entry->d);
+    return err;
 }
 
 static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir,
@@ -597,35 +657,41 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir,
     while (true) {
         if (dir->d.size + lfs_entry_size(entry) <= lfs->cfg->block_size) {
             entry->off = dir->d.size - 4;
-            return lfs_dir_commit(lfs, dir, (struct lfs_region[]){
+
+            lfs_entry_tole32(&entry->d);
+            int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){
                     {entry->off, 0, &entry->d, sizeof(entry->d)},
                     {entry->off, 0, data, entry->d.nlen}
                 }, 2);
+            lfs_entry_fromle32(&entry->d);
+            return err;
         }
 
         // we need to allocate a new dir block
         if (!(0x80000000 & dir->d.size)) {
-            lfs_dir_t newdir;
-            int err = lfs_dir_alloc(lfs, &newdir);
+            lfs_dir_t olddir = *dir;
+            int err = lfs_dir_alloc(lfs, dir);
             if (err) {
                 return err;
             }
 
-            newdir.d.tail[0] = dir->d.tail[0];
-            newdir.d.tail[1] = dir->d.tail[1];
-            entry->off = newdir.d.size - 4;
-            err = lfs_dir_commit(lfs, &newdir, (struct lfs_region[]){
+            dir->d.tail[0] = olddir.d.tail[0];
+            dir->d.tail[1] = olddir.d.tail[1];
+            entry->off = dir->d.size - 4;
+            lfs_entry_tole32(&entry->d);
+            err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){
                     {entry->off, 0, &entry->d, sizeof(entry->d)},
                     {entry->off, 0, data, entry->d.nlen}
                 }, 2);
+            lfs_entry_fromle32(&entry->d);
             if (err) {
                 return err;
             }
 
-            dir->d.size |= 0x80000000;
-            dir->d.tail[0] = newdir.pair[0];
-            dir->d.tail[1] = newdir.pair[1];
-            return lfs_dir_commit(lfs, dir, NULL, 0);
+            olddir.d.size |= 0x80000000;
+            olddir.d.tail[0] = dir->pair[0];
+            olddir.d.tail[1] = dir->pair[1];
+            return lfs_dir_commit(lfs, &olddir, NULL, 0);
         }
 
         int err = lfs_dir_fetch(lfs, dir, dir->d.tail);
@@ -703,6 +769,7 @@ static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) {
 
     int err = lfs_bd_read(lfs, dir->pair[0], dir->off,
             &entry->d, sizeof(entry->d));
+    lfs_entry_fromle32(&entry->d);
     if (err) {
         return err;
     }
@@ -724,6 +791,19 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir,
         pathname += strspn(pathname, "/");
         pathlen = strcspn(pathname, "/");
 
+        // special case for root dir
+        if (pathname[0] == '\0') {
+            *entry = (lfs_entry_t){
+                .d.type = LFS_TYPE_DIR,
+                .d.elen = sizeof(entry->d) - 4,
+                .d.alen = 0,
+                .d.nlen = 0,
+                .d.u.dir[0] = lfs->root[0],
+                .d.u.dir[1] = lfs->root[1],
+            };
+            return 0;
+        }
+
         // skip '.' and root '..'
         if ((pathlen == 1 && memcmp(pathname, ".", 1) == 0) ||
             (pathlen == 2 && memcmp(pathname, "..", 2) == 0)) {
@@ -880,15 +960,6 @@ int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path) {
         return err;
     }
 
-    // check for root, can only be something like '/././../.'
-    if (strspn(path, "/.") == strlen(path)) {
-        dir->head[0] = dir->pair[0];
-        dir->head[1] = dir->pair[1];
-        dir->pos = sizeof(dir->d) - 2;
-        dir->off = sizeof(dir->d);
-        return 0;
-    }
-
     lfs_entry_t entry;
     err = lfs_dir_find(lfs, dir, &entry, &path);
     if (err) {
@@ -1065,11 +1136,12 @@ static int lfs_ctz_find(lfs_t *lfs,
                 lfs_ctz(current));
 
         int err = lfs_cache_read(lfs, rcache, pcache, head, 4*skip, &head, 4);
+        head = lfs_fromle32(head);
         if (err) {
             return err;
         }
 
-        assert(head >= 2 && head <= lfs->cfg->block_count);
+        LFS_ASSERT(head >= 2 && head <= lfs->cfg->block_count);
         current -= 1 << skip;
     }
 
@@ -1089,7 +1161,7 @@ static int lfs_ctz_extend(lfs_t *lfs,
         if (err) {
             return err;
         }
-        assert(nblock >= 2 && nblock <= lfs->cfg->block_count);
+        LFS_ASSERT(nblock >= 2 && nblock <= lfs->cfg->block_count);
 
         if (true) {
             err = lfs_bd_erase(lfs, nblock);
@@ -1140,8 +1212,10 @@ static int lfs_ctz_extend(lfs_t *lfs,
             lfs_size_t skips = lfs_ctz(index) + 1;
 
             for (lfs_off_t i = 0; i < skips; i++) {
+                head = lfs_tole32(head);
                 err = lfs_cache_prog(lfs, pcache, rcache,
                         nblock, 4*i, &head, 4);
+                head = lfs_fromle32(head);
                 if (err) {
                     if (err == LFS_ERR_CORRUPT) {
                         goto relocate;
@@ -1152,12 +1226,13 @@ static int lfs_ctz_extend(lfs_t *lfs,
                 if (i != skips-1) {
                     err = lfs_cache_read(lfs, rcache, NULL,
                             head, 4*i, &head, 4);
+                    head = lfs_fromle32(head);
                     if (err) {
                         return err;
                     }
                 }
 
-                assert(head >= 2 && head <= lfs->cfg->block_count);
+                LFS_ASSERT(head >= 2 && head <= lfs->cfg->block_count);
             }
 
             *block = nblock;
@@ -1196,6 +1271,8 @@ static int lfs_ctz_traverse(lfs_t *lfs,
         lfs_block_t heads[2];
         int count = 2 - (index & 1);
         err = lfs_cache_read(lfs, rcache, pcache, head, 0, &heads, count*4);
+        heads[0] = lfs_fromle32(heads[0]);
+        heads[1] = lfs_fromle32(heads[1]);
         if (err) {
             return err;
         }
@@ -1281,12 +1358,12 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file,
     if (lfs->cfg->file_buffer) {
         file->cache.buffer = lfs->cfg->file_buffer;
     } else if ((file->flags & 3) == LFS_O_RDONLY) {
-        file->cache.buffer = malloc(lfs->cfg->read_size);
+        file->cache.buffer = lfs_malloc(lfs->cfg->read_size);
         if (!file->cache.buffer) {
             return LFS_ERR_NOMEM;
         }
     } else {
-        file->cache.buffer = malloc(lfs->cfg->prog_size);
+        file->cache.buffer = lfs_malloc(lfs->cfg->prog_size);
         if (!file->cache.buffer) {
             return LFS_ERR_NOMEM;
         }
@@ -1312,7 +1389,7 @@ int lfs_file_close(lfs_t *lfs, lfs_file_t *file) {
 
     // clean up memory
     if (!lfs->cfg->file_buffer) {
-        free(file->cache.buffer);
+        lfs_free(file->cache.buffer);
     }
 
     return err;
@@ -1456,11 +1533,12 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) {
         lfs_entry_t entry = {.off = file->poff};
         err = lfs_bd_read(lfs, cwd.pair[0], entry.off,
                 &entry.d, sizeof(entry.d));
+        lfs_entry_fromle32(&entry.d);
         if (err) {
             return err;
         }
 
-        assert(entry.d.type == LFS_TYPE_REG);
+        LFS_ASSERT(entry.d.type == LFS_TYPE_REG);
         entry.d.u.file.head = file->head;
         entry.d.u.file.size = file->size;
 
@@ -1686,7 +1764,7 @@ int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) {
 
         // flush+seek if not already at end
         if (file->pos != oldsize) {
-            int err = lfs_file_seek(lfs, file, 0, SEEK_END);
+            int err = lfs_file_seek(lfs, file, 0, LFS_SEEK_END);
             if (err < 0) {
                 return err;
             }
@@ -1736,14 +1814,6 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) {
 
 /// General fs operations ///
 int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) {
-    // check for root, can only be something like '/././../.'
-    if (strspn(path, "/.") == strlen(path)) {
-        memset(info, 0, sizeof(*info));
-        info->type = LFS_TYPE_DIR;
-        strcpy(info->name, "/");
-        return 0;
-    }
-
     lfs_dir_t cwd;
     int err = lfs_dir_fetch(lfs, &cwd, lfs->root);
     if (err) {
@@ -1762,11 +1832,15 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) {
         info->size = entry.d.u.file.size;
     }
 
-    err = lfs_bd_read(lfs, cwd.pair[0],
-            entry.off + 4+entry.d.elen+entry.d.alen,
-            info->name, entry.d.nlen);
-    if (err) {
-        return err;
+    if (lfs_paircmp(entry.d.u.dir, lfs->root) == 0) {
+        strcpy(info->name, "/");
+    } else {
+        err = lfs_bd_read(lfs, cwd.pair[0],
+                entry.off + 4+entry.d.elen+entry.d.alen,
+                info->name, entry.d.nlen);
+        if (err) {
+            return err;
+        }
     }
 
     return 0;
@@ -1819,7 +1893,7 @@ int lfs_remove(lfs_t *lfs, const char *path) {
             return res;
         }
 
-        assert(res); // must have pred
+        LFS_ASSERT(res); // must have pred
         cwd.d.tail[0] = dir.d.tail[0];
         cwd.d.tail[1] = dir.d.tail[1];
 
@@ -1936,7 +2010,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) {
             return res;
         }
 
-        assert(res); // must have pred
+        LFS_ASSERT(res); // must have pred
         newcwd.d.tail[0] = dir.d.tail[0];
         newcwd.d.tail[1] = dir.d.tail[1];
 
@@ -1959,7 +2033,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) {
     if (lfs->cfg->read_buffer) {
         lfs->rcache.buffer = lfs->cfg->read_buffer;
     } else {
-        lfs->rcache.buffer = malloc(lfs->cfg->read_size);
+        lfs->rcache.buffer = lfs_malloc(lfs->cfg->read_size);
         if (!lfs->rcache.buffer) {
             return LFS_ERR_NOMEM;
         }
@@ -1970,30 +2044,30 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) {
     if (lfs->cfg->prog_buffer) {
         lfs->pcache.buffer = lfs->cfg->prog_buffer;
     } else {
-        lfs->pcache.buffer = malloc(lfs->cfg->prog_size);
+        lfs->pcache.buffer = lfs_malloc(lfs->cfg->prog_size);
         if (!lfs->pcache.buffer) {
             return LFS_ERR_NOMEM;
         }
     }
 
     // setup lookahead, round down to nearest 32-bits
-    assert(lfs->cfg->lookahead % 32 == 0);
-    assert(lfs->cfg->lookahead > 0);
+    LFS_ASSERT(lfs->cfg->lookahead % 32 == 0);
+    LFS_ASSERT(lfs->cfg->lookahead > 0);
     if (lfs->cfg->lookahead_buffer) {
         lfs->free.buffer = lfs->cfg->lookahead_buffer;
     } else {
-        lfs->free.buffer = malloc(lfs->cfg->lookahead/8);
+        lfs->free.buffer = lfs_malloc(lfs->cfg->lookahead/8);
         if (!lfs->free.buffer) {
             return LFS_ERR_NOMEM;
         }
     }
 
     // check that program and read sizes are multiples of the block size
-    assert(lfs->cfg->prog_size % lfs->cfg->read_size == 0);
-    assert(lfs->cfg->block_size % lfs->cfg->prog_size == 0);
+    LFS_ASSERT(lfs->cfg->prog_size % lfs->cfg->read_size == 0);
+    LFS_ASSERT(lfs->cfg->block_size % lfs->cfg->prog_size == 0);
 
     // check that the block size is large enough to fit ctz pointers
-    assert(4*lfs_npw2(0xffffffff / (lfs->cfg->block_size-2*4))
+    LFS_ASSERT(4*lfs_npw2(0xffffffff / (lfs->cfg->block_size-2*4))
             <= lfs->cfg->block_size);
 
     // setup default state
@@ -2009,15 +2083,15 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) {
 static int lfs_deinit(lfs_t *lfs) {
     // free allocated memory
     if (!lfs->cfg->read_buffer) {
-        free(lfs->rcache.buffer);
+        lfs_free(lfs->rcache.buffer);
     }
 
     if (!lfs->cfg->prog_buffer) {
-        free(lfs->pcache.buffer);
+        lfs_free(lfs->pcache.buffer);
     }
 
     if (!lfs->cfg->lookahead_buffer) {
-        free(lfs->free.buffer);
+        lfs_free(lfs->free.buffer);
     }
 
     return 0;
@@ -2031,9 +2105,9 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) {
 
     // create free lookahead
     memset(lfs->free.buffer, 0, lfs->cfg->lookahead/8);
-    lfs->free.begin = 0;
-    lfs->free.size = lfs_min(lfs->cfg->lookahead, lfs->cfg->block_count);
     lfs->free.off = 0;
+    lfs->free.size = lfs_min(lfs->cfg->lookahead, lfs->cfg->block_count);
+    lfs->free.i = 0;
     lfs_alloc_ack(lfs);
 
     // create superblock dir
@@ -2075,6 +2149,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) {
     superdir.d.size = sizeof(superdir.d) + sizeof(superblock.d) + 4;
 
     // write both pairs to be safe
+    lfs_superblock_tole32(&superblock.d);
     bool valid = false;
     for (int i = 0; i < 2; i++) {
         err = lfs_dir_commit(lfs, &superdir, (struct lfs_region[]){
@@ -2109,9 +2184,9 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) {
     }
 
     // setup free lookahead
-    lfs->free.begin = 0;
-    lfs->free.size = 0;
     lfs->free.off = 0;
+    lfs->free.size = 0;
+    lfs->free.i = 0;
     lfs_alloc_ack(lfs);
 
     // load superblock
@@ -2125,6 +2200,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) {
     if (!err) {
         err = lfs_bd_read(lfs, dir.pair[0], sizeof(dir.d),
                 &superblock.d, sizeof(superblock.d));
+        lfs_superblock_fromle32(&superblock.d);
         if (err) {
             return err;
         }
@@ -2134,7 +2210,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) {
     }
 
     if (err || memcmp(superblock.d.magic, "littlefs", 8) != 0) {
-        LFS_ERROR("Invalid superblock at %d %d", dir.pair[0], dir.pair[1]);
+        LFS_ERROR("Invalid superblock at %d %d", 0, 1);
         return LFS_ERR_CORRUPT;
     }
 
@@ -2182,6 +2258,7 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) {
         while (dir.off + sizeof(entry.d) <= (0x7fffffff & dir.d.size)-4) {
             err = lfs_bd_read(lfs, dir.pair[0], dir.off,
                     &entry.d, sizeof(entry.d));
+            lfs_entry_fromle32(&entry.d);
             if (err) {
                 return err;
             }
@@ -2222,7 +2299,7 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) {
             }
         }
     }
-    
+
     return 0;
 }
 
@@ -2256,7 +2333,7 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2],
     if (lfs_pairisnull(lfs->root)) {
         return 0;
     }
-    
+
     parent->d.tail[0] = 0;
     parent->d.tail[1] = 1;
 
@@ -2317,7 +2394,7 @@ static int lfs_moved(lfs_t *lfs, const void *e) {
                 break;
             }
 
-            if (!(0x80 & entry.d.type) && 
+            if (!(0x80 & entry.d.type) &&
                  memcmp(&entry.d.u, e, sizeof(entry.d.u)) == 0) {
                 return true;
             }

lfs.h

@@ -27,7 +27,7 @@
 // Software library version
 // Major (top-nibble), incremented on backwards incompatible changes
 // Minor (bottom-nibble), incremented on feature additions
-#define LFS_VERSION 0x00010002
+#define LFS_VERSION 0x00010003
 #define LFS_VERSION_MAJOR (0xffff & (LFS_VERSION >> 16))
 #define LFS_VERSION_MINOR (0xffff & (LFS_VERSION >>  0))
 
@@ -259,9 +259,9 @@ typedef struct lfs_superblock {
 } lfs_superblock_t;
 
 typedef struct lfs_free {
-    lfs_block_t begin;
-    lfs_block_t size;
     lfs_block_t off;
+    lfs_block_t size;
+    lfs_block_t i;
     lfs_block_t ack;
     uint32_t *buffer;
 } lfs_free_t;
@@ -320,10 +320,6 @@ int lfs_remove(lfs_t *lfs, const char *path);
 // If the destination exists, it must match the source in type.
 // If the destination is a directory, the directory must be empty.
 //
-// Note: If power loss occurs, it is possible that the file or directory
-// will exist in both the oldpath and newpath simultaneously after the
-// next mount.
-//
 // Returns a negative error code on failure.
 int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath);
 

lfs_util.c

@@ -17,7 +17,11 @@
  */
 #include "lfs_util.h"
 
+// Only compile if user does not provide custom config
+#ifndef LFS_CONFIG
 
+
+// Software CRC implementation with small lookup table
 void lfs_crc(uint32_t *restrict crc, const void *buffer, size_t size) {
     static const uint32_t rtable[16] = {
         0x00000000, 0x1db71064, 0x3b6e20c8, 0x26d930ac,
@@ -34,3 +38,5 @@ void lfs_crc(uint32_t *restrict crc, const void *buffer, size_t size) {
     }
 }
 
+
+#endif

lfs_util.h

@@ -18,13 +18,73 @@
 #ifndef LFS_UTIL_H
 #define LFS_UTIL_H
 
-#include <stdlib.h>
+// Users can override lfs_util.h with their own configuration by defining
+// LFS_CONFIG as a header file to include (-DLFS_CONFIG=lfs_config.h).
+//
+// If LFS_CONFIG is used, none of the default utils will be emitted and must be
+// provided by the config file. To start I would suggest copying lfs_util.h and
+// modifying as needed.
+#ifdef LFS_CONFIG
+#define LFS_STRINGIZE(x) LFS_STRINGIZE2(x)
+#define LFS_STRINGIZE2(x) #x
+#include LFS_STRINGIZE(LFS_CONFIG)
+#else
+
+// System includes
 #include <stdint.h>
+#include <stdbool.h>
+#include <string.h>
+
+#ifndef LFS_NO_MALLOC
+#include <stdlib.h>
+#endif
+#ifndef LFS_NO_ASSERT
+#include <assert.h>
+#endif
+#if !defined(LFS_NO_DEBUG) || !defined(LFS_NO_WARN) || !defined(LFS_NO_ERROR)
 #include <stdio.h>
+#endif
 
 
-// Builtin functions, these may be replaced by more
-// efficient implementations in the system
+// Macros, may be replaced by system specific wrappers. Arguments to these
+// macros must not have side-effects as the macros can be removed for a smaller
+// code footprint
+
+// Logging functions
+#ifndef LFS_NO_DEBUG
+#define LFS_DEBUG(fmt, ...) \
+    printf("lfs debug:%d: " fmt "\n", __LINE__, __VA_ARGS__)
+#else
+#define LFS_DEBUG(fmt, ...)
+#endif
+
+#ifndef LFS_NO_WARN
+#define LFS_WARN(fmt, ...) \
+    printf("lfs warn:%d: " fmt "\n", __LINE__, __VA_ARGS__)
+#else
+#define LFS_WARN(fmt, ...)
+#endif
+
+#ifndef LFS_NO_ERROR
+#define LFS_ERROR(fmt, ...) \
+    printf("lfs error:%d: " fmt "\n", __LINE__, __VA_ARGS__)
+#else
+#define LFS_ERROR(fmt, ...)
+#endif
+
+// Runtime assertions
+#ifndef LFS_NO_ASSERT
+#define LFS_ASSERT(test) assert(test)
+#else
+#define LFS_ASSERT(test)
+#endif
+
+
+// Builtin functions, these may be replaced by more efficient
+// toolchain-specific implementations. LFS_NO_INTRINSICS falls back to a more
+// expensive basic C implementation for debugging purposes
+
+// Min/max functions for unsigned 32-bit numbers
 static inline uint32_t lfs_max(uint32_t a, uint32_t b) {
     return (a > b) ? a : b;
 }
@@ -33,31 +93,93 @@ static inline uint32_t lfs_min(uint32_t a, uint32_t b) {
     return (a < b) ? a : b;
 }
 
-static inline uint32_t lfs_ctz(uint32_t a) {
-    return __builtin_ctz(a);
-}
-
+// Find the next smallest power of 2 less than or equal to a
 static inline uint32_t lfs_npw2(uint32_t a) {
+#if !defined(LFS_NO_INTRINSICS) && (defined(__GNUC__) || defined(__CC_ARM))
     return 32 - __builtin_clz(a-1);
+#else
+    uint32_t r = 0;
+    uint32_t s;
+    a -= 1;
+    s = (a > 0xffff) << 4; a >>= s; r |= s;
+    s = (a > 0xff  ) << 3; a >>= s; r |= s;
+    s = (a > 0xf   ) << 2; a >>= s; r |= s;
+    s = (a > 0x3   ) << 1; a >>= s; r |= s;
+    return (r | (a >> 1)) + 1;
+#endif
 }
 
+// Count the number of trailing binary zeros in a
+// lfs_ctz(0) may be undefined
+static inline uint32_t lfs_ctz(uint32_t a) {
+#if !defined(LFS_NO_INTRINSICS) && defined(__GNUC__)
+    return __builtin_ctz(a);
+#else
+    return lfs_npw2((a & -a) + 1) - 1;
+#endif
+}
+
+// Count the number of binary ones in a
 static inline uint32_t lfs_popc(uint32_t a) {
+#if !defined(LFS_NO_INTRINSICS) && (defined(__GNUC__) || defined(__CC_ARM))
     return __builtin_popcount(a);
+#else
+    a = a - ((a >> 1) & 0x55555555);
+    a = (a & 0x33333333) + ((a >> 2) & 0x33333333);
+    return (((a + (a >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24;
+#endif
 }
 
+// Find the sequence comparison of a and b, this is the distance
+// between a and b ignoring overflow
 static inline int lfs_scmp(uint32_t a, uint32_t b) {
     return (int)(unsigned)(a - b);
 }
 
-// CRC-32 with polynomial = 0x04c11db7
+// Convert from 32-bit little-endian to native order
+static inline uint32_t lfs_fromle32(uint32_t a) {
+#if !defined(LFS_NO_INTRINSICS) && ( \
+    (defined(  BYTE_ORDER  ) &&   BYTE_ORDER   ==   ORDER_LITTLE_ENDIAN  ) || \
+    (defined(__BYTE_ORDER  ) && __BYTE_ORDER   == __ORDER_LITTLE_ENDIAN  ) || \
+    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
+    return a;
+#elif !defined(LFS_NO_INTRINSICS) && ( \
+    (defined(  BYTE_ORDER  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
+    (defined(__BYTE_ORDER  ) && __BYTE_ORDER   == __ORDER_BIG_ENDIAN  ) || \
+    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
+    return __builtin_bswap32(a);
+#else
+    return (((uint8_t*)&a)[0] <<  0) |
+           (((uint8_t*)&a)[1] <<  8) |
+           (((uint8_t*)&a)[2] << 16) |
+           (((uint8_t*)&a)[3] << 24);
+#endif
+}
+
+// Convert to 32-bit little-endian from native order
+static inline uint32_t lfs_tole32(uint32_t a) {
+    return lfs_fromle32(a);
+}
+
+// Calculate CRC-32 with polynomial = 0x04c11db7
 void lfs_crc(uint32_t *crc, const void *buffer, size_t size);
 
+// Allocate memory, only used if buffers are not provided to littlefs
+static inline void *lfs_malloc(size_t size) {
+#ifndef LFS_NO_MALLOC
+    return malloc(size);
+#else
+    return NULL;
+#endif
+}
 
-// Logging functions, these may be replaced by system-specific
-// logging functions
-#define LFS_DEBUG(fmt, ...) printf("lfs debug: " fmt "\n", __VA_ARGS__)
-#define LFS_WARN(fmt, ...)  printf("lfs warn: " fmt "\n", __VA_ARGS__)
-#define LFS_ERROR(fmt, ...) printf("lfs error: " fmt "\n", __VA_ARGS__)
+// Deallocate memory, only used if buffers are not provided to littlefs
+static inline void lfs_free(void *p) {
+#ifndef LFS_NO_MALLOC
+    free(p);
+#endif
+}
 
 
 #endif
+#endif

tests/test.py

@@ -33,10 +33,15 @@ def generate(test):
         pass
 
 def compile():
-    subprocess.check_call(['make', '--no-print-directory', '-s'])
+    subprocess.check_call([
+            os.environ.get('MAKE', 'make'),
+            '--no-print-directory', '-s'])
 
 def execute():
-    subprocess.check_call(["./lfs"])
+    if 'EXEC' in os.environ:
+        subprocess.check_call([os.environ['EXEC'], "./lfs"])
+    else:
+        subprocess.check_call(["./lfs"])
 
 def main(test=None):
     if test and not test.startswith('-'):

tests/test_alloc.sh

@@ -110,10 +110,35 @@ lfs_alloc_singleproc multiprocreuse
 lfs_verify multiprocreuse
 lfs_verify singleprocreuse
 
-echo "--- Cleanup ---"
 lfs_remove multiprocreuse
 lfs_remove singleprocreuse
 
+echo "--- Lookahead overflow test ---"
+lfs_mkdir overflow
+for name in bacon eggs pancakes
+do
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+
+//    // setup lookahead to almost overflow
+//    lfs.free.begin = ((lfs_size_t)-1) - 2*$SIZE;
+//    lfs.free.size = 0;
+//    lfs.free.off = 0;
+
+    lfs_file_open(&lfs, &file[0], "overflow/$name",
+            LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0;
+    size = strlen("$name");
+    memcpy(buffer, "$name", size);
+    for (int i = 0; i < $SIZE; i++) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+done
+lfs_verify overflow
+lfs_remove overflow
+
 echo "--- Exhaustion test ---"
 tests/test.py << TEST
     lfs_mount(&lfs, &cfg) => 0;
@@ -289,7 +314,7 @@ tests/test.py << TEST
     }
     lfs_file_close(&lfs, &file[0]) => 0;
 
-    // open whole
+    // open hole
     lfs_remove(&lfs, "bump") => 0;
 
     lfs_mkdir(&lfs, "splitdir") => 0;
@@ -301,5 +326,130 @@ tests/test.py << TEST
     lfs_unmount(&lfs) => 0;
 TEST
 
+echo "--- Outdated lookahead test ---"
+rm -rf blocks
+tests/test.py << TEST
+    lfs_format(&lfs, &cfg) => 0;
+
+    lfs_mount(&lfs, &cfg) => 0;
+
+    // fill completely with two files
+    lfs_file_open(&lfs, &file[0], "exhaustion1",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    size = strlen("blahblahblahblah");
+    memcpy(buffer, "blahblahblahblah", size);
+    for (lfs_size_t i = 0;
+            i < ((cfg.block_count-4)/2)*(cfg.block_size-8);
+            i += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_close(&lfs, &file[0]) => 0;
+
+    lfs_file_open(&lfs, &file[0], "exhaustion2",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    size = strlen("blahblahblahblah");
+    memcpy(buffer, "blahblahblahblah", size);
+    for (lfs_size_t i = 0;
+            i < ((cfg.block_count-4+1)/2)*(cfg.block_size-8);
+            i += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_close(&lfs, &file[0]) => 0;
+
+    // remount to force reset of lookahead
+    lfs_unmount(&lfs) => 0;
+
+    lfs_mount(&lfs, &cfg) => 0;
+
+    // rewrite one file
+    lfs_file_open(&lfs, &file[0], "exhaustion1",
+            LFS_O_WRONLY | LFS_O_TRUNC) => 0;
+    lfs_file_sync(&lfs, &file[0]) => 0;
+    size = strlen("blahblahblahblah");
+    memcpy(buffer, "blahblahblahblah", size);
+    for (lfs_size_t i = 0;
+            i < ((cfg.block_count-4)/2)*(cfg.block_size-8);
+            i += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_close(&lfs, &file[0]) => 0;
+
+    // rewrite second file, this requires lookahead does not
+    // use old population
+    lfs_file_open(&lfs, &file[0], "exhaustion2",
+            LFS_O_WRONLY | LFS_O_TRUNC) => 0;
+    lfs_file_sync(&lfs, &file[0]) => 0;
+    size = strlen("blahblahblahblah");
+    memcpy(buffer, "blahblahblahblah", size);
+    for (lfs_size_t i = 0;
+            i < ((cfg.block_count-4+1)/2)*(cfg.block_size-8);
+            i += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_close(&lfs, &file[0]) => 0;
+TEST
+
+echo "--- Outdated lookahead and split dir test ---"
+rm -rf blocks
+tests/test.py << TEST
+    lfs_format(&lfs, &cfg) => 0;
+    lfs_mount(&lfs, &cfg) => 0;
+
+    // fill completely with two files
+    lfs_file_open(&lfs, &file[0], "exhaustion1",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    size = strlen("blahblahblahblah");
+    memcpy(buffer, "blahblahblahblah", size);
+    for (lfs_size_t i = 0;
+            i < ((cfg.block_count-4)/2)*(cfg.block_size-8);
+            i += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_close(&lfs, &file[0]) => 0;
+
+    lfs_file_open(&lfs, &file[0], "exhaustion2",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    size = strlen("blahblahblahblah");
+    memcpy(buffer, "blahblahblahblah", size);
+    for (lfs_size_t i = 0;
+            i < ((cfg.block_count-4+1)/2)*(cfg.block_size-8);
+            i += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_close(&lfs, &file[0]) => 0;
+
+    // remount to force reset of lookahead
+    lfs_unmount(&lfs) => 0;
+
+    lfs_mount(&lfs, &cfg) => 0;
+
+    // rewrite one file with a hole of one block
+    lfs_file_open(&lfs, &file[0], "exhaustion1",
+            LFS_O_WRONLY | LFS_O_TRUNC) => 0;
+    lfs_file_sync(&lfs, &file[0]) => 0;
+    size = strlen("blahblahblahblah");
+    memcpy(buffer, "blahblahblahblah", size);
+    for (lfs_size_t i = 0;
+            i < ((cfg.block_count-4)/2 - 1)*(cfg.block_size-8);
+            i += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_close(&lfs, &file[0]) => 0;
+
+    // try to allocate a directory, should fail!
+    lfs_mkdir(&lfs, "split") => LFS_ERR_NOSPC;
+
+    // file should not fail
+    lfs_file_open(&lfs, &file[0], "notasplit",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    lfs_file_write(&lfs, &file[0], "hi", 2) => 2;
+    lfs_file_close(&lfs, &file[0]) => 0;
+
+    lfs_unmount(&lfs) => 0;
+TEST
+
+
+
+
 echo "--- Results ---"
 tests/stats.py

tests/test_dirs.sh

@@ -118,6 +118,7 @@ tests/test.py << TEST
         sprintf((char*)buffer, "test%d", i);
         lfs_dir_read(&lfs, &dir[0], &info) => 1;
         strcmp(info.name, (char*)buffer) => 0;
+        info.type => LFS_TYPE_DIR;
     }
     lfs_dir_read(&lfs, &dir[0], &info) => 0;
     lfs_unmount(&lfs) => 0;
@@ -355,5 +356,70 @@ tests/test.py << TEST
     lfs_unmount(&lfs) => 0;
 TEST
 
+echo "--- Multi-block directory with files ---"
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_mkdir(&lfs, "prickly-pear") => 0;
+    for (int i = 0; i < $LARGESIZE; i++) {
+        sprintf((char*)buffer, "prickly-pear/test%d", i);
+        lfs_file_open(&lfs, &file[0], (char*)buffer,
+                LFS_O_WRONLY | LFS_O_CREAT) => 0;
+        size = 6;
+        memcpy(wbuffer, "Hello", size);
+        lfs_file_write(&lfs, &file[0], wbuffer, size) => size;
+        lfs_file_close(&lfs, &file[0]) => 0;
+    }
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_dir_open(&lfs, &dir[0], "prickly-pear") => 0;
+    lfs_dir_read(&lfs, &dir[0], &info) => 1;
+    strcmp(info.name, ".") => 0;
+    info.type => LFS_TYPE_DIR;
+    lfs_dir_read(&lfs, &dir[0], &info) => 1;
+    strcmp(info.name, "..") => 0;
+    info.type => LFS_TYPE_DIR;
+    for (int i = 0; i < $LARGESIZE; i++) {
+        sprintf((char*)buffer, "test%d", i);
+        lfs_dir_read(&lfs, &dir[0], &info) => 1;
+        strcmp(info.name, (char*)buffer) => 0;
+        info.type => LFS_TYPE_REG;
+        info.size => 6;
+    }
+    lfs_dir_read(&lfs, &dir[0], &info) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+
+echo "--- Multi-block remove with files ---"
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_remove(&lfs, "prickly-pear") => LFS_ERR_NOTEMPTY;
+
+    for (int i = 0; i < $LARGESIZE; i++) {
+        sprintf((char*)buffer, "prickly-pear/test%d", i);
+        lfs_remove(&lfs, (char*)buffer) => 0;
+    }
+
+    lfs_remove(&lfs, "prickly-pear") => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_dir_open(&lfs, &dir[0], "/") => 0;
+    lfs_dir_read(&lfs, &dir[0], &info) => 1;
+    strcmp(info.name, ".") => 0;
+    info.type => LFS_TYPE_DIR;
+    lfs_dir_read(&lfs, &dir[0], &info) => 1;
+    strcmp(info.name, "..") => 0;
+    info.type => LFS_TYPE_DIR;
+    lfs_dir_read(&lfs, &dir[0], &info) => 1;
+    strcmp(info.name, "burito") => 0;
+    info.type => LFS_TYPE_REG;
+    lfs_dir_read(&lfs, &dir[0], &info) => 0;
+    lfs_dir_close(&lfs, &dir[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+
 echo "--- Results ---"
 tests/stats.py

tests/test_files.sh

@@ -135,5 +135,24 @@ tests/test.py << TEST
     lfs_unmount(&lfs) => 0;
 TEST
 
+echo "--- Many file test ---"
+tests/test.py << TEST
+    lfs_format(&lfs, &cfg) => 0;
+TEST
+tests/test.py << TEST
+    // Create 300 files of 6 bytes
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_mkdir(&lfs, "directory") => 0;
+    for (unsigned i = 0; i < 300; i++) {
+        snprintf((char*)buffer, sizeof(buffer), "file_%03d", i);
+        lfs_file_open(&lfs, &file[0], (char*)buffer, LFS_O_WRONLY | LFS_O_CREAT) => 0;
+        size = 6;
+        memcpy(wbuffer, "Hello", size);
+        lfs_file_write(&lfs, &file[0], wbuffer, size) => size;
+        lfs_file_close(&lfs, &file[0]) => 0;
+    }
+    lfs_unmount(&lfs) => 0;
+TEST
+
 echo "--- Results ---"
 tests/stats.py

tests/test_interspersed.sh

@@ -1,13 +1,13 @@
 #!/bin/bash
 set -eu
 
-echo "=== Parallel tests ==="
+echo "=== Interspersed tests ==="
 rm -rf blocks
 tests/test.py << TEST
     lfs_format(&lfs, &cfg) => 0;
 TEST
 
-echo "--- Parallel file test ---"
+echo "--- Interspersed file test ---"
 tests/test.py << TEST
     lfs_mount(&lfs, &cfg) => 0;
     lfs_file_open(&lfs, &file[0], "a", LFS_O_WRONLY | LFS_O_CREAT) => 0;
@@ -77,7 +77,7 @@ tests/test.py << TEST
     lfs_unmount(&lfs) => 0;
 TEST
 
-echo "--- Parallel remove file test ---"
+echo "--- Interspersed remove file test ---"
 tests/test.py << TEST
     lfs_mount(&lfs, &cfg) => 0;
     lfs_file_open(&lfs, &file[0], "e", LFS_O_WRONLY | LFS_O_CREAT) => 0;

tests/test_paths.sh

@@ -108,6 +108,10 @@ tests/test.py << TEST
     lfs_stat(&lfs, "/", &info) => 0;
     info.type => LFS_TYPE_DIR;
     strcmp(info.name, "/") => 0;
+
+    lfs_mkdir(&lfs, "/") => LFS_ERR_EXIST;
+    lfs_file_open(&lfs, &file[0], "/", LFS_O_WRONLY | LFS_O_CREAT)
+        => LFS_ERR_ISDIR;
     lfs_unmount(&lfs) => 0;
 TEST