Added scripts/code_size.py, for more in-depth code-size reporting

Inspired by Linux's Bloat-O-Meter, code_size.py wraps nm to provide
function-level code size, and supports detailed comparison between
different builds.

One difference is that code_size.py invokes littlefs's build system
similarly to test.py, creating a duplicate build in the "sizes"
directory. This makes it easy to monitor a cross-compiled build size
while simultaneously testing on the host machine.

.gitignore

@@ -8,3 +8,5 @@ blocks/
 lfs
 test.c
 tests/*.toml.*
+scripts/__pycache__
+.gdb_history

.travis.yml

@@ -208,6 +208,38 @@ jobs:
     script:
       - make test TFLAGS+="-k --valgrind"
 
+  # test compilation in read-only mode
+  - stage: test
+    env:
+      - NAME=littlefs-readonly
+      - CC="arm-linux-gnueabi-gcc --static -mthumb"
+      - CFLAGS="-Werror -DLFS_READONLY"
+    if: branch !~ -prefix$
+    install:
+      - *install-common
+      - sudo apt-get install
+            gcc-arm-linux-gnueabi
+            libc6-dev-armel-cross
+      - arm-linux-gnueabi-gcc --version
+    # report-size will compile littlefs and report the size
+    script: [*report-size]
+
+  # test compilation in thread-safe mode
+  - stage: test
+    env:
+      - NAME=littlefs-threadsafe
+      - CC="arm-linux-gnueabi-gcc --static -mthumb"
+      - CFLAGS="-Werror -DLFS_THREADSAFE"
+    if: branch !~ -prefix$
+    install:
+      - *install-common
+      - sudo apt-get install
+            gcc-arm-linux-gnueabi
+            libc6-dev-armel-cross
+      - arm-linux-gnueabi-gcc --version
+    # report-size will compile littlefs and report the size
+    script: [*report-size]
+
   # self-host with littlefs-fuse for fuzz test
   - stage: test
     env:

Makefile

@@ -6,6 +6,7 @@ endif
 CC ?= gcc
 AR ?= ar
 SIZE ?= size
+NM ?= nm
 
 SRC += $(wildcard *.c bd/*.c)
 OBJ := $(SRC:.c=.o)
@@ -26,11 +27,10 @@ endif
 override CFLAGS += -I.
 override CFLAGS += -std=c99 -Wall -pedantic
 override CFLAGS += -Wextra -Wshadow -Wjump-misses-init -Wundef
-# Remove missing-field-initializers because of GCC bug
-override CFLAGS += -Wno-missing-field-initializers
 
 ifdef VERBOSE
 override TFLAGS += -v
+override SFLAGS += -v
 endif
 
 
@@ -41,6 +41,9 @@ asm: $(ASM)
 size: $(OBJ)
 	$(SIZE) -t $^
 
+code_size:
+	./scripts/code_size.py $(SFLAGS)
+
 test:
 	./scripts/test.py $(TFLAGS)
 .SECONDEXPANSION:
@@ -67,3 +70,4 @@ clean:
 	rm -f $(DEP)
 	rm -f $(ASM)
 	rm -f tests/*.toml.*
+	rm -f sizes/*

README.md

@@ -115,6 +115,9 @@ the filesystem until sync or close is called on the file.
 
 ## Other notes
 
+Littlefs is written in C, and specifically should compile with any compiler
+that conforms to the `C99` standard.
+
 All littlefs calls have the potential to return a negative error code. The
 errors can be either one of those found in the `enum lfs_error` in
 [lfs.h](lfs.h), or an error returned by the user's block device operations.
@@ -219,6 +222,11 @@ License Identifiers that are here available: http://spdx.org/licenses/
   want this, but it is handy for demos.  You can see it in action
   [here][littlefs-js-demo].
   
+- [littlefs-python] - A Python wrapper for littlefs. The project allows you
+  to create images of the filesystem on your PC. Check if littlefs will fit
+  your needs, create images for a later download to the target memory or
+  inspect the content of a binary image of the target memory.
+
 - [mklfs] - A command line tool built by the [Lua RTOS] guys for making
   littlefs images from a host PC. Supports Windows, Mac OS, and Linux.
 
@@ -247,3 +255,4 @@ License Identifiers that are here available: http://spdx.org/licenses/
 [LittleFileSystem]: https://os.mbed.com/docs/mbed-os/v5.12/apis/littlefilesystem.html
 [SPIFFS]: https://github.com/pellepl/spiffs
 [Dhara]: https://github.com/dlbeer/dhara
+[littlefs-python]: https://pypi.org/project/littlefs-python/

SPEC.md

@@ -289,7 +289,7 @@ Layout of the name tag:
 ```
         tag                          data
 [--      32      --][---        variable length        ---]
-[1| 3| 8 | 10 | 10 ][---            (size)             ---]
+[1| 3| 8 | 10 | 10 ][---          (size * 8)           ---]
  ^  ^  ^    ^    ^- size                   ^- file name
  |  |  |    '------ id
  |  |  '----------- file type
@@ -470,7 +470,7 @@ Layout of the inline-struct tag:
 ```
         tag                          data
 [--      32      --][---        variable length        ---]
-[1|- 11 -| 10 | 10 ][---            (size)             ---]
+[1|- 11 -| 10 | 10 ][---           (size * 8)          ---]
  ^    ^     ^    ^- size                    ^- inline data
  |    |     '------ id
  |    '------------ type (0x201)
@@ -556,7 +556,7 @@ Layout of the user-attr tag:
 ```
         tag                          data
 [--      32      --][---        variable length        ---]
-[1| 3| 8 | 10 | 10 ][---            (size)             ---]
+[1| 3| 8 | 10 | 10 ][---           (size * 8)          ---]
  ^  ^  ^    ^    ^- size                    ^- attr data
  |  |  |    '------ id
  |  |  '----------- attr type
@@ -764,9 +764,9 @@ Layout of the CRC tag:
 ```
         tag                                    data
 [--      32      --][--      32      --|---        variable length        ---]
-[1| 3| 8 | 10 | 10 ][--      32      --|---            (size)             ---]
+[1| 3| 8 | 10 | 10 ][--      32      --|---        (size * 8 - 32)        ---]
  ^  ^  ^    ^    ^            ^- crc                             ^- padding
- |  |  |    |    '- size (12)
+ |  |  |    |    '- size
  |  |  |    '------ id (0x3ff)
  |  |  '----------- valid state
  |  '-------------- type1 (0x5)

bd/lfs_filebd.c

@@ -12,7 +12,7 @@
 
 int lfs_filebd_createcfg(const struct lfs_config *cfg, const char *path,
         const struct lfs_filebd_config *bdcfg) {
-    LFS_TRACE("lfs_filebd_createcfg(%p {.context=%p, "
+    LFS_FILEBD_TRACE("lfs_filebd_createcfg(%p {.context=%p, "
                 ".read=%p, .prog=%p, .erase=%p, .sync=%p, "
                 ".read_size=%"PRIu32", .prog_size=%"PRIu32", "
                 ".block_size=%"PRIu32", .block_count=%"PRIu32"}, "
@@ -30,16 +30,16 @@ int lfs_filebd_createcfg(const struct lfs_config *cfg, const char *path,
     bd->fd = open(path, O_RDWR | O_CREAT, 0666);
     if (bd->fd < 0) {
         int err = -errno;
-        LFS_TRACE("lfs_filebd_createcfg -> %d", err);
+        LFS_FILEBD_TRACE("lfs_filebd_createcfg -> %d", err);
         return err;
     }
 
-    LFS_TRACE("lfs_filebd_createcfg -> %d", 0);
+    LFS_FILEBD_TRACE("lfs_filebd_createcfg -> %d", 0);
     return 0;
 }
 
 int lfs_filebd_create(const struct lfs_config *cfg, const char *path) {
-    LFS_TRACE("lfs_filebd_create(%p {.context=%p, "
+    LFS_FILEBD_TRACE("lfs_filebd_create(%p {.context=%p, "
                 ".read=%p, .prog=%p, .erase=%p, .sync=%p, "
                 ".read_size=%"PRIu32", .prog_size=%"PRIu32", "
                 ".block_size=%"PRIu32", .block_count=%"PRIu32"}, "
@@ -51,26 +51,27 @@ int lfs_filebd_create(const struct lfs_config *cfg, const char *path) {
             path);
     static const struct lfs_filebd_config defaults = {.erase_value=-1};
     int err = lfs_filebd_createcfg(cfg, path, &defaults);
-    LFS_TRACE("lfs_filebd_create -> %d", err);
+    LFS_FILEBD_TRACE("lfs_filebd_create -> %d", err);
     return err;
 }
 
 int lfs_filebd_destroy(const struct lfs_config *cfg) {
-    LFS_TRACE("lfs_filebd_destroy(%p)", (void*)cfg);
+    LFS_FILEBD_TRACE("lfs_filebd_destroy(%p)", (void*)cfg);
     lfs_filebd_t *bd = cfg->context;
     int err = close(bd->fd);
     if (err < 0) {
         err = -errno;
-        LFS_TRACE("lfs_filebd_destroy -> %d", err);
+        LFS_FILEBD_TRACE("lfs_filebd_destroy -> %d", err);
         return err;
     }
-    LFS_TRACE("lfs_filebd_destroy -> %d", 0);
+    LFS_FILEBD_TRACE("lfs_filebd_destroy -> %d", 0);
     return 0;
 }
 
 int lfs_filebd_read(const struct lfs_config *cfg, lfs_block_t block,
         lfs_off_t off, void *buffer, lfs_size_t size) {
-    LFS_TRACE("lfs_filebd_read(%p, 0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
+    LFS_FILEBD_TRACE("lfs_filebd_read(%p, "
+                "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
             (void*)cfg, block, off, buffer, size);
     lfs_filebd_t *bd = cfg->context;
 
@@ -89,24 +90,24 @@ int lfs_filebd_read(const struct lfs_config *cfg, lfs_block_t block,
             (off_t)block*cfg->block_size + (off_t)off, SEEK_SET);
     if (res1 < 0) {
         int err = -errno;
-        LFS_TRACE("lfs_filebd_read -> %d", err);
+        LFS_FILEBD_TRACE("lfs_filebd_read -> %d", err);
         return err;
     }
 
     ssize_t res2 = read(bd->fd, buffer, size);
     if (res2 < 0) {
         int err = -errno;
-        LFS_TRACE("lfs_filebd_read -> %d", err);
+        LFS_FILEBD_TRACE("lfs_filebd_read -> %d", err);
         return err;
     }
 
-    LFS_TRACE("lfs_filebd_read -> %d", 0);
+    LFS_FILEBD_TRACE("lfs_filebd_read -> %d", 0);
     return 0;
 }
 
 int lfs_filebd_prog(const struct lfs_config *cfg, lfs_block_t block,
         lfs_off_t off, const void *buffer, lfs_size_t size) {
-    LFS_TRACE("lfs_filebd_prog(%p, 0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
+    LFS_FILEBD_TRACE("lfs_filebd_prog(%p, 0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
             (void*)cfg, block, off, buffer, size);
     lfs_filebd_t *bd = cfg->context;
 
@@ -121,7 +122,7 @@ int lfs_filebd_prog(const struct lfs_config *cfg, lfs_block_t block,
                 (off_t)block*cfg->block_size + (off_t)off, SEEK_SET);
         if (res1 < 0) {
             int err = -errno;
-            LFS_TRACE("lfs_filebd_prog -> %d", err);
+            LFS_FILEBD_TRACE("lfs_filebd_prog -> %d", err);
             return err;
         }
 
@@ -130,7 +131,7 @@ int lfs_filebd_prog(const struct lfs_config *cfg, lfs_block_t block,
             ssize_t res2 = read(bd->fd, &c, 1);
             if (res2 < 0) {
                 int err = -errno;
-                LFS_TRACE("lfs_filebd_prog -> %d", err);
+                LFS_FILEBD_TRACE("lfs_filebd_prog -> %d", err);
                 return err;
             }
 
@@ -143,23 +144,23 @@ int lfs_filebd_prog(const struct lfs_config *cfg, lfs_block_t block,
             (off_t)block*cfg->block_size + (off_t)off, SEEK_SET);
     if (res1 < 0) {
         int err = -errno;
-        LFS_TRACE("lfs_filebd_prog -> %d", err);
+        LFS_FILEBD_TRACE("lfs_filebd_prog -> %d", err);
         return err;
     }
 
     ssize_t res2 = write(bd->fd, buffer, size);
     if (res2 < 0) {
         int err = -errno;
-        LFS_TRACE("lfs_filebd_prog -> %d", err);
+        LFS_FILEBD_TRACE("lfs_filebd_prog -> %d", err);
         return err;
     }
 
-    LFS_TRACE("lfs_filebd_prog -> %d", 0);
+    LFS_FILEBD_TRACE("lfs_filebd_prog -> %d", 0);
     return 0;
 }
 
 int lfs_filebd_erase(const struct lfs_config *cfg, lfs_block_t block) {
-    LFS_TRACE("lfs_filebd_erase(%p, 0x%"PRIx32")", (void*)cfg, block);
+    LFS_FILEBD_TRACE("lfs_filebd_erase(%p, 0x%"PRIx32")", (void*)cfg, block);
     lfs_filebd_t *bd = cfg->context;
 
     // check if erase is valid
@@ -170,7 +171,7 @@ int lfs_filebd_erase(const struct lfs_config *cfg, lfs_block_t block) {
         off_t res1 = lseek(bd->fd, (off_t)block*cfg->block_size, SEEK_SET);
         if (res1 < 0) {
             int err = -errno;
-            LFS_TRACE("lfs_filebd_erase -> %d", err);
+            LFS_FILEBD_TRACE("lfs_filebd_erase -> %d", err);
             return err;
         }
 
@@ -178,27 +179,27 @@ int lfs_filebd_erase(const struct lfs_config *cfg, lfs_block_t block) {
             ssize_t res2 = write(bd->fd, &(uint8_t){bd->cfg->erase_value}, 1);
             if (res2 < 0) {
                 int err = -errno;
-                LFS_TRACE("lfs_filebd_erase -> %d", err);
+                LFS_FILEBD_TRACE("lfs_filebd_erase -> %d", err);
                 return err;
             }
         }
     }
 
-    LFS_TRACE("lfs_filebd_erase -> %d", 0);
+    LFS_FILEBD_TRACE("lfs_filebd_erase -> %d", 0);
     return 0;
 }
 
 int lfs_filebd_sync(const struct lfs_config *cfg) {
-    LFS_TRACE("lfs_filebd_sync(%p)", (void*)cfg);
+    LFS_FILEBD_TRACE("lfs_filebd_sync(%p)", (void*)cfg);
     // file sync
     lfs_filebd_t *bd = cfg->context;
     int err = fsync(bd->fd);
     if (err) {
         err = -errno;
-        LFS_TRACE("lfs_filebd_sync -> %d", 0);
+        LFS_FILEBD_TRACE("lfs_filebd_sync -> %d", 0);
         return err;
     }
 
-    LFS_TRACE("lfs_filebd_sync -> %d", 0);
+    LFS_FILEBD_TRACE("lfs_filebd_sync -> %d", 0);
     return 0;
 }

bd/lfs_filebd.h

@@ -15,6 +15,14 @@ extern "C"
 {
 #endif
 
+
+// Block device specific tracing
+#ifdef LFS_FILEBD_YES_TRACE
+#define LFS_FILEBD_TRACE(...) LFS_TRACE(__VA_ARGS__)
+#else
+#define LFS_FILEBD_TRACE(...)
+#endif
+
 // filebd config (optional)
 struct lfs_filebd_config {
     // 8-bit erase value to use for simulating erases. -1 does not simulate

bd/lfs_rambd.c

@@ -8,7 +8,7 @@
 
 int lfs_rambd_createcfg(const struct lfs_config *cfg,
         const struct lfs_rambd_config *bdcfg) {
-    LFS_TRACE("lfs_rambd_createcfg(%p {.context=%p, "
+    LFS_RAMBD_TRACE("lfs_rambd_createcfg(%p {.context=%p, "
                 ".read=%p, .prog=%p, .erase=%p, .sync=%p, "
                 ".read_size=%"PRIu32", .prog_size=%"PRIu32", "
                 ".block_size=%"PRIu32", .block_count=%"PRIu32"}, "
@@ -27,7 +27,7 @@ int lfs_rambd_createcfg(const struct lfs_config *cfg,
     } else {
         bd->buffer = lfs_malloc(cfg->block_size * cfg->block_count);
         if (!bd->buffer) {
-            LFS_TRACE("lfs_rambd_createcfg -> %d", LFS_ERR_NOMEM);
+            LFS_RAMBD_TRACE("lfs_rambd_createcfg -> %d", LFS_ERR_NOMEM);
             return LFS_ERR_NOMEM;
         }
     }
@@ -38,12 +38,12 @@ int lfs_rambd_createcfg(const struct lfs_config *cfg,
                 cfg->block_size * cfg->block_count);
     }
 
-    LFS_TRACE("lfs_rambd_createcfg -> %d", 0);
+    LFS_RAMBD_TRACE("lfs_rambd_createcfg -> %d", 0);
     return 0;
 }
 
 int lfs_rambd_create(const struct lfs_config *cfg) {
-    LFS_TRACE("lfs_rambd_create(%p {.context=%p, "
+    LFS_RAMBD_TRACE("lfs_rambd_create(%p {.context=%p, "
                 ".read=%p, .prog=%p, .erase=%p, .sync=%p, "
                 ".read_size=%"PRIu32", .prog_size=%"PRIu32", "
                 ".block_size=%"PRIu32", .block_count=%"PRIu32"})",
@@ -53,24 +53,25 @@ int lfs_rambd_create(const struct lfs_config *cfg) {
             cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count);
     static const struct lfs_rambd_config defaults = {.erase_value=-1};
     int err = lfs_rambd_createcfg(cfg, &defaults);
-    LFS_TRACE("lfs_rambd_create -> %d", err);
+    LFS_RAMBD_TRACE("lfs_rambd_create -> %d", err);
     return err;
 }
 
 int lfs_rambd_destroy(const struct lfs_config *cfg) {
-    LFS_TRACE("lfs_rambd_destroy(%p)", (void*)cfg);
+    LFS_RAMBD_TRACE("lfs_rambd_destroy(%p)", (void*)cfg);
     // clean up memory
     lfs_rambd_t *bd = cfg->context;
     if (!bd->cfg->buffer) {
         lfs_free(bd->buffer);
     }
-    LFS_TRACE("lfs_rambd_destroy -> %d", 0);
+    LFS_RAMBD_TRACE("lfs_rambd_destroy -> %d", 0);
     return 0;
 }
 
 int lfs_rambd_read(const struct lfs_config *cfg, lfs_block_t block,
         lfs_off_t off, void *buffer, lfs_size_t size) {
-    LFS_TRACE("lfs_rambd_read(%p, 0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
+    LFS_RAMBD_TRACE("lfs_rambd_read(%p, "
+                "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
             (void*)cfg, block, off, buffer, size);
     lfs_rambd_t *bd = cfg->context;
 
@@ -82,13 +83,14 @@ int lfs_rambd_read(const struct lfs_config *cfg, lfs_block_t block,
     // read data
     memcpy(buffer, &bd->buffer[block*cfg->block_size + off], size);
 
-    LFS_TRACE("lfs_rambd_read -> %d", 0);
+    LFS_RAMBD_TRACE("lfs_rambd_read -> %d", 0);
     return 0;
 }
 
 int lfs_rambd_prog(const struct lfs_config *cfg, lfs_block_t block,
         lfs_off_t off, const void *buffer, lfs_size_t size) {
-    LFS_TRACE("lfs_rambd_prog(%p, 0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
+    LFS_RAMBD_TRACE("lfs_rambd_prog(%p, "
+                "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
             (void*)cfg, block, off, buffer, size);
     lfs_rambd_t *bd = cfg->context;
 
@@ -108,12 +110,12 @@ int lfs_rambd_prog(const struct lfs_config *cfg, lfs_block_t block,
     // program data
     memcpy(&bd->buffer[block*cfg->block_size + off], buffer, size);
 
-    LFS_TRACE("lfs_rambd_prog -> %d", 0);
+    LFS_RAMBD_TRACE("lfs_rambd_prog -> %d", 0);
     return 0;
 }
 
 int lfs_rambd_erase(const struct lfs_config *cfg, lfs_block_t block) {
-    LFS_TRACE("lfs_rambd_erase(%p, 0x%"PRIx32")", (void*)cfg, block);
+    LFS_RAMBD_TRACE("lfs_rambd_erase(%p, 0x%"PRIx32")", (void*)cfg, block);
     lfs_rambd_t *bd = cfg->context;
 
     // check if erase is valid
@@ -125,14 +127,14 @@ int lfs_rambd_erase(const struct lfs_config *cfg, lfs_block_t block) {
                 bd->cfg->erase_value, cfg->block_size);
     }
 
-    LFS_TRACE("lfs_rambd_erase -> %d", 0);
+    LFS_RAMBD_TRACE("lfs_rambd_erase -> %d", 0);
     return 0;
 }
 
 int lfs_rambd_sync(const struct lfs_config *cfg) {
-    LFS_TRACE("lfs_rambd_sync(%p)", (void*)cfg);
+    LFS_RAMBD_TRACE("lfs_rambd_sync(%p)", (void*)cfg);
     // sync does nothing because we aren't backed by anything real
     (void)cfg;
-    LFS_TRACE("lfs_rambd_sync -> %d", 0);
+    LFS_RAMBD_TRACE("lfs_rambd_sync -> %d", 0);
     return 0;
 }

bd/lfs_rambd.h

@@ -15,6 +15,14 @@ extern "C"
 {
 #endif
 
+
+// Block device specific tracing
+#ifdef LFS_RAMBD_YES_TRACE
+#define LFS_RAMBD_TRACE(...) LFS_TRACE(__VA_ARGS__)
+#else
+#define LFS_RAMBD_TRACE(...)
+#endif
+
 // rambd config (optional)
 struct lfs_rambd_config {
     // 8-bit erase value to simulate erasing with. -1 indicates no erase

bd/lfs_testbd.c

@@ -12,7 +12,7 @@
 
 int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path,
         const struct lfs_testbd_config *bdcfg) {
-    LFS_TRACE("lfs_testbd_createcfg(%p {.context=%p, "
+    LFS_TESTBD_TRACE("lfs_testbd_createcfg(%p {.context=%p, "
                 ".read=%p, .prog=%p, .erase=%p, .sync=%p, "
                 ".read_size=%"PRIu32", .prog_size=%"PRIu32", "
                 ".block_size=%"PRIu32", .block_count=%"PRIu32"}, "
@@ -38,9 +38,9 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path,
         if (bd->cfg->wear_buffer) {
             bd->wear = bd->cfg->wear_buffer;
         } else {
-            bd->wear = lfs_malloc(sizeof(lfs_testbd_wear_t) * cfg->block_count);
+            bd->wear = lfs_malloc(sizeof(lfs_testbd_wear_t)*cfg->block_count);
             if (!bd->wear) {
-                LFS_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM);
+                LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM);
                 return LFS_ERR_NOMEM;
             }
         }
@@ -54,7 +54,7 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path,
             .erase_value = bd->cfg->erase_value,
         };
         int err = lfs_filebd_createcfg(cfg, path, &bd->u.file.cfg);
-        LFS_TRACE("lfs_testbd_createcfg -> %d", err);
+        LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", err);
         return err;
     } else {
         bd->u.ram.cfg = (struct lfs_rambd_config){
@@ -62,13 +62,13 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path,
             .buffer = bd->cfg->buffer,
         };
         int err = lfs_rambd_createcfg(cfg, &bd->u.ram.cfg);
-        LFS_TRACE("lfs_testbd_createcfg -> %d", err);
+        LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", err);
         return err;
     }
 }
 
 int lfs_testbd_create(const struct lfs_config *cfg, const char *path) {
-    LFS_TRACE("lfs_testbd_create(%p {.context=%p, "
+    LFS_TESTBD_TRACE("lfs_testbd_create(%p {.context=%p, "
                 ".read=%p, .prog=%p, .erase=%p, .sync=%p, "
                 ".read_size=%"PRIu32", .prog_size=%"PRIu32", "
                 ".block_size=%"PRIu32", .block_count=%"PRIu32"}, "
@@ -80,12 +80,12 @@ int lfs_testbd_create(const struct lfs_config *cfg, const char *path) {
             path);
     static const struct lfs_testbd_config defaults = {.erase_value=-1};
     int err = lfs_testbd_createcfg(cfg, path, &defaults);
-    LFS_TRACE("lfs_testbd_create -> %d", err);
+    LFS_TESTBD_TRACE("lfs_testbd_create -> %d", err);
     return err;
 }
 
 int lfs_testbd_destroy(const struct lfs_config *cfg) {
-    LFS_TRACE("lfs_testbd_destroy(%p)", (void*)cfg);
+    LFS_TESTBD_TRACE("lfs_testbd_destroy(%p)", (void*)cfg);
     lfs_testbd_t *bd = cfg->context;
     if (bd->cfg->erase_cycles && !bd->cfg->wear_buffer) {
         lfs_free(bd->wear);
@@ -93,11 +93,11 @@ int lfs_testbd_destroy(const struct lfs_config *cfg) {
 
     if (bd->persist) {
         int err = lfs_filebd_destroy(cfg);
-        LFS_TRACE("lfs_testbd_destroy -> %d", err);
+        LFS_TESTBD_TRACE("lfs_testbd_destroy -> %d", err);
         return err;
     } else {
         int err = lfs_rambd_destroy(cfg);
-        LFS_TRACE("lfs_testbd_destroy -> %d", err);
+        LFS_TESTBD_TRACE("lfs_testbd_destroy -> %d", err);
         return err;
     }
 }
@@ -145,7 +145,8 @@ static int lfs_testbd_rawsync(const struct lfs_config *cfg) {
 /// block device API ///
 int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block,
         lfs_off_t off, void *buffer, lfs_size_t size) {
-    LFS_TRACE("lfs_testbd_read(%p, 0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
+    LFS_TESTBD_TRACE("lfs_testbd_read(%p, "
+                "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
             (void*)cfg, block, off, buffer, size);
     lfs_testbd_t *bd = cfg->context;
 
@@ -157,19 +158,20 @@ int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block,
     // block bad?
     if (bd->cfg->erase_cycles && bd->wear[block] >= bd->cfg->erase_cycles &&
             bd->cfg->badblock_behavior == LFS_TESTBD_BADBLOCK_READERROR) {
-        LFS_TRACE("lfs_testbd_read -> %d", LFS_ERR_CORRUPT);
+        LFS_TESTBD_TRACE("lfs_testbd_read -> %d", LFS_ERR_CORRUPT);
         return LFS_ERR_CORRUPT;
     }
 
     // read
     int err = lfs_testbd_rawread(cfg, block, off, buffer, size);
-    LFS_TRACE("lfs_testbd_read -> %d", err);
+    LFS_TESTBD_TRACE("lfs_testbd_read -> %d", err);
     return err;
 }
 
 int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block,
         lfs_off_t off, const void *buffer, lfs_size_t size) {
-    LFS_TRACE("lfs_testbd_prog(%p, 0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
+    LFS_TESTBD_TRACE("lfs_testbd_prog(%p, "
+                "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
             (void*)cfg, block, off, buffer, size);
     lfs_testbd_t *bd = cfg->context;
 
@@ -182,13 +184,13 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block,
     if (bd->cfg->erase_cycles && bd->wear[block] >= bd->cfg->erase_cycles) {
         if (bd->cfg->badblock_behavior ==
                 LFS_TESTBD_BADBLOCK_PROGERROR) {
-            LFS_TRACE("lfs_testbd_prog -> %d", LFS_ERR_CORRUPT);
+            LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", LFS_ERR_CORRUPT);
             return LFS_ERR_CORRUPT;
         } else if (bd->cfg->badblock_behavior ==
                 LFS_TESTBD_BADBLOCK_PROGNOOP ||
                 bd->cfg->badblock_behavior ==
                 LFS_TESTBD_BADBLOCK_ERASENOOP) {
-            LFS_TRACE("lfs_testbd_prog -> %d", 0);
+            LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0);
             return 0;
         }
     }
@@ -196,7 +198,7 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block,
     // prog
     int err = lfs_testbd_rawprog(cfg, block, off, buffer, size);
     if (err) {
-        LFS_TRACE("lfs_testbd_prog -> %d", err);
+        LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err);
         return err;
     }
 
@@ -205,18 +207,18 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block,
         bd->power_cycles -= 1;
         if (bd->power_cycles == 0) {
             // sync to make sure we persist the last changes
-            assert(lfs_testbd_rawsync(cfg) == 0);
+            LFS_ASSERT(lfs_testbd_rawsync(cfg) == 0);
             // simulate power loss
             exit(33);
         }
     }
 
-    LFS_TRACE("lfs_testbd_prog -> %d", 0);
+    LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0);
     return 0;
 }
 
 int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) {
-    LFS_TRACE("lfs_testbd_erase(%p, 0x%"PRIx32")", (void*)cfg, block);
+    LFS_TESTBD_TRACE("lfs_testbd_erase(%p, 0x%"PRIx32")", (void*)cfg, block);
     lfs_testbd_t *bd = cfg->context;
 
     // check if erase is valid
@@ -227,11 +229,11 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) {
         if (bd->wear[block] >= bd->cfg->erase_cycles) {
             if (bd->cfg->badblock_behavior ==
                     LFS_TESTBD_BADBLOCK_ERASEERROR) {
-                LFS_TRACE("lfs_testbd_erase -> %d", LFS_ERR_CORRUPT);
+                LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", LFS_ERR_CORRUPT);
                 return LFS_ERR_CORRUPT;
             } else if (bd->cfg->badblock_behavior ==
                     LFS_TESTBD_BADBLOCK_ERASENOOP) {
-                LFS_TRACE("lfs_testbd_erase -> %d", 0);
+                LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", 0);
                 return 0;
             }
         } else {
@@ -243,7 +245,7 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) {
     // erase
     int err = lfs_testbd_rawerase(cfg, block);
     if (err) {
-        LFS_TRACE("lfs_testbd_erase -> %d", err);
+        LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err);
         return err;
     }
 
@@ -252,20 +254,20 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) {
         bd->power_cycles -= 1;
         if (bd->power_cycles == 0) {
             // sync to make sure we persist the last changes
-            assert(lfs_testbd_rawsync(cfg) == 0);
+            LFS_ASSERT(lfs_testbd_rawsync(cfg) == 0);
             // simulate power loss
             exit(33);
         }
     }
 
-    LFS_TRACE("lfs_testbd_prog -> %d", 0);
+    LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0);
     return 0;
 }
 
 int lfs_testbd_sync(const struct lfs_config *cfg) {
-    LFS_TRACE("lfs_testbd_sync(%p)", (void*)cfg);
+    LFS_TESTBD_TRACE("lfs_testbd_sync(%p)", (void*)cfg);
     int err = lfs_testbd_rawsync(cfg);
-    LFS_TRACE("lfs_testbd_sync -> %d", err);
+    LFS_TESTBD_TRACE("lfs_testbd_sync -> %d", err);
     return err;
 }
 
@@ -273,20 +275,20 @@ int lfs_testbd_sync(const struct lfs_config *cfg) {
 /// simulated wear operations ///
 lfs_testbd_swear_t lfs_testbd_getwear(const struct lfs_config *cfg,
         lfs_block_t block) {
-    LFS_TRACE("lfs_testbd_getwear(%p, %"PRIu32")", (void*)cfg, block);
+    LFS_TESTBD_TRACE("lfs_testbd_getwear(%p, %"PRIu32")", (void*)cfg, block);
     lfs_testbd_t *bd = cfg->context;
 
     // check if block is valid
     LFS_ASSERT(bd->cfg->erase_cycles);
     LFS_ASSERT(block < cfg->block_count);
 
-    LFS_TRACE("lfs_testbd_getwear -> %"PRIu32, bd->wear[block]);
+    LFS_TESTBD_TRACE("lfs_testbd_getwear -> %"PRIu32, bd->wear[block]);
     return bd->wear[block];
 }
 
 int lfs_testbd_setwear(const struct lfs_config *cfg,
         lfs_block_t block, lfs_testbd_wear_t wear) {
-    LFS_TRACE("lfs_testbd_setwear(%p, %"PRIu32")", (void*)cfg, block);
+    LFS_TESTBD_TRACE("lfs_testbd_setwear(%p, %"PRIu32")", (void*)cfg, block);
     lfs_testbd_t *bd = cfg->context;
 
     // check if block is valid
@@ -295,6 +297,6 @@ int lfs_testbd_setwear(const struct lfs_config *cfg,
 
     bd->wear[block] = wear;
 
-    LFS_TRACE("lfs_testbd_setwear -> %d", 0);
+    LFS_TESTBD_TRACE("lfs_testbd_setwear -> %d", 0);
     return 0;
 }

bd/lfs_testbd.h

@@ -19,6 +19,13 @@ extern "C"
 #endif
 
 
+// Block device specific tracing
+#ifdef LFS_TESTBD_YES_TRACE
+#define LFS_TESTBD_TRACE(...) LFS_TRACE(__VA_ARGS__)
+#else
+#define LFS_TESTBD_TRACE(...)
+#endif
+
 // Mode determining how "bad blocks" behave during testing. This simulates
 // some real-world circumstances such as progs not sticking (prog-noop),
 // a readonly disk (erase-noop), and ECC failures (read-error).

lfs.h

@@ -9,6 +9,7 @@
 
 #include <stdint.h>
 #include <stdbool.h>
+#include "lfs_util.h"
 
 #ifdef __cplusplus
 extern "C"
@@ -21,7 +22,7 @@ extern "C"
 // Software library version
 // Major (top-nibble), incremented on backwards incompatible changes
 // Minor (bottom-nibble), incremented on feature additions
-#define LFS_VERSION 0x00020001
+#define LFS_VERSION 0x00020003
 #define LFS_VERSION_MAJOR (0xffff & (LFS_VERSION >> 16))
 #define LFS_VERSION_MINOR (0xffff & (LFS_VERSION >>  0))
 
@@ -123,20 +124,25 @@ enum lfs_type {
 enum lfs_open_flags {
     // open flags
     LFS_O_RDONLY = 1,         // Open a file as read only
+#ifndef LFS_READONLY
     LFS_O_WRONLY = 2,         // Open a file as write only
     LFS_O_RDWR   = 3,         // Open a file as read and write
     LFS_O_CREAT  = 0x0100,    // Create a file if it does not exist
     LFS_O_EXCL   = 0x0200,    // Fail if a file already exists
     LFS_O_TRUNC  = 0x0400,    // Truncate the existing file to zero size
     LFS_O_APPEND = 0x0800,    // Move to end of file on every write
+#endif
 
     // internally used flags
+#ifndef LFS_READONLY
     LFS_F_DIRTY   = 0x010000, // File does not match storage
     LFS_F_WRITING = 0x020000, // File has been written since last flush
+#endif
     LFS_F_READING = 0x040000, // File has been read since last flush
-    LFS_F_ERRED   = 0x080000, // An error occured during write
+#ifndef LFS_READONLY
+    LFS_F_ERRED   = 0x080000, // An error occurred during write
+#endif
     LFS_F_INLINE  = 0x100000, // Currently inlined in directory entry
-    LFS_F_OPENED  = 0x200000, // File has been opened
 };
 
 // File seek flags
@@ -174,6 +180,16 @@ struct lfs_config {
     // are propogated to the user.
     int (*sync)(const struct lfs_config *c);
 
+#ifdef LFS_THREADSAFE
+    // Lock the underlying block device. Negative error codes
+    // are propogated to the user.
+    int (*lock)(const struct lfs_config *c);
+
+    // Unlock the underlying block device. Negative error codes
+    // are propogated to the user.
+    int (*unlock)(const struct lfs_config *c);
+#endif
+
     // Minimum size of a block read. All read operations will be a
     // multiple of this value.
     lfs_size_t read_size;
@@ -399,6 +415,7 @@ typedef struct lfs {
 
 /// Filesystem functions ///
 
+#ifndef LFS_READONLY
 // Format a block device with the littlefs
 //
 // Requires a littlefs object and config struct. This clobbers the littlefs
@@ -407,6 +424,7 @@ typedef struct lfs {
 //
 // Returns a negative error code on failure.
 int lfs_format(lfs_t *lfs, const struct lfs_config *config);
+#endif
 
 // Mounts a littlefs
 //
@@ -426,12 +444,15 @@ int lfs_unmount(lfs_t *lfs);
 
 /// General operations ///
 
+#ifndef LFS_READONLY
 // Removes a file or directory
 //
 // If removing a directory, the directory must be empty.
 // Returns a negative error code on failure.
 int lfs_remove(lfs_t *lfs, const char *path);
+#endif
 
+#ifndef LFS_READONLY
 // Rename or move a file or directory
 //
 // If the destination exists, it must match the source in type.
@@ -439,6 +460,7 @@ int lfs_remove(lfs_t *lfs, const char *path);
 //
 // Returns a negative error code on failure.
 int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath);
+#endif
 
 // Find info about a file or directory
 //
@@ -461,6 +483,7 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info);
 lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path,
         uint8_t type, void *buffer, lfs_size_t size);
 
+#ifndef LFS_READONLY
 // Set custom attributes
 //
 // Custom attributes are uniquely identified by an 8-bit type and limited
@@ -470,13 +493,16 @@ lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path,
 // Returns a negative error code on failure.
 int lfs_setattr(lfs_t *lfs, const char *path,
         uint8_t type, const void *buffer, lfs_size_t size);
+#endif
 
+#ifndef LFS_READONLY
 // Removes a custom attribute
 //
 // If an attribute is not found, nothing happens.
 //
 // Returns a negative error code on failure.
 int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type);
+#endif
 
 
 /// File operations ///
@@ -525,6 +551,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file);
 lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file,
         void *buffer, lfs_size_t size);
 
+#ifndef LFS_READONLY
 // Write data to file
 //
 // Takes a buffer and size indicating the data to write. The file will not
@@ -533,6 +560,7 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file,
 // Returns the number of bytes written, or a negative error code on failure.
 lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file,
         const void *buffer, lfs_size_t size);
+#endif
 
 // Change the position of the file
 //
@@ -541,10 +569,12 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file,
 lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file,
         lfs_soff_t off, int whence);
 
+#ifndef LFS_READONLY
 // Truncates the size of the file to the specified size
 //
 // Returns a negative error code on failure.
 int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size);
+#endif
 
 // Return the position of the file
 //
@@ -567,10 +597,12 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file);
 
 /// Directory operations ///
 
+#ifndef LFS_READONLY
 // Create a directory
 //
 // Returns a negative error code on failure.
 int lfs_mkdir(lfs_t *lfs, const char *path);
+#endif
 
 // Open a directory
 //
@@ -632,6 +664,7 @@ lfs_ssize_t lfs_fs_size(lfs_t *lfs);
 // Returns a negative error code on failure.
 int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data);
 
+#ifndef LFS_READONLY
 #ifdef LFS_MIGRATE
 // Attempts to migrate a previous version of littlefs
 //
@@ -646,6 +679,7 @@ int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data);
 // Returns a negative error code on failure.
 int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg);
 #endif
+#endif
 
 
 #ifdef __cplusplus

lfs_util.h

@@ -50,31 +50,35 @@ extern "C"
 
 // Logging functions
 #ifdef LFS_YES_TRACE
-#define LFS_TRACE(fmt, ...) \
+#define LFS_TRACE_(fmt, ...) \
-    printf("%s:%d:trace: " fmt "\n", __FILE__, __LINE__, __VA_ARGS__)
+    printf("%s:%d:trace: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__)
+#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")
 #else
-#define LFS_TRACE(fmt, ...)
+#define LFS_TRACE(...)
 #endif
 
 #ifndef LFS_NO_DEBUG
-#define LFS_DEBUG(fmt, ...) \
+#define LFS_DEBUG_(fmt, ...) \
-    printf("%s:%d:debug: " fmt "\n", __FILE__, __LINE__, __VA_ARGS__)
+    printf("%s:%d:debug: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__)
+#define LFS_DEBUG(...) LFS_DEBUG_(__VA_ARGS__, "")
 #else
-#define LFS_DEBUG(fmt, ...)
+#define LFS_DEBUG(...)
 #endif
 
 #ifndef LFS_NO_WARN
-#define LFS_WARN(fmt, ...) \
+#define LFS_WARN_(fmt, ...) \
-    printf("%s:%d:warn: " fmt "\n", __FILE__, __LINE__, __VA_ARGS__)
+    printf("%s:%d:warn: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__)
+#define LFS_WARN(...) LFS_WARN_(__VA_ARGS__, "")
 #else
-#define LFS_WARN(fmt, ...)
+#define LFS_WARN(...)
 #endif
 
 #ifndef LFS_NO_ERROR
-#define LFS_ERROR(fmt, ...) \
+#define LFS_ERROR_(fmt, ...) \
-    printf("%s:%d:error: " fmt "\n", __FILE__, __LINE__, __VA_ARGS__)
+    printf("%s:%d:error: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__)
+#define LFS_ERROR(...) LFS_ERROR_(__VA_ARGS__, "")
 #else
-#define LFS_ERROR(fmt, ...)
+#define LFS_ERROR(...)
 #endif
 
 // Runtime assertions
@@ -107,7 +111,7 @@ static inline uint32_t lfs_alignup(uint32_t a, uint32_t alignment) {
     return lfs_aligndown(a + alignment-1, alignment);
 }
 
-// Find the next smallest power of 2 less than or equal to a
+// Find the smallest power of 2 greater than or equal to a
 static inline uint32_t lfs_npw2(uint32_t a) {
 #if !defined(LFS_NO_INTRINSICS) && (defined(__GNUC__) || defined(__CC_ARM))
     return 32 - __builtin_clz(a-1);

scripts/code_size.py

@@ -0,0 +1,328 @@
+#!/usr/bin/env python3
+#
+# This script finds the code size at the function level, with/without
+# static functions, and has some conveniences for comparing different
+# versions. It's basically one big wrapper around nm, and may or may
+# not have been written out of jealousy of Linux's Bloat-O-Meter.
+#
+# Here's a useful bash script to use while developing:
+# ./scripts/code_size.py -qo old.csv
+# while true ; do ./code_scripts/size.py -d old.csv ; inotifywait -rqe modify * ; done
+#
+# Or even better, to automatically update results on commit:
+# ./scripts/code_size.py -qo commit.csv
+# while true ; do ./scripts/code_size.py -d commit.csv -o current.csv ; git diff --exit-code --quiet && cp current.csv commit.csv ; inotifywait -rqe modify * ; done
+#
+# Or my personal favorite:
+# ./scripts/code_size.py -qo master.csv && cp master.csv commit.csv
+# while true ; do ( ./scripts/code_size.py -i commit.csv -d master.csv -s ; ./scripts/code_size.py -i current.csv -d master.csv -s ; ./scripts/code_size.py -d master.csv -o current.csv -s ) | awk 'BEGIN {printf "%-16s %7s %7s %7s\n","","old","new","diff"} (NR==2 && $1="commit") || (NR==4 && $1="prev") || (NR==6 && $1="current") {printf "%-16s %7s %7s %7s %s\n",$1,$2,$3,$5,$6}' ; git diff --exit-code --quiet && cp current.csv commit.csv ; inotifywait -rqe modify * ; done
+#
+
+import os
+import itertools as it
+import subprocess as sp
+import shlex
+import re
+import csv
+import collections as co
+
+SIZEDIR = 'sizes'
+RULES = """
+define FLATTEN
+%(sizedir)s/%(build)s.$(subst /,.,$(target)): $(target)
+    ( echo "#line 1 \\"$$<\\"" ; %(cat)s $$< ) > $$@
+%(sizedir)s/%(build)s.$(subst /,.,$(target:.c=.size)): \\
+        %(sizedir)s/%(build)s.$(subst /,.,$(target:.c=.o))
+    $(NM) --size-sort $$^ | sed 's/^/$(subst /,\\/,$(target:.c=.o)):/' > $$@
+endef
+$(foreach target,$(SRC),$(eval $(FLATTEN)))
+
+-include %(sizedir)s/*.d
+.SECONDARY:
+
+%%.size: $(foreach t,$(subst /,.,$(SRC:.c=.size)),%%.$t)
+    cat $^ > $@
+"""
+CATS = {
+    'code': 'cat',
+    'code_inlined': 'sed \'s/^static\( inline\)\?//\'',
+}
+
+def build(**args):
+    # mkdir -p sizedir
+    os.makedirs(args['sizedir'], exist_ok=True)
+
+    if args.get('inlined', False):
+        builds = ['code', 'code_inlined']
+    else:
+        builds = ['code']
+
+    # write makefiles for the different types of builds
+    makefiles = []
+    targets = []
+    for build in builds:
+        path = args['sizedir'] + '/' + build
+        with open(path + '.mk', 'w') as mk:
+            mk.write(RULES.replace(4*' ', '\t') % dict(
+                sizedir=args['sizedir'],
+                build=build,
+                cat=CATS[build]))
+            mk.write('\n')
+
+            # pass on defines
+            for d in args['D']:
+                mk.write('%s: override CFLAGS += -D%s\n' % (
+                    path+'.size', d))
+
+        makefiles.append(path + '.mk')
+        targets.append(path + '.size')
+
+    # build in parallel
+    cmd = (['make', '-f', 'Makefile'] +
+        list(it.chain.from_iterable(['-f', m] for m in makefiles)) +
+        [target for target in targets])
+    if args.get('verbose', False):
+        print(' '.join(shlex.quote(c) for c in cmd))
+    proc = sp.Popen(cmd,
+        stdout=sp.DEVNULL if not args.get('verbose', False) else None)
+    proc.wait()
+    if proc.returncode != 0:
+        sys.exit(-1)
+
+    # find results
+    build_results = co.defaultdict(lambda: 0)
+    # notes
+    # - filters type
+    # - discards internal/debug functions (leading __)
+    pattern = re.compile(
+        '^(?P<file>[^:]+)' +
+        ':(?P<size>[0-9a-fA-F]+)' +
+        ' (?P<type>[%s])' % re.escape(args['type']) +
+        ' (?!__)(?P<name>.+?)$')
+    for build in builds:
+        path = args['sizedir'] + '/' + build
+        with open(path + '.size') as size:
+            for line in size:
+                match = pattern.match(line)
+                if match:
+                    file = match.group('file')
+                    # discard .8449 suffixes created by optimizer
+                    name = re.sub('\.[0-9]+', '', match.group('name'))
+                    size = int(match.group('size'), 16)
+                    build_results[(build, file, name)] += size
+
+    results = []
+    for (build, file, name), size in build_results.items():
+        if build == 'code':
+            results.append((file, name, size, False))
+        elif (build == 'code_inlined' and
+                ('inlined', file, name) not in results):
+            results.append((file, name, size, True))
+
+    return results
+
+def main(**args):
+    # find results
+    if not args.get('input', None):
+        results = build(**args)
+    else:
+        with open(args['input']) as f:
+            r = csv.DictReader(f)
+            results = [
+                (   result['file'],
+                    result['name'],
+                    int(result['size']),
+                    bool(int(result.get('inlined', 0))))
+                for result in r
+                if (not bool(int(result.get('inlined', 0))) or
+                    args.get('inlined', False))]
+
+    total = 0
+    for _, _, size, inlined in results:
+        if not inlined:
+            total += size
+
+    # find previous results?
+    if args.get('diff', None):
+        with open(args['diff']) as f:
+            r = csv.DictReader(f)
+            prev_results = [
+                (   result['file'],
+                    result['name'],
+                    int(result['size']),
+                    bool(int(result.get('inlined', 0))))
+                for result in r
+                if (not bool(int(result.get('inlined', 0))) or
+                    args.get('inlined', False))]
+
+        prev_total = 0
+        for _, _, size, inlined in prev_results:
+            if not inlined:
+                prev_total += size
+
+    # write results to CSV
+    if args.get('output', None):
+        results.sort(key=lambda x: (-x[2], x))
+        with open(args['output'], 'w') as f:
+            w = csv.writer(f)
+            if args.get('inlined', False):
+                w.writerow(['file', 'name', 'size', 'inlined'])
+                for file, name, size, inlined in results:
+                    w.writerow((file, name, size, int(inlined)))
+            else:
+                w.writerow(['file', 'name', 'size'])
+                for file, name, size, inlined in results:
+                    w.writerow((file, name, size))
+
+    # print results
+    def dedup_functions(results):
+        functions = co.defaultdict(lambda: (0, True))
+        for _, name, size, inlined in results:
+            if not inlined:
+                functions[name] = (functions[name][0] + size, False)
+        for _, name, size, inlined in results:
+            if inlined and functions[name][1]:
+                functions[name] = (functions[name][0] + size, True)
+        return functions
+
+    def dedup_files(results):
+        files = co.defaultdict(lambda: 0)
+        for file, _, size, inlined in results:
+            if not inlined:
+                files[file] += size
+        return files
+
+    def diff_sizes(olds, news):
+        diff = co.defaultdict(lambda: (None, None, None))
+        for name, new in news.items():
+            diff[name] = (None, new, new)
+        for name, old in olds.items():
+            new = diff[name][1] or 0
+            diff[name] = (old, new, new-old)
+        return diff
+
+    def print_header(name=''):
+        if not args.get('diff', False):
+            print('%-40s %7s' % (name, 'size'))
+        else:
+            print('%-40s %7s %7s %7s' % (name, 'old', 'new', 'diff'))
+
+    def print_functions():
+        functions = dedup_functions(results)
+        functions = {
+            name+' (inlined)' if inlined else name: size
+            for name, (size, inlined) in functions.items()}
+
+        if not args.get('diff', None):
+            print_header('function')
+            for name, size in sorted(functions.items(),
+                    key=lambda x: (-x[1], x)):
+                print("%-40s %7d" % (name, size))
+        else:
+            prev_functions = dedup_functions(prev_results)
+            prev_functions = {
+                name+' (inlined)' if inlined else name: size
+                for name, (size, inlined) in prev_functions.items()}
+            diff = diff_sizes(functions, prev_functions)
+            print_header('function (%d added, %d removed)' % (
+                sum(1 for old, _, _ in diff.values() if not old),
+                sum(1 for _, new, _ in diff.values() if not new)))
+            for name, (old, new, diff) in sorted(diff.items(),
+                    key=lambda x: (-(x[1][2] or 0), x)):
+                if diff or args.get('all', False):
+                    print("%-40s %7s %7s %+7d%s" % (
+                        name, old or "-", new or "-", diff,
+                        ' (%+.2f%%)' % (100*((new-old)/old))
+                        if old and new else
+                        ''))
+
+    def print_files():
+        files = dedup_files(results)
+
+        if not args.get('diff', None):
+            print_header('file')
+            for file, size in sorted(files.items(),
+                    key=lambda x: (-x[1], x)):
+                print("%-40s %7d" % (file, size))
+        else:
+            prev_files = dedup_files(prev_results)
+            diff = diff_sizes(files, prev_files)
+            print_header('file (%d added, %d removed)' % (
+                sum(1 for old, _, _ in diff.values() if not old),
+                sum(1 for _, new, _ in diff.values() if not new)))
+            for name, (old, new, diff) in sorted(diff.items(),
+                    key=lambda x: (-(x[1][2] or 0), x)):
+                if diff or args.get('all', False):
+                    print("%-40s %7s %7s %+7d%s" % (
+                        name, old or "-", new or "-", diff,
+                        ' (%+.2f%%)' % (100*((new-old)/old))
+                        if old and new else
+                        ''))
+
+    def print_totals():
+        if not args.get('diff', None):
+            print("%-40s %7d" % ('TOTALS', total))
+        else:
+            print("%-40s %7s %7s %+7d%s" % (
+                'TOTALS', prev_total, total, total-prev_total,
+                ' (%+.2f%%)' % (100*((total-prev_total)/total))
+                if prev_total and total else
+                ''))
+
+    def print_status():
+        if not args.get('diff', None):
+            print(total)
+        else:
+            print("%d (%+.2f%%)" % (total, 100*((total-prev_total)/total)))
+
+    if args.get('quiet', False):
+        pass
+    elif args.get('status', False):
+        print_status()
+    elif args.get('summary', False):
+        print_header()
+        print_totals()
+    elif args.get('files', False):
+        print_files()
+        print_totals()
+    else:
+        print_functions()
+        print_totals()
+
+if __name__ == "__main__":
+    import argparse
+    import sys
+    parser = argparse.ArgumentParser(
+        description="Find code size at the function level.")
+    parser.add_argument('sizedir', nargs='?', default=SIZEDIR,
+        help="Directory to store intermediary results. Defaults "
+            "to \"%s\"." % SIZEDIR)
+    parser.add_argument('-D', action='append', default=[],
+        help="Specify compile-time define.")
+    parser.add_argument('-v', '--verbose', action='store_true',
+        help="Output commands that run behind the scenes.")
+    parser.add_argument('-i', '--input',
+        help="Don't compile and find code sizes, instead use this CSV file.")
+    parser.add_argument('-o', '--output',
+        help="Specify CSV file to store results.")
+    parser.add_argument('-d', '--diff',
+        help="Specify CSV file to diff code size against.")
+    parser.add_argument('-a', '--all', action='store_true',
+        help="Show all functions, not just the ones that changed.")
+    parser.add_argument('--inlined', action='store_true',
+        help="Run a second compilation to find the sizes of functions normally "
+            "removed by optimizations. These will be shown as \"*.inlined\" "
+            "functions, and will not be included in the total.")
+    parser.add_argument('--files', action='store_true',
+        help="Show file-level code sizes. Note this does not include padding! "
+            "So sizes may differ from other tools.")
+    parser.add_argument('-s', '--summary', action='store_true',
+        help="Only show the total code size.")
+    parser.add_argument('-S', '--status', action='store_true',
+        help="Show minimum info useful for a single-line status.")
+    parser.add_argument('-q', '--quiet', action='store_true',
+        help="Don't show anything, useful with -o.")
+    parser.add_argument('--type', default='tTrRdDbB',
+        help="Type of symbols to report, this uses the same single-character "
+            "type-names emitted by nm. Defaults to %(default)r.")
+    sys.exit(main(**vars(parser.parse_args())))

scripts/explode_asserts.py

@@ -166,8 +166,8 @@ def mkassert(type, comp, lh, rh, size=None):
         'type': type.lower(), 'TYPE': type.upper(),
         'comp': comp.lower(), 'COMP': comp.upper(),
         'prefix': PREFIX.lower(), 'PREFIX': PREFIX.upper(),
-        'lh': lh.strip(),
+        'lh': lh.strip(' '),
-        'rh': rh.strip(),
+        'rh': rh.strip(' '),
         'size': size,
     }
     if size:

scripts/readmdir.py

@@ -233,8 +233,8 @@ class MetadataPair:
 
     def __lt__(self, other):
         # corrupt blocks don't count
-        if not self and other:
+        if not self or not other:
-            return True
+            return bool(other)
 
         # use sequence arithmetic to avoid overflow
         return not ((other.rev - self.rev) & 0x80000000)
@@ -318,6 +318,24 @@ def main(args):
 
     # find most recent pair
     mdir = MetadataPair(blocks)
+
+    try:
+        mdir.tail = mdir[Tag('tail', 0, 0)]
+        if mdir.tail.size != 8 or mdir.tail.data == 8*b'\xff':
+            mdir.tail = None
+    except KeyError:
+        mdir.tail = None
+
+    print("mdir {%s} rev %d%s%s%s" % (
+        ', '.join('%#x' % b
+            for b in [args.block1, args.block2]
+            if b is not None),
+        mdir.rev,
+        ' (was %s)' % ', '.join('%d' % m.rev for m in mdir.pair[1:])
+        if len(mdir.pair) > 1 else '',
+        ' (corrupted!)' if not mdir else '',
+        ' -> {%#x, %#x}' % struct.unpack('<II', mdir.tail.data)
+        if mdir.tail else ''))
     if args.all:
         mdir.dump_all(truncate=not args.no_truncate)
     elif args.log:

scripts/readtree.py

@@ -7,79 +7,14 @@ import io
 import itertools as it
 from readmdir import Tag, MetadataPair
 
-def popc(x):
-    return bin(x).count('1')
-
-def ctz(x):
-    return len(bin(x)) - len(bin(x).rstrip('0'))
-
-def dumpentries(args, mdir, f):
-    for k, id_ in enumerate(mdir.ids):
-        name = mdir[Tag('name', id_, 0)]
-        struct_ = mdir[Tag('struct', id_, 0)]
-
-        f.write("id %d %s %s" % (
-            id_, name.typerepr(),
-            json.dumps(name.data.decode('utf8'))))
-        if struct_.is_('dirstruct'):
-            f.write(" dir {%#x, %#x}" % struct.unpack(
-                '<II', struct_.data[:8].ljust(8, b'\xff')))
-        if struct_.is_('ctzstruct'):
-            f.write(" ctz {%#x} size %d" % struct.unpack(
-                '<II', struct_.data[:8].ljust(8, b'\xff')))
-        if struct_.is_('inlinestruct'):
-            f.write(" inline size %d" % struct_.size)
-        f.write("\n")
-
-        if args.data and struct_.is_('inlinestruct'):
-            for i in range(0, len(struct_.data), 16):
-                f.write("  %08x: %-47s  %-16s\n" % (
-                    i, ' '.join('%02x' % c for c in struct_.data[i:i+16]),
-                    ''.join(c if c >= ' ' and c <= '~' else '.'
-                        for c in map(chr, struct_.data[i:i+16]))))
-        elif args.data and struct_.is_('ctzstruct'):
-            block, size = struct.unpack(
-                '<II', struct_.data[:8].ljust(8, b'\xff'))
-            data = []
-            i = 0 if size == 0 else (size-1) // (args.block_size - 8)
-            if i != 0:
-                i = ((size-1) - 4*popc(i-1)+2) // (args.block_size - 8)
-            with open(args.disk, 'rb') as f2:
-                while i >= 0:
-                    f2.seek(block * args.block_size)
-                    dat = f2.read(args.block_size)
-                    data.append(dat[4*(ctz(i)+1) if i != 0 else 0:])
-                    block, = struct.unpack('<I', dat[:4].ljust(4, b'\xff'))
-                    i -= 1
-
-            data = bytes(it.islice(
-                it.chain.from_iterable(reversed(data)), size))
-            for i in range(0, min(len(data), 256)
-                    if not args.no_truncate else len(data), 16):
-                f.write("  %08x: %-47s  %-16s\n" % (
-                    i, ' '.join('%02x' % c for c in data[i:i+16]),
-                    ''.join(c if c >= ' ' and c <= '~' else '.'
-                        for c in map(chr, data[i:i+16]))))
-
-        for tag in mdir.tags:
-            if tag.id==id_ and tag.is_('userattr'):
-                f.write("id %d %s size %d\n" % (
-                    id_, tag.typerepr(), tag.size))
-
-                if args.data:
-                    for i in range(0, len(tag.data), 16):
-                        f.write("  %-47s  %-16s\n" % (
-                            ' '.join('%02x' % c for c in tag.data[i:i+16]),
-                            ''.join(c if c >= ' ' and c <= '~' else '.'
-                                for c in map(chr, tag.data[i:i+16]))))
-
 def main(args):
-    with open(args.disk, 'rb') as f:
-        dirs = []
     superblock = None
-        gstate = b''
+    gstate = b'\0\0\0\0\0\0\0\0\0\0\0\0'
+    dirs = []
     mdirs = []
+    corrupted = []
     cycle = False
+    with open(args.disk, 'rb') as f:
         tail = (args.block1, args.block2)
         hard = False
         while True:
@@ -126,6 +61,10 @@ def main(args):
             except KeyError:
                 pass
 
+            # corrupted?
+            if not mdir:
+                corrupted.append(mdir)
+
             # add to directories
             mdirs.append(mdir)
             if mdir.tail is None or not mdir.tail.is_('hardtail'):
@@ -160,54 +99,44 @@ def main(args):
 
         dir[0].path = path.replace('//', '/')
 
-    # dump tree
+    # print littlefs + version info
-    if not args.superblock and not args.gstate and not args.mdirs:
+    version = ('?', '?')
-        args.superblock = True
+    if superblock:
-        args.gstate = True
+        version = tuple(reversed(
-        args.mdirs = True
+            struct.unpack('<HH', superblock[1].data[0:4].ljust(4, b'\xff'))))
+    print("%-47s%s" % ("littlefs v%s.%s" % version,
+        "data (truncated, if it fits)"
+        if not any([args.no_truncate, args.log, args.all]) else ""))
 
-    if args.superblock and superblock:
+    # print gstate
-        print("superblock %s v%d.%d" % (
-            json.dumps(superblock[0].data.decode('utf8')),
-            struct.unpack('<H', superblock[1].data[2:2+2])[0],
-            struct.unpack('<H', superblock[1].data[0:0+2])[0]))
-        print(
-            "  block_size %d\n"
-            "  block_count %d\n"
-            "  name_max %d\n"
-            "  file_max %d\n"
-            "  attr_max %d" % struct.unpack(
-                '<IIIII', superblock[1].data[4:4+20].ljust(20, b'\xff')))
-
-    if args.gstate and gstate:
     print("gstate 0x%s" % ''.join('%02x' % c for c in gstate))
     tag = Tag(struct.unpack('<I', gstate[0:4].ljust(4, b'\xff'))[0])
     blocks = struct.unpack('<II', gstate[4:4+8].ljust(8, b'\xff'))
-        if tag.size:
+    if tag.size or not tag.isvalid:
-            print("  orphans %d" % tag.size)
+        print("  orphans >=%d" % max(tag.size, 1))
     if tag.type:
         print("  move dir {%#x, %#x} id %d" % (
             blocks[0], blocks[1], tag.id))
 
-    if args.mdirs:
+    # print mdir info
     for i, dir in enumerate(dirs):
         print("dir %s" % (json.dumps(dir[0].path)
             if hasattr(dir[0], 'path') else '(orphan)'))
 
         for j, mdir in enumerate(dir):
-                print("mdir {%#x, %#x} rev %d%s" % (
+            print("mdir {%#x, %#x} rev %d (was %d)%s%s" % (
-                    mdir.blocks[0], mdir.blocks[1], mdir.rev,
+                mdir.blocks[0], mdir.blocks[1], mdir.rev, mdir.pair[1].rev,
-                    ' (corrupted)' if not mdir else ''))
+                ' (corrupted!)' if not mdir else '',
+                ' -> {%#x, %#x}' % struct.unpack('<II', mdir.tail.data)
+                if mdir.tail else ''))
 
             f = io.StringIO()
-                if args.tags:
+            if args.log:
-                    mdir.dump_tags(f, truncate=not args.no_truncate)
-                elif args.log:
                 mdir.dump_log(f, truncate=not args.no_truncate)
             elif args.all:
                 mdir.dump_all(f, truncate=not args.no_truncate)
             else:
-                    dumpentries(args, mdir, f)
+                mdir.dump_tags(f, truncate=not args.no_truncate)
 
             lines = list(filter(None, f.getvalue().split('\n')))
             for k, line in enumerate(lines):
@@ -217,15 +146,18 @@ def main(args):
                     '|',
                     line))
 
-    if cycle:
+    errcode = 0
-        print("*** cycle detected! -> {%#x, %#x} ***" % (cycle[0], cycle[1]))
+    for mdir in corrupted:
+        errcode = errcode or 1
+        print("*** corrupted mdir {%#x, %#x}! ***" % (
+            mdir.blocks[0], mdir.blocks[1]))
 
     if cycle:
-        return 2
+        errcode = errcode or 2
-    elif not all(mdir for dir in dirs for mdir in dir):
+        print("*** cycle detected {%#x, %#x}! ***" % (
-        return 1
+            cycle[0], cycle[1]))
-    else:
+
-        return 0;
+    return errcode
 
 if __name__ == "__main__":
     import argparse
@@ -238,24 +170,14 @@ if __name__ == "__main__":
         help="Size of a block in bytes.")
     parser.add_argument('block1', nargs='?', default=0,
         type=lambda x: int(x, 0),
-        help="Optional first block address for finding the root.")
+        help="Optional first block address for finding the superblock.")
     parser.add_argument('block2', nargs='?', default=1,
         type=lambda x: int(x, 0),
-        help="Optional second block address for finding the root.")
+        help="Optional second block address for finding the superblock.")
-    parser.add_argument('-s', '--superblock', action='store_true',
-        help="Show contents of the superblock.")
-    parser.add_argument('-g', '--gstate', action='store_true',
-        help="Show contents of global-state.")
-    parser.add_argument('-m', '--mdirs', action='store_true',
-        help="Show contents of metadata-pairs/directories.")
-    parser.add_argument('-t', '--tags', action='store_true',
-        help="Show metadata tags instead of reconstructing entries.")
     parser.add_argument('-l', '--log', action='store_true',
         help="Show tags in log.")
     parser.add_argument('-a', '--all', action='store_true',
         help="Show all tags in log, included tags in corrupted commits.")
-    parser.add_argument('-d', '--data', action='store_true',
-        help="Also show the raw contents of files/attrs/tags.")
     parser.add_argument('-T', '--no-truncate', action='store_true',
-        help="Don't truncate large amounts of data.")
+        help="Show the full contents of files/attrs/tags.")
     sys.exit(main(parser.parse_args()))

scripts/test.py

@@ -231,7 +231,7 @@ class TestCase:
                 ncmd.extend(['-ex', 'r'])
                 if failure.assert_:
                     ncmd.extend(['-ex', 'up 2'])
-            elif gdb == 'start':
+            elif gdb == 'main':
                 ncmd.extend([
                     '-ex', 'b %s:%d' % (self.suite.path, self.code_lineno),
                     '-ex', 'r'])
@@ -760,7 +760,7 @@ if __name__ == "__main__":
         help="Store disk image in a file.")
     parser.add_argument('-b', '--build', action='store_true',
         help="Only build the tests, do not execute.")
-    parser.add_argument('-g', '--gdb', choices=['init', 'start', 'assert'],
+    parser.add_argument('-g', '--gdb', choices=['init', 'main', 'assert'],
         nargs='?', const='assert',
         help="Drop into gdb on test failure.")
     parser.add_argument('--no-internal', action='store_true',

tests/test_alloc.toml

@@ -323,6 +323,90 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
+[[case]] # what if we have a bad block during an allocation scan?
+in = "lfs.c"
+define.LFS_ERASE_CYCLES = 0xffffffff
+define.LFS_BADBLOCK_BEHAVIOR = 'LFS_TESTBD_BADBLOCK_READERROR'
+code = '''
+    lfs_format(&lfs, &cfg) => 0;
+    lfs_mount(&lfs, &cfg) => 0;
+    // first fill to exhaustion to find available space
+    lfs_file_open(&lfs, &file, "pacman", LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    strcpy((char*)buffer, "waka");
+    size = strlen("waka");
+    lfs_size_t filesize = 0;
+    while (true) {
+        lfs_ssize_t res = lfs_file_write(&lfs, &file, buffer, size);
+        assert(res == (lfs_ssize_t)size || res == LFS_ERR_NOSPC);
+        if (res == LFS_ERR_NOSPC) {
+            break;
+        }
+        filesize += size;
+    }
+    lfs_file_close(&lfs, &file) => 0;
+    // now fill all but a couple of blocks of the filesystem with data
+    filesize -= 3*LFS_BLOCK_SIZE;
+    lfs_file_open(&lfs, &file, "pacman", LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    strcpy((char*)buffer, "waka");
+    size = strlen("waka");
+    for (lfs_size_t i = 0; i < filesize/size; i++) {
+        lfs_file_write(&lfs, &file, buffer, size) => size;
+    }
+    lfs_file_close(&lfs, &file) => 0;
+    // also save head of file so we can error during lookahead scan
+    lfs_block_t fileblock = file.ctz.head;
+    lfs_unmount(&lfs) => 0;
+
+    // remount to force an alloc scan
+    lfs_mount(&lfs, &cfg) => 0;
+
+    // but mark the head of our file as a "bad block", this is force our
+    // scan to bail early
+    lfs_testbd_setwear(&cfg, fileblock, 0xffffffff) => 0;
+    lfs_file_open(&lfs, &file, "ghost", LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    strcpy((char*)buffer, "chomp");
+    size = strlen("chomp");
+    while (true) {
+        lfs_ssize_t res = lfs_file_write(&lfs, &file, buffer, size);
+        assert(res == (lfs_ssize_t)size || res == LFS_ERR_CORRUPT);
+        if (res == LFS_ERR_CORRUPT) {
+            break;
+        }
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    // now reverse the "bad block" and try to write the file again until we
+    // run out of space
+    lfs_testbd_setwear(&cfg, fileblock, 0) => 0;
+    lfs_file_open(&lfs, &file, "ghost", LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    strcpy((char*)buffer, "chomp");
+    size = strlen("chomp");
+    while (true) {
+        lfs_ssize_t res = lfs_file_write(&lfs, &file, buffer, size);
+        assert(res == (lfs_ssize_t)size || res == LFS_ERR_NOSPC);
+        if (res == LFS_ERR_NOSPC) {
+            break;
+        }
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    lfs_unmount(&lfs) => 0;
+
+    // check that the disk isn't hurt
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file, "pacman", LFS_O_RDONLY) => 0;
+    strcpy((char*)buffer, "waka");
+    size = strlen("waka");
+    for (lfs_size_t i = 0; i < filesize/size; i++) {
+        uint8_t rbuffer[4];
+        lfs_file_read(&lfs, &file, rbuffer, size) => size;
+        assert(memcmp(rbuffer, buffer, size) == 0);
+    }
+    lfs_file_close(&lfs, &file) => 0;
+    lfs_unmount(&lfs) => 0;
+'''
+
+
 # Below, I don't like these tests. They're fragile and depend _heavily_
 # on the geometry of the block device. But they are valuable. Eventually they
 # should be removed and replaced with generalized tests.

tests/test_evil.toml

@@ -0,0 +1,288 @@
+# Tests for recovering from conditions which shouldn't normally
+# happen during normal operation of littlefs
+
+# invalid pointer tests (outside of block_count)
+
+[[case]] # invalid tail-pointer test
+define.TAIL_TYPE = ['LFS_TYPE_HARDTAIL', 'LFS_TYPE_SOFTTAIL']
+define.INVALSET = [0x3, 0x1, 0x2]
+in = "lfs.c"
+code = '''
+    // create littlefs
+    lfs_format(&lfs, &cfg) => 0;
+
+    // change tail-pointer to invalid pointers
+    lfs_init(&lfs, &cfg) => 0;
+    lfs_mdir_t mdir;
+    lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0;
+    lfs_dir_commit(&lfs, &mdir, LFS_MKATTRS(
+            {LFS_MKTAG(LFS_TYPE_HARDTAIL, 0x3ff, 8),
+                (lfs_block_t[2]){
+                    (INVALSET & 0x1) ? 0xcccccccc : 0,
+                    (INVALSET & 0x2) ? 0xcccccccc : 0}})) => 0;
+    lfs_deinit(&lfs) => 0;
+
+    // test that mount fails gracefully
+    lfs_mount(&lfs, &cfg) => LFS_ERR_CORRUPT;
+'''
+
+[[case]] # invalid dir pointer test
+define.INVALSET = [0x3, 0x1, 0x2]
+in = "lfs.c"
+code = '''
+    // create littlefs
+    lfs_format(&lfs, &cfg) => 0;
+    // make a dir
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_mkdir(&lfs, "dir_here") => 0;
+    lfs_unmount(&lfs) => 0;
+
+    // change the dir pointer to be invalid
+    lfs_init(&lfs, &cfg) => 0;
+    lfs_mdir_t mdir;
+    lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0;
+    // make sure id 1 == our directory
+    lfs_dir_get(&lfs, &mdir,
+            LFS_MKTAG(0x700, 0x3ff, 0),
+            LFS_MKTAG(LFS_TYPE_NAME, 1, strlen("dir_here")), buffer)
+                => LFS_MKTAG(LFS_TYPE_DIR, 1, strlen("dir_here"));
+    assert(memcmp((char*)buffer, "dir_here", strlen("dir_here")) == 0);
+    // change dir pointer
+    lfs_dir_commit(&lfs, &mdir, LFS_MKATTRS(
+            {LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 1, 8),
+                (lfs_block_t[2]){
+                    (INVALSET & 0x1) ? 0xcccccccc : 0,
+                    (INVALSET & 0x2) ? 0xcccccccc : 0}})) => 0;
+    lfs_deinit(&lfs) => 0;
+
+    // test that accessing our bad dir fails, note there's a number
+    // of ways to access the dir, some can fail, but some don't
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_stat(&lfs, "dir_here", &info) => 0;
+    assert(strcmp(info.name, "dir_here") == 0);
+    assert(info.type == LFS_TYPE_DIR);
+
+    lfs_dir_open(&lfs, &dir, "dir_here") => LFS_ERR_CORRUPT;
+    lfs_stat(&lfs, "dir_here/file_here", &info) => LFS_ERR_CORRUPT;
+    lfs_dir_open(&lfs, &dir, "dir_here/dir_here") => LFS_ERR_CORRUPT;
+    lfs_file_open(&lfs, &file, "dir_here/file_here",
+            LFS_O_RDONLY) => LFS_ERR_CORRUPT;
+    lfs_file_open(&lfs, &file, "dir_here/file_here",
+            LFS_O_WRONLY | LFS_O_CREAT) => LFS_ERR_CORRUPT;
+    lfs_unmount(&lfs) => 0;
+'''
+
+[[case]] # invalid file pointer test
+in = "lfs.c"
+define.SIZE = [10, 1000, 100000] # faked file size
+code = '''
+    // create littlefs
+    lfs_format(&lfs, &cfg) => 0;
+    // make a file
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file, "file_here",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    lfs_file_close(&lfs, &file) => 0;
+    lfs_unmount(&lfs) => 0;
+
+    // change the file pointer to be invalid
+    lfs_init(&lfs, &cfg) => 0;
+    lfs_mdir_t mdir;
+    lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0;
+    // make sure id 1 == our file
+    lfs_dir_get(&lfs, &mdir,
+            LFS_MKTAG(0x700, 0x3ff, 0),
+            LFS_MKTAG(LFS_TYPE_NAME, 1, strlen("file_here")), buffer)
+                => LFS_MKTAG(LFS_TYPE_REG, 1, strlen("file_here"));
+    assert(memcmp((char*)buffer, "file_here", strlen("file_here")) == 0);
+    // change file pointer
+    lfs_dir_commit(&lfs, &mdir, LFS_MKATTRS(
+            {LFS_MKTAG(LFS_TYPE_CTZSTRUCT, 1, sizeof(struct lfs_ctz)),
+                &(struct lfs_ctz){0xcccccccc, lfs_tole32(SIZE)}})) => 0;
+    lfs_deinit(&lfs) => 0;
+
+    // test that accessing our bad file fails, note there's a number
+    // of ways to access the dir, some can fail, but some don't
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_stat(&lfs, "file_here", &info) => 0;
+    assert(strcmp(info.name, "file_here") == 0);
+    assert(info.type == LFS_TYPE_REG);
+    assert(info.size == SIZE);
+
+    lfs_file_open(&lfs, &file, "file_here", LFS_O_RDONLY) => 0;
+    lfs_file_read(&lfs, &file, buffer, SIZE) => LFS_ERR_CORRUPT;
+    lfs_file_close(&lfs, &file) => 0;
+
+    // any allocs that traverse CTZ must unfortunately must fail
+    if (SIZE > 2*LFS_BLOCK_SIZE) {
+        lfs_mkdir(&lfs, "dir_here") => LFS_ERR_CORRUPT;
+    }
+    lfs_unmount(&lfs) => 0;
+'''
+
+[[case]] # invalid pointer in CTZ skip-list test
+define.SIZE = ['2*LFS_BLOCK_SIZE', '3*LFS_BLOCK_SIZE', '4*LFS_BLOCK_SIZE']
+in = "lfs.c"
+code = '''
+    // create littlefs
+    lfs_format(&lfs, &cfg) => 0;
+    // make a file
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file, "file_here",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    for (int i = 0; i < SIZE; i++) {
+        char c = 'c';
+        lfs_file_write(&lfs, &file, &c, 1) => 1;
+    }
+    lfs_file_close(&lfs, &file) => 0;
+    lfs_unmount(&lfs) => 0;
+    // change pointer in CTZ skip-list to be invalid
+    lfs_init(&lfs, &cfg) => 0;
+    lfs_mdir_t mdir;
+    lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0;
+    // make sure id 1 == our file and get our CTZ structure
+    lfs_dir_get(&lfs, &mdir,
+            LFS_MKTAG(0x700, 0x3ff, 0),
+            LFS_MKTAG(LFS_TYPE_NAME, 1, strlen("file_here")), buffer)
+                => LFS_MKTAG(LFS_TYPE_REG, 1, strlen("file_here"));
+    assert(memcmp((char*)buffer, "file_here", strlen("file_here")) == 0);
+    struct lfs_ctz ctz;
+    lfs_dir_get(&lfs, &mdir,
+            LFS_MKTAG(0x700, 0x3ff, 0),
+            LFS_MKTAG(LFS_TYPE_STRUCT, 1, sizeof(struct lfs_ctz)), &ctz)
+                => LFS_MKTAG(LFS_TYPE_CTZSTRUCT, 1, sizeof(struct lfs_ctz));
+    lfs_ctz_fromle32(&ctz);
+    // rewrite block to contain bad pointer
+    uint8_t bbuffer[LFS_BLOCK_SIZE];
+    cfg.read(&cfg, ctz.head, 0, bbuffer, LFS_BLOCK_SIZE) => 0;
+    uint32_t bad = lfs_tole32(0xcccccccc);
+    memcpy(&bbuffer[0], &bad, sizeof(bad));
+    memcpy(&bbuffer[4], &bad, sizeof(bad));
+    cfg.erase(&cfg, ctz.head) => 0;
+    cfg.prog(&cfg, ctz.head, 0, bbuffer, LFS_BLOCK_SIZE) => 0;
+    lfs_deinit(&lfs) => 0;
+
+    // test that accessing our bad file fails, note there's a number
+    // of ways to access the dir, some can fail, but some don't
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_stat(&lfs, "file_here", &info) => 0;
+    assert(strcmp(info.name, "file_here") == 0);
+    assert(info.type == LFS_TYPE_REG);
+    assert(info.size == SIZE);
+
+    lfs_file_open(&lfs, &file, "file_here", LFS_O_RDONLY) => 0;
+    lfs_file_read(&lfs, &file, buffer, SIZE) => LFS_ERR_CORRUPT;
+    lfs_file_close(&lfs, &file) => 0;
+
+    // any allocs that traverse CTZ must unfortunately must fail
+    if (SIZE > 2*LFS_BLOCK_SIZE) {
+        lfs_mkdir(&lfs, "dir_here") => LFS_ERR_CORRUPT;
+    }
+    lfs_unmount(&lfs) => 0;
+'''
+
+
+[[case]] # invalid gstate pointer
+define.INVALSET = [0x3, 0x1, 0x2]
+in = "lfs.c"
+code = '''
+    // create littlefs
+    lfs_format(&lfs, &cfg) => 0;
+
+    // create an invalid gstate
+    lfs_init(&lfs, &cfg) => 0;
+    lfs_mdir_t mdir;
+    lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0;
+    lfs_fs_prepmove(&lfs, 1, (lfs_block_t [2]){
+            (INVALSET & 0x1) ? 0xcccccccc : 0,
+            (INVALSET & 0x2) ? 0xcccccccc : 0});
+    lfs_dir_commit(&lfs, &mdir, NULL, 0) => 0;
+    lfs_deinit(&lfs) => 0;
+
+    // test that mount fails gracefully
+    // mount may not fail, but our first alloc should fail when
+    // we try to fix the gstate
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_mkdir(&lfs, "should_fail") => LFS_ERR_CORRUPT;
+    lfs_unmount(&lfs) => 0;
+'''
+
+# cycle detection/recovery tests
+
+[[case]] # metadata-pair threaded-list loop test
+in = "lfs.c"
+code = '''
+    // create littlefs
+    lfs_format(&lfs, &cfg) => 0;
+
+    // change tail-pointer to point to ourself
+    lfs_init(&lfs, &cfg) => 0;
+    lfs_mdir_t mdir;
+    lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0;
+    lfs_dir_commit(&lfs, &mdir, LFS_MKATTRS(
+            {LFS_MKTAG(LFS_TYPE_HARDTAIL, 0x3ff, 8),
+                (lfs_block_t[2]){0, 1}})) => 0;
+    lfs_deinit(&lfs) => 0;
+
+    // test that mount fails gracefully
+    lfs_mount(&lfs, &cfg) => LFS_ERR_CORRUPT;
+'''
+
+[[case]] # metadata-pair threaded-list 2-length loop test
+in = "lfs.c"
+code = '''
+    // create littlefs with child dir
+    lfs_format(&lfs, &cfg) => 0;
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_mkdir(&lfs, "child") => 0;
+    lfs_unmount(&lfs) => 0;
+
+    // find child
+    lfs_init(&lfs, &cfg) => 0;
+    lfs_mdir_t mdir;
+    lfs_block_t pair[2];
+    lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0;
+    lfs_dir_get(&lfs, &mdir,
+            LFS_MKTAG(0x7ff, 0x3ff, 0),
+            LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 1, sizeof(pair)), pair)
+                => LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 1, sizeof(pair));
+    lfs_pair_fromle32(pair);
+    // change tail-pointer to point to root
+    lfs_dir_fetch(&lfs, &mdir, pair) => 0;
+    lfs_dir_commit(&lfs, &mdir, LFS_MKATTRS(
+            {LFS_MKTAG(LFS_TYPE_HARDTAIL, 0x3ff, 8),
+                (lfs_block_t[2]){0, 1}})) => 0;
+    lfs_deinit(&lfs) => 0;
+
+    // test that mount fails gracefully
+    lfs_mount(&lfs, &cfg) => LFS_ERR_CORRUPT;
+'''
+
+[[case]] # metadata-pair threaded-list 1-length child loop test
+in = "lfs.c"
+code = '''
+    // create littlefs with child dir
+    lfs_format(&lfs, &cfg) => 0;
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_mkdir(&lfs, "child") => 0;
+    lfs_unmount(&lfs) => 0;
+
+    // find child
+    lfs_init(&lfs, &cfg) => 0;
+    lfs_mdir_t mdir;
+    lfs_block_t pair[2];
+    lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0;
+    lfs_dir_get(&lfs, &mdir,
+            LFS_MKTAG(0x7ff, 0x3ff, 0),
+            LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 1, sizeof(pair)), pair)
+                => LFS_MKTAG(LFS_TYPE_DIRSTRUCT, 1, sizeof(pair));
+    lfs_pair_fromle32(pair);
+    // change tail-pointer to point to ourself
+    lfs_dir_fetch(&lfs, &mdir, pair) => 0;
+    lfs_dir_commit(&lfs, &mdir, LFS_MKATTRS(
+            {LFS_MKTAG(LFS_TYPE_HARDTAIL, 0x3ff, 8), pair})) => 0;
+    lfs_deinit(&lfs) => 0;
+
+    // test that mount fails gracefully
+    lfs_mount(&lfs, &cfg) => LFS_ERR_CORRUPT;
+'''

tests/test_superblocks.toml

@@ -27,41 +27,55 @@ code = '''
 '''
 
 [[case]] # expanding superblock
-define.BLOCK_CYCLES = [32, 33, 1]
+define.LFS_BLOCK_CYCLES = [32, 33, 1]
 define.N = [10, 100, 1000]
 code = '''
     lfs_format(&lfs, &cfg) => 0;
     lfs_mount(&lfs, &cfg) => 0;
     for (int i = 0; i < N; i++) {
-        lfs_mkdir(&lfs, "dummy") => 0;
+        lfs_file_open(&lfs, &file, "dummy",
+                LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
+        lfs_file_close(&lfs, &file) => 0;
         lfs_stat(&lfs, "dummy", &info) => 0;
         assert(strcmp(info.name, "dummy") == 0);
+        assert(info.type == LFS_TYPE_REG);
         lfs_remove(&lfs, "dummy") => 0;
     }
     lfs_unmount(&lfs) => 0;
 
     // one last check after power-cycle
     lfs_mount(&lfs, &cfg) => 0;
-    lfs_mkdir(&lfs, "dummy") => 0;
+    lfs_file_open(&lfs, &file, "dummy",
+            LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
+    lfs_file_close(&lfs, &file) => 0;
     lfs_stat(&lfs, "dummy", &info) => 0;
     assert(strcmp(info.name, "dummy") == 0);
+    assert(info.type == LFS_TYPE_REG);
     lfs_unmount(&lfs) => 0;
 '''
 
 [[case]] # expanding superblock with power cycle
-define.BLOCK_CYCLES = [32, 33, 1]
+define.LFS_BLOCK_CYCLES = [32, 33, 1]
 define.N = [10, 100, 1000]
 code = '''
     lfs_format(&lfs, &cfg) => 0;
     for (int i = 0; i < N; i++) {
         lfs_mount(&lfs, &cfg) => 0;
         // remove lingering dummy?
-        err = lfs_remove(&lfs, "dummy");
+        err = lfs_stat(&lfs, "dummy", &info);
         assert(err == 0 || (err == LFS_ERR_NOENT && i == 0));
+        if (!err) {
+            assert(strcmp(info.name, "dummy") == 0);
+            assert(info.type == LFS_TYPE_REG);
+            lfs_remove(&lfs, "dummy") => 0;
+        }
 
-        lfs_mkdir(&lfs, "dummy") => 0;
+        lfs_file_open(&lfs, &file, "dummy",
+                LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
+        lfs_file_close(&lfs, &file) => 0;
         lfs_stat(&lfs, "dummy", &info) => 0;
         assert(strcmp(info.name, "dummy") == 0);
+        assert(info.type == LFS_TYPE_REG);
         lfs_unmount(&lfs) => 0;
     }
 
@@ -69,11 +83,12 @@ code = '''
     lfs_mount(&lfs, &cfg) => 0;
     lfs_stat(&lfs, "dummy", &info) => 0;
     assert(strcmp(info.name, "dummy") == 0);
+    assert(info.type == LFS_TYPE_REG);
     lfs_unmount(&lfs) => 0;
 '''
 
 [[case]] # reentrant expanding superblock
-define.BLOCK_CYCLES = [2, 1]
+define.LFS_BLOCK_CYCLES = [2, 1]
 define.N = 24
 reentrant = true
 code = '''
@@ -85,12 +100,20 @@ code = '''
 
     for (int i = 0; i < N; i++) {
         // remove lingering dummy?
-        err = lfs_remove(&lfs, "dummy");
+        err = lfs_stat(&lfs, "dummy", &info);
         assert(err == 0 || (err == LFS_ERR_NOENT && i == 0));
+        if (!err) {
+            assert(strcmp(info.name, "dummy") == 0);
+            assert(info.type == LFS_TYPE_REG);
+            lfs_remove(&lfs, "dummy") => 0;
+        }
 
-        lfs_mkdir(&lfs, "dummy") => 0;
+        lfs_file_open(&lfs, &file, "dummy",
+                LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
+        lfs_file_close(&lfs, &file) => 0;
         lfs_stat(&lfs, "dummy", &info) => 0;
         assert(strcmp(info.name, "dummy") == 0);
+        assert(info.type == LFS_TYPE_REG);
     }
 
     lfs_unmount(&lfs) => 0;
@@ -99,5 +122,6 @@ code = '''
     lfs_mount(&lfs, &cfg) => 0;
     lfs_stat(&lfs, "dummy", &info) => 0;
     assert(strcmp(info.name, "dummy") == 0);
+    assert(info.type == LFS_TYPE_REG);
     lfs_unmount(&lfs) => 0;
 '''