Fixed single unchecked bit during commit verification

This bug was exposed by the bad-block tests due to changes to block
allocation, but could have been hit before these changes.

In flash, when blocks fail, they don't fail in a predictable manner. To
account for this, the bad-block tests check a number of failure
behaviors. The interesting one here is "LFS_TESTBD_BADBLOCK_ERASENOOP",
in which bad blocks can not be erased or programmed, and are stuck with
the data written at the time the blocks go bad.

This is actually a pretty realistic failure behavior, since flash needs a
large voltage to force the electrons of the floating gates. Though
realistically, such a failure would like corrupt the data a bit, not leave the
underlying data perfectly intact.

LFS_TESTBD_BADBLOCK_ERASENOOP is rather interesting to test for because it
means bad blocks can end up with perfectly valid CRCs after a failed write,
confusing littlefs.

---

In this case, we had the perfect series of operations such that a test
was repeatedly writing the same sequence of metadata commits to the same
block, which eventually goes bad, leaving the block stuck with metadata
that occurs later in the sequence.

What this means is that after the first commit, the metadata block
contained both the first and second commits, even though the loop in the
test hadn't reached that point yet.

expected       actual
.----------.  .----------.
| commit 1 |  | commit 1 |
| crc 1    |  | crc 1    |
|          |  | commit 2 <-- (from previous iteration)
|          |  | crc 2    |
'----------'  '----------'

To protect against this, littlefs normally compares the written CRC
against the expected CRC, but because this was the exact same data that
it was going to write, this CRCs end up the same.

Ah! But doesn't littlefs also encode the state of the next page to keep
track of if the next page has been erased or not? Wouldn't that change
between iterations?

It does! In a single bit in the CRC-tag. But thanks to some incorrect
logic attempting to avoid an extra condition in the loop for writing out
padding commits, the CRC that littlefs checked against was the CRC
immediately before we include the "is-next-page-erased" bit.

Changing the verification check to use the same CRC as what is used to
verify commits on fetch solves this problem.

.gitignore

@@ -8,3 +8,5 @@ blocks/
 lfs
 test.c
 tests/*.toml.*
+scripts/__pycache__
+.gdb_history

Makefile

@@ -26,8 +26,6 @@ endif
 override CFLAGS += -I.
 override CFLAGS += -std=c99 -Wall -pedantic
 override CFLAGS += -Wextra -Wshadow -Wjump-misses-init -Wundef
-# Remove missing-field-initializers because of GCC bug
-override CFLAGS += -Wno-missing-field-initializers
 
 ifdef VERBOSE
 override TFLAGS += -v

README.md

@@ -115,6 +115,9 @@ the filesystem until sync or close is called on the file.
 
 ## Other notes
 
+Littlefs is written in C, and specifically should compile with any compiler
+that conforms to the `C99` standard.
+
 All littlefs calls have the potential to return a negative error code. The
 errors can be either one of those found in the `enum lfs_error` in
 [lfs.h](lfs.h), or an error returned by the user's block device operations.

SPEC.md

@@ -289,8 +289,8 @@ Layout of the name tag:
 ```
         tag                          data
 [--      32      --][---        variable length        ---]
-[1| 3| 8 | 10 | 10 ][---            (size)             ---]
- ^  ^  ^    ^    ^- size               ^- file name
+[1| 3| 8 | 10 | 10 ][---          (size * 8)           ---]
+ ^  ^  ^    ^    ^- size                   ^- file name
  |  |  |    '------ id
  |  |  '----------- file type
  |  '-------------- type1 (0x0)
@@ -470,8 +470,8 @@ Layout of the inline-struct tag:
 ```
         tag                          data
 [--      32      --][---        variable length        ---]
-[1|- 11 -| 10 | 10 ][---            (size)             ---]
- ^    ^     ^    ^- size               ^- inline data
+[1|- 11 -| 10 | 10 ][---           (size * 8)          ---]
+ ^    ^     ^    ^- size                    ^- inline data
  |    |     '------ id
  |    '------------ type (0x201)
  '----------------- valid bit
@@ -556,8 +556,8 @@ Layout of the user-attr tag:
 ```
         tag                          data
 [--      32      --][---        variable length        ---]
-[1| 3| 8 | 10 | 10 ][---            (size)             ---]
- ^  ^  ^    ^    ^- size               ^- attr data
+[1| 3| 8 | 10 | 10 ][---           (size * 8)          ---]
+ ^  ^  ^    ^    ^- size                    ^- attr data
  |  |  |    '------ id
  |  |  '----------- attr type
  |  '-------------- type1 (0x3)
@@ -764,9 +764,9 @@ Layout of the CRC tag:
 ```
         tag                                    data
 [--      32      --][--      32      --|---        variable length        ---]
-[1| 3| 8 | 10 | 10 ][--      32      --|---            (size)             ---]
- ^  ^  ^    ^    ^            ^- crc                      ^- padding
- |  |  |    |    '- size (12)
+[1| 3| 8 | 10 | 10 ][--      32      --|---        (size * 8 - 32)        ---]
+ ^  ^  ^    ^    ^            ^- crc                             ^- padding
+ |  |  |    |    '- size
  |  |  |    '------ id (0x3ff)
  |  |  '----------- valid state
  |  '-------------- type1 (0x5)

bd/lfs_filebd.c

@@ -12,7 +12,7 @@
 
 int lfs_filebd_createcfg(const struct lfs_config *cfg, const char *path,
         const struct lfs_filebd_config *bdcfg) {
-    LFS_TRACE("lfs_filebd_createcfg(%p {.context=%p, "
+    LFS_FILEBD_TRACE("lfs_filebd_createcfg(%p {.context=%p, "
                 ".read=%p, .prog=%p, .erase=%p, .sync=%p, "
                 ".read_size=%"PRIu32", .prog_size=%"PRIu32", "
                 ".block_size=%"PRIu32", .block_count=%"PRIu32"}, "
@@ -30,16 +30,16 @@ int lfs_filebd_createcfg(const struct lfs_config *cfg, const char *path,
     bd->fd = open(path, O_RDWR | O_CREAT, 0666);
     if (bd->fd < 0) {
         int err = -errno;
-        LFS_TRACE("lfs_filebd_createcfg -> %d", err);
+        LFS_FILEBD_TRACE("lfs_filebd_createcfg -> %d", err);
         return err;
     }
 
-    LFS_TRACE("lfs_filebd_createcfg -> %d", 0);
+    LFS_FILEBD_TRACE("lfs_filebd_createcfg -> %d", 0);
     return 0;
 }
 
 int lfs_filebd_create(const struct lfs_config *cfg, const char *path) {
-    LFS_TRACE("lfs_filebd_create(%p {.context=%p, "
+    LFS_FILEBD_TRACE("lfs_filebd_create(%p {.context=%p, "
                 ".read=%p, .prog=%p, .erase=%p, .sync=%p, "
                 ".read_size=%"PRIu32", .prog_size=%"PRIu32", "
                 ".block_size=%"PRIu32", .block_count=%"PRIu32"}, "
@@ -51,26 +51,27 @@ int lfs_filebd_create(const struct lfs_config *cfg, const char *path) {
             path);
     static const struct lfs_filebd_config defaults = {.erase_value=-1};
     int err = lfs_filebd_createcfg(cfg, path, &defaults);
-    LFS_TRACE("lfs_filebd_create -> %d", err);
+    LFS_FILEBD_TRACE("lfs_filebd_create -> %d", err);
     return err;
 }
 
 int lfs_filebd_destroy(const struct lfs_config *cfg) {
-    LFS_TRACE("lfs_filebd_destroy(%p)", (void*)cfg);
+    LFS_FILEBD_TRACE("lfs_filebd_destroy(%p)", (void*)cfg);
     lfs_filebd_t *bd = cfg->context;
     int err = close(bd->fd);
     if (err < 0) {
         err = -errno;
-        LFS_TRACE("lfs_filebd_destroy -> %d", err);
+        LFS_FILEBD_TRACE("lfs_filebd_destroy -> %d", err);
         return err;
     }
-    LFS_TRACE("lfs_filebd_destroy -> %d", 0);
+    LFS_FILEBD_TRACE("lfs_filebd_destroy -> %d", 0);
     return 0;
 }
 
 int lfs_filebd_read(const struct lfs_config *cfg, lfs_block_t block,
         lfs_off_t off, void *buffer, lfs_size_t size) {
-    LFS_TRACE("lfs_filebd_read(%p, 0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
+    LFS_FILEBD_TRACE("lfs_filebd_read(%p, "
+                "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
             (void*)cfg, block, off, buffer, size);
     lfs_filebd_t *bd = cfg->context;
 
@@ -89,24 +90,24 @@ int lfs_filebd_read(const struct lfs_config *cfg, lfs_block_t block,
             (off_t)block*cfg->block_size + (off_t)off, SEEK_SET);
     if (res1 < 0) {
         int err = -errno;
-        LFS_TRACE("lfs_filebd_read -> %d", err);
+        LFS_FILEBD_TRACE("lfs_filebd_read -> %d", err);
         return err;
     }
 
     ssize_t res2 = read(bd->fd, buffer, size);
     if (res2 < 0) {
         int err = -errno;
-        LFS_TRACE("lfs_filebd_read -> %d", err);
+        LFS_FILEBD_TRACE("lfs_filebd_read -> %d", err);
         return err;
     }
 
-    LFS_TRACE("lfs_filebd_read -> %d", 0);
+    LFS_FILEBD_TRACE("lfs_filebd_read -> %d", 0);
     return 0;
 }
 
 int lfs_filebd_prog(const struct lfs_config *cfg, lfs_block_t block,
         lfs_off_t off, const void *buffer, lfs_size_t size) {
-    LFS_TRACE("lfs_filebd_prog(%p, 0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
+    LFS_FILEBD_TRACE("lfs_filebd_prog(%p, 0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
             (void*)cfg, block, off, buffer, size);
     lfs_filebd_t *bd = cfg->context;
 
@@ -121,7 +122,7 @@ int lfs_filebd_prog(const struct lfs_config *cfg, lfs_block_t block,
                 (off_t)block*cfg->block_size + (off_t)off, SEEK_SET);
         if (res1 < 0) {
             int err = -errno;
-            LFS_TRACE("lfs_filebd_prog -> %d", err);
+            LFS_FILEBD_TRACE("lfs_filebd_prog -> %d", err);
             return err;
         }
 
@@ -130,7 +131,7 @@ int lfs_filebd_prog(const struct lfs_config *cfg, lfs_block_t block,
             ssize_t res2 = read(bd->fd, &c, 1);
             if (res2 < 0) {
                 int err = -errno;
-                LFS_TRACE("lfs_filebd_prog -> %d", err);
+                LFS_FILEBD_TRACE("lfs_filebd_prog -> %d", err);
                 return err;
             }
 
@@ -143,23 +144,23 @@ int lfs_filebd_prog(const struct lfs_config *cfg, lfs_block_t block,
             (off_t)block*cfg->block_size + (off_t)off, SEEK_SET);
     if (res1 < 0) {
         int err = -errno;
-        LFS_TRACE("lfs_filebd_prog -> %d", err);
+        LFS_FILEBD_TRACE("lfs_filebd_prog -> %d", err);
         return err;
     }
 
     ssize_t res2 = write(bd->fd, buffer, size);
     if (res2 < 0) {
         int err = -errno;
-        LFS_TRACE("lfs_filebd_prog -> %d", err);
+        LFS_FILEBD_TRACE("lfs_filebd_prog -> %d", err);
         return err;
     }
 
-    LFS_TRACE("lfs_filebd_prog -> %d", 0);
+    LFS_FILEBD_TRACE("lfs_filebd_prog -> %d", 0);
     return 0;
 }
 
 int lfs_filebd_erase(const struct lfs_config *cfg, lfs_block_t block) {
-    LFS_TRACE("lfs_filebd_erase(%p, 0x%"PRIx32")", (void*)cfg, block);
+    LFS_FILEBD_TRACE("lfs_filebd_erase(%p, 0x%"PRIx32")", (void*)cfg, block);
     lfs_filebd_t *bd = cfg->context;
 
     // check if erase is valid
@@ -170,7 +171,7 @@ int lfs_filebd_erase(const struct lfs_config *cfg, lfs_block_t block) {
         off_t res1 = lseek(bd->fd, (off_t)block*cfg->block_size, SEEK_SET);
         if (res1 < 0) {
             int err = -errno;
-            LFS_TRACE("lfs_filebd_erase -> %d", err);
+            LFS_FILEBD_TRACE("lfs_filebd_erase -> %d", err);
             return err;
         }
 
@@ -178,27 +179,27 @@ int lfs_filebd_erase(const struct lfs_config *cfg, lfs_block_t block) {
             ssize_t res2 = write(bd->fd, &(uint8_t){bd->cfg->erase_value}, 1);
             if (res2 < 0) {
                 int err = -errno;
-                LFS_TRACE("lfs_filebd_erase -> %d", err);
+                LFS_FILEBD_TRACE("lfs_filebd_erase -> %d", err);
                 return err;
             }
         }
     }
 
-    LFS_TRACE("lfs_filebd_erase -> %d", 0);
+    LFS_FILEBD_TRACE("lfs_filebd_erase -> %d", 0);
     return 0;
 }
 
 int lfs_filebd_sync(const struct lfs_config *cfg) {
-    LFS_TRACE("lfs_filebd_sync(%p)", (void*)cfg);
+    LFS_FILEBD_TRACE("lfs_filebd_sync(%p)", (void*)cfg);
     // file sync
     lfs_filebd_t *bd = cfg->context;
     int err = fsync(bd->fd);
     if (err) {
         err = -errno;
-        LFS_TRACE("lfs_filebd_sync -> %d", 0);
+        LFS_FILEBD_TRACE("lfs_filebd_sync -> %d", 0);
         return err;
     }
 
-    LFS_TRACE("lfs_filebd_sync -> %d", 0);
+    LFS_FILEBD_TRACE("lfs_filebd_sync -> %d", 0);
     return 0;
 }

bd/lfs_filebd.h

@@ -15,6 +15,14 @@ extern "C"
 {
 #endif
 
+
+// Block device specific tracing
+#ifdef LFS_FILEBD_YES_TRACE
+#define LFS_FILEBD_TRACE(...) LFS_TRACE(__VA_ARGS__)
+#else
+#define LFS_FILEBD_TRACE(...)
+#endif
+
 // filebd config (optional)
 struct lfs_filebd_config {
     // 8-bit erase value to use for simulating erases. -1 does not simulate

bd/lfs_rambd.c

@@ -8,7 +8,7 @@
 
 int lfs_rambd_createcfg(const struct lfs_config *cfg,
         const struct lfs_rambd_config *bdcfg) {
-    LFS_TRACE("lfs_rambd_createcfg(%p {.context=%p, "
+    LFS_RAMBD_TRACE("lfs_rambd_createcfg(%p {.context=%p, "
                 ".read=%p, .prog=%p, .erase=%p, .sync=%p, "
                 ".read_size=%"PRIu32", .prog_size=%"PRIu32", "
                 ".block_size=%"PRIu32", .block_count=%"PRIu32"}, "
@@ -27,7 +27,7 @@ int lfs_rambd_createcfg(const struct lfs_config *cfg,
     } else {
         bd->buffer = lfs_malloc(cfg->block_size * cfg->block_count);
         if (!bd->buffer) {
-            LFS_TRACE("lfs_rambd_createcfg -> %d", LFS_ERR_NOMEM);
+            LFS_RAMBD_TRACE("lfs_rambd_createcfg -> %d", LFS_ERR_NOMEM);
             return LFS_ERR_NOMEM;
         }
     }
@@ -38,12 +38,12 @@ int lfs_rambd_createcfg(const struct lfs_config *cfg,
                 cfg->block_size * cfg->block_count);
     }
 
-    LFS_TRACE("lfs_rambd_createcfg -> %d", 0);
+    LFS_RAMBD_TRACE("lfs_rambd_createcfg -> %d", 0);
     return 0;
 }
 
 int lfs_rambd_create(const struct lfs_config *cfg) {
-    LFS_TRACE("lfs_rambd_create(%p {.context=%p, "
+    LFS_RAMBD_TRACE("lfs_rambd_create(%p {.context=%p, "
                 ".read=%p, .prog=%p, .erase=%p, .sync=%p, "
                 ".read_size=%"PRIu32", .prog_size=%"PRIu32", "
                 ".block_size=%"PRIu32", .block_count=%"PRIu32"})",
@@ -53,24 +53,25 @@ int lfs_rambd_create(const struct lfs_config *cfg) {
             cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count);
     static const struct lfs_rambd_config defaults = {.erase_value=-1};
     int err = lfs_rambd_createcfg(cfg, &defaults);
-    LFS_TRACE("lfs_rambd_create -> %d", err);
+    LFS_RAMBD_TRACE("lfs_rambd_create -> %d", err);
     return err;
 }
 
 int lfs_rambd_destroy(const struct lfs_config *cfg) {
-    LFS_TRACE("lfs_rambd_destroy(%p)", (void*)cfg);
+    LFS_RAMBD_TRACE("lfs_rambd_destroy(%p)", (void*)cfg);
     // clean up memory
     lfs_rambd_t *bd = cfg->context;
     if (!bd->cfg->buffer) {
         lfs_free(bd->buffer);
     }
-    LFS_TRACE("lfs_rambd_destroy -> %d", 0);
+    LFS_RAMBD_TRACE("lfs_rambd_destroy -> %d", 0);
     return 0;
 }
 
 int lfs_rambd_read(const struct lfs_config *cfg, lfs_block_t block,
         lfs_off_t off, void *buffer, lfs_size_t size) {
-    LFS_TRACE("lfs_rambd_read(%p, 0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
+    LFS_RAMBD_TRACE("lfs_rambd_read(%p, "
+                "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
             (void*)cfg, block, off, buffer, size);
     lfs_rambd_t *bd = cfg->context;
 
@@ -82,13 +83,14 @@ int lfs_rambd_read(const struct lfs_config *cfg, lfs_block_t block,
     // read data
     memcpy(buffer, &bd->buffer[block*cfg->block_size + off], size);
 
-    LFS_TRACE("lfs_rambd_read -> %d", 0);
+    LFS_RAMBD_TRACE("lfs_rambd_read -> %d", 0);
     return 0;
 }
 
 int lfs_rambd_prog(const struct lfs_config *cfg, lfs_block_t block,
         lfs_off_t off, const void *buffer, lfs_size_t size) {
-    LFS_TRACE("lfs_rambd_prog(%p, 0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
+    LFS_RAMBD_TRACE("lfs_rambd_prog(%p, "
+                "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
             (void*)cfg, block, off, buffer, size);
     lfs_rambd_t *bd = cfg->context;
 
@@ -108,12 +110,12 @@ int lfs_rambd_prog(const struct lfs_config *cfg, lfs_block_t block,
     // program data
     memcpy(&bd->buffer[block*cfg->block_size + off], buffer, size);
 
-    LFS_TRACE("lfs_rambd_prog -> %d", 0);
+    LFS_RAMBD_TRACE("lfs_rambd_prog -> %d", 0);
     return 0;
 }
 
 int lfs_rambd_erase(const struct lfs_config *cfg, lfs_block_t block) {
-    LFS_TRACE("lfs_rambd_erase(%p, 0x%"PRIx32")", (void*)cfg, block);
+    LFS_RAMBD_TRACE("lfs_rambd_erase(%p, 0x%"PRIx32")", (void*)cfg, block);
     lfs_rambd_t *bd = cfg->context;
 
     // check if erase is valid
@@ -125,14 +127,14 @@ int lfs_rambd_erase(const struct lfs_config *cfg, lfs_block_t block) {
                 bd->cfg->erase_value, cfg->block_size);
     }
 
-    LFS_TRACE("lfs_rambd_erase -> %d", 0);
+    LFS_RAMBD_TRACE("lfs_rambd_erase -> %d", 0);
     return 0;
 }
 
 int lfs_rambd_sync(const struct lfs_config *cfg) {
-    LFS_TRACE("lfs_rambd_sync(%p)", (void*)cfg);
+    LFS_RAMBD_TRACE("lfs_rambd_sync(%p)", (void*)cfg);
     // sync does nothing because we aren't backed by anything real
     (void)cfg;
-    LFS_TRACE("lfs_rambd_sync -> %d", 0);
+    LFS_RAMBD_TRACE("lfs_rambd_sync -> %d", 0);
     return 0;
 }

bd/lfs_rambd.h

@@ -15,6 +15,14 @@ extern "C"
 {
 #endif
 
+
+// Block device specific tracing
+#ifdef LFS_RAMBD_YES_TRACE
+#define LFS_RAMBD_TRACE(...) LFS_TRACE(__VA_ARGS__)
+#else
+#define LFS_RAMBD_TRACE(...)
+#endif
+
 // rambd config (optional)
 struct lfs_rambd_config {
     // 8-bit erase value to simulate erasing with. -1 indicates no erase

bd/lfs_testbd.c

@@ -12,7 +12,7 @@
 
 int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path,
         const struct lfs_testbd_config *bdcfg) {
-    LFS_TRACE("lfs_testbd_createcfg(%p {.context=%p, "
+    LFS_TESTBD_TRACE("lfs_testbd_createcfg(%p {.context=%p, "
                 ".read=%p, .prog=%p, .erase=%p, .sync=%p, "
                 ".read_size=%"PRIu32", .prog_size=%"PRIu32", "
                 ".block_size=%"PRIu32", .block_count=%"PRIu32"}, "
@@ -38,9 +38,9 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path,
         if (bd->cfg->wear_buffer) {
             bd->wear = bd->cfg->wear_buffer;
         } else {
-            bd->wear = lfs_malloc(sizeof(lfs_testbd_wear_t) * cfg->block_count);
+            bd->wear = lfs_malloc(sizeof(lfs_testbd_wear_t)*cfg->block_count);
             if (!bd->wear) {
-                LFS_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM);
+                LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", LFS_ERR_NOMEM);
                 return LFS_ERR_NOMEM;
             }
         }
@@ -54,7 +54,7 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path,
             .erase_value = bd->cfg->erase_value,
         };
         int err = lfs_filebd_createcfg(cfg, path, &bd->u.file.cfg);
-        LFS_TRACE("lfs_testbd_createcfg -> %d", err);
+        LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", err);
         return err;
     } else {
         bd->u.ram.cfg = (struct lfs_rambd_config){
@@ -62,13 +62,13 @@ int lfs_testbd_createcfg(const struct lfs_config *cfg, const char *path,
             .buffer = bd->cfg->buffer,
         };
         int err = lfs_rambd_createcfg(cfg, &bd->u.ram.cfg);
-        LFS_TRACE("lfs_testbd_createcfg -> %d", err);
+        LFS_TESTBD_TRACE("lfs_testbd_createcfg -> %d", err);
         return err;
     }
 }
 
 int lfs_testbd_create(const struct lfs_config *cfg, const char *path) {
-    LFS_TRACE("lfs_testbd_create(%p {.context=%p, "
+    LFS_TESTBD_TRACE("lfs_testbd_create(%p {.context=%p, "
                 ".read=%p, .prog=%p, .erase=%p, .sync=%p, "
                 ".read_size=%"PRIu32", .prog_size=%"PRIu32", "
                 ".block_size=%"PRIu32", .block_count=%"PRIu32"}, "
@@ -80,12 +80,12 @@ int lfs_testbd_create(const struct lfs_config *cfg, const char *path) {
             path);
     static const struct lfs_testbd_config defaults = {.erase_value=-1};
     int err = lfs_testbd_createcfg(cfg, path, &defaults);
-    LFS_TRACE("lfs_testbd_create -> %d", err);
+    LFS_TESTBD_TRACE("lfs_testbd_create -> %d", err);
     return err;
 }
 
 int lfs_testbd_destroy(const struct lfs_config *cfg) {
-    LFS_TRACE("lfs_testbd_destroy(%p)", (void*)cfg);
+    LFS_TESTBD_TRACE("lfs_testbd_destroy(%p)", (void*)cfg);
     lfs_testbd_t *bd = cfg->context;
     if (bd->cfg->erase_cycles && !bd->cfg->wear_buffer) {
         lfs_free(bd->wear);
@@ -93,11 +93,11 @@ int lfs_testbd_destroy(const struct lfs_config *cfg) {
 
     if (bd->persist) {
         int err = lfs_filebd_destroy(cfg);
-        LFS_TRACE("lfs_testbd_destroy -> %d", err);
+        LFS_TESTBD_TRACE("lfs_testbd_destroy -> %d", err);
         return err;
     } else {
         int err = lfs_rambd_destroy(cfg);
-        LFS_TRACE("lfs_testbd_destroy -> %d", err);
+        LFS_TESTBD_TRACE("lfs_testbd_destroy -> %d", err);
         return err;
     }
 }
@@ -145,7 +145,8 @@ static int lfs_testbd_rawsync(const struct lfs_config *cfg) {
 /// block device API ///
 int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block,
         lfs_off_t off, void *buffer, lfs_size_t size) {
-    LFS_TRACE("lfs_testbd_read(%p, 0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
+    LFS_TESTBD_TRACE("lfs_testbd_read(%p, "
+                "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
             (void*)cfg, block, off, buffer, size);
     lfs_testbd_t *bd = cfg->context;
 
@@ -157,19 +158,20 @@ int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block,
     // block bad?
     if (bd->cfg->erase_cycles && bd->wear[block] >= bd->cfg->erase_cycles &&
             bd->cfg->badblock_behavior == LFS_TESTBD_BADBLOCK_READERROR) {
-        LFS_TRACE("lfs_testbd_read -> %d", LFS_ERR_CORRUPT);
+        LFS_TESTBD_TRACE("lfs_testbd_read -> %d", LFS_ERR_CORRUPT);
         return LFS_ERR_CORRUPT;
     }
 
     // read
     int err = lfs_testbd_rawread(cfg, block, off, buffer, size);
-    LFS_TRACE("lfs_testbd_read -> %d", err);
+    LFS_TESTBD_TRACE("lfs_testbd_read -> %d", err);
     return err;
 }
 
 int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block,
         lfs_off_t off, const void *buffer, lfs_size_t size) {
-    LFS_TRACE("lfs_testbd_prog(%p, 0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
+    LFS_TESTBD_TRACE("lfs_testbd_prog(%p, "
+                "0x%"PRIx32", %"PRIu32", %p, %"PRIu32")",
             (void*)cfg, block, off, buffer, size);
     lfs_testbd_t *bd = cfg->context;
 
@@ -182,13 +184,13 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block,
     if (bd->cfg->erase_cycles && bd->wear[block] >= bd->cfg->erase_cycles) {
         if (bd->cfg->badblock_behavior ==
                 LFS_TESTBD_BADBLOCK_PROGERROR) {
-            LFS_TRACE("lfs_testbd_prog -> %d", LFS_ERR_CORRUPT);
+            LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", LFS_ERR_CORRUPT);
             return LFS_ERR_CORRUPT;
         } else if (bd->cfg->badblock_behavior ==
                 LFS_TESTBD_BADBLOCK_PROGNOOP ||
                 bd->cfg->badblock_behavior ==
                 LFS_TESTBD_BADBLOCK_ERASENOOP) {
-            LFS_TRACE("lfs_testbd_prog -> %d", 0);
+            LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0);
             return 0;
         }
     }
@@ -196,7 +198,7 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block,
     // prog
     int err = lfs_testbd_rawprog(cfg, block, off, buffer, size);
     if (err) {
-        LFS_TRACE("lfs_testbd_prog -> %d", err);
+        LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", err);
         return err;
     }
 
@@ -211,12 +213,12 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block,
         }
     }
 
-    LFS_TRACE("lfs_testbd_prog -> %d", 0);
+    LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0);
     return 0;
 }
 
 int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) {
-    LFS_TRACE("lfs_testbd_erase(%p, 0x%"PRIx32")", (void*)cfg, block);
+    LFS_TESTBD_TRACE("lfs_testbd_erase(%p, 0x%"PRIx32")", (void*)cfg, block);
     lfs_testbd_t *bd = cfg->context;
 
     // check if erase is valid
@@ -227,11 +229,11 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) {
         if (bd->wear[block] >= bd->cfg->erase_cycles) {
             if (bd->cfg->badblock_behavior ==
                     LFS_TESTBD_BADBLOCK_ERASEERROR) {
-                LFS_TRACE("lfs_testbd_erase -> %d", LFS_ERR_CORRUPT);
+                LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", LFS_ERR_CORRUPT);
                 return LFS_ERR_CORRUPT;
             } else if (bd->cfg->badblock_behavior ==
                     LFS_TESTBD_BADBLOCK_ERASENOOP) {
-                LFS_TRACE("lfs_testbd_erase -> %d", 0);
+                LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", 0);
                 return 0;
             }
         } else {
@@ -243,7 +245,7 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) {
     // erase
     int err = lfs_testbd_rawerase(cfg, block);
     if (err) {
-        LFS_TRACE("lfs_testbd_erase -> %d", err);
+        LFS_TESTBD_TRACE("lfs_testbd_erase -> %d", err);
         return err;
     }
 
@@ -258,14 +260,14 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) {
         }
     }
 
-    LFS_TRACE("lfs_testbd_prog -> %d", 0);
+    LFS_TESTBD_TRACE("lfs_testbd_prog -> %d", 0);
     return 0;
 }
 
 int lfs_testbd_sync(const struct lfs_config *cfg) {
-    LFS_TRACE("lfs_testbd_sync(%p)", (void*)cfg);
+    LFS_TESTBD_TRACE("lfs_testbd_sync(%p)", (void*)cfg);
     int err = lfs_testbd_rawsync(cfg);
-    LFS_TRACE("lfs_testbd_sync -> %d", err);
+    LFS_TESTBD_TRACE("lfs_testbd_sync -> %d", err);
     return err;
 }
 
@@ -273,20 +275,20 @@ int lfs_testbd_sync(const struct lfs_config *cfg) {
 /// simulated wear operations ///
 lfs_testbd_swear_t lfs_testbd_getwear(const struct lfs_config *cfg,
         lfs_block_t block) {
-    LFS_TRACE("lfs_testbd_getwear(%p, %"PRIu32")", (void*)cfg, block);
+    LFS_TESTBD_TRACE("lfs_testbd_getwear(%p, %"PRIu32")", (void*)cfg, block);
     lfs_testbd_t *bd = cfg->context;
 
     // check if block is valid
     LFS_ASSERT(bd->cfg->erase_cycles);
     LFS_ASSERT(block < cfg->block_count);
 
-    LFS_TRACE("lfs_testbd_getwear -> %"PRIu32, bd->wear[block]);
+    LFS_TESTBD_TRACE("lfs_testbd_getwear -> %"PRIu32, bd->wear[block]);
     return bd->wear[block];
 }
 
 int lfs_testbd_setwear(const struct lfs_config *cfg,
         lfs_block_t block, lfs_testbd_wear_t wear) {
-    LFS_TRACE("lfs_testbd_setwear(%p, %"PRIu32")", (void*)cfg, block);
+    LFS_TESTBD_TRACE("lfs_testbd_setwear(%p, %"PRIu32")", (void*)cfg, block);
     lfs_testbd_t *bd = cfg->context;
 
     // check if block is valid
@@ -295,6 +297,6 @@ int lfs_testbd_setwear(const struct lfs_config *cfg,
 
     bd->wear[block] = wear;
 
-    LFS_TRACE("lfs_testbd_setwear -> %d", 0);
+    LFS_TESTBD_TRACE("lfs_testbd_setwear -> %d", 0);
     return 0;
 }

bd/lfs_testbd.h

@@ -19,6 +19,13 @@ extern "C"
 #endif
 
 
+// Block device specific tracing
+#ifdef LFS_TESTBD_YES_TRACE
+#define LFS_TESTBD_TRACE(...) LFS_TRACE(__VA_ARGS__)
+#else
+#define LFS_TESTBD_TRACE(...)
+#endif
+
 // Mode determining how "bad blocks" behave during testing. This simulates
 // some real-world circumstances such as progs not sticking (prog-noop),
 // a readonly disk (erase-noop), and ECC failures (read-error).

lfs.c

@@ -71,6 +71,21 @@ static int lfs_bd_read(lfs_t *lfs,
             diff = lfs_min(diff, rcache->off-off);
         }
 
+        if (size >= hint && off % lfs->cfg->read_size == 0 &&
+                size >= lfs->cfg->read_size) {
+            // bypass cache?
+            diff = lfs_aligndown(diff, lfs->cfg->read_size);
+            int err = lfs->cfg->read(lfs->cfg, block, off, data, diff);
+            if (err) {
+                return err;
+            }
+
+            data += diff;
+            off += diff;
+            size -= diff;
+            continue;
+        }
+
         // load to cache, first condition can no longer fail
         LFS_ASSERT(block < lfs->cfg->block_count);
         rcache->block = block;
@@ -437,6 +452,21 @@ static int lfs_alloc_lookahead(void *p, lfs_block_t block) {
     return 0;
 }
 
+// indicate allocated blocks have been committed into the filesystem, this
+// is to prevent blocks from being garbage collected in the middle of a
+// commit operation
+static void lfs_alloc_ack(lfs_t *lfs) {
+    lfs->free.ack = lfs->cfg->block_count;
+}
+
+// drop the lookahead buffer, this is done during mounting and failed
+// traversals in order to avoid invalid lookahead state
+static void lfs_alloc_drop(lfs_t *lfs) {
+    lfs->free.size = 0;
+    lfs->free.i = 0;
+    lfs_alloc_ack(lfs);
+}
+
 static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) {
     while (true) {
         while (lfs->free.i != lfs->free.size) {
@@ -477,16 +507,12 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) {
         memset(lfs->free.buffer, 0, lfs->cfg->lookahead_size);
         int err = lfs_fs_traverseraw(lfs, lfs_alloc_lookahead, lfs, true);
         if (err) {
+            lfs_alloc_drop(lfs);
             return err;
         }
     }
 }
 
-static void lfs_alloc_ack(lfs_t *lfs) {
-    lfs->free.ack = lfs->cfg->block_count;
-}
-
-
 /// Metadata pair and directory operations ///
 static lfs_stag_t lfs_dir_getslice(lfs_t *lfs, const lfs_mdir_t *dir,
         lfs_tag_t gmask, lfs_tag_t gtag,
@@ -846,8 +872,10 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
                 ptag ^= (lfs_tag_t)(lfs_tag_chunk(tag) & 1U) << 31;
 
                 // toss our crc into the filesystem seed for
-                // pseudorandom numbers
-                lfs->seed ^= crc;
+                // pseudorandom numbers, note we use another crc here
+                // as a collection function because it is sufficiently
+                // random and convenient
+                lfs->seed = lfs_crc(lfs->seed, &crc, sizeof(crc));
 
                 // update with what's found so far
                 besttag = tempbesttag;
@@ -970,7 +998,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
         dir->rev = revs[(r+1)%2];
     }
 
-    LFS_ERROR("Corrupted dir pair at %"PRIx32" %"PRIx32,
+    LFS_ERROR("Corrupted dir pair at {0x%"PRIx32", 0x%"PRIx32"}",
             dir->pair[0], dir->pair[1]);
     return LFS_ERR_CORRUPT;
 }
@@ -1237,12 +1265,13 @@ static int lfs_dir_commitattr(lfs_t *lfs, struct lfs_commit *commit,
 }
 
 static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) {
-    const lfs_off_t off1 = commit->off;
-    const uint32_t crc1 = commit->crc;
     // align to program units
-    const lfs_off_t end = lfs_alignup(off1 + 2*sizeof(uint32_t),
+    const lfs_off_t end = lfs_alignup(commit->off + 2*sizeof(uint32_t),
             lfs->cfg->prog_size);
 
+    lfs_off_t off1 = 0;
+    uint32_t crc1 = 0;
+
     // create crc tags to fill up remainder of commit, note that
     // padding is not crced, which lets fetches skip padding but
     // makes committing a bit more complicated
@@ -1278,6 +1307,12 @@ static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) {
             return err;
         }
 
+        // keep track of non-padding checksum to verify
+        if (off1 == 0) {
+            off1 = commit->off + sizeof(uint32_t);
+            crc1 = commit->crc;
+        }
+
         commit->off += sizeof(tag)+lfs_tag_size(tag);
         commit->ptag = tag ^ ((lfs_tag_t)reset << 31);
         commit->crc = 0xffffffff; // reset crc for next "commit"
@@ -1291,7 +1326,7 @@ static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) {
 
     // successful commit, check checksums to make sure
     lfs_off_t off = commit->begin;
-    lfs_off_t noff = off1 + sizeof(uint32_t);
+    lfs_off_t noff = off1;
     while (off < end) {
         uint32_t crc = 0xffffffff;
         for (lfs_off_t i = off; i < noff+sizeof(uint32_t); i++) {
@@ -1658,12 +1693,13 @@ relocate:
         relocated = true;
         lfs_cache_drop(lfs, &lfs->pcache);
         if (!tired) {
-            LFS_DEBUG("Bad block at %"PRIx32, dir->pair[1]);
+            LFS_DEBUG("Bad block at 0x%"PRIx32, dir->pair[1]);
         }
 
         // can't relocate superblock, filesystem is now frozen
         if (lfs_pair_cmp(dir->pair, (const lfs_block_t[2]){0, 1}) == 0) {
-            LFS_WARN("Superblock %"PRIx32" has become unwritable", dir->pair[1]);
+            LFS_WARN("Superblock 0x%"PRIx32" has become unwritable",
+                    dir->pair[1]);
             return LFS_ERR_NOSPC;
         }
 
@@ -1679,7 +1715,8 @@ relocate:
 
     if (relocated) {
         // update references if we relocated
-        LFS_DEBUG("Relocating %"PRIx32" %"PRIx32" -> %"PRIx32" %"PRIx32,
+        LFS_DEBUG("Relocating {0x%"PRIx32", 0x%"PRIx32"} "
+                    "-> {0x%"PRIx32", 0x%"PRIx32"}",
                 oldpair[0], oldpair[1], dir->pair[0], dir->pair[1]);
         int err = lfs_fs_relocate(lfs, oldpair, dir->pair);
         if (err) {
@@ -2302,7 +2339,7 @@ static int lfs_ctz_extend(lfs_t *lfs,
         }
 
 relocate:
-        LFS_DEBUG("Bad block at %"PRIx32, nblock);
+        LFS_DEBUG("Bad block at 0x%"PRIx32, nblock);
 
         // just clear cache and try a new block
         lfs_cache_drop(lfs, pcache);
@@ -2406,9 +2443,9 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file,
 
         // get next slot and create entry to remember name
         err = lfs_dir_commit(lfs, &file->m, LFS_MKATTRS(
-                {LFS_MKTAG(LFS_TYPE_CREATE, file->id, 0)},
+                {LFS_MKTAG(LFS_TYPE_CREATE, file->id, 0), NULL},
                 {LFS_MKTAG(LFS_TYPE_REG, file->id, nlen), path},
-                {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, file->id, 0)}));
+                {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, file->id, 0), NULL}));
         if (err) {
             err = LFS_ERR_NAMETOOLONG;
             goto cleanup;
@@ -2606,7 +2643,7 @@ static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) {
         return 0;
 
 relocate:
-        LFS_DEBUG("Bad block at %"PRIx32, nblock);
+        LFS_DEBUG("Bad block at 0x%"PRIx32, nblock);
 
         // just clear cache and try a new block
         lfs_cache_drop(lfs, &lfs->pcache);
@@ -2683,7 +2720,7 @@ static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) {
                 break;
 
 relocate:
-                LFS_DEBUG("Bad block at %"PRIx32, file->block);
+                LFS_DEBUG("Bad block at 0x%"PRIx32, file->block);
                 err = lfs_file_relocate(lfs, file);
                 if (err) {
                     return err;
@@ -3174,7 +3211,7 @@ int lfs_remove(lfs_t *lfs, const char *path) {
 
     // delete the entry
     err = lfs_dir_commit(lfs, &cwd, LFS_MKATTRS(
-            {LFS_MKTAG(LFS_TYPE_DELETE, lfs_tag_id(tag), 0)}));
+            {LFS_MKTAG(LFS_TYPE_DELETE, lfs_tag_id(tag), 0), NULL}));
     if (err) {
         lfs->mlist = dir.next;
         LFS_TRACE("lfs_remove -> %d", err);
@@ -3300,12 +3337,12 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) {
     // move over all attributes
     err = lfs_dir_commit(lfs, &newcwd, LFS_MKATTRS(
             {LFS_MKTAG_IF(prevtag != LFS_ERR_NOENT,
-                LFS_TYPE_DELETE, newid, 0)},
-            {LFS_MKTAG(LFS_TYPE_CREATE, newid, 0)},
+                LFS_TYPE_DELETE, newid, 0), NULL},
+            {LFS_MKTAG(LFS_TYPE_CREATE, newid, 0), NULL},
             {LFS_MKTAG(lfs_tag_type3(oldtag), newid, strlen(newpath)), newpath},
             {LFS_MKTAG(LFS_FROM_MOVE, newid, lfs_tag_id(oldtag)), &oldcwd},
             {LFS_MKTAG_IF(samepair,
-                LFS_TYPE_DELETE, newoldid, 0)}));
+                LFS_TYPE_DELETE, newoldid, 0), NULL}));
     if (err) {
         lfs->mlist = prevdir.next;
         LFS_TRACE("lfs_rename -> %d", err);
@@ -3318,7 +3355,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) {
         // prep gstate and delete move id
         lfs_fs_prepmove(lfs, 0x3ff, NULL);
         err = lfs_dir_commit(lfs, &oldcwd, LFS_MKATTRS(
-                {LFS_MKTAG(LFS_TYPE_DELETE, lfs_tag_id(oldtag), 0)}));
+                {LFS_MKTAG(LFS_TYPE_DELETE, lfs_tag_id(oldtag), 0), NULL}));
         if (err) {
             lfs->mlist = prevdir.next;
             LFS_TRACE("lfs_rename -> %d", err);
@@ -3610,7 +3647,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) {
 
         lfs_superblock_tole32(&superblock);
         err = lfs_dir_commit(lfs, &root, LFS_MKATTRS(
-                {LFS_MKTAG(LFS_TYPE_CREATE, 0, 0)},
+                {LFS_MKTAG(LFS_TYPE_CREATE, 0, 0), NULL},
                 {LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), "littlefs"},
                 {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)),
                     &superblock}));
@@ -3707,7 +3744,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) {
             uint16_t minor_version = (0xffff & (superblock.version >>  0));
             if ((major_version != LFS_DISK_VERSION_MAJOR ||
                  minor_version > LFS_DISK_VERSION_MINOR)) {
-                LFS_ERROR("Invalid version %"PRIu16".%"PRIu16,
+                LFS_ERROR("Invalid version v%"PRIu16".%"PRIu16,
                         major_version, minor_version);
                 err = LFS_ERR_INVAL;
                 goto cleanup;
@@ -3763,7 +3800,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) {
 
     // update littlefs with gstate
     if (!lfs_gstate_iszero(&lfs->gstate)) {
-        LFS_DEBUG("Found pending gstate %08"PRIx32" %08"PRIx32" %08"PRIx32,
+        LFS_DEBUG("Found pending gstate 0x%08"PRIx32"%08"PRIx32"%08"PRIx32,
                 lfs->gstate.tag,
                 lfs->gstate.pair[0],
                 lfs->gstate.pair[1]);
@@ -3771,11 +3808,10 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) {
     lfs->gstate.tag += !lfs_tag_isvalid(lfs->gstate.tag);
     lfs->gdisk = lfs->gstate;
 
-    // setup free lookahead
-    lfs->free.off = lfs->seed % lfs->cfg->block_size;
-    lfs->free.size = 0;
-    lfs->free.i = 0;
-    lfs_alloc_ack(lfs);
+    // setup free lookahead, to distribute allocations uniformly across
+    // boots, we start the allocator at a random location
+    lfs->free.off = lfs->seed % lfs->cfg->block_count;
+    lfs_alloc_drop(lfs);
 
     LFS_TRACE("lfs_mount -> %d", 0);
     return 0;
@@ -3981,8 +4017,6 @@ static int lfs_fs_relocate(lfs_t *lfs,
         const lfs_block_t oldpair[2], lfs_block_t newpair[2]) {
     // update internal root
     if (lfs_pair_cmp(oldpair, lfs->root) == 0) {
-        LFS_DEBUG("Relocating root %"PRIx32" %"PRIx32,
-                newpair[0], newpair[1]);
         lfs->root[0] = newpair[0];
         lfs->root[1] = newpair[1];
     }
@@ -4018,7 +4052,7 @@ static int lfs_fs_relocate(lfs_t *lfs,
         if (lfs_gstate_hasmovehere(&lfs->gstate, parent.pair)) {
             moveid = lfs_tag_id(lfs->gstate.tag);
             LFS_DEBUG("Fixing move while relocating "
-                    "%"PRIx32" %"PRIx32" %"PRIx16"\n",
+                    "{0x%"PRIx32", 0x%"PRIx32"} 0x%"PRIx16"\n",
                     parent.pair[0], parent.pair[1], moveid);
             lfs_fs_prepmove(lfs, 0x3ff, NULL);
             if (moveid < lfs_tag_id(tag)) {
@@ -4029,7 +4063,7 @@ static int lfs_fs_relocate(lfs_t *lfs,
         lfs_pair_tole32(newpair);
         int err = lfs_dir_commit(lfs, &parent, LFS_MKATTRS(
                 {LFS_MKTAG_IF(moveid != 0x3ff,
-                    LFS_TYPE_DELETE, moveid, 0)},
+                    LFS_TYPE_DELETE, moveid, 0), NULL},
                 {tag, newpair}));
         lfs_pair_fromle32(newpair);
         if (err) {
@@ -4054,7 +4088,7 @@ static int lfs_fs_relocate(lfs_t *lfs,
         if (lfs_gstate_hasmovehere(&lfs->gstate, parent.pair)) {
             moveid = lfs_tag_id(lfs->gstate.tag);
             LFS_DEBUG("Fixing move while relocating "
-                    "%"PRIx32" %"PRIx32" %"PRIx16"\n",
+                    "{0x%"PRIx32", 0x%"PRIx32"} 0x%"PRIx16"\n",
                     parent.pair[0], parent.pair[1], moveid);
             lfs_fs_prepmove(lfs, 0x3ff, NULL);
         }
@@ -4063,7 +4097,7 @@ static int lfs_fs_relocate(lfs_t *lfs,
         lfs_pair_tole32(newpair);
         err = lfs_dir_commit(lfs, &parent, LFS_MKATTRS(
                 {LFS_MKTAG_IF(moveid != 0x3ff,
-                    LFS_TYPE_DELETE, moveid, 0)},
+                    LFS_TYPE_DELETE, moveid, 0), NULL},
                 {LFS_MKTAG(LFS_TYPE_TAIL + parent.split, 0x3ff, 8), newpair}));
         lfs_pair_fromle32(newpair);
         if (err) {
@@ -4095,7 +4129,7 @@ static int lfs_fs_demove(lfs_t *lfs) {
     }
 
     // Fix bad moves
-    LFS_DEBUG("Fixing move %"PRIx32" %"PRIx32" %"PRIx16,
+    LFS_DEBUG("Fixing move {0x%"PRIx32", 0x%"PRIx32"} 0x%"PRIx16,
             lfs->gdisk.pair[0],
             lfs->gdisk.pair[1],
             lfs_tag_id(lfs->gdisk.tag));
@@ -4111,7 +4145,7 @@ static int lfs_fs_demove(lfs_t *lfs) {
     uint16_t moveid = lfs_tag_id(lfs->gdisk.tag);
     lfs_fs_prepmove(lfs, 0x3ff, NULL);
     err = lfs_dir_commit(lfs, &movedir, LFS_MKATTRS(
-            {LFS_MKTAG(LFS_TYPE_DELETE, moveid, 0)}));
+            {LFS_MKTAG(LFS_TYPE_DELETE, moveid, 0), NULL}));
     if (err) {
         return err;
     }
@@ -4146,7 +4180,7 @@ static int lfs_fs_deorphan(lfs_t *lfs) {
 
             if (tag == LFS_ERR_NOENT) {
                 // we are an orphan
-                LFS_DEBUG("Fixing orphan %"PRIx32" %"PRIx32,
+                LFS_DEBUG("Fixing orphan {0x%"PRIx32", 0x%"PRIx32"}",
                         pdir.tail[0], pdir.tail[1]);
 
                 err = lfs_dir_drop(lfs, &pdir, &dir);
@@ -4168,8 +4202,8 @@ static int lfs_fs_deorphan(lfs_t *lfs) {
 
             if (!lfs_pair_sync(pair, pdir.tail)) {
                 // we have desynced
-                LFS_DEBUG("Fixing half-orphan "
-                        "%"PRIx32" %"PRIx32" -> %"PRIx32" %"PRIx32,
+                LFS_DEBUG("Fixing half-orphan {0x%"PRIx32", 0x%"PRIx32"} "
+                            "-> {0x%"PRIx32", 0x%"PRIx32"}",
                         pdir.tail[0], pdir.tail[1], pair[0], pair[1]);
 
                 lfs_pair_tole32(pair);
@@ -4432,7 +4466,7 @@ static int lfs1_dir_fetch(lfs_t *lfs,
     }
 
     if (!valid) {
-        LFS_ERROR("Corrupted dir pair at %" PRIx32 " %" PRIx32 ,
+        LFS_ERROR("Corrupted dir pair at {0x%"PRIx32", 0x%"PRIx32"}",
                 tpair[0], tpair[1]);
         return LFS_ERR_CORRUPT;
     }
@@ -4620,7 +4654,8 @@ static int lfs1_mount(lfs_t *lfs, struct lfs1 *lfs1,
         }
 
         if (err || memcmp(superblock.d.magic, "littlefs", 8) != 0) {
-            LFS_ERROR("Invalid superblock at %d %d", 0, 1);
+            LFS_ERROR("Invalid superblock at {0x%"PRIx32", 0x%"PRIx32"}",
+                    0, 1);
             err = LFS_ERR_CORRUPT;
             goto cleanup;
         }
@@ -4629,7 +4664,7 @@ static int lfs1_mount(lfs_t *lfs, struct lfs1 *lfs1,
         uint16_t minor_version = (0xffff & (superblock.d.version >>  0));
         if ((major_version != LFS1_DISK_VERSION_MAJOR ||
              minor_version > LFS1_DISK_VERSION_MINOR)) {
-            LFS_ERROR("Invalid version %d.%d", major_version, minor_version);
+            LFS_ERROR("Invalid version v%d.%d", major_version, minor_version);
             err = LFS_ERR_INVAL;
             goto cleanup;
         }
@@ -4795,7 +4830,8 @@ int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg) {
 
             // Copy over first block to thread into fs. Unfortunately
             // if this fails there is not much we can do.
-            LFS_DEBUG("Migrating %"PRIx32" %"PRIx32" -> %"PRIx32" %"PRIx32,
+            LFS_DEBUG("Migrating {0x%"PRIx32", 0x%"PRIx32"} "
+                        "-> {0x%"PRIx32", 0x%"PRIx32"}",
                     lfs->root[0], lfs->root[1], dir1.head[0], dir1.head[1]);
 
             err = lfs_bd_erase(lfs, dir1.head[1]);

lfs.h

@@ -21,7 +21,7 @@ extern "C"
 // Software library version
 // Major (top-nibble), incremented on backwards incompatible changes
 // Minor (bottom-nibble), incremented on feature additions
-#define LFS_VERSION 0x00020001
+#define LFS_VERSION 0x00020002
 #define LFS_VERSION_MAJOR (0xffff & (LFS_VERSION >> 16))
 #define LFS_VERSION_MINOR (0xffff & (LFS_VERSION >>  0))
 

lfs_util.h

@@ -50,31 +50,35 @@ extern "C"
 
 // Logging functions
 #ifdef LFS_YES_TRACE
-#define LFS_TRACE(fmt, ...) \
-    printf("%s:%d:trace: " fmt "\n", __FILE__, __LINE__, __VA_ARGS__)
+#define LFS_TRACE_(fmt, ...) \
+    printf("%s:%d:trace: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__)
+#define LFS_TRACE(...) LFS_TRACE_(__VA_ARGS__, "")
 #else
-#define LFS_TRACE(fmt, ...)
+#define LFS_TRACE(...)
 #endif
 
 #ifndef LFS_NO_DEBUG
-#define LFS_DEBUG(fmt, ...) \
-    printf("%s:%d:debug: " fmt "\n", __FILE__, __LINE__, __VA_ARGS__)
+#define LFS_DEBUG_(fmt, ...) \
+    printf("%s:%d:debug: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__)
+#define LFS_DEBUG(...) LFS_DEBUG_(__VA_ARGS__, "")
 #else
-#define LFS_DEBUG(fmt, ...)
+#define LFS_DEBUG(...)
 #endif
 
 #ifndef LFS_NO_WARN
-#define LFS_WARN(fmt, ...) \
-    printf("%s:%d:warn: " fmt "\n", __FILE__, __LINE__, __VA_ARGS__)
+#define LFS_WARN_(fmt, ...) \
+    printf("%s:%d:warn: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__)
+#define LFS_WARN(...) LFS_WARN_(__VA_ARGS__, "")
 #else
-#define LFS_WARN(fmt, ...)
+#define LFS_WARN(...)
 #endif
 
 #ifndef LFS_NO_ERROR
-#define LFS_ERROR(fmt, ...) \
-    printf("%s:%d:error: " fmt "\n", __FILE__, __LINE__, __VA_ARGS__)
+#define LFS_ERROR_(fmt, ...) \
+    printf("%s:%d:error: " fmt "%s\n", __FILE__, __LINE__, __VA_ARGS__)
+#define LFS_ERROR(...) LFS_ERROR_(__VA_ARGS__, "")
 #else
-#define LFS_ERROR(fmt, ...)
+#define LFS_ERROR(...)
 #endif
 
 // Runtime assertions
@@ -107,7 +111,7 @@ static inline uint32_t lfs_alignup(uint32_t a, uint32_t alignment) {
     return lfs_aligndown(a + alignment-1, alignment);
 }
 
-// Find the next smallest power of 2 less than or equal to a
+// Find the smallest power of 2 greater than or equal to a
 static inline uint32_t lfs_npw2(uint32_t a) {
 #if !defined(LFS_NO_INTRINSICS) && (defined(__GNUC__) || defined(__CC_ARM))
     return 32 - __builtin_clz(a-1);

scripts/readmdir.py

@@ -233,8 +233,8 @@ class MetadataPair:
 
     def __lt__(self, other):
         # corrupt blocks don't count
-        if not self and other:
-            return True
+        if not self or not other:
+            return bool(other)
 
         # use sequence arithmetic to avoid overflow
         return not ((other.rev - self.rev) & 0x80000000)
@@ -318,14 +318,24 @@ def main(args):
 
     # find most recent pair
     mdir = MetadataPair(blocks)
-    print("mdir {%s} rev %d%s%s" % (
+
+    try:
+        mdir.tail = mdir[Tag('tail', 0, 0)]
+        if mdir.tail.size != 8 or mdir.tail.data == 8*b'\xff':
+            mdir.tail = None
+    except KeyError:
+        mdir.tail = None
+
+    print("mdir {%s} rev %d%s%s%s" % (
         ', '.join('%#x' % b
             for b in [args.block1, args.block2]
             if b is not None),
         mdir.rev,
         ' (was %s)' % ', '.join('%d' % m.rev for m in mdir.pair[1:])
         if len(mdir.pair) > 1 else '',
-        ' (corrupted)' if not mdir else ''))
+        ' (corrupted!)' if not mdir else '',
+        ' -> {%#x, %#x}' % struct.unpack('<II', mdir.tail.data)
+        if mdir.tail else ''))
     if args.all:
         mdir.dump_all(truncate=not args.no_truncate)
     elif args.log:

scripts/readtree.py

@@ -7,97 +7,14 @@ import io
 import itertools as it
 from readmdir import Tag, MetadataPair
 
-def popc(x):
-    return bin(x).count('1')
-
-def ctz(x):
-    return len(bin(x)) - len(bin(x).rstrip('0'))
-
-def dumpentries(args, mdir, f):
-    for k, id_ in enumerate(mdir.ids):
-        name = mdir[Tag('name', id_, 0)]
-        struct_ = mdir[Tag('struct', id_, 0)]
-
-        desc = "id %d %s %s" % (
-            id_, name.typerepr(),
-            json.dumps(name.data.decode('utf8')))
-        if struct_.is_('dirstruct'):
-            desc += " dir {%#x, %#x}" % struct.unpack(
-                '<II', struct_.data[:8].ljust(8, b'\xff'))
-        if struct_.is_('ctzstruct'):
-            desc += " ctz {%#x} size %d" % struct.unpack(
-                '<II', struct_.data[:8].ljust(8, b'\xff'))
-        if struct_.is_('inlinestruct'):
-            desc += " inline size %d" % struct_.size
-
-        data = None
-        if struct_.is_('inlinestruct'):
-            data = struct_.data
-        elif struct_.is_('ctzstruct'):
-            block, size = struct.unpack(
-                '<II', struct_.data[:8].ljust(8, b'\xff'))
-            data = []
-            i = 0 if size == 0 else (size-1) // (args.block_size - 8)
-            if i != 0:
-                i = ((size-1) - 4*popc(i-1)+2) // (args.block_size - 8)
-            with open(args.disk, 'rb') as f2:
-                while i >= 0:
-                    f2.seek(block * args.block_size)
-                    dat = f2.read(args.block_size)
-                    data.append(dat[4*(ctz(i)+1) if i != 0 else 0:])
-                    block, = struct.unpack('<I', dat[:4].ljust(4, b'\xff'))
-                    i -= 1
-            data = bytes(it.islice(
-                it.chain.from_iterable(reversed(data)), size))
-
-        f.write("%-45s%s\n" % (desc,
-            "%-23s  %-8s" % (
-                ' '.join('%02x' % c for c in data[:8]),
-                ''.join(c if c >= ' ' and c <= '~' else '.'
-                    for c in map(chr, data[:8])))
-            if not args.no_truncate and len(desc) < 45
-                and data is not None else ""))
-
-        if name.is_('superblock') and struct_.is_('inlinestruct'):
-            f.write(
-                "  block_size %d\n"
-                "  block_count %d\n"
-                "  name_max %d\n"
-                "  file_max %d\n"
-                "  attr_max %d\n" % struct.unpack(
-                    '<IIIII', struct_.data[4:4+20].ljust(20, b'\xff')))
-
-        for tag in mdir.tags:
-            if tag.id==id_ and tag.is_('userattr'):
-                desc = "%s size %d" % (tag.typerepr(), tag.size)
-                f.write("  %-43s%s\n" % (desc,
-                    "%-23s  %-8s" % (
-                        ' '.join('%02x' % c for c in tag.data[:8]),
-                        ''.join(c if c >= ' ' and c <= '~' else '.'
-                            for c in map(chr, tag.data[:8])))
-                    if not args.no_truncate and len(desc) < 43 else ""))
-
-                if args.no_truncate:
-                    for i in range(0, len(tag.data), 16):
-                        f.write("    %08x: %-47s  %-16s\n" % (
-                            i, ' '.join('%02x' % c for c in tag.data[i:i+16]),
-                            ''.join(c if c >= ' ' and c <= '~' else '.'
-                                for c in map(chr, tag.data[i:i+16]))))
-
-        if args.no_truncate and data is not None:
-            for i in range(0, len(data), 16):
-                f.write("  %08x: %-47s  %-16s\n" % (
-                    i, ' '.join('%02x' % c for c in data[i:i+16]),
-                    ''.join(c if c >= ' ' and c <= '~' else '.'
-                        for c in map(chr, data[i:i+16]))))
-
 def main(args):
+    superblock = None
+    gstate = b'\0\0\0\0\0\0\0\0\0\0\0\0'
+    dirs = []
+    mdirs = []
+    corrupted = []
+    cycle = False
     with open(args.disk, 'rb') as f:
-        dirs = []
-        superblock = None
-        gstate = b''
-        mdirs = []
-        cycle = False
         tail = (args.block1, args.block2)
         hard = False
         while True:
@@ -144,6 +61,10 @@ def main(args):
             except KeyError:
                 pass
 
+            # corrupted?
+            if not mdir:
+                corrupted.append(mdir)
+
             # add to directories
             mdirs.append(mdir)
             if mdir.tail is None or not mdir.tail.is_('hardtail'):
@@ -178,62 +99,65 @@ def main(args):
 
         dir[0].path = path.replace('//', '/')
 
-    # dump tree
+    # print littlefs + version info
     version = ('?', '?')
     if superblock:
         version = tuple(reversed(
             struct.unpack('<HH', superblock[1].data[0:4].ljust(4, b'\xff'))))
     print("%-47s%s" % ("littlefs v%s.%s" % version,
         "data (truncated, if it fits)"
-        if not any([args.no_truncate, args.tags, args.log, args.all]) else ""))
+        if not any([args.no_truncate, args.log, args.all]) else ""))
 
-    if gstate:
-        print("gstate 0x%s" % ''.join('%02x' % c for c in gstate))
-        tag = Tag(struct.unpack('<I', gstate[0:4].ljust(4, b'\xff'))[0])
-        blocks = struct.unpack('<II', gstate[4:4+8].ljust(8, b'\xff'))
-        if tag.size or not tag.isvalid:
-            print("  orphans >=%d" % max(tag.size, 1))
-        if tag.type:
-            print("  move dir {%#x, %#x} id %d" % (
-                blocks[0], blocks[1], tag.id))
+    # print gstate
+    print("gstate 0x%s" % ''.join('%02x' % c for c in gstate))
+    tag = Tag(struct.unpack('<I', gstate[0:4].ljust(4, b'\xff'))[0])
+    blocks = struct.unpack('<II', gstate[4:4+8].ljust(8, b'\xff'))
+    if tag.size or not tag.isvalid:
+        print("  orphans >=%d" % max(tag.size, 1))
+    if tag.type:
+        print("  move dir {%#x, %#x} id %d" % (
+            blocks[0], blocks[1], tag.id))
 
+    # print mdir info
     for i, dir in enumerate(dirs):
         print("dir %s" % (json.dumps(dir[0].path)
             if hasattr(dir[0], 'path') else '(orphan)'))
 
         for j, mdir in enumerate(dir):
-            print("mdir {%#x, %#x} rev %d%s" % (
-                mdir.blocks[0], mdir.blocks[1], mdir.rev,
-                ' (corrupted)' if not mdir else ''))
+            print("mdir {%#x, %#x} rev %d (was %d)%s%s" % (
+                mdir.blocks[0], mdir.blocks[1], mdir.rev, mdir.pair[1].rev,
+                ' (corrupted!)' if not mdir else '',
+                ' -> {%#x, %#x}' % struct.unpack('<II', mdir.tail.data)
+                if mdir.tail else ''))
 
             f = io.StringIO()
-            if args.tags:
-                mdir.dump_tags(f, truncate=not args.no_truncate)
-            elif args.log:
+            if args.log:
                 mdir.dump_log(f, truncate=not args.no_truncate)
             elif args.all:
                 mdir.dump_all(f, truncate=not args.no_truncate)
             else:
-                dumpentries(args, mdir, f)
+                mdir.dump_tags(f, truncate=not args.no_truncate)
 
             lines = list(filter(None, f.getvalue().split('\n')))
             for k, line in enumerate(lines):
                 print("%s %s" % (
-                    ' ' if i == len(dirs)-1 and j == len(dir)-1 else
+                    ' ' if j == len(dir)-1 else
                     'v' if k == len(lines)-1 else
-                    '.' if j == len(dir)-1 else
                     '|',
                     line))
 
-    if cycle:
-        print("*** cycle detected! -> {%#x, %#x} ***" % (cycle[0], cycle[1]))
+    errcode = 0
+    for mdir in corrupted:
+        errcode = errcode or 1
+        print("*** corrupted mdir {%#x, %#x}! ***" % (
+            mdir.blocks[0], mdir.blocks[1]))
 
     if cycle:
-        return 2
-    elif not all(mdir for dir in dirs for mdir in dir):
-        return 1
-    else:
-        return 0;
+        errcode = errcode or 2
+        print("*** cycle detected {%#x, %#x}! ***" % (
+            cycle[0], cycle[1]))
+
+    return errcode
 
 if __name__ == "__main__":
     import argparse
@@ -246,12 +170,10 @@ if __name__ == "__main__":
         help="Size of a block in bytes.")
     parser.add_argument('block1', nargs='?', default=0,
         type=lambda x: int(x, 0),
-        help="Optional first block address for finding the root.")
+        help="Optional first block address for finding the superblock.")
     parser.add_argument('block2', nargs='?', default=1,
         type=lambda x: int(x, 0),
-        help="Optional second block address for finding the root.")
-    parser.add_argument('-t', '--tags', action='store_true',
-        help="Show metadata tags instead of reconstructing entries.")
+        help="Optional second block address for finding the superblock.")
     parser.add_argument('-l', '--log', action='store_true',
         help="Show tags in log.")
     parser.add_argument('-a', '--all', action='store_true',

scripts/test.py

@@ -231,7 +231,7 @@ class TestCase:
                 ncmd.extend(['-ex', 'r'])
                 if failure.assert_:
                     ncmd.extend(['-ex', 'up 2'])
-            elif gdb == 'start':
+            elif gdb == 'main':
                 ncmd.extend([
                     '-ex', 'b %s:%d' % (self.suite.path, self.code_lineno),
                     '-ex', 'r'])
@@ -760,7 +760,7 @@ if __name__ == "__main__":
         help="Store disk image in a file.")
     parser.add_argument('-b', '--build', action='store_true',
         help="Only build the tests, do not execute.")
-    parser.add_argument('-g', '--gdb', choices=['init', 'start', 'assert'],
+    parser.add_argument('-g', '--gdb', choices=['init', 'main', 'assert'],
         nargs='?', const='assert',
         help="Drop into gdb on test failure.")
     parser.add_argument('--no-internal', action='store_true',

tests/test_alloc.toml

@@ -323,6 +323,90 @@ code = '''
     lfs_unmount(&lfs) => 0;
 '''
 
+[[case]] # what if we have a bad block during an allocation scan?
+in = "lfs.c"
+define.LFS_ERASE_CYCLES = 0xffffffff
+define.LFS_BADBLOCK_BEHAVIOR = 'LFS_TESTBD_BADBLOCK_READERROR'
+code = '''
+    lfs_format(&lfs, &cfg) => 0;
+    lfs_mount(&lfs, &cfg) => 0;
+    // first fill to exhaustion to find available space
+    lfs_file_open(&lfs, &file, "pacman", LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    strcpy((char*)buffer, "waka");
+    size = strlen("waka");
+    lfs_size_t filesize = 0;
+    while (true) {
+        lfs_ssize_t res = lfs_file_write(&lfs, &file, buffer, size);
+        assert(res == (lfs_ssize_t)size || res == LFS_ERR_NOSPC);
+        if (res == LFS_ERR_NOSPC) {
+            break;
+        }
+        filesize += size;
+    }
+    lfs_file_close(&lfs, &file) => 0;
+    // now fill all but a couple of blocks of the filesystem with data
+    filesize -= 3*LFS_BLOCK_SIZE;
+    lfs_file_open(&lfs, &file, "pacman", LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    strcpy((char*)buffer, "waka");
+    size = strlen("waka");
+    for (lfs_size_t i = 0; i < filesize/size; i++) {
+        lfs_file_write(&lfs, &file, buffer, size) => size;
+    }
+    lfs_file_close(&lfs, &file) => 0;
+    // also save head of file so we can error during lookahead scan
+    lfs_block_t fileblock = file.ctz.head;
+    lfs_unmount(&lfs) => 0;
+
+    // remount to force an alloc scan
+    lfs_mount(&lfs, &cfg) => 0;
+
+    // but mark the head of our file as a "bad block", this is force our
+    // scan to bail early
+    lfs_testbd_setwear(&cfg, fileblock, 0xffffffff) => 0;
+    lfs_file_open(&lfs, &file, "ghost", LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    strcpy((char*)buffer, "chomp");
+    size = strlen("chomp");
+    while (true) {
+        lfs_ssize_t res = lfs_file_write(&lfs, &file, buffer, size);
+        assert(res == (lfs_ssize_t)size || res == LFS_ERR_CORRUPT);
+        if (res == LFS_ERR_CORRUPT) {
+            break;
+        }
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    // now reverse the "bad block" and try to write the file again until we
+    // run out of space
+    lfs_testbd_setwear(&cfg, fileblock, 0) => 0;
+    lfs_file_open(&lfs, &file, "ghost", LFS_O_WRONLY | LFS_O_CREAT) => 0;
+    strcpy((char*)buffer, "chomp");
+    size = strlen("chomp");
+    while (true) {
+        lfs_ssize_t res = lfs_file_write(&lfs, &file, buffer, size);
+        assert(res == (lfs_ssize_t)size || res == LFS_ERR_NOSPC);
+        if (res == LFS_ERR_NOSPC) {
+            break;
+        }
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    lfs_unmount(&lfs) => 0;
+
+    // check that the disk isn't hurt
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file, "pacman", LFS_O_RDONLY) => 0;
+    strcpy((char*)buffer, "waka");
+    size = strlen("waka");
+    for (lfs_size_t i = 0; i < filesize/size; i++) {
+        uint8_t rbuffer[4];
+        lfs_file_read(&lfs, &file, rbuffer, size) => size;
+        assert(memcmp(rbuffer, buffer, size) == 0);
+    }
+    lfs_file_close(&lfs, &file) => 0;
+    lfs_unmount(&lfs) => 0;
+'''
+
+
 # Below, I don't like these tests. They're fragile and depend _heavily_
 # on the geometry of the block device. But they are valuable. Eventually they
 # should be removed and replaced with generalized tests.