Artificially limited number of file ids per metadata block

This is an expirement to determine which field in the tag structure is
the most critical: tag id or tag size.

This came from looking at NAND storage and discussions around behaviour of
large prog_sizes. Initial exploration indicates that prog_sizes around
2KiB are not _that_ uncommon, and the 1KiB limitation is surprising.

It's possible to increase the lfs_tag size to 12-bits (4096), but at the
cost of only 8-bit ids (256).

  [----            32             ----]
a [1|-3-|-- 8 --|--  10  --|--  10  --]
b [1|-3-|-- 8 --|-- 8 --|--   12    --]

This requires more investigation, but in order to allow us to change
the tag sizes with minimal impact I've artificially limited the number
of file ids to 0xfe (255) different file ids per metadata pair. If
12-bit lengths turn out to be a bad idea, we can remove the artificial
limit without backwards incompatible changes.

To avoid breaking users already on v2-alpha, this change will refuse
_creating_ file ids > 255, but should read file ids > 255 without
issues.

lfs.c

@@ -29,7 +29,7 @@ static inline void lfs_cache_drop(lfs_t *lfs, lfs_cache_t *rcache) {
 
 static inline void lfs_cache_zero(lfs_t *lfs, lfs_cache_t *pcache) {
     // zero to avoid information leak
-    memset(pcache->buffer, 0xff, lfs->cfg->cache_size);
+    memset(pcache->buffer, 0xff, lfs->cfg->prog_size);
     pcache->block = 0xffffffff;
 }
 
@@ -827,7 +827,8 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
             // next commit not yet programmed or we're not in valid range
             if (!lfs_tag_isvalid(tag) ||
                     off + lfs_tag_dsize(tag) > lfs->cfg->block_size) {
-                dir->erased = (lfs_tag_type1(ptag) == LFS_TYPE_CRC);
+                dir->erased = (lfs_tag_type1(ptag) == LFS_TYPE_CRC &&
+                        dir->off % lfs->cfg->prog_size == 0);
                 break;
             }
 
@@ -854,7 +855,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
                 }
 
                 // reset the next bit if we need to
-                ptag ^= (lfs_tag_chunk(tag) & 1) << 31;
+                ptag ^= (lfs_tag_chunk(tag) & 1U) << 31;
 
                 // toss our crc into the filesystem seed for
                 // pseudorandom numbers
@@ -1436,8 +1437,10 @@ static int lfs_dir_compact(lfs_t *lfs,
         // space is complicated, we need room for tail, crc, gstate,
         // cleanup delete, and we cap at half a block to give room
         // for metadata updates.
-        if (size <= lfs_min(lfs->cfg->block_size - 36,
-                lfs_alignup(lfs->cfg->block_size/2, lfs->cfg->prog_size))) {
+        if (end - begin < 0xff &&
+                size <= lfs_min(lfs->cfg->block_size - 36,
+                    lfs_alignup(lfs->cfg->block_size/2,
+                        lfs->cfg->prog_size))) {
             break;
         }
 
@@ -1497,7 +1500,7 @@ static int lfs_dir_compact(lfs_t *lfs,
 
     // begin loop to commit compaction to blocks until a compact sticks
     while (true) {
-        if (true) {
+        {
             // There's nothing special about our global delta, so feed it into
             // our local global delta
             int err = lfs_dir_getgstate(lfs, dir, &lfs->gdelta);
@@ -1596,13 +1599,12 @@ static int lfs_dir_compact(lfs_t *lfs,
             dir->count = end - begin;
             dir->off = commit.off;
             dir->etag = commit.ptag;
-            dir->erased = true;
+            dir->erased = (dir->off % lfs->cfg->prog_size == 0);
             // note we able to have already handled move here
             if (lfs_gstate_hasmovehere(&lfs->gpending, dir->pair)) {
                 lfs_gstate_xormove(&lfs->gpending,
                     &lfs->gpending, 0x3ff, NULL);
             }
-
         }
         break;
 
@@ -1711,7 +1713,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir,
         }
     }
 
-    if (dir->erased) {
+    if (dir->erased || dir->count >= 0xff) {
         // try to commit
         struct lfs_commit commit = {
             .block = dir->pair[0],
@@ -2122,7 +2124,7 @@ static int lfs_ctz_extend(lfs_t *lfs,
         }
         LFS_ASSERT(nblock >= 2 && nblock <= lfs->cfg->block_count);
 
-        if (true) {
+        {
             err = lfs_bd_erase(lfs, nblock);
             if (err) {
                 if (err == LFS_ERR_CORRUPT) {
@@ -2381,7 +2383,8 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file,
         if (file->ctz.size > 0) {
             lfs_stag_t res = lfs_dir_get(lfs, &file->m,
                     LFS_MKTAG(0x700, 0x3ff, 0),
-                    LFS_MKTAG(LFS_TYPE_STRUCT, file->id, file->cache.size),
+                    LFS_MKTAG(LFS_TYPE_STRUCT, file->id,
+                        lfs_min(file->cache.size, 0x3fe)),
                     file->cache.buffer);
             if (res < 0) {
                 err = res;
@@ -2576,6 +2579,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) {
     while (true) {
         int err = lfs_file_flush(lfs, file);
         if (err) {
+            file->flags |= LFS_F_ERRED;
             return err;
         }
 
@@ -2611,6 +2615,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) {
                 if (err == LFS_ERR_NOSPC && (file->flags & LFS_F_INLINE)) {
                     goto relocate;
                 }
+                file->flags |= LFS_F_ERRED;
                 return err;
             }
 
@@ -2624,6 +2629,7 @@ relocate:
         file->off = file->pos;
         err = lfs_file_relocate(lfs, file);
         if (err) {
+            file->flags |= LFS_F_ERRED;
             return err;
         }
     }
@@ -3223,8 +3229,9 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) {
     lfs_cache_zero(lfs, &lfs->pcache);
 
     // setup lookahead, must be multiple of 64-bits
-    LFS_ASSERT(lfs->cfg->lookahead_size % 8 == 0);
     LFS_ASSERT(lfs->cfg->lookahead_size > 0);
+    LFS_ASSERT(lfs->cfg->lookahead_size % 8 == 0 &&
+            (uintptr_t)lfs->cfg->lookahead_buffer % 8 == 0);
     if (lfs->cfg->lookahead_buffer) {
         lfs->free.buffer = lfs->cfg->lookahead_buffer;
     } else {
@@ -3292,7 +3299,7 @@ static int lfs_deinit(lfs_t *lfs) {
 
 int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) {
     int err = 0;
-    if (true) {
+    {
         err = lfs_init(lfs, cfg);
         if (err) {
             return err;
@@ -4176,7 +4183,7 @@ static int lfs1_moved(lfs_t *lfs, const void *e) {
 static int lfs1_mount(lfs_t *lfs, struct lfs1 *lfs1,
         const struct lfs_config *cfg) {
     int err = 0;
-    if (true) {
+    {
         err = lfs_init(lfs, cfg);
         if (err) {
             return err;
@@ -4247,7 +4254,7 @@ int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg) {
         return err;
     }
 
-    if (true) {
+    {
         // iterate through each directory, copying over entries
         // into new directory
         lfs1_dir_t dir1;

lfs.h

@@ -215,8 +215,9 @@ struct lfs_config {
     // By default lfs_malloc is used to allocate this buffer.
     void *prog_buffer;
 
-    // Optional statically allocated program buffer. Must be lookahead_size.
-    // By default lfs_malloc is used to allocate this buffer.
+    // Optional statically allocated lookahead buffer. Must be lookahead_size
+    // and aligned to a 64-bit boundary. By default lfs_malloc is used to
+    // allocate this buffer.
     void *lookahead_buffer;
 
     // Optional upper limit on length of file names in bytes. No downside for

lfs_util.h

@@ -192,6 +192,7 @@ static inline uint32_t lfs_tobe32(uint32_t a) {
 uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size);
 
 // Allocate memory, only used if buffers are not provided to littlefs
+// Note, memory must be 64-bit aligned
 static inline void *lfs_malloc(size_t size) {
 #ifndef LFS_NO_MALLOC
     return malloc(size);