Switched to separate-tag encoding of forward-looking CRCs

Previously forward-looking CRCs was just two new CRC types, one for
commits with forward-looking CRCs, one without. These both contained the
CRC needed to complete the current commit (note that the commit CRC
must come last!).

         [--   32   --|--   32   --|--   32   --|--   32   --]
with:    [  crc3 tag  | nprog size |  nprog crc | commit crc ]
without: [  crc2 tag  | commit crc ]

This meant there had to be several checks for the two possible structure
sizes, messying up the implementation.

         [--   32   --|--   32   --|--   32   --|--   32   --|--   32   --]
with:    [nprogcrc tag| nprog size |  nprog crc | commit tag | commit crc ]
without: [ commit tag | commit crc ]

But we already have a mechanism for storing optional metadata! The
different metadata tags! So why not use a separate tage for the
forward-looking CRC, separate from the commit CRC?

I wasn't sure this would actually help that much, there are still
necessary conditions for wether or not a forward-looking CRC is there,
but in the end it simplified the code quite nicely, and resulted in a ~200 byte
code-cost saving.

bd/lfs_filebd.c

@@ -80,11 +80,6 @@ int lfs_filebd_read(const struct lfs_config *cfg, lfs_block_t block,
     LFS_ASSERT(size % cfg->read_size == 0);
     LFS_ASSERT(block < cfg->block_count);
 
-    // zero for reproducability (in case file is truncated)
-    if (bd->cfg->erase_value != -1) {
-        memset(buffer, bd->cfg->erase_value, size);
-    }
-
     // read
     off_t res1 = lseek(bd->fd,
             (off_t)block*cfg->block_size + (off_t)off, SEEK_SET);
@@ -101,6 +96,11 @@ int lfs_filebd_read(const struct lfs_config *cfg, lfs_block_t block,
         return err;
     }
 
+    // file truncated? zero for reproducability
+    if ((lfs_size_t)res2 < size) {
+        memset((uint8_t*)buffer + res2, 0, size-res2);
+    }
+
     LFS_FILEBD_TRACE("lfs_filebd_read -> %d", 0);
     return 0;
 }

bd/lfs_rambd.c

@@ -32,11 +32,8 @@ int lfs_rambd_createcfg(const struct lfs_config *cfg,
         }
     }
 
-    // zero for reproducability?
-    if (bd->cfg->erase_value != -1) {
-        memset(bd->buffer, bd->cfg->erase_value,
-                cfg->block_size * cfg->block_count);
-    }
+    // zero for reproducability (this matches filebd)
+    memset(bd->buffer, 0, cfg->block_size * cfg->block_count);
 
     LFS_RAMBD_TRACE("lfs_rambd_createcfg -> %d", 0);
     return 0;

lfs.c

@@ -122,16 +122,15 @@ static int lfs_bd_cmp(lfs_t *lfs,
 
     for (lfs_off_t i = 0; i < size; i += diff) {
         uint8_t dat[8];
-
         diff = lfs_min(size-i, sizeof(dat));
-        int res = lfs_bd_read(lfs,
+        int err = lfs_bd_read(lfs,
                 pcache, rcache, hint-i,
                 block, off+i, &dat, diff);
-        if (res) {
-            return res;
+        if (err) {
+            return err;
         }
 
-        res = memcmp(dat, data + i, diff);
+        int res = memcmp(dat, data + i, diff);
         if (res) {
             return res < 0 ? LFS_CMP_LT : LFS_CMP_GT;
         }
@@ -140,6 +139,27 @@ static int lfs_bd_cmp(lfs_t *lfs,
     return LFS_CMP_EQ;
 }
 
+static int lfs_bd_crc(lfs_t *lfs,
+        const lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_size_t hint,
+        lfs_block_t block, lfs_off_t off, lfs_size_t size, uint32_t *crc) {
+    lfs_size_t diff = 0;
+
+    for (lfs_off_t i = 0; i < size; i += diff) {
+        uint8_t dat[8];
+        diff = lfs_min(size-i, sizeof(dat));
+        int err = lfs_bd_read(lfs,
+                pcache, rcache, hint-i,
+                block, off+i, &dat, diff);
+        if (err) {
+            return err;
+        }
+
+        *crc = lfs_crc(*crc, &dat, diff);
+    }
+
+    return 0;
+}
+
 #ifndef LFS_READONLY
 static int lfs_bd_flush(lfs_t *lfs,
         lfs_cache_t *pcache, lfs_cache_t *rcache, bool validate) {
@@ -310,6 +330,10 @@ static inline uint16_t lfs_tag_type1(lfs_tag_t tag) {
     return (tag & 0x70000000) >> 20;
 }
 
+static inline uint16_t lfs_tag_type2(lfs_tag_t tag) {
+    return (tag & 0x78000000) >> 20;
+}
+
 static inline uint16_t lfs_tag_type3(lfs_tag_t tag) {
     return (tag & 0x7ff00000) >> 20;
 }
@@ -394,6 +418,22 @@ static inline void lfs_gstate_tole32(lfs_gstate_t *a) {
     a->pair[1] = lfs_tole32(a->pair[1]);
 }
 
+// operations on estate in CRC tags
+struct lfs_estate {
+    lfs_size_t size;
+    uint32_t crc;
+};
+
+static void lfs_estate_fromle32(struct lfs_estate *estate) {
+    estate->size = lfs_fromle32(estate->size);
+    estate->crc = lfs_fromle32(estate->crc);
+}
+
+static void lfs_estate_tole32(struct lfs_estate *estate) {
+    estate->size = lfs_tole32(estate->size);
+    estate->crc = lfs_tole32(estate->crc);
+}
+
 // other endianness operations
 static void lfs_ctz_fromle32(struct lfs_ctz *ctz) {
     ctz->head = lfs_fromle32(ctz->head);
@@ -880,6 +920,9 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
         bool tempsplit = false;
         lfs_stag_t tempbesttag = besttag;
 
+        bool hasestate = false;
+        struct lfs_estate estate;
+
         dir->rev = lfs_tole32(dir->rev);
         uint32_t crc = lfs_crc(0xffffffff, &dir->rev, sizeof(dir->rev));
         dir->rev = lfs_fromle32(dir->rev);
@@ -901,21 +944,17 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
             }
 
             crc = lfs_crc(crc, &tag, sizeof(tag));
-            tag = lfs_frombe32(tag) ^ ptag;
+            tag = (lfs_frombe32(tag) ^ ptag) & 0x7fffffff;
 
-            // next commit not yet programmed or we're not in valid range
-            if (!lfs_tag_isvalid(tag)) {
-                dir->erased = (lfs_tag_type1(ptag) == LFS_TYPE_CRC &&
-                        dir->off % lfs->cfg->prog_size == 0);
-                break;
-            } else if (off + lfs_tag_dsize(tag) > lfs->cfg->block_size) {
+            // out of range?
+            if (off + lfs_tag_dsize(tag) > lfs->cfg->block_size) {
                 dir->erased = false;
                 break;
             }
 
             ptag = tag;
 
-            if (lfs_tag_type1(tag) == LFS_TYPE_CRC) {
+            if (lfs_tag_type2(tag) == LFS_TYPE_CRC) {
                 // check the crc attr
                 uint32_t dcrc;
                 err = lfs_bd_read(lfs,
@@ -935,9 +974,6 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
                     break;
                 }
 
-                // reset the next bit if we need to
-                ptag ^= (lfs_tag_t)(lfs_tag_chunk(tag) & 1U) << 31;
-
                 // toss our crc into the filesystem seed for
                 // pseudorandom numbers, note we use another crc here
                 // as a collection function because it is sufficiently
@@ -953,17 +989,61 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
                 dir->tail[1] = temptail[1];
                 dir->split = tempsplit;
 
+                if (hasestate) {
+                    // check if the next page is erased
+                    //
+                    // this may look inefficient, but since cache_size is
+                    // probably > prog_size, the data will always remain in
+                    // cache for the next iteration
+
+                    // first we get a tag-worth of bits, this is so we can
+                    // tweak our current tag to force future writes to be
+                    // different than the erased state
+                    lfs_tag_t etag;
+                    err = lfs_bd_read(lfs,
+                            NULL, &lfs->rcache, lfs->cfg->block_size,
+                            dir->pair[0], dir->off, &etag, sizeof(etag));
+                    if (err && err != LFS_ERR_CORRUPT) {
+                        return err;
+                    }
+
+                    // perturb valid bit?
+                    dir->etag |= 0x80000000 & ~lfs_frombe32(etag);
+
+                    // crc the rest the full prog_size, including etag in case
+                    // the actual esize is < tag size (though this shouldn't
+                    // happen normally)
+                    uint32_t tcrc = 0xffffffff;
+                    err = lfs_bd_crc(lfs,
+                            NULL, &lfs->rcache, lfs->cfg->block_size,
+                            dir->pair[0], dir->off, estate.size, &tcrc);
+                    if (err && err != LFS_ERR_CORRUPT) {
+                        return err;
+                    }
+
+                    if (tcrc == estate.crc) {
+                        dir->erased = true;
+                        break;
+                    }
+                } else if (lfs_tag_chunk(tag) < 2) {
+                    // for backwards compatibility we fall through here on
+                    // unrecognized tags, leaving it up to the CRC to reject
+                    // bad commits
+                } else {
+                    // end of block commit
+                    dir->erased = false;
+                    break;
+                }
+
                 // reset crc
                 crc = 0xffffffff;
-                continue;
-            }
-
-            // crc the entry first, hopefully leaving it in the cache
-            for (lfs_off_t j = sizeof(tag); j < lfs_tag_dsize(tag); j++) {
-                uint8_t dat;
-                err = lfs_bd_read(lfs,
+                hasestate = false;
+            } else {
+                // crc the entry first, hopefully leaving it in the cache
+                err = lfs_bd_crc(lfs,
                         NULL, &lfs->rcache, lfs->cfg->block_size,
-                        dir->pair[0], off+j, &dat, 1);
+                        dir->pair[0], off+sizeof(tag),
+                        lfs_tag_dsize(tag)-sizeof(tag), &crc);
                 if (err) {
                     if (err == LFS_ERR_CORRUPT) {
                         dir->erased = false;
@@ -972,64 +1052,77 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
                     return err;
                 }
 
-                crc = lfs_crc(crc, &dat, 1);
-            }
-
-            // directory modification tags?
-            if (lfs_tag_type1(tag) == LFS_TYPE_NAME) {
-                // increase count of files if necessary
-                if (lfs_tag_id(tag) >= tempcount) {
-                    tempcount = lfs_tag_id(tag) + 1;
-                }
-            } else if (lfs_tag_type1(tag) == LFS_TYPE_SPLICE) {
-                tempcount += lfs_tag_splice(tag);
-
-                if (tag == (LFS_MKTAG(LFS_TYPE_DELETE, 0, 0) |
-                        (LFS_MKTAG(0, 0x3ff, 0) & tempbesttag))) {
-                    tempbesttag |= 0x80000000;
-                } else if (tempbesttag != -1 &&
-                        lfs_tag_id(tag) <= lfs_tag_id(tempbesttag)) {
-                    tempbesttag += LFS_MKTAG(0, lfs_tag_splice(tag), 0);
-                }
-            } else if (lfs_tag_type1(tag) == LFS_TYPE_TAIL) {
-                tempsplit = (lfs_tag_chunk(tag) & 1);
-
-                err = lfs_bd_read(lfs,
-                        NULL, &lfs->rcache, lfs->cfg->block_size,
-                        dir->pair[0], off+sizeof(tag), &temptail, 8);
-                if (err) {
-                    if (err == LFS_ERR_CORRUPT) {
-                        dir->erased = false;
-                        break;
+                // directory modification tags?
+                if (lfs_tag_type1(tag) == LFS_TYPE_NAME) {
+                    // increase count of files if necessary
+                    if (lfs_tag_id(tag) >= tempcount) {
+                        tempcount = lfs_tag_id(tag) + 1;
                     }
-                }
-                lfs_pair_fromle32(temptail);
-            }
+                } else if (lfs_tag_type1(tag) == LFS_TYPE_SPLICE) {
+                    tempcount += lfs_tag_splice(tag);
 
-            // found a match for our fetcher?
-            if ((fmask & tag) == (fmask & ftag)) {
-                int res = cb(data, tag, &(struct lfs_diskoff){
-                        dir->pair[0], off+sizeof(tag)});
-                if (res < 0) {
-                    if (res == LFS_ERR_CORRUPT) {
-                        dir->erased = false;
-                        break;
+                    if (tag == (LFS_MKTAG(LFS_TYPE_DELETE, 0, 0) |
+                            (LFS_MKTAG(0, 0x3ff, 0) & tempbesttag))) {
+                        tempbesttag |= 0x80000000;
+                    } else if (tempbesttag != -1 &&
+                            lfs_tag_id(tag) <= lfs_tag_id(tempbesttag)) {
+                        tempbesttag += LFS_MKTAG(0, lfs_tag_splice(tag), 0);
                     }
-                    return res;
+                } else if (lfs_tag_type1(tag) == LFS_TYPE_TAIL) {
+                    tempsplit = (lfs_tag_chunk(tag) & 1);
+
+                    err = lfs_bd_read(lfs,
+                            NULL, &lfs->rcache, lfs->cfg->block_size,
+                            dir->pair[0], off+sizeof(tag),
+                            &temptail, sizeof(temptail));
+                    if (err) {
+                        if (err == LFS_ERR_CORRUPT) {
+                            dir->erased = false;
+                            break;
+                        }
+                    }
+                    lfs_pair_fromle32(temptail);
+                } else if (lfs_tag_type1(tag) == LFS_TYPE_NPROGCRC) {
+                    err = lfs_bd_read(lfs,
+                            NULL, &lfs->rcache, lfs->cfg->block_size,
+                            dir->pair[0], off+sizeof(tag),
+                            &estate, sizeof(estate));
+                    if (err) {
+                        if (err == LFS_ERR_CORRUPT) {
+                            dir->erased = false;
+                            break;
+                        }
+                    }
+                    lfs_estate_fromle32(&estate);
+                    hasestate = true;
                 }
 
-                if (res == LFS_CMP_EQ) {
-                    // found a match
-                    tempbesttag = tag;
-                } else if ((LFS_MKTAG(0x7ff, 0x3ff, 0) & tag) ==
-                        (LFS_MKTAG(0x7ff, 0x3ff, 0) & tempbesttag)) {
-                    // found an identical tag, but contents didn't match
-                    // this must mean that our besttag has been overwritten
-                    tempbesttag = -1;
-                } else if (res == LFS_CMP_GT &&
-                        lfs_tag_id(tag) <= lfs_tag_id(tempbesttag)) {
-                    // found a greater match, keep track to keep things sorted
-                    tempbesttag = tag | 0x80000000;
+                // found a match for our fetcher?
+                if ((fmask & tag) == (fmask & ftag)) {
+                    int res = cb(data, tag, &(struct lfs_diskoff){
+                            dir->pair[0], off+sizeof(tag)});
+                    if (res < 0) {
+                        if (res == LFS_ERR_CORRUPT) {
+                            dir->erased = false;
+                            break;
+                        }
+                        return res;
+                    }
+
+                    if (res == LFS_CMP_EQ) {
+                        // found a match
+                        tempbesttag = tag;
+                    } else if ((LFS_MKTAG(0x7ff, 0x3ff, 0) & tag) ==
+                            (LFS_MKTAG(0x7ff, 0x3ff, 0) & tempbesttag)) {
+                        // found an identical tag, but contents didn't match
+                        // this must mean that our besttag has been overwritten
+                        tempbesttag = -1;
+                    } else if (res == LFS_CMP_GT &&
+                            lfs_tag_id(tag) <= lfs_tag_id(tempbesttag)) {
+                        // found a greater match, keep track to keep
+                        // things sorted
+                        tempbesttag = tag | 0x80000000;
+                    }
                 }
             }
         }
@@ -1338,7 +1431,12 @@ static int lfs_dir_commitattr(lfs_t *lfs, struct lfs_commit *commit,
 #ifndef LFS_READONLY
 static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) {
     // align to program units
-    const lfs_off_t end = lfs_alignup(commit->off + 2*sizeof(uint32_t),
+    //
+    // this gets a bit complex as we have two types of crcs:
+    // - 4-word crc with estate to check following prog (middle of block)
+    // - 2-word crc with no following prog (end of block)
+    const lfs_off_t end = lfs_alignup(
+            lfs_min(commit->off + 5*sizeof(uint32_t), lfs->cfg->block_size),
             lfs->cfg->prog_size);
 
     lfs_off_t off1 = 0;
@@ -1354,40 +1452,71 @@ static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) {
             noff = lfs_min(noff, end - 2*sizeof(uint32_t));
         }
 
-        // read erased state from next program unit
-        lfs_tag_t tag = 0xffffffff;
-        int err = lfs_bd_read(lfs,
-                NULL, &lfs->rcache, sizeof(tag),
-                commit->block, noff, &tag, sizeof(tag));
-        if (err && err != LFS_ERR_CORRUPT) {
-            return err;
+        // clamp erase size to tag size, this gives us the full tag as potential
+        // to intentionally invalidate erase CRCs
+        const lfs_size_t esize = lfs_max(
+                lfs->cfg->prog_size, sizeof(lfs_tag_t));
+        lfs_tag_t etag = 0;
+        // space for estate? also only emit on last commit in padding commits
+        if (noff <= lfs->cfg->block_size - esize) {
+            // first we get a tag-worth of bits, this is so we can
+            // tweak our current tag to force future writes to be
+            // different than the erased state
+            int err = lfs_bd_read(lfs,
+                    NULL, &lfs->rcache, esize,
+                    commit->block, noff, &etag, sizeof(etag));
+            if (err && err != LFS_ERR_CORRUPT) {
+                return err;
+            }
+
+            // find expected erased state
+            uint32_t ecrc = 0xffffffff;
+            err = lfs_bd_crc(lfs,
+                    NULL, &lfs->rcache, esize,
+                    commit->block, noff, esize, &ecrc);
+            if (err && err != LFS_ERR_CORRUPT) {
+                return err;
+            }
+
+            struct lfs_estate estate = {.size = esize, .crc = ecrc};
+            lfs_estate_tole32(&estate);
+            err = lfs_dir_commitattr(lfs, commit,
+                    LFS_MKTAG(LFS_TYPE_NPROGCRC, 0x3ff, sizeof(estate)),
+                    &estate);
+            if (err) {
+                return err;
+            }
         }
 
-        // build crc tag
-        bool reset = ~lfs_frombe32(tag) >> 31;
-        tag = LFS_MKTAG(LFS_TYPE_CRC + reset, 0x3ff, noff - off);
+        // build crc state
+        off = commit->off + sizeof(lfs_tag_t);
+        struct {
+            lfs_tag_t tag;
+            uint32_t crc;
+        } cstate;
+        lfs_tag_t ctag = LFS_MKTAG(LFS_TYPE_COMMITCRC, 0x3ff, noff-off);
+        cstate.tag = lfs_tobe32(ctag ^ commit->ptag);
+        commit->crc = lfs_crc(commit->crc, &cstate.tag, sizeof(cstate.tag));
+        cstate.crc = lfs_tole32(commit->crc);
 
-        // write out crc
-        uint32_t footer[2];
-        footer[0] = lfs_tobe32(tag ^ commit->ptag);
-        commit->crc = lfs_crc(commit->crc, &footer[0], sizeof(footer[0]));
-        footer[1] = lfs_tole32(commit->crc);
-        err = lfs_bd_prog(lfs,
+        int err = lfs_bd_prog(lfs,
                 &lfs->pcache, &lfs->rcache, false,
-                commit->block, commit->off, &footer, sizeof(footer));
+                commit->block, commit->off, &cstate, sizeof(cstate));
         if (err) {
             return err;
         }
 
         // keep track of non-padding checksum to verify
         if (off1 == 0) {
-            off1 = commit->off + sizeof(uint32_t);
+            off1 = off;
             crc1 = commit->crc;
         }
 
-        commit->off += sizeof(tag)+lfs_tag_size(tag);
-        commit->ptag = tag ^ ((lfs_tag_t)reset << 31);
-        commit->crc = 0xffffffff; // reset crc for next "commit"
+        commit->off = noff;
+        // perturb valid bit?
+        commit->ptag = ctag | (0x80000000 & ~lfs_frombe32(etag));
+        // reset crc for next commit
+        commit->crc = 0xffffffff;
     }
 
     // flush buffers
@@ -1397,40 +1526,34 @@ static int lfs_dir_commitcrc(lfs_t *lfs, struct lfs_commit *commit) {
     }
 
     // successful commit, check checksums to make sure
+    //
+    // note that we don't need to check padding commits, worst
+    // case if they are corrupted we would have had to compact anyways
     lfs_off_t off = commit->begin;
-    lfs_off_t noff = off1;
-    while (off < end) {
-        uint32_t crc = 0xffffffff;
-        for (lfs_off_t i = off; i < noff+sizeof(uint32_t); i++) {
-            // check against written crc, may catch blocks that
-            // become readonly and match our commit size exactly
-            if (i == off1 && crc != crc1) {
-                return LFS_ERR_CORRUPT;
-            }
+    uint32_t crc = 0xffffffff;
+    err = lfs_bd_crc(lfs,
+            NULL, &lfs->rcache, off1+sizeof(uint32_t),
+            commit->block, off, off1-off, &crc);
+    if (err) {
+        return err;
+    }
 
-            // leave it up to caching to make this efficient
-            uint8_t dat;
-            err = lfs_bd_read(lfs,
-                    NULL, &lfs->rcache, noff+sizeof(uint32_t)-i,
-                    commit->block, i, &dat, 1);
-            if (err) {
-                return err;
-            }
+    // check against known crc for non-padding commits
+    if (crc != crc1) {
+        return LFS_ERR_CORRUPT;
+    }
 
-            crc = lfs_crc(crc, &dat, 1);
-        }
+    // make sure to check crc in case we happened to pick
+    // up an unrelated crc (frozen block?)
+    err = lfs_bd_crc(lfs,
+            NULL, &lfs->rcache, sizeof(uint32_t),
+            commit->block, off1, sizeof(uint32_t), &crc);
+    if (err) {
+        return err;
+    }
 
-        // detected write error?
-        if (crc != 0) {
-            return LFS_ERR_CORRUPT;
-        }
-
-        // skip padding
-        off = lfs_min(end - noff, 0x3fe) + noff;
-        if (off < end) {
-            off = lfs_min(off, end - 2*sizeof(uint32_t));
-        }
-        noff = off + sizeof(uint32_t);
+    if (crc != 0) {
+        return LFS_ERR_CORRUPT;
     }
 
     return 0;

lfs.h

@@ -113,6 +113,8 @@ enum lfs_type {
     LFS_TYPE_SOFTTAIL       = 0x600,
     LFS_TYPE_HARDTAIL       = 0x601,
     LFS_TYPE_MOVESTATE      = 0x7ff,
+    LFS_TYPE_COMMITCRC      = 0x502,
+    LFS_TYPE_NPROGCRC       = 0x5ff,
 
     // internal chip sources
     LFS_FROM_NOOP           = 0x000,

scripts/readmdir.py

@@ -24,6 +24,7 @@ TAG_TYPES = {
     'gstate':       (0x700, 0x700),
     'movestate':    (0x7ff, 0x7ff),
     'crc':          (0x700, 0x500),
+    'nprogcrc':     (0x7ff, 0x5ff),
 }
 
 class Tag:
@@ -99,7 +100,16 @@ class Tag:
         return struct.unpack('b', struct.pack('B', self.chunk))[0]
 
     def is_(self, type):
-        return (self.type & TAG_TYPES[type][0]) == TAG_TYPES[type][1]
+        try:
+            if ' ' in type:
+                type1, type3 = type.split()
+                return (self.is_(type1) and
+                    (self.type & ~TAG_TYPES[type1][0]) == int(type3, 0))
+
+            return self.type == int(type, 0)
+
+        except (ValueError, KeyError):
+            return (self.type & TAG_TYPES[type][0]) == TAG_TYPES[type][1]
 
     def mkmask(self):
         return Tag(
@@ -109,14 +119,20 @@ class Tag:
 
     def chid(self, nid):
         ntag = Tag(self.type, nid, self.size)
-        if hasattr(self, 'off'):  ntag.off  = self.off
-        if hasattr(self, 'data'): ntag.data = self.data
-        if hasattr(self, 'crc'):  ntag.crc  = self.crc
+        if hasattr(self, 'off'):    ntag.off    = self.off
+        if hasattr(self, 'data'):   ntag.data   = self.data
+        if hasattr(self, 'crc'):    ntag.crc    = self.crc
+        if hasattr(self, 'erased'): ntag.erased = self.erased
         return ntag
 
     def typerepr(self):
-        if self.is_('crc') and getattr(self, 'crc', 0xffffffff) != 0xffffffff:
-            return 'crc (bad)'
+        if (self.is_('crc') and not self.is_('nprogcrc') and
+                getattr(self, 'crc', 0xffffffff) != 0xffffffff):
+            crc_status = ' (bad)'
+        elif self.is_('nprogcrc') and getattr(self, 'erased', False):
+            crc_status = ' (era)'
+        else:
+            crc_status = ''
 
         reverse_types = {v: k for k, v in TAG_TYPES.items()}
         for prefix in range(12):
@@ -124,12 +140,12 @@ class Tag:
             if (mask, self.type & mask) in reverse_types:
                 type = reverse_types[mask, self.type & mask]
                 if prefix > 0:
-                    return '%s %#0*x' % (
-                        type, prefix//4, self.type & ((1 << prefix)-1))
+                    return '%s %#x%s' % (
+                        type, self.type & ((1 << prefix)-1), crc_status)
                 else:
-                    return type
+                    return '%s%s' % (type, crc_status)
         else:
-            return '%02x' % self.type
+            return '%02x%s' % (self.type, crc_status)
 
     def idrepr(self):
         return repr(self.id) if self.id != 0x3ff else '.'
@@ -172,6 +188,8 @@ class MetadataPair:
 
         self.rev, = struct.unpack('<I', block[0:4])
         crc = binascii.crc32(block[0:4])
+        etag = None
+        estate = None
 
         # parse tags
         corrupt = False
@@ -182,11 +200,11 @@ class MetadataPair:
         while len(block) - off >= 4:
             ntag, = struct.unpack('>I', block[off:off+4])
 
-            tag = Tag(int(tag) ^ ntag)
+            tag = Tag((int(tag) ^ ntag) & 0x7fffffff)
             tag.off = off + 4
             tag.data = block[off+4:off+tag.dsize]
-            if tag.is_('crc'):
-                crc = binascii.crc32(block[off:off+4+4], crc)
+            if tag.is_('crc') and not tag.is_('nprogcrc'):
+                crc = binascii.crc32(block[off:off+2*4], crc)
             else:
                 crc = binascii.crc32(block[off:off+tag.dsize], crc)
             tag.crc = crc
@@ -194,16 +212,29 @@ class MetadataPair:
 
             self.all_.append(tag)
 
-            if tag.is_('crc'):
+            if tag.is_('nprogcrc') and len(tag.data) == 8:
+                etag = tag
+                estate = struct.unpack('<II', tag.data)
+            elif tag.is_('crc'):
                 # is valid commit?
                 if crc != 0xffffffff:
                     corrupt = True
                 if not corrupt:
                     self.log = self.all_.copy()
+                    # end of commit?
+                    if estate:
+                        esize, ecrc = estate
+                        dcrc = 0xffffffff ^ binascii.crc32(block[off:off+esize])
+                        if ecrc == dcrc:
+                            etag.erased = True
+                            corrupt = True
+                    elif not (tag.is_('crc 0x0') or tag.is_('crc 0x1')):
+                        corrupt = True
 
                 # reset tag parsing
                 crc = 0
-                tag = Tag(int(tag) ^ ((tag.type & 1) << 31))
+                etag = None
+                estate = None
 
         # find active ids
         self.ids = list(it.takewhile(
@@ -280,7 +311,7 @@ class MetadataPair:
         f.write('\n')
 
         for tag in tags:
-            f.write("%08x: %08x  %-13s %4s %4s" % (
+            f.write("%08x: %08x  %-14s %3s %4s" % (
                 tag.off, tag,
                 tag.typerepr(), tag.idrepr(), tag.sizerepr()))
             if truncate:

tests/test_powerloss.toml

@@ -0,0 +1,172 @@
+# There are already a number of tests that test general operations under
+# power-loss (see the reentrant attribute). These tests are for explicitly
+# testing specific corner cases.
+
+[[case]] # only a revision count
+code = '''
+    lfs_format(&lfs, &cfg) => 0;
+
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_mkdir(&lfs, "notebook") => 0;
+    lfs_file_open(&lfs, &file, "notebook/paper",
+            LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0;
+    strcpy((char*)buffer, "hello");
+    size = strlen("hello");
+    for (int i = 0; i < 5; i++) {
+        lfs_file_write(&lfs, &file, buffer, size) => size;
+        lfs_file_sync(&lfs, &file) => 0;
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    char rbuffer[256];
+    lfs_file_open(&lfs, &file, "notebook/paper", LFS_O_RDONLY) => 0;
+    for (int i = 0; i < 5; i++) {
+        lfs_file_read(&lfs, &file, rbuffer, size) => size;
+        assert(memcmp(rbuffer, buffer, size) == 0);
+    }
+    lfs_file_close(&lfs, &file) => 0;
+    lfs_unmount(&lfs) => 0;
+
+    // get pair/rev count
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_dir_open(&lfs, &dir, "notebook") => 0;
+    lfs_block_t pair[2] = {dir.m.pair[0], dir.m.pair[1]};
+    uint32_t rev = dir.m.rev;
+    lfs_dir_close(&lfs, &dir) => 0;
+    lfs_unmount(&lfs) => 0;
+
+    // write just the revision count
+    uint8_t bbuffer[LFS_BLOCK_SIZE];
+    cfg.read(&cfg, pair[1], 0, bbuffer, LFS_BLOCK_SIZE) => 0;
+
+    memcpy(bbuffer, &(uint32_t){lfs_tole32(rev+1)}, sizeof(uint32_t));
+
+    cfg.erase(&cfg, pair[1]) => 0;
+    cfg.prog(&cfg, pair[1], 0, bbuffer, LFS_BLOCK_SIZE) => 0;
+
+    lfs_mount(&lfs, &cfg) => 0;
+
+    // can read?
+    lfs_file_open(&lfs, &file, "notebook/paper", LFS_O_RDONLY) => 0;
+    for (int i = 0; i < 5; i++) {
+        lfs_file_read(&lfs, &file, rbuffer, size) => size;
+        assert(memcmp(rbuffer, buffer, size) == 0);
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    // can write?
+    lfs_file_open(&lfs, &file, "notebook/paper",
+            LFS_O_WRONLY | LFS_O_APPEND) => 0;
+    strcpy((char*)buffer, "goodbye");
+    size = strlen("goodbye");
+    for (int i = 0; i < 5; i++) {
+        lfs_file_write(&lfs, &file, buffer, size) => size;
+        lfs_file_sync(&lfs, &file) => 0;
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    lfs_file_open(&lfs, &file, "notebook/paper", LFS_O_RDONLY) => 0;
+    strcpy((char*)buffer, "hello");
+    size = strlen("hello");
+    for (int i = 0; i < 5; i++) {
+        lfs_file_read(&lfs, &file, rbuffer, size) => size;
+        assert(memcmp(rbuffer, buffer, size) == 0);
+    }
+    strcpy((char*)buffer, "goodbye");
+    size = strlen("goodbye");
+    for (int i = 0; i < 5; i++) {
+        lfs_file_read(&lfs, &file, rbuffer, size) => size;
+        assert(memcmp(rbuffer, buffer, size) == 0);
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    lfs_unmount(&lfs) => 0;
+'''
+
+[[case]] # partial prog, may not be byte in order!
+if = "LFS_PROG_SIZE < LFS_BLOCK_SIZE"
+define.BYTE_OFF = ["0", "LFS_PROG_SIZE-1", "LFS_PROG_SIZE/2"]
+define.BYTE_VALUE = [0x33, 0xcc]
+in = "lfs.c"
+code = '''
+    lfs_format(&lfs, &cfg) => 0;
+
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_mkdir(&lfs, "notebook") => 0;
+    lfs_file_open(&lfs, &file, "notebook/paper",
+            LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0;
+    strcpy((char*)buffer, "hello");
+    size = strlen("hello");
+    for (int i = 0; i < 5; i++) {
+        lfs_file_write(&lfs, &file, buffer, size) => size;
+        lfs_file_sync(&lfs, &file) => 0;
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    char rbuffer[256];
+    lfs_file_open(&lfs, &file, "notebook/paper", LFS_O_RDONLY) => 0;
+    for (int i = 0; i < 5; i++) {
+        lfs_file_read(&lfs, &file, rbuffer, size) => size;
+        assert(memcmp(rbuffer, buffer, size) == 0);
+    }
+    lfs_file_close(&lfs, &file) => 0;
+    lfs_unmount(&lfs) => 0;
+
+    // imitate a partial prog, value should not matter, if littlefs
+    // doesn't notice the partial prog testbd will assert
+
+    // get offset to next prog
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_dir_open(&lfs, &dir, "notebook") => 0;
+    lfs_block_t block = dir.m.pair[0];
+    lfs_off_t off = dir.m.off;
+    lfs_dir_close(&lfs, &dir) => 0;
+    lfs_unmount(&lfs) => 0;
+
+    // tweak byte
+    uint8_t bbuffer[LFS_BLOCK_SIZE];
+    cfg.read(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0;
+
+    bbuffer[off + BYTE_OFF] = BYTE_VALUE;
+
+    cfg.erase(&cfg, block) => 0;
+    cfg.prog(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0;
+
+    lfs_mount(&lfs, &cfg) => 0;
+
+    // can read?
+    lfs_file_open(&lfs, &file, "notebook/paper", LFS_O_RDONLY) => 0;
+    for (int i = 0; i < 5; i++) {
+        lfs_file_read(&lfs, &file, rbuffer, size) => size;
+        assert(memcmp(rbuffer, buffer, size) == 0);
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    // can write?
+    lfs_file_open(&lfs, &file, "notebook/paper",
+            LFS_O_WRONLY | LFS_O_APPEND) => 0;
+    strcpy((char*)buffer, "goodbye");
+    size = strlen("goodbye");
+    for (int i = 0; i < 5; i++) {
+        lfs_file_write(&lfs, &file, buffer, size) => size;
+        lfs_file_sync(&lfs, &file) => 0;
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    lfs_file_open(&lfs, &file, "notebook/paper", LFS_O_RDONLY) => 0;
+    strcpy((char*)buffer, "hello");
+    size = strlen("hello");
+    for (int i = 0; i < 5; i++) {
+        lfs_file_read(&lfs, &file, rbuffer, size) => size;
+        assert(memcmp(rbuffer, buffer, size) == 0);
+    }
+    strcpy((char*)buffer, "goodbye");
+    size = strlen("goodbye");
+    for (int i = 0; i < 5; i++) {
+        lfs_file_read(&lfs, &file, rbuffer, size) => size;
+        assert(memcmp(rbuffer, buffer, size) == 0);
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    lfs_unmount(&lfs) => 0;
+'''