Switched to separate-tag encoding of forward-looking CRCs

Previously forward-looking CRCs was just two new CRC types, one for
commits with forward-looking CRCs, one without. These both contained the
CRC needed to complete the current commit (note that the commit CRC
must come last!).

         [--   32   --|--   32   --|--   32   --|--   32   --]
with:    [  crc3 tag  | nprog size |  nprog crc | commit crc ]
without: [  crc2 tag  | commit crc ]

This meant there had to be several checks for the two possible structure
sizes, messying up the implementation.

         [--   32   --|--   32   --|--   32   --|--   32   --|--   32   --]
with:    [nprogcrc tag| nprog size |  nprog crc | commit tag | commit crc ]
without: [ commit tag | commit crc ]

But we already have a mechanism for storing optional metadata! The
different metadata tags! So why not use a separate tage for the
forward-looking CRC, separate from the commit CRC?

I wasn't sure this would actually help that much, there are still
necessary conditions for wether or not a forward-looking CRC is there,
but in the end it simplified the code quite nicely, and resulted in a ~200 byte
code-cost saving.

bd/lfs_filebd.c

@@ -80,11 +80,6 @@ int lfs_filebd_read(const struct lfs_config *cfg, lfs_block_t block,
     LFS_ASSERT(size % cfg->read_size == 0);
     LFS_ASSERT(block < cfg->block_count);
 
-    // zero for reproducability (in case file is truncated)
-    if (bd->cfg->erase_value != -1) {
-        memset(buffer, bd->cfg->erase_value, size);
-    }
-
     // read
     off_t res1 = lseek(bd->fd,
             (off_t)block*cfg->block_size + (off_t)off, SEEK_SET);
@@ -101,6 +96,11 @@ int lfs_filebd_read(const struct lfs_config *cfg, lfs_block_t block,
         return err;
     }
 
+    // file truncated? zero for reproducability
+    if ((lfs_size_t)res2 < size) {
+        memset((uint8_t*)buffer + res2, 0, size-res2);
+    }
+
     LFS_FILEBD_TRACE("lfs_filebd_read -> %d", 0);
     return 0;
 }

bd/lfs_rambd.c

@@ -32,11 +32,8 @@ int lfs_rambd_createcfg(const struct lfs_config *cfg,
         }
     }
 
-    // zero for reproducability?
-    if (bd->cfg->erase_value != -1) {
-        memset(bd->buffer, bd->cfg->erase_value,
-                cfg->block_size * cfg->block_count);
-    }
+    // zero for reproducability (this matches filebd)
+    memset(bd->buffer, 0, cfg->block_size * cfg->block_count);
 
     LFS_RAMBD_TRACE("lfs_rambd_createcfg -> %d", 0);
     return 0;

lfs.h

@@ -113,6 +113,8 @@ enum lfs_type {
     LFS_TYPE_SOFTTAIL       = 0x600,
     LFS_TYPE_HARDTAIL       = 0x601,
     LFS_TYPE_MOVESTATE      = 0x7ff,
+    LFS_TYPE_COMMITCRC      = 0x502,
+    LFS_TYPE_NPROGCRC       = 0x5ff,
 
     // internal chip sources
     LFS_FROM_NOOP           = 0x000,

scripts/readmdir.py

@@ -24,6 +24,7 @@ TAG_TYPES = {
     'gstate':       (0x700, 0x700),
     'movestate':    (0x7ff, 0x7ff),
     'crc':          (0x700, 0x500),
+    'nprogcrc':     (0x7ff, 0x5ff),
 }
 
 class Tag:
@@ -99,7 +100,16 @@ class Tag:
         return struct.unpack('b', struct.pack('B', self.chunk))[0]
 
     def is_(self, type):
-        return (self.type & TAG_TYPES[type][0]) == TAG_TYPES[type][1]
+        try:
+            if ' ' in type:
+                type1, type3 = type.split()
+                return (self.is_(type1) and
+                    (self.type & ~TAG_TYPES[type1][0]) == int(type3, 0))
+
+            return self.type == int(type, 0)
+
+        except (ValueError, KeyError):
+            return (self.type & TAG_TYPES[type][0]) == TAG_TYPES[type][1]
 
     def mkmask(self):
         return Tag(
@@ -109,14 +119,20 @@ class Tag:
 
     def chid(self, nid):
         ntag = Tag(self.type, nid, self.size)
-        if hasattr(self, 'off'):  ntag.off  = self.off
-        if hasattr(self, 'data'): ntag.data = self.data
-        if hasattr(self, 'crc'):  ntag.crc  = self.crc
+        if hasattr(self, 'off'):    ntag.off    = self.off
+        if hasattr(self, 'data'):   ntag.data   = self.data
+        if hasattr(self, 'crc'):    ntag.crc    = self.crc
+        if hasattr(self, 'erased'): ntag.erased = self.erased
         return ntag
 
     def typerepr(self):
-        if self.is_('crc') and getattr(self, 'crc', 0xffffffff) != 0xffffffff:
-            return 'crc (bad)'
+        if (self.is_('crc') and not self.is_('nprogcrc') and
+                getattr(self, 'crc', 0xffffffff) != 0xffffffff):
+            crc_status = ' (bad)'
+        elif self.is_('nprogcrc') and getattr(self, 'erased', False):
+            crc_status = ' (era)'
+        else:
+            crc_status = ''
 
         reverse_types = {v: k for k, v in TAG_TYPES.items()}
         for prefix in range(12):
@@ -124,12 +140,12 @@ class Tag:
             if (mask, self.type & mask) in reverse_types:
                 type = reverse_types[mask, self.type & mask]
                 if prefix > 0:
-                    return '%s %#0*x' % (
-                        type, prefix//4, self.type & ((1 << prefix)-1))
+                    return '%s %#x%s' % (
+                        type, self.type & ((1 << prefix)-1), crc_status)
                 else:
-                    return type
+                    return '%s%s' % (type, crc_status)
         else:
-            return '%02x' % self.type
+            return '%02x%s' % (self.type, crc_status)
 
     def idrepr(self):
         return repr(self.id) if self.id != 0x3ff else '.'
@@ -172,6 +188,8 @@ class MetadataPair:
 
         self.rev, = struct.unpack('<I', block[0:4])
         crc = binascii.crc32(block[0:4])
+        etag = None
+        estate = None
 
         # parse tags
         corrupt = False
@@ -182,11 +200,11 @@ class MetadataPair:
         while len(block) - off >= 4:
             ntag, = struct.unpack('>I', block[off:off+4])
 
-            tag = Tag(int(tag) ^ ntag)
+            tag = Tag((int(tag) ^ ntag) & 0x7fffffff)
             tag.off = off + 4
             tag.data = block[off+4:off+tag.dsize]
-            if tag.is_('crc'):
-                crc = binascii.crc32(block[off:off+4+4], crc)
+            if tag.is_('crc') and not tag.is_('nprogcrc'):
+                crc = binascii.crc32(block[off:off+2*4], crc)
             else:
                 crc = binascii.crc32(block[off:off+tag.dsize], crc)
             tag.crc = crc
@@ -194,16 +212,29 @@ class MetadataPair:
 
             self.all_.append(tag)
 
-            if tag.is_('crc'):
+            if tag.is_('nprogcrc') and len(tag.data) == 8:
+                etag = tag
+                estate = struct.unpack('<II', tag.data)
+            elif tag.is_('crc'):
                 # is valid commit?
                 if crc != 0xffffffff:
                     corrupt = True
                 if not corrupt:
                     self.log = self.all_.copy()
+                    # end of commit?
+                    if estate:
+                        esize, ecrc = estate
+                        dcrc = 0xffffffff ^ binascii.crc32(block[off:off+esize])
+                        if ecrc == dcrc:
+                            etag.erased = True
+                            corrupt = True
+                    elif not (tag.is_('crc 0x0') or tag.is_('crc 0x1')):
+                        corrupt = True
 
                 # reset tag parsing
                 crc = 0
-                tag = Tag(int(tag) ^ ((tag.type & 1) << 31))
+                etag = None
+                estate = None
 
         # find active ids
         self.ids = list(it.takewhile(
@@ -280,7 +311,7 @@ class MetadataPair:
         f.write('\n')
 
         for tag in tags:
-            f.write("%08x: %08x  %-13s %4s %4s" % (
+            f.write("%08x: %08x  %-14s %3s %4s" % (
                 tag.off, tag,
                 tag.typerepr(), tag.idrepr(), tag.sizerepr()))
             if truncate:

scripts/test.py

@@ -746,9 +746,9 @@ if __name__ == "__main__":
     parser.add_argument('testpaths', nargs='*', default=[TESTDIR],
         help="Description of test(s) to run. By default, this is all tests \
             found in the \"{0}\" directory. Here, you can specify a different \
-            directory of tests, a specific file, a suite by name, and even \
-            specific test cases and permutations. For example \
-            \"test_dirs#1\" or \"{0}/test_dirs.toml#1#1\".".format(TESTDIR))
+            directory of tests, a specific file, a suite by name, and even a \
+            specific test case by adding brackets. For example \
+            \"test_dirs[0]\" or \"{0}/test_dirs.toml[0]\".".format(TESTDIR))
     parser.add_argument('-D', action='append', default=[],
         help="Overriding parameter definitions.")
     parser.add_argument('-v', '--verbose', action='store_true',

tests/test_powerloss.toml

@@ -0,0 +1,172 @@
+# There are already a number of tests that test general operations under
+# power-loss (see the reentrant attribute). These tests are for explicitly
+# testing specific corner cases.
+
+[[case]] # only a revision count
+code = '''
+    lfs_format(&lfs, &cfg) => 0;
+
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_mkdir(&lfs, "notebook") => 0;
+    lfs_file_open(&lfs, &file, "notebook/paper",
+            LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0;
+    strcpy((char*)buffer, "hello");
+    size = strlen("hello");
+    for (int i = 0; i < 5; i++) {
+        lfs_file_write(&lfs, &file, buffer, size) => size;
+        lfs_file_sync(&lfs, &file) => 0;
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    char rbuffer[256];
+    lfs_file_open(&lfs, &file, "notebook/paper", LFS_O_RDONLY) => 0;
+    for (int i = 0; i < 5; i++) {
+        lfs_file_read(&lfs, &file, rbuffer, size) => size;
+        assert(memcmp(rbuffer, buffer, size) == 0);
+    }
+    lfs_file_close(&lfs, &file) => 0;
+    lfs_unmount(&lfs) => 0;
+
+    // get pair/rev count
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_dir_open(&lfs, &dir, "notebook") => 0;
+    lfs_block_t pair[2] = {dir.m.pair[0], dir.m.pair[1]};
+    uint32_t rev = dir.m.rev;
+    lfs_dir_close(&lfs, &dir) => 0;
+    lfs_unmount(&lfs) => 0;
+
+    // write just the revision count
+    uint8_t bbuffer[LFS_BLOCK_SIZE];
+    cfg.read(&cfg, pair[1], 0, bbuffer, LFS_BLOCK_SIZE) => 0;
+
+    memcpy(bbuffer, &(uint32_t){lfs_tole32(rev+1)}, sizeof(uint32_t));
+
+    cfg.erase(&cfg, pair[1]) => 0;
+    cfg.prog(&cfg, pair[1], 0, bbuffer, LFS_BLOCK_SIZE) => 0;
+
+    lfs_mount(&lfs, &cfg) => 0;
+
+    // can read?
+    lfs_file_open(&lfs, &file, "notebook/paper", LFS_O_RDONLY) => 0;
+    for (int i = 0; i < 5; i++) {
+        lfs_file_read(&lfs, &file, rbuffer, size) => size;
+        assert(memcmp(rbuffer, buffer, size) == 0);
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    // can write?
+    lfs_file_open(&lfs, &file, "notebook/paper",
+            LFS_O_WRONLY | LFS_O_APPEND) => 0;
+    strcpy((char*)buffer, "goodbye");
+    size = strlen("goodbye");
+    for (int i = 0; i < 5; i++) {
+        lfs_file_write(&lfs, &file, buffer, size) => size;
+        lfs_file_sync(&lfs, &file) => 0;
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    lfs_file_open(&lfs, &file, "notebook/paper", LFS_O_RDONLY) => 0;
+    strcpy((char*)buffer, "hello");
+    size = strlen("hello");
+    for (int i = 0; i < 5; i++) {
+        lfs_file_read(&lfs, &file, rbuffer, size) => size;
+        assert(memcmp(rbuffer, buffer, size) == 0);
+    }
+    strcpy((char*)buffer, "goodbye");
+    size = strlen("goodbye");
+    for (int i = 0; i < 5; i++) {
+        lfs_file_read(&lfs, &file, rbuffer, size) => size;
+        assert(memcmp(rbuffer, buffer, size) == 0);
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    lfs_unmount(&lfs) => 0;
+'''
+
+[[case]] # partial prog, may not be byte in order!
+if = "LFS_PROG_SIZE < LFS_BLOCK_SIZE"
+define.BYTE_OFF = ["0", "LFS_PROG_SIZE-1", "LFS_PROG_SIZE/2"]
+define.BYTE_VALUE = [0x33, 0xcc]
+in = "lfs.c"
+code = '''
+    lfs_format(&lfs, &cfg) => 0;
+
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_mkdir(&lfs, "notebook") => 0;
+    lfs_file_open(&lfs, &file, "notebook/paper",
+            LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0;
+    strcpy((char*)buffer, "hello");
+    size = strlen("hello");
+    for (int i = 0; i < 5; i++) {
+        lfs_file_write(&lfs, &file, buffer, size) => size;
+        lfs_file_sync(&lfs, &file) => 0;
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    char rbuffer[256];
+    lfs_file_open(&lfs, &file, "notebook/paper", LFS_O_RDONLY) => 0;
+    for (int i = 0; i < 5; i++) {
+        lfs_file_read(&lfs, &file, rbuffer, size) => size;
+        assert(memcmp(rbuffer, buffer, size) == 0);
+    }
+    lfs_file_close(&lfs, &file) => 0;
+    lfs_unmount(&lfs) => 0;
+
+    // imitate a partial prog, value should not matter, if littlefs
+    // doesn't notice the partial prog testbd will assert
+
+    // get offset to next prog
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_dir_open(&lfs, &dir, "notebook") => 0;
+    lfs_block_t block = dir.m.pair[0];
+    lfs_off_t off = dir.m.off;
+    lfs_dir_close(&lfs, &dir) => 0;
+    lfs_unmount(&lfs) => 0;
+
+    // tweak byte
+    uint8_t bbuffer[LFS_BLOCK_SIZE];
+    cfg.read(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0;
+
+    bbuffer[off + BYTE_OFF] = BYTE_VALUE;
+
+    cfg.erase(&cfg, block) => 0;
+    cfg.prog(&cfg, block, 0, bbuffer, LFS_BLOCK_SIZE) => 0;
+
+    lfs_mount(&lfs, &cfg) => 0;
+
+    // can read?
+    lfs_file_open(&lfs, &file, "notebook/paper", LFS_O_RDONLY) => 0;
+    for (int i = 0; i < 5; i++) {
+        lfs_file_read(&lfs, &file, rbuffer, size) => size;
+        assert(memcmp(rbuffer, buffer, size) == 0);
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    // can write?
+    lfs_file_open(&lfs, &file, "notebook/paper",
+            LFS_O_WRONLY | LFS_O_APPEND) => 0;
+    strcpy((char*)buffer, "goodbye");
+    size = strlen("goodbye");
+    for (int i = 0; i < 5; i++) {
+        lfs_file_write(&lfs, &file, buffer, size) => size;
+        lfs_file_sync(&lfs, &file) => 0;
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    lfs_file_open(&lfs, &file, "notebook/paper", LFS_O_RDONLY) => 0;
+    strcpy((char*)buffer, "hello");
+    size = strlen("hello");
+    for (int i = 0; i < 5; i++) {
+        lfs_file_read(&lfs, &file, rbuffer, size) => size;
+        assert(memcmp(rbuffer, buffer, size) == 0);
+    }
+    strcpy((char*)buffer, "goodbye");
+    size = strlen("goodbye");
+    for (int i = 0; i < 5; i++) {
+        lfs_file_read(&lfs, &file, rbuffer, size) => size;
+        assert(memcmp(rbuffer, buffer, size) == 0);
+    }
+    lfs_file_close(&lfs, &file) => 0;
+
+    lfs_unmount(&lfs) => 0;
+'''