WIP Stage 1 of B-trees with separate child slots

Startingto run into some problems, not using separate child slots may
end up working better, even if it it's imperfect

lfs.h

@@ -88,32 +88,34 @@ enum lfs_error {
 // File types
 enum lfs_type {
     // file types
-    LFS_TYPE_REG            = 0x001,
-    LFS_TYPE_DIR            = 0x002,
+    LFS_TYPE_REG            = 0x004,
+    LFS_TYPE_DIR            = 0x005,
 
     // internally used types
     LFS_TYPE_USER           = 0x100,
-    LFS_TYPE_SUPERBLOCK     = 0x011,
-    LFS_TYPE_ROOT           = 0x012,
+    LFS_TYPE_SUPERBLOCK     = 0x002,
+    LFS_TYPE_ROOT           = 0x003,
+    LFS_TYPE_CHILD          = 0x001,
     LFS_TYPE_NAME           = 0x000,
-    LFS_TYPE_DELETE         = 0x030,
-    LFS_TYPE_STRUCT         = 0x040,
-    LFS_TYPE_GLOBALS        = 0x080,
-    LFS_TYPE_TAIL           = 0x0c0,
-    LFS_TYPE_SOFTTAIL       = 0x0c0,
-    LFS_TYPE_HARDTAIL       = 0x0c1,
-    LFS_TYPE_CRC            = 0x0f0,
+    LFS_TYPE_DELETE         = 0x060,
+    LFS_TYPE_STRUCT         = 0x020,
+    LFS_TYPE_GLOBALS        = 0x0e0,
+    LFS_TYPE_TAIL           = 0x080,
+    LFS_TYPE_SOFTTAIL       = 0x080,
+    LFS_TYPE_HARDTAIL       = 0x081,
+    LFS_TYPE_CRC            = 0x0a0,
 
-    LFS_TYPE_DIRSTRUCT      = 0x040,
-    LFS_TYPE_INLINESTRUCT   = 0x041,
-    LFS_TYPE_CTZSTRUCT      = 0x042,
+    LFS_TYPE_DIRSTRUCT      = 0x020,
+    LFS_TYPE_INLINESTRUCT   = 0x021,
+    LFS_TYPE_CTZSTRUCT      = 0x022,
 
     // internal chip sources
     LFS_FROM_REGION         = 0x000,
     LFS_FROM_DISK           = 0x200,
-    LFS_FROM_MOVE           = 0x050,
-    LFS_FROM_ATTRS          = 0x060,
-    LFS_FROM_SUPERBLOCK     = 0x070,
+    LFS_FROM_MOVE           = 0x040,
+    LFS_FROM_COMPACT        = 0x041,
+    LFS_FROM_SPLIT          = 0x042,
+    LFS_FROM_ATTRS          = 0x043,
 };
 
 // File open flags
@@ -302,8 +304,13 @@ typedef union lfs_global {
     struct {
         lfs_block_t movepair[2];
         uint16_t moveid;
-        bool deorphaned;
-    } s;
+        uint8_t deorphaned;
+    } l;
+    struct {
+        lfs_block_t movepair[2];
+        uint16_t moveid;
+        uint8_t orphans;
+    } g;
 } lfs_global_t;
 
 typedef struct lfs_mdir {
@@ -624,35 +631,6 @@ lfs_ssize_t lfs_fs_size(lfs_t *lfs);
 // Returns a negative error code on failure.
 int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data);
 
-// Get custom attributes on the filesystem
-//
-// Custom attributes are uniquely identified by an 8-bit type and limited
-// to LFS_ATTR_MAX bytes. When read, if the stored attribute is smaller than
-// the buffer, it will be padded with zeros. If the stored attribute is larger,
-// then it will be silently truncated.
-//
-// Note, filesystem-level attributes are not available for wear-leveling
-//
-// Returns the size of the attribute, or a negative error code on failure.
-// Note, the returned size is the size of the attribute on disk, irrespective
-// of the size of the buffer. This can be used to dynamically allocate a buffer
-// or check for existance.
-lfs_ssize_t lfs_fs_getattr(lfs_t *lfs,
-        uint8_t type, void *buffer, lfs_size_t size);
-
-// Set custom attributes on the filesystem
-//
-// Custom attributes are uniquely identified by an 8-bit type and limited
-// to LFS_ATTR_MAX bytes. If an attribute is not found, it will be
-// implicitly created, and setting the size of an attribute to zero deletes
-// the attribute.
-//
-// Note, filesystem-level attributes are not available for wear-leveling
-//
-// Returns a negative error code on failure.
-int lfs_fs_setattr(lfs_t *lfs,
-        uint8_t type, const void *buffer, lfs_size_t size);
-
 
 #ifdef __cplusplus
 } /* extern "C" */

tests/corrupt.py

@@ -11,7 +11,7 @@ def corrupt(block):
         file.read(4)
 
         # go to last commit
-        tag = 0
+        tag = 0xffffffff
         while True:
             try:
                 ntag, = struct.unpack('<I', file.read(4))

tests/debug.py

@@ -4,18 +4,19 @@ import struct
 import binascii
 
 TYPES = {
-    (0x1ff, 0x001): 'reg',
-    (0x1ff, 0x002): 'dir',
-    (0x1ff, 0x011): 'superblock',
-    (0x1ff, 0x012): 'root',
-    (0x1ff, 0x030): 'delete',
-    (0x1f0, 0x080): 'globals',
-    (0x1ff, 0x0c0): 'tail soft',
-    (0x1ff, 0x0c1): 'tail hard',
-    (0x1ff, 0x0f0): 'crc',
-    (0x1ff, 0x040): 'struct dir',
-    (0x1ff, 0x041): 'struct inline',
-    (0x1ff, 0x042): 'struct ctz',
+    (0x1ff, 0x004): 'reg',
+    (0x1ff, 0x005): 'dir',
+    (0x1ff, 0x002): 'superblock',
+    (0x1ff, 0x003): 'root',
+    (0x1ff, 0x001): 'child',
+    (0x1ff, 0x060): 'delete',
+    (0x1f0, 0x0e0): 'globals',
+    (0x1ff, 0x080): 'tail soft',
+    (0x1ff, 0x081): 'tail hard',
+    (0x1f0, 0x0a0): 'crc',
+    (0x1ff, 0x020): 'struct dir',
+    (0x1ff, 0x021): 'struct inline',
+    (0x1ff, 0x022): 'struct ctz',
     (0x100, 0x100): 'attr',
 }
 
@@ -50,16 +51,20 @@ def main(*blocks):
                 crc = ncrc
 
             versions.append((nrev, '%s (rev %d)' % (block, nrev)))
-        except IOError:
+        except (IOError, struct.error):
             pass
 
+    if not file:
+        print 'Bad metadata pair {%s}' % ', '.join(blocks)
+        return 1
+
     print "--- %s ---" % ', '.join(v for _,v in sorted(versions, reverse=True))
 
     # go through each tag, print useful information
     print "%-4s  %-8s  %-14s  %3s  %3s  %s" % (
         'off', 'tag', 'type', 'id', 'len', 'dump')
 
-    tag = 0
+    tag = 0xffffffff
     off = 4
     while True:
         try:
@@ -75,24 +80,28 @@ def main(*blocks):
         type = (tag & 0x7fc00000) >> 22
         id   = (tag & 0x003ff000) >> 12
         size = (tag & 0x00000fff) >> 0
+        iscrc = (type & 0x1f0) == 0x0f0
 
         data = file.read(size)
-        if type == 0x0f0:
+        if iscrc:
             crc = binascii.crc32(data[:4], crc)
         else:
             crc = binascii.crc32(data, crc)
 
         print '%04x: %08x  %-14s  %3s  %3d  %-23s  %-8s' % (
             off, tag,
-            typeof(type) + (' bad!' if type == 0x0f0 and ~crc else ''),
+            typeof(type) + (' bad!' if iscrc and ~crc else ''),
             id if id != 0x3ff else '.', size,
             ' '.join('%02x' % ord(c) for c in data[:8]),
             ''.join(c if c >= ' ' and c <= '~' else '.' for c in data[:8]))
 
         off += tag & 0xfff
-        if type == 0x0f0:
+        if iscrc:
             crc = 0
+            tag ^= (type & 1) << 31
+
+    return 0
 
 if __name__ == "__main__":
     import sys
-    main(*sys.argv[1:])
+    sys.exit(main(*sys.argv[1:]))

tests/test_alloc.sh

@@ -341,6 +341,10 @@ tests/test.py << TEST
     }
     lfs_file_close(&lfs, &file[0]) => 0;
 
+    // remount to force reset of lookahead
+    lfs_unmount(&lfs) => 0;
+    lfs_mount(&lfs, &cfg) => 0;
+
     // open hole
     lfs_remove(&lfs, "bump") => 0;
 
@@ -350,7 +354,7 @@ tests/test.py << TEST
     for (lfs_size_t i = 0; i < cfg.block_size; i += 2) {
         memcpy(&buffer[i], "hi", 2);
     }
-    lfs_file_write(&lfs, &file[0], buffer, cfg.block_size) => LFS_ERR_NOSPC;
+    lfs_file_write(&lfs, &file[0], buffer, 2*cfg.block_size) => LFS_ERR_NOSPC;
     lfs_file_close(&lfs, &file[0]) => 0;
 
     lfs_unmount(&lfs) => 0;
@@ -388,7 +392,6 @@ tests/test.py << TEST
 
     // remount to force reset of lookahead
     lfs_unmount(&lfs) => 0;
-
     lfs_mount(&lfs, &cfg) => 0;
 
     // rewrite one file
@@ -451,7 +454,6 @@ tests/test.py << TEST
 
     // remount to force reset of lookahead
     lfs_unmount(&lfs) => 0;
-
     lfs_mount(&lfs, &cfg) => 0;
 
     // rewrite one file with a hole of one block

tests/test_attrs.sh

@@ -77,55 +77,55 @@ tests/test.py << TEST
     lfs_unmount(&lfs) => 0;
 TEST
 
-echo "--- Set/get fs attribute ---"
+echo "--- Set/get root attribute ---"
 tests/test.py << TEST
     lfs_mount(&lfs, &cfg) => 0;
-    lfs_fs_setattr(&lfs, 'A', "aaaa",   4) => 0;
-    lfs_fs_setattr(&lfs, 'B', "bbbbbb", 6) => 0;
-    lfs_fs_setattr(&lfs, 'C', "ccccc",  5) => 0;
-    lfs_fs_getattr(&lfs, 'A', buffer,    4) => 4;
-    lfs_fs_getattr(&lfs, 'B', buffer+4,  6) => 6;
-    lfs_fs_getattr(&lfs, 'C', buffer+10, 5) => 5;
+    lfs_setattr(&lfs, "/", 'A', "aaaa",   4) => 0;
+    lfs_setattr(&lfs, "/", 'B', "bbbbbb", 6) => 0;
+    lfs_setattr(&lfs, "/", 'C', "ccccc",  5) => 0;
+    lfs_getattr(&lfs, "/", 'A', buffer,    4) => 4;
+    lfs_getattr(&lfs, "/", 'B', buffer+4,  6) => 6;
+    lfs_getattr(&lfs, "/", 'C', buffer+10, 5) => 5;
     memcmp(buffer,    "aaaa",   4) => 0;
     memcmp(buffer+4,  "bbbbbb", 6) => 0;
     memcmp(buffer+10, "ccccc",  5) => 0;
 
-    lfs_fs_setattr(&lfs, 'B', "", 0) => 0;
-    lfs_fs_getattr(&lfs, 'A', buffer,    4) => 4;
-    lfs_fs_getattr(&lfs, 'B', buffer+4,  6) => 0;
-    lfs_fs_getattr(&lfs, 'C', buffer+10, 5) => 5;
+    lfs_setattr(&lfs, "/", 'B', "", 0) => 0;
+    lfs_getattr(&lfs, "/", 'A', buffer,    4) => 4;
+    lfs_getattr(&lfs, "/", 'B', buffer+4,  6) => 0;
+    lfs_getattr(&lfs, "/", 'C', buffer+10, 5) => 5;
     memcmp(buffer,    "aaaa",         4) => 0;
     memcmp(buffer+4,  "\0\0\0\0\0\0", 6) => 0;
     memcmp(buffer+10, "ccccc",        5) => 0;
 
-    lfs_fs_setattr(&lfs, 'B', "dddddd", 6) => 0;
-    lfs_fs_getattr(&lfs, 'A', buffer,    4) => 4;
-    lfs_fs_getattr(&lfs, 'B', buffer+4,  6) => 6;
-    lfs_fs_getattr(&lfs, 'C', buffer+10, 5) => 5;
+    lfs_setattr(&lfs, "/", 'B', "dddddd", 6) => 0;
+    lfs_getattr(&lfs, "/", 'A', buffer,    4) => 4;
+    lfs_getattr(&lfs, "/", 'B', buffer+4,  6) => 6;
+    lfs_getattr(&lfs, "/", 'C', buffer+10, 5) => 5;
     memcmp(buffer,    "aaaa",   4) => 0;
     memcmp(buffer+4,  "dddddd", 6) => 0;
     memcmp(buffer+10, "ccccc",  5) => 0;
 
-    lfs_fs_setattr(&lfs, 'B', "eee", 3) => 0;
-    lfs_fs_getattr(&lfs, 'A', buffer,    4) => 4;
-    lfs_fs_getattr(&lfs, 'B', buffer+4,  6) => 3;
-    lfs_fs_getattr(&lfs, 'C', buffer+10, 5) => 5;
+    lfs_setattr(&lfs, "/", 'B', "eee", 3) => 0;
+    lfs_getattr(&lfs, "/", 'A', buffer,    4) => 4;
+    lfs_getattr(&lfs, "/", 'B', buffer+4,  6) => 3;
+    lfs_getattr(&lfs, "/", 'C', buffer+10, 5) => 5;
     memcmp(buffer,    "aaaa",      4) => 0;
     memcmp(buffer+4,  "eee\0\0\0", 6) => 0;
     memcmp(buffer+10, "ccccc",     5) => 0;
 
-    lfs_fs_setattr(&lfs, 'A', buffer, LFS_ATTR_MAX+1) => LFS_ERR_NOSPC;
-    lfs_fs_setattr(&lfs, 'B', "fffffffff", 9) => 0;
-    lfs_fs_getattr(&lfs, 'A', buffer,    4) => 4;
-    lfs_fs_getattr(&lfs, 'B', buffer+4,  6) => 9;
-    lfs_fs_getattr(&lfs, 'C', buffer+10, 5) => 5;
+    lfs_setattr(&lfs, "/", 'A', buffer, LFS_ATTR_MAX+1) => LFS_ERR_NOSPC;
+    lfs_setattr(&lfs, "/", 'B', "fffffffff", 9) => 0;
+    lfs_getattr(&lfs, "/", 'A', buffer,    4) => 4;
+    lfs_getattr(&lfs, "/", 'B', buffer+4,  6) => 9;
+    lfs_getattr(&lfs, "/", 'C', buffer+10, 5) => 5;
     lfs_unmount(&lfs) => 0;
 TEST
 tests/test.py << TEST
     lfs_mount(&lfs, &cfg) => 0;
-    lfs_fs_getattr(&lfs, 'A', buffer,    4) => 4;
-    lfs_fs_getattr(&lfs, 'B', buffer+4,  9) => 9;
-    lfs_fs_getattr(&lfs, 'C', buffer+13, 5) => 5;
+    lfs_getattr(&lfs, "/", 'A', buffer,    4) => 4;
+    lfs_getattr(&lfs, "/", 'B', buffer+4,  9) => 9;
+    lfs_getattr(&lfs, "/", 'C', buffer+13, 5) => 5;
     memcmp(buffer,    "aaaa",      4) => 0;
     memcmp(buffer+4,  "fffffffff", 9) => 0;
     memcmp(buffer+13, "ccccc",     5) => 0;

tests/test_format.sh

@@ -22,7 +22,7 @@ echo "--- Invalid superblocks ---"
 ln -f -s /dev/zero blocks/0
 ln -f -s /dev/zero blocks/1
 tests/test.py << TEST
-    lfs_format(&lfs, &cfg) => LFS_ERR_CORRUPT;
+    lfs_format(&lfs, &cfg) => LFS_ERR_NOSPC;
 TEST
 rm blocks/0 blocks/1