Stop wear-leveling during migration

Stop proactively relocate blocks during migrations, this can cause a number of
failure states such: clobbering the v1 superblock if we relocate root, and
invalidating directory pointers if we relocate the head of a directory. On top
of this, relocations increase the overall complexity of lfs_migration, which is
already a delicate operation.

Makefile

@@ -55,6 +55,7 @@ test: \
 	test_attrs \
 	test_move \
 	test_orphan \
+	test_relocations \
 	test_corrupt
 	@rm test.c
 test_%: tests/test_%.sh

README.md

@@ -53,6 +53,7 @@ const struct lfs_config cfg = {
     .block_count = 128,
     .cache_size = 16,
     .lookahead_size = 16,
+    .block_cycles = 500,
 };
 
 // entry point

lfs.c

@@ -977,7 +977,7 @@ static int lfs_dir_fetch(lfs_t *lfs,
         lfs_mdir_t *dir, const lfs_block_t pair[2]) {
     // note, mask=-1, tag=0 can never match a tag since this
     // pattern has the invalid bit set
-    return lfs_dir_fetchmatch(lfs, dir, pair, -1, 0, NULL, NULL, NULL);
+    return (int)lfs_dir_fetchmatch(lfs, dir, pair, -1, 0, NULL, NULL, NULL);
 }
 
 static int lfs_dir_getgstate(lfs_t *lfs, const lfs_mdir_t *dir,
@@ -1010,7 +1010,7 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir,
     lfs_stag_t tag = lfs_dir_get(lfs, dir, LFS_MKTAG(0x780, 0x3ff, 0),
             LFS_MKTAG(LFS_TYPE_NAME, id, lfs->name_max+1), info->name);
     if (tag < 0) {
-        return tag;
+        return (int)tag;
     }
 
     info->type = lfs_tag_type3(tag);
@@ -1019,7 +1019,7 @@ static int lfs_dir_getinfo(lfs_t *lfs, lfs_mdir_t *dir,
     tag = lfs_dir_get(lfs, dir, LFS_MKTAG(0x700, 0x3ff, 0),
             LFS_MKTAG(LFS_TYPE_STRUCT, id, sizeof(ctz)), &ctz);
     if (tag < 0) {
-        return tag;
+        return (int)tag;
     }
     lfs_ctz_fromle32(&ctz);
 
@@ -1062,7 +1062,7 @@ static int lfs_dir_find_match(void *data,
     return LFS_CMP_EQ;
 }
 
-static int lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir,
+static lfs_stag_t lfs_dir_find(lfs_t *lfs, lfs_mdir_t *dir,
         const char **path, uint16_t *id) {
     // we reduce path to a single name if we can find it
     const char *name = *path;
@@ -1503,9 +1503,13 @@ static int lfs_dir_compact(lfs_t *lfs,
                 }
             }
 #ifdef LFS_MIGRATE
-        } else if (lfs_pair_cmp(dir->pair, lfs->root) == 0 && lfs->lfs1) {
+        } else if (lfs->lfs1) {
-            // we can't relocate our root during migrations, as this would
+            // do not proactively relocate blocks during migrations, this
-            // cause the superblock to get updated, which would clobber v1
+            // can cause a number of failure states such: clobbering the
+            // v1 superblock if we relocate root, and invalidating directory
+            // pointers if we relocate the head of a directory. On top of
+            // this, relocations increase the overall complexity of
+            // lfs_migration, which is already a delicate operation.
 #endif
         } else {
             // we're writing too much, time to relocate
@@ -2103,8 +2107,6 @@ int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir) {
         return err;
     }
 
-    dir->m.pair[0] = dir->head[0];
-    dir->m.pair[1] = dir->head[1];
     dir->id = 0;
     dir->pos = 0;
     LFS_TRACE("lfs_dir_rewind -> %d", 0);
@@ -3006,10 +3008,10 @@ int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) {
     } else if (size > oldsize) {
         // flush+seek if not already at end
         if (file->pos != oldsize) {
-            int err = lfs_file_seek(lfs, file, 0, LFS_SEEK_END);
+            lfs_soff_t res = lfs_file_seek(lfs, file, 0, LFS_SEEK_END);
-            if (err < 0) {
+            if (res < 0) {
-                LFS_TRACE("lfs_file_truncate -> %d", err);
+                LFS_TRACE("lfs_file_truncate -> %d", res);
-                return err;
+                return (int)res;
             }
         }
 
@@ -3018,16 +3020,16 @@ int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) {
             lfs_ssize_t res = lfs_file_write(lfs, file, &(uint8_t){0}, 1);
             if (res < 0) {
                 LFS_TRACE("lfs_file_truncate -> %d", res);
-                return res;
+                return (int)res;
             }
         }
     }
 
     // restore pos
-    int err = lfs_file_seek(lfs, file, pos, LFS_SEEK_SET);
+    lfs_soff_t res = lfs_file_seek(lfs, file, pos, LFS_SEEK_SET);
-    if (err < 0) {
+    if (res < 0) {
-      LFS_TRACE("lfs_file_truncate -> %d", err);
+      LFS_TRACE("lfs_file_truncate -> %d", res);
-      return err;
+      return (int)res;
     }
 
     LFS_TRACE("lfs_file_truncate -> %d", 0);
@@ -3047,7 +3049,7 @@ int lfs_file_rewind(lfs_t *lfs, lfs_file_t *file) {
     lfs_soff_t res = lfs_file_seek(lfs, file, 0, LFS_SEEK_SET);
     if (res < 0) {
         LFS_TRACE("lfs_file_rewind -> %d", res);
-        return res;
+        return (int)res;
     }
 
     LFS_TRACE("lfs_file_rewind -> %d", 0);
@@ -3076,7 +3078,7 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info) {
     lfs_stag_t tag = lfs_dir_find(lfs, &cwd, &path, NULL);
     if (tag < 0) {
         LFS_TRACE("lfs_stat -> %d", tag);
-        return tag;
+        return (int)tag;
     }
 
     int err = lfs_dir_getinfo(lfs, &cwd, lfs_tag_id(tag), info);
@@ -3097,7 +3099,7 @@ int lfs_remove(lfs_t *lfs, const char *path) {
     lfs_stag_t tag = lfs_dir_find(lfs, &cwd, &path, NULL);
     if (tag < 0 || lfs_tag_id(tag) == 0x3ff) {
         LFS_TRACE("lfs_remove -> %d", (tag < 0) ? tag : LFS_ERR_INVAL);
-        return (tag < 0) ? tag : LFS_ERR_INVAL;
+        return (tag < 0) ? (int)tag : LFS_ERR_INVAL;
     }
 
     lfs_mdir_t dir;
@@ -3108,7 +3110,7 @@ int lfs_remove(lfs_t *lfs, const char *path) {
                 LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair);
         if (res < 0) {
             LFS_TRACE("lfs_remove -> %d", res);
-            return res;
+            return (int)res;
         }
         lfs_pair_fromle32(pair);
 
@@ -3171,7 +3173,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) {
     lfs_stag_t oldtag = lfs_dir_find(lfs, &oldcwd, &oldpath, NULL);
     if (oldtag < 0 || lfs_tag_id(oldtag) == 0x3ff) {
         LFS_TRACE("lfs_rename -> %d", (oldtag < 0) ? oldtag : LFS_ERR_INVAL);
-        return (oldtag < 0) ? oldtag : LFS_ERR_INVAL;
+        return (oldtag < 0) ? (int)oldtag : LFS_ERR_INVAL;
     }
 
     // find new entry
@@ -3181,7 +3183,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) {
     if ((prevtag < 0 || lfs_tag_id(prevtag) == 0x3ff) &&
             !(prevtag == LFS_ERR_NOENT && newid != 0x3ff)) {
         LFS_TRACE("lfs_rename -> %d", (prevtag < 0) ? prevtag : LFS_ERR_INVAL);
-        return (prevtag < 0) ? prevtag : LFS_ERR_INVAL;
+        return (prevtag < 0) ? (int)prevtag : LFS_ERR_INVAL;
     }
 
     lfs_mdir_t prevdir;
@@ -3202,7 +3204,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) {
                 LFS_MKTAG(LFS_TYPE_STRUCT, newid, 8), prevpair);
         if (res < 0) {
             LFS_TRACE("lfs_rename -> %d", res);
-            return res;
+            return (int)res;
         }
         lfs_pair_fromle32(prevpair);
 
@@ -3887,6 +3889,12 @@ static int lfs_fs_relocate(lfs_t *lfs,
             d->m.pair[0] = newpair[0];
             d->m.pair[1] = newpair[1];
         }
+
+        if (d->type == LFS_TYPE_DIR &&
+                lfs_pair_cmp(oldpair, ((lfs_dir_t*)d)->head) == 0) {
+            ((lfs_dir_t*)d)->head[0] = newpair[0];
+            ((lfs_dir_t*)d)->head[1] = newpair[1];
+        }
     }
 
     // find parent

tests/test_paths.sh

@@ -179,7 +179,7 @@ echo "--- Really big path test ---"
 scripts/test.py << TEST
     lfs_mount(&lfs, &cfg) => 0;
     memset(path, 'w', LFS_NAME_MAX);
-    path[LFS_NAME_MAX+1] = '\0';
+    path[LFS_NAME_MAX] = '\0';
     lfs_mkdir(&lfs, path) => 0;
     lfs_remove(&lfs, path) => 0;
     lfs_file_open(&lfs, &file, path,
@@ -189,7 +189,7 @@ scripts/test.py << TEST
 
     memcpy(path, "coffee/", strlen("coffee/"));
     memset(path+strlen("coffee/"), 'w', LFS_NAME_MAX);
-    path[strlen("coffee/")+LFS_NAME_MAX+1] = '\0';
+    path[strlen("coffee/")+LFS_NAME_MAX] = '\0';
     lfs_mkdir(&lfs, path) => 0;
     lfs_remove(&lfs, path) => 0;
     lfs_file_open(&lfs, &file, path,

tests/test_relocations.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+set -eu
+export TEST_FILE=$0
+trap 'export TEST_LINE=$LINENO' DEBUG
+
+ITERATIONS=20
+COUNT=10
+
+echo "=== Relocation tests ==="
+rm -rf blocks
+scripts/test.py << TEST
+    lfs_format(&lfs, &cfg) => 0;
+    // fill up filesystem so only ~16 blocks are left
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file, "padding", LFS_O_CREAT | LFS_O_WRONLY) => 0;
+    memset(buffer, 0, 512);
+    while (LFS_BLOCK_COUNT - lfs_fs_size(&lfs) > 16) {
+        lfs_file_write(&lfs, &file, buffer, 512) => 512;
+    }
+    lfs_file_close(&lfs, &file) => 0;
+    // make a child dir to use in bounded space
+    lfs_mkdir(&lfs, "child") => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+
+echo "--- Outdated head test ---"
+scripts/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    for (int j = 0; j < $ITERATIONS; j++) {
+        for (int i = 0; i < $COUNT; i++) {
+            sprintf(path, "child/test%03d_loooooooooooooooooong_name", i);
+            lfs_file_open(&lfs, &file, path, LFS_O_CREAT | LFS_O_WRONLY) => 0;
+            lfs_file_close(&lfs, &file) => 0;
+        }
+
+        lfs_dir_open(&lfs, &dir, "child") => 0;
+        lfs_dir_read(&lfs, &dir, &info) => 1;
+        lfs_dir_read(&lfs, &dir, &info) => 1;
+        for (int i = 0; i < $COUNT; i++) {
+            sprintf(path, "test%03d_loooooooooooooooooong_name", i);
+            lfs_dir_read(&lfs, &dir, &info) => 1;
+            strcmp(info.name, path) => 0;
+            info.size => 0;
+
+            sprintf(path, "child/test%03d_loooooooooooooooooong_name", i);
+            lfs_file_open(&lfs, &file, path, LFS_O_WRONLY) => 0;
+            lfs_file_write(&lfs, &file, "hi", 2) => 2;
+            lfs_file_close(&lfs, &file) => 0;
+        }
+        lfs_dir_read(&lfs, &dir, &info) => 0;
+
+        lfs_dir_rewind(&lfs, &dir) => 0;
+        lfs_dir_read(&lfs, &dir, &info) => 1;
+        lfs_dir_read(&lfs, &dir, &info) => 1;
+        for (int i = 0; i < $COUNT; i++) {
+            sprintf(path, "test%03d_loooooooooooooooooong_name", i);
+            lfs_dir_read(&lfs, &dir, &info) => 1;
+            strcmp(info.name, path) => 0;
+            info.size => 2;
+
+            sprintf(path, "child/test%03d_loooooooooooooooooong_name", i);
+            lfs_file_open(&lfs, &file, path, LFS_O_WRONLY) => 0;
+            lfs_file_write(&lfs, &file, "hi", 2) => 2;
+            lfs_file_close(&lfs, &file) => 0;
+        }
+        lfs_dir_read(&lfs, &dir, &info) => 0;
+
+        lfs_dir_rewind(&lfs, &dir) => 0;
+        lfs_dir_read(&lfs, &dir, &info) => 1;
+        lfs_dir_read(&lfs, &dir, &info) => 1;
+        for (int i = 0; i < $COUNT; i++) {
+            sprintf(path, "test%03d_loooooooooooooooooong_name", i);
+            lfs_dir_read(&lfs, &dir, &info) => 1;
+            strcmp(info.name, path) => 0;
+            info.size => 2;
+        }
+        lfs_dir_read(&lfs, &dir, &info) => 0;
+        lfs_dir_close(&lfs, &dir) => 0;
+
+        for (int i = 0; i < $COUNT; i++) {
+            sprintf(path, "child/test%03d_loooooooooooooooooong_name", i);
+            lfs_remove(&lfs, path) => 0;
+        }
+    }
+    lfs_unmount(&lfs) => 0;
+TEST
+
+scripts/results.py