fixup for 5c87b6864

Duplicate test_relocation.toml into a variant with removeall.

README.md

@@ -192,7 +192,7 @@ More details on how littlefs works can be found in [DESIGN.md](DESIGN.md) and
 ## Testing
 
 The littlefs comes with a test suite designed to run on a PC using the
-[emulated block device](bd/lfs_testbd.h) found in the `bd` directory.
+[emulated block device](emubd/lfs_emubd.h) found in the emubd directory.
 The tests assume a Linux environment and can be started with make:
 
 ``` bash

SPEC.md

@@ -233,19 +233,19 @@ Metadata tag fields:
    into a 3-bit abstract type and an 8-bit chunk field. Note that the value
    `0x000` is invalid and not assigned a type.
 
-    1. **Type1 (3-bits)** - Abstract type of the tag. Groups the tags into
-       8 categories that facilitate bitmasked lookups.
+3. **Type1 (3-bits)** - Abstract type of the tag. Groups the tags into
+   8 categories that facilitate bitmasked lookups.
 
-    2. **Chunk (8-bits)** - Chunk field used for various purposes by the different
-       abstract types.  type1+chunk+id form a unique identifier for each tag in the
-       metadata block.
+4. **Chunk (8-bits)** - Chunk field used for various purposes by the different
+   abstract types.  type1+chunk+id form a unique identifier for each tag in the
+   metadata block.
 
-3. **Id (10-bits)** - File id associated with the tag. Each file in a metadata
+5. **Id (10-bits)** - File id associated with the tag. Each file in a metadata
    block gets a unique id which is used to associate tags with that file. The
    special value `0x3ff` is used for any tags that are not associated with a
    file, such as directory and global metadata.
 
-4. **Length (10-bits)** - Length of the data in bytes. The special value
+6. **Length (10-bits)** - Length of the data in bytes. The special value
    `0x3ff` indicates that this tag has been deleted.
 
 ## Metadata types

bd/lfs_filebd.c

@@ -80,7 +80,7 @@ int lfs_filebd_read(const struct lfs_config *cfg, lfs_block_t block,
     LFS_ASSERT(size % cfg->read_size == 0);
     LFS_ASSERT(block < cfg->block_count);
 
-    // zero for reproducibility (in case file is truncated)
+    // zero for reproducability (in case file is truncated)
     if (bd->cfg->erase_value != -1) {
         memset(buffer, bd->cfg->erase_value, size);
     }

bd/lfs_rambd.c

@@ -32,12 +32,10 @@ int lfs_rambd_createcfg(const struct lfs_config *cfg,
         }
     }
 
-    // zero for reproducibility?
+    // zero for reproducability?
     if (bd->cfg->erase_value != -1) {
         memset(bd->buffer, bd->cfg->erase_value,
                 cfg->block_size * cfg->block_count);
-    } else {
-        memset(bd->buffer, 0, cfg->block_size * cfg->block_count);
     }
 
     LFS_RAMBD_TRACE("lfs_rambd_createcfg -> %d", 0);

lfs.c

@@ -11,7 +11,6 @@
 #define LFS_BLOCK_INLINE ((lfs_block_t)-2)
 
 /// Caching block device operations ///
-
 static inline void lfs_cache_drop(lfs_t *lfs, lfs_cache_t *rcache) {
     // do not zero, cheaper if cache is readonly or only going to be
     // written with identical data (during relocates)
@@ -269,26 +268,22 @@ static inline int lfs_pair_cmp(
              paira[0] == pairb[1] || paira[1] == pairb[0]);
 }
 
-#ifndef LFS_READONLY
 static inline bool lfs_pair_sync(
         const lfs_block_t paira[2],
         const lfs_block_t pairb[2]) {
     return (paira[0] == pairb[0] && paira[1] == pairb[1]) ||
            (paira[0] == pairb[1] && paira[1] == pairb[0]);
 }
-#endif
 
 static inline void lfs_pair_fromle32(lfs_block_t pair[2]) {
     pair[0] = lfs_fromle32(pair[0]);
     pair[1] = lfs_fromle32(pair[1]);
 }
 
-#ifndef LFS_READONLY
 static inline void lfs_pair_tole32(lfs_block_t pair[2]) {
     pair[0] = lfs_tole32(pair[0]);
     pair[1] = lfs_tole32(pair[1]);
 }
-#endif
 
 // operations on 32-bit entry tags
 typedef uint32_t lfs_tag_t;
@@ -370,7 +365,6 @@ static inline bool lfs_gstate_iszero(const lfs_gstate_t *a) {
     return true;
 }
 
-#ifndef LFS_READONLY
 static inline bool lfs_gstate_hasorphans(const lfs_gstate_t *a) {
     return lfs_tag_size(a->tag);
 }
@@ -382,7 +376,6 @@ static inline uint8_t lfs_gstate_getorphans(const lfs_gstate_t *a) {
 static inline bool lfs_gstate_hasmove(const lfs_gstate_t *a) {
     return lfs_tag_type1(a->tag);
 }
-#endif
 
 static inline bool lfs_gstate_hasmovehere(const lfs_gstate_t *a,
         const lfs_block_t *pair) {
@@ -395,13 +388,11 @@ static inline void lfs_gstate_fromle32(lfs_gstate_t *a) {
     a->pair[1] = lfs_fromle32(a->pair[1]);
 }
 
-#ifndef LFS_READONLY
 static inline void lfs_gstate_tole32(lfs_gstate_t *a) {
     a->tag     = lfs_tole32(a->tag);
     a->pair[0] = lfs_tole32(a->pair[0]);
     a->pair[1] = lfs_tole32(a->pair[1]);
 }
-#endif
 
 // other endianness operations
 static void lfs_ctz_fromle32(struct lfs_ctz *ctz) {
@@ -425,7 +416,6 @@ static inline void lfs_superblock_fromle32(lfs_superblock_t *superblock) {
     superblock->attr_max    = lfs_fromle32(superblock->attr_max);
 }
 
-#ifndef LFS_READONLY
 static inline void lfs_superblock_tole32(lfs_superblock_t *superblock) {
     superblock->version     = lfs_tole32(superblock->version);
     superblock->block_size  = lfs_tole32(superblock->block_size);
@@ -434,7 +424,6 @@ static inline void lfs_superblock_tole32(lfs_superblock_t *superblock) {
     superblock->file_max    = lfs_tole32(superblock->file_max);
     superblock->attr_max    = lfs_tole32(superblock->attr_max);
 }
-#endif
 
 #ifndef LFS_NO_ASSERT
 static bool lfs_mlist_isopen(struct lfs_mlist *head,
@@ -1460,7 +1449,7 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir) {
         }
     }
 
-    // zero for reproducibility in case initial block is unreadable
+    // zero for reproducability in case initial block is unreadable
     dir->rev = 0;
 
     // rather than clobbering one of the blocks we just pretend
@@ -1520,6 +1509,7 @@ static int lfs_dir_split(lfs_t *lfs,
         lfs_mdir_t *dir, const struct lfs_mattr *attrs, int attrcount,
         lfs_mdir_t *source, uint16_t split, uint16_t end) {
     // create tail directory
+    lfs_alloc_ack(lfs);
     lfs_mdir_t tail;
     int err = lfs_dir_alloc(lfs, &tail);
     if (err) {
@@ -2740,6 +2730,7 @@ static int lfs_file_outline(lfs_t *lfs, lfs_file_t *file) {
 }
 #endif
 
+#ifndef LFS_READONLY
 static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) {
     if (file->flags & LFS_F_READING) {
         if (!(file->flags & LFS_F_INLINE)) {
@@ -2748,7 +2739,6 @@ static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) {
         file->flags &= ~LFS_F_READING;
     }
 
-#ifndef LFS_READONLY
     if (file->flags & LFS_F_WRITING) {
         lfs_off_t pos = file->pos;
 
@@ -2815,10 +2805,10 @@ relocate:
 
         file->pos = pos;
     }
-#endif
 
     return 0;
 }
+#endif
 
 #ifndef LFS_READONLY
 static int lfs_file_rawsync(lfs_t *lfs, lfs_file_t *file) {
@@ -3091,11 +3081,13 @@ static lfs_soff_t lfs_file_rawseek(lfs_t *lfs, lfs_file_t *file,
         return npos;
     }
 
+#ifndef LFS_READONLY
     // write out everything beforehand, may be noop if rdonly
     int err = lfs_file_flush(lfs, file);
     if (err) {
         return err;
     }
+#endif
 
     // update pos
     file->pos = npos;
@@ -3198,7 +3190,106 @@ static int lfs_rawstat(lfs_t *lfs, const char *path, struct lfs_info *info) {
 }
 
 #ifndef LFS_READONLY
-static int lfs_rawremove(lfs_t *lfs, const char *path) {
+typedef int (*lfs_dir_prep_helper_t)(lfs_t *, struct lfs_mlist *, lfs_gstate_t *);
+
+static int lfs_dir_prep_remove_nonempty_folders(lfs_t *lfs, struct lfs_mlist *dir,
+        lfs_gstate_t *tmp_gstate)
+{
+    lfs_gstate_t split_gstate;
+    uint16_t id = 0;
+
+    // Walk tags stored in this directory and check for any directory
+    // tags.  Removal of directories with a directory in them can lead
+    // to additional orphans in the filesystem, so we return
+    // LFS_ERR_NOTEMPTY in this case.  Otherwise, leave the loaded
+    // directory for the tail end of the directory split to leave a proper
+    // view of the filesystem after removal.
+    while (true) {
+        if (dir->m.count == id) {
+            if (!dir->m.split) {
+                // We have iterated through the folder to the last
+                // tag.
+                break;
+            }
+
+            // Before we fetch the next block, update our fetched gstate xor
+            lfs_dir_getgstate(lfs, &dir->m, &split_gstate);
+            lfs_gstate_xor(tmp_gstate, &split_gstate);
+
+            int err = lfs_dir_fetch(lfs, &dir->m, dir->m.tail);
+            if (err) {
+                return err;
+            }
+
+            id = 0;
+        }
+
+        lfs_stag_t tag = lfs_dir_get(lfs, &dir->m, LFS_MKTAG(0x780, 0x3ff, 0),
+                LFS_MKTAG(LFS_TYPE_NAME, id, 0), NULL);
+
+        if (tag < 0) {
+            return tag;
+        }
+
+        if (lfs_tag_type3(tag) == LFS_TYPE_DIR) {
+            return LFS_ERR_NOTEMPTY;
+        }
+
+        id += 1;
+    }
+
+    return 0;
+}
+
+static int lfs_dir_prep_removal(lfs_t *lfs, struct lfs_mlist *dir,
+        lfs_mdir_t *newcwd, uint16_t newid, lfs_block_t *pair,
+        lfs_gstate_t *tmp_gstate, lfs_dir_prep_helper_t helper)
+{
+    lfs_stag_t res = lfs_dir_get(lfs, newcwd, LFS_MKTAG(0x700, 0x3ff, 0),
+            LFS_MKTAG(LFS_TYPE_STRUCT, newid, 8), pair);
+    if (res < 0) {
+        return (int)res;
+    }
+    lfs_pair_fromle32(pair);
+
+    memset(tmp_gstate, 0, sizeof(*tmp_gstate));
+
+    int err = lfs_dir_fetch(lfs, &dir->m, pair);
+    if (err) {
+        return err;
+    }
+
+    if (dir->m.count > 0 || dir->m.split) {
+        // Normal POSIX behavior wouldn't allow a non-empty
+        // folder to be removed/renamed into in this manner
+        if (NULL == helper) {
+            return LFS_ERR_NOTEMPTY;
+        }
+
+        err = helper(lfs, dir, tmp_gstate);
+        if (err) {
+            return err;
+        }
+    }
+
+    // mark fs as orphaned
+    err = lfs_fs_preporphans(lfs, +1);
+    if (err) {
+        return err;
+    }
+
+    // I know it's crazy but yes, dir can be changed by our parent's
+    // commit (if predecessor is child)
+    dir->type = 0;
+    dir->id = 0;
+    lfs->mlist = dir;
+
+    return 0;
+}
+#endif
+
+#ifndef LFS_READONLY
+static int lfs_rawremove(lfs_t *lfs, const char *path, lfs_dir_prep_helper_t helper) {
     // deorphan if we haven't yet, needed at most once after poweron
     int err = lfs_fs_forceconsistency(lfs);
     if (err) {
@@ -3212,37 +3303,16 @@ static int lfs_rawremove(lfs_t *lfs, const char *path) {
     }
 
     struct lfs_mlist dir;
+    lfs_block_t pair[2];
+    lfs_gstate_t tmp_gstate;
     dir.next = lfs->mlist;
     if (lfs_tag_type3(tag) == LFS_TYPE_DIR) {
-        // must be empty before removal
-        lfs_block_t pair[2];
-        lfs_stag_t res = lfs_dir_get(lfs, &cwd, LFS_MKTAG(0x700, 0x3ff, 0),
-                LFS_MKTAG(LFS_TYPE_STRUCT, lfs_tag_id(tag), 8), pair);
-        if (res < 0) {
-            return (int)res;
-        }
-        lfs_pair_fromle32(pair);
-
-        err = lfs_dir_fetch(lfs, &dir.m, pair);
-        if (err) {
+        // must be empty before removal to prevent orphans
+        err = lfs_dir_prep_removal(lfs, &dir, &cwd, lfs_tag_id(tag),
+                pair, &tmp_gstate, helper);
+        if (err < 0) {
             return err;
         }
-
-        if (dir.m.count > 0 || dir.m.split) {
-            return LFS_ERR_NOTEMPTY;
-        }
-
-        // mark fs as orphaned
-        err = lfs_fs_preporphans(lfs, +1);
-        if (err) {
-            return err;
-        }
-
-        // I know it's crazy but yes, dir can be changed by our parent's
-        // commit (if predecessor is child)
-        dir.type = 0;
-        dir.id = 0;
-        lfs->mlist = &dir;
     }
 
     // delete the entry
@@ -3261,11 +3331,15 @@ static int lfs_rawremove(lfs_t *lfs, const char *path) {
             return err;
         }
 
-        err = lfs_fs_pred(lfs, dir.m.pair, &cwd);
+        err = lfs_fs_pred(lfs, pair, &cwd);
         if (err) {
             return err;
         }
 
+        // Merge in gstate from first block splits within the directory;
+        // lfs_dir_drop will pick up the last gstate entry.
+        lfs_gstate_xor(&lfs->gdelta, &tmp_gstate);
+
         err = lfs_dir_drop(lfs, &cwd, &dir.m);
         if (err) {
             return err;
@@ -3277,7 +3351,8 @@ static int lfs_rawremove(lfs_t *lfs, const char *path) {
 #endif
 
 #ifndef LFS_READONLY
-static int lfs_rawrename(lfs_t *lfs, const char *oldpath, const char *newpath) {
+static int lfs_rawrename(lfs_t *lfs, const char *oldpath,
+        const char *newpath, lfs_dir_prep_helper_t helper) {
     // deorphan if we haven't yet, needed at most once after poweron
     int err = lfs_fs_forceconsistency(lfs);
     if (err) {
@@ -3305,6 +3380,8 @@ static int lfs_rawrename(lfs_t *lfs, const char *oldpath, const char *newpath) {
     uint16_t newoldid = lfs_tag_id(oldtag);
 
     struct lfs_mlist prevdir;
+    lfs_block_t dir_pair[2];
+    lfs_gstate_t tmp_gstate;
     prevdir.next = lfs->mlist;
     if (prevtag == LFS_ERR_NOENT) {
         // check that name fits
@@ -3325,36 +3402,12 @@ static int lfs_rawrename(lfs_t *lfs, const char *oldpath, const char *newpath) {
         // we're renaming to ourselves??
         return 0;
     } else if (lfs_tag_type3(prevtag) == LFS_TYPE_DIR) {
-        // must be empty before removal
-        lfs_block_t prevpair[2];
-        lfs_stag_t res = lfs_dir_get(lfs, &newcwd, LFS_MKTAG(0x700, 0x3ff, 0),
-                LFS_MKTAG(LFS_TYPE_STRUCT, newid, 8), prevpair);
-        if (res < 0) {
-            return (int)res;
-        }
-        lfs_pair_fromle32(prevpair);
-
-        // must be empty before removal
-        err = lfs_dir_fetch(lfs, &prevdir.m, prevpair);
+        // must be empty before removal to prevent orphans
+        err = lfs_dir_prep_removal(lfs, &prevdir, &newcwd, newid,
+                dir_pair, &tmp_gstate, helper);
         if (err) {
             return err;
         }
-
-        if (prevdir.m.count > 0 || prevdir.m.split) {
-            return LFS_ERR_NOTEMPTY;
-        }
-
-        // mark fs as orphaned
-        err = lfs_fs_preporphans(lfs, +1);
-        if (err) {
-            return err;
-        }
-
-        // I know it's crazy but yes, dir can be changed by our parent's
-        // commit (if predecessor is child)
-        prevdir.type = 0;
-        prevdir.id = 0;
-        lfs->mlist = &prevdir;
     }
 
     if (!samepair) {
@@ -3396,11 +3449,15 @@ static int lfs_rawrename(lfs_t *lfs, const char *oldpath, const char *newpath) {
             return err;
         }
 
-        err = lfs_fs_pred(lfs, prevdir.m.pair, &newcwd);
+        err = lfs_fs_pred(lfs, dir_pair, &newcwd);
         if (err) {
             return err;
         }
 
+        // Merge in gstate from first split blocks in the directory;
+        // lfs_dir_drop will pick up the other gstate entries.
+        lfs_gstate_xor(&lfs->gdelta, &tmp_gstate);
+
         err = lfs_dir_drop(lfs, &newcwd, &prevdir.m);
         if (err) {
             return err;
@@ -4082,7 +4139,7 @@ static int lfs_fs_relocate(lfs_t *lfs,
             lfs_fs_prepmove(lfs, 0x3ff, NULL);
         }
 
-        // replace bad pair, either we clean up desync, or no desync occurred
+        // replace bad pair, either we clean up desync, or no desync occured
         lfs_pair_tole32(newpair);
         err = lfs_dir_commit(lfs, &parent, LFS_MKATTRS(
                 {LFS_MKTAG_IF(moveid != 0x3ff,
@@ -5002,7 +5059,7 @@ int lfs_remove(lfs_t *lfs, const char *path) {
     }
     LFS_TRACE("lfs_remove(%p, \"%s\")", (void*)lfs, path);
 
-    err = lfs_rawremove(lfs, path);
+    err = lfs_rawremove(lfs, path, NULL);
 
     LFS_TRACE("lfs_remove -> %d", err);
     LFS_UNLOCK(lfs->cfg);
@@ -5010,6 +5067,24 @@ int lfs_remove(lfs_t *lfs, const char *path) {
 }
 #endif
 
+#ifndef LFS_READONLY
+int lfs_removeall(lfs_t *lfs, const char *path) {
+    int err = LFS_LOCK(lfs->cfg);
+    if (err) {
+        return err;
+    }
+    LFS_TRACE("lfs_removeall(%p, \"%s\")", (void*)lfs, path);
+
+    // Note: We pass in a helper pointer here so that this extra
+    //       logic can be dropped if it is never referenced
+    err = lfs_rawremove(lfs, path, lfs_dir_prep_remove_nonempty_folders);
+
+    LFS_TRACE("lfs_removeall -> %d", err);
+    LFS_UNLOCK(lfs->cfg);
+    return err;
+}
+#endif
+
 #ifndef LFS_READONLY
 int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) {
     int err = LFS_LOCK(lfs->cfg);
@@ -5018,7 +5093,25 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) {
     }
     LFS_TRACE("lfs_rename(%p, \"%s\", \"%s\")", (void*)lfs, oldpath, newpath);
 
-    err = lfs_rawrename(lfs, oldpath, newpath);
+    err = lfs_rawrename(lfs, oldpath, newpath, NULL);
+
+    LFS_TRACE("lfs_rename -> %d", err);
+    LFS_UNLOCK(lfs->cfg);
+    return err;
+}
+#endif
+
+#ifndef LFS_READONLY
+int lfs_rename_with_removeall(lfs_t *lfs, const char *oldpath, const char *newpath) {
+    int err = LFS_LOCK(lfs->cfg);
+    if (err) {
+        return err;
+    }
+    LFS_TRACE("lfs_rename(%p, \"%s\", \"%s\")", (void*)lfs, oldpath, newpath);
+
+    // Note: We pass in a helper pointer here so that this extra
+    //       logic can be dropped if it is never referenced
+    err = lfs_rawrename(lfs, oldpath, newpath, lfs_dir_prep_remove_nonempty_folders);
 
     LFS_TRACE("lfs_rename -> %d", err);
     LFS_UNLOCK(lfs->cfg);

lfs.h

@@ -159,49 +159,49 @@ struct lfs_config {
     // information to the block device operations
     void *context;
 
-    // Read a region in a block. Negative error codes are propagated
+    // Read a region in a block. Negative error codes are propogated
     // to the user.
     int (*read)(const struct lfs_config *c, lfs_block_t block,
             lfs_off_t off, void *buffer, lfs_size_t size);
 
     // Program a region in a block. The block must have previously
-    // been erased. Negative error codes are propagated to the user.
+    // been erased. Negative error codes are propogated to the user.
     // May return LFS_ERR_CORRUPT if the block should be considered bad.
     int (*prog)(const struct lfs_config *c, lfs_block_t block,
             lfs_off_t off, const void *buffer, lfs_size_t size);
 
     // Erase a block. A block must be erased before being programmed.
     // The state of an erased block is undefined. Negative error codes
-    // are propagated to the user.
+    // are propogated to the user.
     // May return LFS_ERR_CORRUPT if the block should be considered bad.
     int (*erase)(const struct lfs_config *c, lfs_block_t block);
 
     // Sync the state of the underlying block device. Negative error codes
-    // are propagated to the user.
+    // are propogated to the user.
     int (*sync)(const struct lfs_config *c);
 
 #ifdef LFS_THREADSAFE
     // Lock the underlying block device. Negative error codes
-    // are propagated to the user.
+    // are propogated to the user.
     int (*lock)(const struct lfs_config *c);
 
     // Unlock the underlying block device. Negative error codes
-    // are propagated to the user.
+    // are propogated to the user.
     int (*unlock)(const struct lfs_config *c);
 #endif
 
-    // Minimum size of a block read in bytes. All read operations will be a
+    // Minimum size of a block read. All read operations will be a
     // multiple of this value.
     lfs_size_t read_size;
 
-    // Minimum size of a block program in bytes. All program operations will be
-    // a multiple of this value.
+    // Minimum size of a block program. All program operations will be a
+    // multiple of this value.
     lfs_size_t prog_size;
 
-    // Size of an erasable block in bytes. This does not impact ram consumption
-    // and may be larger than the physical erase size. However, non-inlined
-    // files take up at minimum one block. Must be a multiple of the read and
-    // program sizes.
+    // Size of an erasable block. This does not impact ram consumption and
+    // may be larger than the physical erase size. However, non-inlined files
+    // take up at minimum one block. Must be a multiple of the read
+    // and program sizes.
     lfs_size_t block_size;
 
     // Number of erasable blocks on the device.
@@ -215,11 +215,11 @@ struct lfs_config {
     // Set to -1 to disable block-level wear-leveling.
     int32_t block_cycles;
 
-    // Size of block caches in bytes. Each cache buffers a portion of a block in
-    // RAM. The littlefs needs a read cache, a program cache, and one additional
+    // Size of block caches. Each cache buffers a portion of a block in RAM.
+    // The littlefs needs a read cache, a program cache, and one additional
     // cache per file. Larger caches can improve performance by storing more
-    // data and reducing the number of disk accesses. Must be a multiple of the
-    // read and program sizes, and a factor of the block size.
+    // data and reducing the number of disk accesses. Must be a multiple of
+    // the read and program sizes, and a factor of the block size.
     lfs_size_t cache_size;
 
     // Size of the lookahead buffer in bytes. A larger lookahead buffer
@@ -458,6 +458,17 @@ int lfs_unmount(lfs_t *lfs);
 int lfs_remove(lfs_t *lfs, const char *path);
 #endif
 
+#ifndef LFS_READONLY
+// Removes a file or directory
+//
+// If removing a directory, the directory must not have
+// any directories but it may contain files.  This is
+// non-POSIX behavior, and thus is a different call
+// than lfs_remove(...)
+// Returns a negative error code on failure.
+int lfs_removeall(lfs_t *lfs, const char *path);
+#endif
+
 #ifndef LFS_READONLY
 // Rename or move a file or directory
 //
@@ -468,6 +479,19 @@ int lfs_remove(lfs_t *lfs, const char *path);
 int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath);
 #endif
 
+#ifndef LFS_READONLY
+// Rename or move a file or directory
+//
+// If the destination exists, it must match the source in type.
+// If the destination is a directory, it may not contain
+// any directories but it may contain files.  This is
+// non-POSIX behavior, and thus is a different call
+// than lfs_rename(...)
+//
+// Returns a negative error code on failure.
+int lfs_rename_with_removeall(lfs_t *lfs, const char *oldpath, const char *newpath);
+#endif
+
 // Find info about a file or directory
 //
 // Fills out the info structure, based on the specified file or directory.
@@ -485,7 +509,7 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info);
 // Returns the size of the attribute, or a negative error code on failure.
 // Note, the returned size is the size of the attribute on disk, irrespective
 // of the size of the buffer. This can be used to dynamically allocate a buffer
-// or check for existence.
+// or check for existance.
 lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path,
         uint8_t type, void *buffer, lfs_size_t size);
 

scripts/test.py

@@ -292,8 +292,6 @@ class TestCase:
                     if e.errno == errno.EIO:
                         break
                     raise
-                if not line:
-                    break;
                 stdout.append(line)
                 if args.get('verbose'):
                     sys.stdout.write(line)
@@ -565,7 +563,7 @@ class TestSuite:
                     path=self.path))
                 mk.write('\n')
 
-            # add truly global defines globally
+            # add truely global defines globally
             for k, v in sorted(self.defines.items()):
                 mk.write('%s.test: override CFLAGS += -D%s=%r\n'
                     % (self.path, k, v))
@@ -656,7 +654,7 @@ def main(**args):
         for path in glob.glob(testpath):
             suites.append(TestSuite(path, classes, defines, filter, **args))
 
-    # sort for reproducibility
+    # sort for reproducability
     suites = sorted(suites)
 
     # generate permutations
@@ -689,8 +687,6 @@ def main(**args):
             if e.errno == errno.EIO:
                 break
             raise
-        if not line:
-            break;
         stdout.append(line)
         if args.get('verbose'):
             sys.stdout.write(line)

tests/test_relocations_with_removeall.toml

@@ -0,0 +1,305 @@
+# specific corner cases worth explicitly testing for
+[[case]] # dangling split dir test
+define.ITERATIONS = 20
+define.COUNT = 10
+define.LFS_BLOCK_CYCLES = [8, 1]
+code = '''
+    lfs_format(&lfs, &cfg) => 0;
+    // fill up filesystem so only ~16 blocks are left
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file, "padding", LFS_O_CREAT | LFS_O_WRONLY) => 0;
+    memset(buffer, 0, 512);
+    while (LFS_BLOCK_COUNT - lfs_fs_size(&lfs) > 16) {
+        lfs_file_write(&lfs, &file, buffer, 512) => 512;
+    }
+    lfs_file_close(&lfs, &file) => 0;
+    // make a child dir to use in bounded space
+    lfs_mkdir(&lfs, "child") => 0;
+    lfs_unmount(&lfs) => 0;
+
+    lfs_mount(&lfs, &cfg) => 0;
+    for (int j = 0; j < ITERATIONS; j++) {
+        for (int i = 0; i < COUNT; i++) {
+            sprintf(path, "child/test%03d_loooooooooooooooooong_name", i);
+            lfs_file_open(&lfs, &file, path, LFS_O_CREAT | LFS_O_WRONLY) => 0;
+            lfs_file_close(&lfs, &file) => 0;
+        }
+
+        lfs_dir_open(&lfs, &dir, "child") => 0;
+        lfs_dir_read(&lfs, &dir, &info) => 1;
+        lfs_dir_read(&lfs, &dir, &info) => 1;
+        for (int i = 0; i < COUNT; i++) {
+            sprintf(path, "test%03d_loooooooooooooooooong_name", i);
+            lfs_dir_read(&lfs, &dir, &info) => 1;
+            strcmp(info.name, path) => 0;
+        }
+        lfs_dir_read(&lfs, &dir, &info) => 0;
+        lfs_dir_close(&lfs, &dir) => 0;
+
+        if (j == ITERATIONS-1) {
+            break;
+        }
+
+        for (int i = 0; i < COUNT; i++) {
+            sprintf(path, "child/test%03d_loooooooooooooooooong_name", i);
+            lfs_removeall(&lfs, path) => 0;
+        }
+    }
+    lfs_unmount(&lfs) => 0;
+
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_dir_open(&lfs, &dir, "child") => 0;
+    lfs_dir_read(&lfs, &dir, &info) => 1;
+    lfs_dir_read(&lfs, &dir, &info) => 1;
+    for (int i = 0; i < COUNT; i++) {
+        sprintf(path, "test%03d_loooooooooooooooooong_name", i);
+        lfs_dir_read(&lfs, &dir, &info) => 1;
+        strcmp(info.name, path) => 0;
+    }
+    lfs_dir_read(&lfs, &dir, &info) => 0;
+    lfs_dir_close(&lfs, &dir) => 0;
+    for (int i = 0; i < COUNT; i++) {
+        sprintf(path, "child/test%03d_loooooooooooooooooong_name", i);
+        lfs_removeall(&lfs, path) => 0;
+    }
+    lfs_unmount(&lfs) => 0;
+'''
+
+[[case]] # outdated head test
+define.ITERATIONS = 20
+define.COUNT = 10
+define.LFS_BLOCK_CYCLES = [8, 1]
+code = '''
+    lfs_format(&lfs, &cfg) => 0;
+    // fill up filesystem so only ~16 blocks are left
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file, "padding", LFS_O_CREAT | LFS_O_WRONLY) => 0;
+    memset(buffer, 0, 512);
+    while (LFS_BLOCK_COUNT - lfs_fs_size(&lfs) > 16) {
+        lfs_file_write(&lfs, &file, buffer, 512) => 512;
+    }
+    lfs_file_close(&lfs, &file) => 0;
+    // make a child dir to use in bounded space
+    lfs_mkdir(&lfs, "child") => 0;
+    lfs_unmount(&lfs) => 0;
+
+    lfs_mount(&lfs, &cfg) => 0;
+    for (int j = 0; j < ITERATIONS; j++) {
+        for (int i = 0; i < COUNT; i++) {
+            sprintf(path, "child/test%03d_loooooooooooooooooong_name", i);
+            lfs_file_open(&lfs, &file, path, LFS_O_CREAT | LFS_O_WRONLY) => 0;
+            lfs_file_close(&lfs, &file) => 0;
+        }
+
+        lfs_dir_open(&lfs, &dir, "child") => 0;
+        lfs_dir_read(&lfs, &dir, &info) => 1;
+        lfs_dir_read(&lfs, &dir, &info) => 1;
+        for (int i = 0; i < COUNT; i++) {
+            sprintf(path, "test%03d_loooooooooooooooooong_name", i);
+            lfs_dir_read(&lfs, &dir, &info) => 1;
+            strcmp(info.name, path) => 0;
+            info.size => 0;
+
+            sprintf(path, "child/test%03d_loooooooooooooooooong_name", i);
+            lfs_file_open(&lfs, &file, path, LFS_O_WRONLY) => 0;
+            lfs_file_write(&lfs, &file, "hi", 2) => 2;
+            lfs_file_close(&lfs, &file) => 0;
+        }
+        lfs_dir_read(&lfs, &dir, &info) => 0;
+
+        lfs_dir_rewind(&lfs, &dir) => 0;
+        lfs_dir_read(&lfs, &dir, &info) => 1;
+        lfs_dir_read(&lfs, &dir, &info) => 1;
+        for (int i = 0; i < COUNT; i++) {
+            sprintf(path, "test%03d_loooooooooooooooooong_name", i);
+            lfs_dir_read(&lfs, &dir, &info) => 1;
+            strcmp(info.name, path) => 0;
+            info.size => 2;
+
+            sprintf(path, "child/test%03d_loooooooooooooooooong_name", i);
+            lfs_file_open(&lfs, &file, path, LFS_O_WRONLY) => 0;
+            lfs_file_write(&lfs, &file, "hi", 2) => 2;
+            lfs_file_close(&lfs, &file) => 0;
+        }
+        lfs_dir_read(&lfs, &dir, &info) => 0;
+
+        lfs_dir_rewind(&lfs, &dir) => 0;
+        lfs_dir_read(&lfs, &dir, &info) => 1;
+        lfs_dir_read(&lfs, &dir, &info) => 1;
+        for (int i = 0; i < COUNT; i++) {
+            sprintf(path, "test%03d_loooooooooooooooooong_name", i);
+            lfs_dir_read(&lfs, &dir, &info) => 1;
+            strcmp(info.name, path) => 0;
+            info.size => 2;
+        }
+        lfs_dir_read(&lfs, &dir, &info) => 0;
+        lfs_dir_close(&lfs, &dir) => 0;
+
+        for (int i = 0; i < COUNT; i++) {
+            sprintf(path, "child/test%03d_loooooooooooooooooong_name", i);
+            lfs_removeall(&lfs, path) => 0;
+        }
+    }
+    lfs_unmount(&lfs) => 0;
+'''
+
+[[case]] # reentrant testing for relocations, this is the same as the
+         # orphan testing, except here we also set block_cycles so that
+         # almost every tree operation needs a relocation
+reentrant = true
+# TODO fix this case, caused by non-DAG trees
+if = '!(DEPTH == 3 && LFS_CACHE_SIZE != 64)'
+define = [
+    {FILES=6,  DEPTH=1, CYCLES=20, LFS_BLOCK_CYCLES=1},
+    {FILES=26, DEPTH=1, CYCLES=20, LFS_BLOCK_CYCLES=1},
+    {FILES=3,  DEPTH=3, CYCLES=20, LFS_BLOCK_CYCLES=1},
+]
+code = '''
+    err = lfs_mount(&lfs, &cfg);
+    if (err) {
+        lfs_format(&lfs, &cfg) => 0;
+        lfs_mount(&lfs, &cfg) => 0;
+    }
+
+    srand(1);
+    const char alpha[] = "abcdefghijklmnopqrstuvwxyz";
+    for (int i = 0; i < CYCLES; i++) {
+        // create random path
+        char full_path[256];
+        for (int d = 0; d < DEPTH; d++) {
+            sprintf(&full_path[2*d], "/%c", alpha[rand() % FILES]);
+        }
+
+        // if it does not exist, we create it, else we destroy
+        int res = lfs_stat(&lfs, full_path, &info);
+        if (res == LFS_ERR_NOENT) {
+            // create each directory in turn, ignore if dir already exists
+            for (int d = 0; d < DEPTH; d++) {
+                strcpy(path, full_path);
+                path[2*d+2] = '\0';
+                err = lfs_mkdir(&lfs, path);
+                assert(!err || err == LFS_ERR_EXIST);
+            }
+
+            for (int d = 0; d < DEPTH; d++) {
+                strcpy(path, full_path);
+                path[2*d+2] = '\0';
+                lfs_stat(&lfs, path, &info) => 0;
+                assert(strcmp(info.name, &path[2*d+1]) == 0);
+                assert(info.type == LFS_TYPE_DIR);
+            }
+        } else {
+            // is valid dir?
+            assert(strcmp(info.name, &full_path[2*(DEPTH-1)+1]) == 0);
+            assert(info.type == LFS_TYPE_DIR);
+
+            // try to delete path in reverse order, ignore if dir is not empty
+            for (int d = DEPTH-1; d >= 0; d--) {
+                strcpy(path, full_path);
+                path[2*d+2] = '\0';
+                err = lfs_removeall(&lfs, path);
+                assert(!err || err == LFS_ERR_NOTEMPTY);
+            }
+
+            lfs_stat(&lfs, full_path, &info) => LFS_ERR_NOENT;
+        }
+    }
+    lfs_unmount(&lfs) => 0;
+'''
+
+[[case]] # reentrant testing for relocations, but now with random renames!
+reentrant = true
+# TODO fix this case, caused by non-DAG trees
+if = '!(DEPTH == 3 && LFS_CACHE_SIZE != 64)'
+define = [
+    {FILES=6,  DEPTH=1, CYCLES=20, LFS_BLOCK_CYCLES=1},
+    {FILES=26, DEPTH=1, CYCLES=20, LFS_BLOCK_CYCLES=1},
+    {FILES=3,  DEPTH=3, CYCLES=20, LFS_BLOCK_CYCLES=1},
+]
+code = '''
+    err = lfs_mount(&lfs, &cfg);
+    if (err) {
+        lfs_format(&lfs, &cfg) => 0;
+        lfs_mount(&lfs, &cfg) => 0;
+    }
+
+    srand(1);
+    const char alpha[] = "abcdefghijklmnopqrstuvwxyz";
+    for (int i = 0; i < CYCLES; i++) {
+        // create random path
+        char full_path[256];
+        for (int d = 0; d < DEPTH; d++) {
+            sprintf(&full_path[2*d], "/%c", alpha[rand() % FILES]);
+        }
+
+        // if it does not exist, we create it, else we destroy
+        int res = lfs_stat(&lfs, full_path, &info);
+        assert(!res || res == LFS_ERR_NOENT);
+        if (res == LFS_ERR_NOENT) {
+            // create each directory in turn, ignore if dir already exists
+            for (int d = 0; d < DEPTH; d++) {
+                strcpy(path, full_path);
+                path[2*d+2] = '\0';
+                err = lfs_mkdir(&lfs, path);
+                assert(!err || err == LFS_ERR_EXIST);
+            }
+
+            for (int d = 0; d < DEPTH; d++) {
+                strcpy(path, full_path);
+                path[2*d+2] = '\0';
+                lfs_stat(&lfs, path, &info) => 0;
+                assert(strcmp(info.name, &path[2*d+1]) == 0);
+                assert(info.type == LFS_TYPE_DIR);
+            }
+        } else {
+            assert(strcmp(info.name, &full_path[2*(DEPTH-1)+1]) == 0);
+            assert(info.type == LFS_TYPE_DIR);
+
+            // create new random path
+            char new_path[256];
+            for (int d = 0; d < DEPTH; d++) {
+                sprintf(&new_path[2*d], "/%c", alpha[rand() % FILES]);
+            }
+
+            // if new path does not exist, rename, otherwise destroy
+            res = lfs_stat(&lfs, new_path, &info);
+            assert(!res || res == LFS_ERR_NOENT);
+            if (res == LFS_ERR_NOENT) {
+                // stop once some dir is renamed
+                for (int d = 0; d < DEPTH; d++) {
+                    strcpy(&path[2*d], &full_path[2*d]);
+                    path[2*d+2] = '\0';
+                    strcpy(&path[128+2*d], &new_path[2*d]);
+                    path[128+2*d+2] = '\0';
+                    err = lfs_rename(&lfs, path, path+128);
+                    assert(!err || err == LFS_ERR_NOTEMPTY);
+                    if (!err) {
+                        strcpy(path, path+128);
+                    }
+                }
+
+                for (int d = 0; d < DEPTH; d++) {
+                    strcpy(path, new_path);
+                    path[2*d+2] = '\0';
+                    lfs_stat(&lfs, path, &info) => 0;
+                    assert(strcmp(info.name, &path[2*d+1]) == 0);
+                    assert(info.type == LFS_TYPE_DIR);
+                }
+                
+                lfs_stat(&lfs, full_path, &info) => LFS_ERR_NOENT;
+            } else {
+                // try to delete path in reverse order,
+                // ignore if dir is not empty
+                for (int d = DEPTH-1; d >= 0; d--) {
+                    strcpy(path, full_path);
+                    path[2*d+2] = '\0';
+                    err = lfs_removeall(&lfs, path);
+                    assert(!err || err == LFS_ERR_NOTEMPTY);
+                }
+
+                lfs_stat(&lfs, full_path, &info) => LFS_ERR_NOENT;
+            }
+        }
+    }
+    lfs_unmount(&lfs) => 0;
+'''