Fixed issue with writes following a truncate

The problem was not setting the file state correctly after the truncate.
To truncate < size, we end up using the cache to traverse the ctz
skip-list far away from where our file->pos is.

We can leave the last block in the cache in case we're going to append
to the file, but if we do this we need to set up file->block+file->off
to tell use where we are in the file, and set the LFS_F_READING flag to
indicate that our cache contains read data.

Note this is different than the LFS_F_DIRTY, which we need also. The
purpose of the flags are as follows:
- LFS_F_DIRTY - file ctz skip-list branch is out of sync with
  filesystem, need to update metadata
- LFS_F_READING - file cache is in use for reading, need to drop cache
- LFS_F_WRITING - file cache is in use for writing, need to write out
  cache to disk

The difference between flags is subtle but important because read/prog
caches are handled differently. Prog caches have asserts in place to
catch programs without erases (the infamous pcache->block == 0xffffffff
assert).

Though maybe the names deserve an update...

Found by ebinans

.travis.yml

@@ -136,10 +136,11 @@ jobs:
       env:
         - STAGE=test
         - NAME=littlefs-migration
+      if: branch !~ -prefix$
       install:
         - sudo apt-get install libfuse-dev
         - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v2-alpha v2
-        - git clone --depth 1 https://github.com/geky/littlefs-fuse v1
+        - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v1 v1
         - fusermount -V
         - gcc --version
       before_script:

lfs.c

@@ -80,21 +80,6 @@ static int lfs_bd_read(lfs_t *lfs,
             diff = lfs_min(diff, rcache->off-off);
         }
 
-        if (size >= hint && off % lfs->cfg->read_size == 0 &&
-                size >= lfs->cfg->read_size) {
-            // bypass cache?
-            diff = lfs_aligndown(diff, lfs->cfg->read_size);
-            int err = lfs->cfg->read(lfs->cfg, block, off, data, diff);
-            if (err) {
-                return err;
-            }
-
-            data += diff;
-            off += diff;
-            size -= diff;
-            continue;
-        }
-
         // load to cache, first condition can no longer fail
         LFS_ASSERT(block < lfs->cfg->block_count);
         rcache->block = block;
@@ -827,7 +812,8 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
             // next commit not yet programmed or we're not in valid range
             if (!lfs_tag_isvalid(tag) ||
                     off + lfs_tag_dsize(tag) > lfs->cfg->block_size) {
-                dir->erased = (lfs_tag_type1(ptag) == LFS_TYPE_CRC);
+                dir->erased = (lfs_tag_type1(ptag) == LFS_TYPE_CRC &&
+                        dir->off % lfs->cfg->prog_size == 0);
                 break;
             }
 
@@ -854,7 +840,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs,
                 }
 
                 // reset the next bit if we need to
-                ptag ^= (lfs_tag_chunk(tag) & 1) << 31;
+                ptag ^= (lfs_tag_chunk(tag) & 1U) << 31;
 
                 // toss our crc into the filesystem seed for
                 // pseudorandom numbers
@@ -1436,8 +1422,10 @@ static int lfs_dir_compact(lfs_t *lfs,
         // space is complicated, we need room for tail, crc, gstate,
         // cleanup delete, and we cap at half a block to give room
         // for metadata updates.
-        if (size <= lfs_min(lfs->cfg->block_size - 36,
-                lfs_alignup(lfs->cfg->block_size/2, lfs->cfg->prog_size))) {
+        if (end - begin < 0xff &&
+                size <= lfs_min(lfs->cfg->block_size - 36,
+                    lfs_alignup(lfs->cfg->block_size/2,
+                        lfs->cfg->prog_size))) {
             break;
         }
 
@@ -1497,7 +1485,7 @@ static int lfs_dir_compact(lfs_t *lfs,
 
     // begin loop to commit compaction to blocks until a compact sticks
     while (true) {
-        if (true) {
+        {
             // There's nothing special about our global delta, so feed it into
             // our local global delta
             int err = lfs_dir_getgstate(lfs, dir, &lfs->gdelta);
@@ -1596,13 +1584,12 @@ static int lfs_dir_compact(lfs_t *lfs,
             dir->count = end - begin;
             dir->off = commit.off;
             dir->etag = commit.ptag;
-            dir->erased = true;
+            dir->erased = (dir->off % lfs->cfg->prog_size == 0);
             // note we able to have already handled move here
             if (lfs_gstate_hasmovehere(&lfs->gpending, dir->pair)) {
                 lfs_gstate_xormove(&lfs->gpending,
                     &lfs->gpending, 0x3ff, NULL);
             }
-
         }
         break;
 
@@ -1711,7 +1698,7 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir,
         }
     }
 
-    if (dir->erased) {
+    if (dir->erased || dir->count >= 0xff) {
         // try to commit
         struct lfs_commit commit = {
             .block = dir->pair[0],
@@ -2122,7 +2109,7 @@ static int lfs_ctz_extend(lfs_t *lfs,
         }
         LFS_ASSERT(nblock >= 2 && nblock <= lfs->cfg->block_count);
 
-        if (true) {
+        {
             err = lfs_bd_erase(lfs, nblock);
             if (err) {
                 if (err == LFS_ERR_CORRUPT) {
@@ -2381,7 +2368,8 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file,
         if (file->ctz.size > 0) {
             lfs_stag_t res = lfs_dir_get(lfs, &file->m,
                     LFS_MKTAG(0x700, 0x3ff, 0),
-                    LFS_MKTAG(LFS_TYPE_STRUCT, file->id, file->cache.size),
+                    LFS_MKTAG(LFS_TYPE_STRUCT, file->id,
+                        lfs_min(file->cache.size, 0x3fe)),
                     file->cache.buffer);
             if (res < 0) {
                 err = res;
@@ -2576,6 +2564,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) {
     while (true) {
         int err = lfs_file_flush(lfs, file);
         if (err) {
+            file->flags |= LFS_F_ERRED;
             return err;
         }
 
@@ -2611,6 +2600,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) {
                 if (err == LFS_ERR_NOSPC && (file->flags & LFS_F_INLINE)) {
                     goto relocate;
                 }
+                file->flags |= LFS_F_ERRED;
                 return err;
             }
 
@@ -2624,6 +2614,7 @@ relocate:
         file->off = file->pos;
         err = lfs_file_relocate(lfs, file);
         if (err) {
+            file->flags |= LFS_F_ERRED;
             return err;
         }
     }
@@ -2858,6 +2849,10 @@ int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) {
         return LFS_ERR_BADF;
     }
 
+    if (size > LFS_FILE_MAX) {
+        return LFS_ERR_INVAL;
+    }
+
     lfs_off_t oldsize = lfs_file_size(lfs, file);
     if (size < oldsize) {
         // need to flush since directly changing metadata
@@ -2869,13 +2864,14 @@ int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) {
         // lookup new head in ctz skip list
         err = lfs_ctz_find(lfs, NULL, &file->cache,
                 file->ctz.head, file->ctz.size,
-                size, &file->ctz.head, &(lfs_off_t){0});
+                size, &file->block, &file->off);
         if (err) {
             return err;
         }
 
+        file->ctz.head = file->block;
         file->ctz.size = size;
-        file->flags |= LFS_F_DIRTY;
+        file->flags |= LFS_F_DIRTY | LFS_F_READING;
     } else if (size > oldsize) {
         lfs_off_t pos = file->pos;
 
@@ -3223,8 +3219,9 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) {
     lfs_cache_zero(lfs, &lfs->pcache);
 
     // setup lookahead, must be multiple of 64-bits
-    LFS_ASSERT(lfs->cfg->lookahead_size % 8 == 0);
     LFS_ASSERT(lfs->cfg->lookahead_size > 0);
+    LFS_ASSERT(lfs->cfg->lookahead_size % 8 == 0 &&
+            (uintptr_t)lfs->cfg->lookahead_buffer % 8 == 0);
     if (lfs->cfg->lookahead_buffer) {
         lfs->free.buffer = lfs->cfg->lookahead_buffer;
     } else {
@@ -3292,7 +3289,7 @@ static int lfs_deinit(lfs_t *lfs) {
 
 int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) {
     int err = 0;
-    if (true) {
+    {
         err = lfs_init(lfs, cfg);
         if (err) {
             return err;
@@ -4176,7 +4173,7 @@ static int lfs1_moved(lfs_t *lfs, const void *e) {
 static int lfs1_mount(lfs_t *lfs, struct lfs1 *lfs1,
         const struct lfs_config *cfg) {
     int err = 0;
-    if (true) {
+    {
         err = lfs_init(lfs, cfg);
         if (err) {
             return err;
@@ -4247,7 +4244,7 @@ int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg) {
         return err;
     }
 
-    if (true) {
+    {
         // iterate through each directory, copying over entries
         // into new directory
         lfs1_dir_t dir1;

lfs.h

@@ -215,8 +215,9 @@ struct lfs_config {
     // By default lfs_malloc is used to allocate this buffer.
     void *prog_buffer;
 
-    // Optional statically allocated program buffer. Must be lookahead_size.
-    // By default lfs_malloc is used to allocate this buffer.
+    // Optional statically allocated lookahead buffer. Must be lookahead_size
+    // and aligned to a 64-bit boundary. By default lfs_malloc is used to
+    // allocate this buffer.
     void *lookahead_buffer;
 
     // Optional upper limit on length of file names in bytes. No downside for
@@ -577,7 +578,8 @@ int lfs_dir_close(lfs_t *lfs, lfs_dir_t *dir);
 // Read an entry in the directory
 //
 // Fills out the info structure, based on the specified file or directory.
-// Returns a negative error code on failure.
+// Returns a positive value on success, 0 at the end of directory,
+// or a negative error code on failure.
 int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info);
 
 // Change the position of the directory

lfs_util.h

@@ -192,6 +192,7 @@ static inline uint32_t lfs_tobe32(uint32_t a) {
 uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size);
 
 // Allocate memory, only used if buffers are not provided to littlefs
+// Note, memory must be 64-bit aligned
 static inline void *lfs_malloc(size_t size) {
 #ifndef LFS_NO_MALLOC
     return malloc(size);

tests/test_truncate.sh

@@ -11,6 +11,150 @@ tests/test.py << TEST
     lfs_format(&lfs, &cfg) => 0;
 TEST
 
+echo "--- Simple truncate ---"
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldynoop",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+
+    strcpy((char*)buffer, "hair");
+    size = strlen((char*)buffer);
+    for (lfs_off_t j = 0; j < $LARGESIZE; j += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldynoop", LFS_O_RDWR) => 0;
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_truncate(&lfs, &file[0], $MEDIUMSIZE) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldynoop", LFS_O_RDONLY) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    size = strlen("hair");
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_read(&lfs, &file[0], buffer, size) => size;
+        memcmp(buffer, "hair", size) => 0;
+    }
+    lfs_file_read(&lfs, &file[0], buffer, size) => 0;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+
+echo "--- Truncate and read ---"
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldyread",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+
+    strcpy((char*)buffer, "hair");
+    size = strlen((char*)buffer);
+    for (lfs_off_t j = 0; j < $LARGESIZE; j += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldyread", LFS_O_RDWR) => 0;
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_truncate(&lfs, &file[0], $MEDIUMSIZE) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    size = strlen("hair");
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_read(&lfs, &file[0], buffer, size) => size;
+        memcmp(buffer, "hair", size) => 0;
+    }
+    lfs_file_read(&lfs, &file[0], buffer, size) => 0;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldyread", LFS_O_RDONLY) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    size = strlen("hair");
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_read(&lfs, &file[0], buffer, size) => size;
+        memcmp(buffer, "hair", size) => 0;
+    }
+    lfs_file_read(&lfs, &file[0], buffer, size) => 0;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+
+echo "--- Truncate and write ---"
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldywrite",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+
+    strcpy((char*)buffer, "hair");
+    size = strlen((char*)buffer);
+    for (lfs_off_t j = 0; j < $LARGESIZE; j += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldywrite", LFS_O_RDWR) => 0;
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_truncate(&lfs, &file[0], $MEDIUMSIZE) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    strcpy((char*)buffer, "bald");
+    size = strlen((char*)buffer);
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldywrite", LFS_O_RDONLY) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    size = strlen("bald");
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_read(&lfs, &file[0], buffer, size) => size;
+        memcmp(buffer, "bald", size) => 0;
+    }
+    lfs_file_read(&lfs, &file[0], buffer, size) => 0;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+
+# More aggressive general truncation tests
 truncate_test() {
 STARTSIZES="$1"
 STARTSEEKS="$2"