Fixed issue with writes following a truncate

The problem was not setting the file state correctly after the truncate.
To truncate < size, we end up using the cache to traverse the ctz
skip-list far away from where our file->pos is.

We can leave the last block in the cache in case we're going to append
to the file, but if we do this we need to set up file->block+file->off
to tell use where we are in the file, and set the LFS_F_READING flag to
indicate that our cache contains read data.

Note this is different than the LFS_F_DIRTY, which we need also. The
purpose of the flags are as follows:
- LFS_F_DIRTY - file ctz skip-list branch is out of sync with
  filesystem, need to update metadata
- LFS_F_READING - file cache is in use for reading, need to drop cache
- LFS_F_WRITING - file cache is in use for writing, need to write out
  cache to disk

The difference between flags is subtle but important because read/prog
caches are handled differently. Prog caches have asserts in place to
catch programs without erases (the infamous pcache->block == 0xffffffff
assert).

Though maybe the names deserve an update...

Found by ebinans

.travis.yml

@@ -66,10 +66,7 @@ jobs:
         - CC="arm-linux-gnueabi-gcc --static -mthumb"
         - EXEC="qemu-arm"
       install:
-        - sudo apt-get install
-              gcc-arm-linux-gnueabi
-              libc6-dev-armel-cross
-              qemu-user
+        - sudo apt-get install gcc-arm-linux-gnueabi qemu-user
         - arm-linux-gnueabi-gcc --version
         - qemu-arm -version
 
@@ -81,10 +78,7 @@ jobs:
         - CC="powerpc-linux-gnu-gcc --static"
         - EXEC="qemu-ppc"
       install:
-        - sudo apt-get install
-              gcc-powerpc-linux-gnu
-              libc6-dev-powerpc-cross
-              qemu-user
+        - sudo apt-get install gcc-powerpc-linux-gnu qemu-user
         - powerpc-linux-gnu-gcc --version
         - qemu-ppc -version
 
@@ -96,10 +90,9 @@ jobs:
         - CC="mips-linux-gnu-gcc --static"
         - EXEC="qemu-mips"
       install:
-        - sudo apt-get install
-              gcc-mips-linux-gnu
-              libc6-dev-mips-cross
-              qemu-user
+        - sudo add-apt-repository -y "deb http://archive.ubuntu.com/ubuntu/ xenial main universe"
+        - sudo apt-get -qq update
+        - sudo apt-get install gcc-mips-linux-gnu qemu-user
         - mips-linux-gnu-gcc --version
         - qemu-mips -version
 

lfs.c

@@ -84,12 +84,9 @@ static int lfs_bd_read(lfs_t *lfs,
         LFS_ASSERT(block < lfs->cfg->block_count);
         rcache->block = block;
         rcache->off = lfs_aligndown(off, lfs->cfg->read_size);
-        rcache->size = lfs_min(
-                lfs_min(
-                    lfs_alignup(off+hint, lfs->cfg->read_size),
-                    lfs->cfg->block_size)
-                - rcache->off,
-                lfs->cfg->cache_size);
+        rcache->size = lfs_min(lfs_alignup(off+hint, lfs->cfg->read_size),
+                lfs_min(lfs->cfg->block_size - rcache->off,
+                    lfs->cfg->cache_size));
         int err = lfs->cfg->read(lfs->cfg, rcache->block,
                 rcache->off, rcache->buffer, rcache->size);
         if (err) {
@@ -194,7 +191,7 @@ static int lfs_bd_prog(lfs_t *lfs,
             off += diff;
             size -= diff;
 
-            pcache->size = lfs_max(pcache->size, off - pcache->off);
+            pcache->size = off - pcache->off;
             if (pcache->size == lfs->cfg->cache_size) {
                 // eagerly flush out pcache if we fill up
                 int err = lfs_bd_flush(lfs, pcache, rcache, validate);
@@ -415,7 +412,6 @@ static int lfs_dir_compact(lfs_t *lfs,
         lfs_mdir_t *dir, const struct lfs_mattr *attrs, int attrcount,
         lfs_mdir_t *source, uint16_t begin, uint16_t end);
 static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file);
-static int lfs_file_outline(lfs_t *lfs, lfs_file_t *file);
 static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file);
 static void lfs_fs_preporphans(lfs_t *lfs, int8_t orphans);
 static void lfs_fs_prepmove(lfs_t *lfs,
@@ -618,7 +614,7 @@ static int lfs_dir_getread(lfs_t *lfs, const lfs_mdir_t *dir,
                 lfs->cfg->cache_size);
         int err = lfs_dir_getslice(lfs, dir, gmask, gtag,
                 rcache->off, rcache->buffer, rcache->size);
-        if (err < 0) {
+        if (err) {
             return err;
         }
     }
@@ -634,9 +630,8 @@ static int lfs_dir_traverse_filter(void *p,
     // check for redundancy
     uint32_t mask = LFS_MKTAG(0x7ff, 0x3ff, 0);
     if ((mask & tag) == (mask & *filtertag) ||
-            lfs_tag_isdelete(*filtertag) ||
-            (mask & tag) == (LFS_MKTAG(LFS_TYPE_DELETE, 0, 0) |
-                (LFS_MKTAG(0, 0x3ff, 0) & *filtertag))) {
+        (mask & tag) == (LFS_MKTAG(LFS_TYPE_DELETE, 0, 0) |
+            (LFS_MKTAG(0, 0x3ff, 0) & *filtertag))) {
         return true;
     }
 
@@ -1645,7 +1640,11 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_mdir_t *dir,
         if (dir != &f->m && lfs_pair_cmp(f->m.pair, dir->pair) == 0 &&
                 f->type == LFS_TYPE_REG && (f->flags & LFS_F_INLINE) &&
                 f->ctz.size > lfs->cfg->cache_size) {
-            int err = lfs_file_outline(lfs, f);
+            f->flags &= ~LFS_F_READING;
+            f->off = 0;
+
+            lfs_alloc_ack(lfs);
+            int err = lfs_file_relocate(lfs, f);
             if (err) {
                 return err;
             }
@@ -2471,6 +2470,7 @@ static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) {
         lfs_cache_zero(lfs, &lfs->pcache);
 
         file->block = nblock;
+        file->flags &= ~LFS_F_INLINE;
         file->flags |= LFS_F_WRITING;
         return 0;
 
@@ -2482,19 +2482,6 @@ relocate:
     }
 }
 
-static int lfs_file_outline(lfs_t *lfs, lfs_file_t *file) {
-    file->off = file->pos;
-    lfs_alloc_ack(lfs);
-    int err = lfs_file_relocate(lfs, file);
-    if (err) {
-        return err;
-    }
-
-    file->flags &= ~LFS_F_INLINE;
-    file->flags |= LFS_F_OUTLINE;
-    return 0;
-}
-
 static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) {
     if (file->flags & LFS_F_READING) {
         if (!(file->flags & LFS_F_INLINE)) {
@@ -2558,7 +2545,7 @@ relocate:
                 }
             }
         } else {
-            file->pos = lfs_max(file->pos, file->ctz.size);
+            file->ctz.size = lfs_max(file->pos, file->ctz.size);
         }
 
         // actual file updates
@@ -2606,9 +2593,6 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) {
 
             // commit file data and attributes
             err = lfs_dir_commit(lfs, &file->m, LFS_MKATTRS(
-                    {file->flags & LFS_F_OUTLINE
-                        ? LFS_MKTAG(LFS_TYPE_INLINESTRUCT, file->id, 0x3ff)
-                        : LFS_MKTAG(LFS_FROM_NOOP, 0, 0), NULL},
                     {LFS_MKTAG(type, file->id, size), buffer},
                     {LFS_MKTAG(LFS_FROM_USERATTRS, file->id,
                         file->cfg->attr_count), file->cfg->attrs}));
@@ -2620,14 +2604,15 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) {
                 return err;
             }
 
-            file->flags &= ~LFS_F_DIRTY & ~LFS_F_OUTLINE;
+            file->flags &= ~LFS_F_DIRTY;
         }
 
         return 0;
 
 relocate:
         // inline file doesn't fit anymore
-        err = lfs_file_outline(lfs, file);
+        file->off = file->pos;
+        err = lfs_file_relocate(lfs, file);
         if (err) {
             file->flags |= LFS_F_ERRED;
             return err;
@@ -2749,10 +2734,12 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file,
 
     if ((file->flags & LFS_F_INLINE) &&
             lfs_max(file->pos+nsize, file->ctz.size) >
-            lfs_min(0x3fe, lfs_min(
+            lfs_min(LFS_ATTR_MAX, lfs_min(
                 lfs->cfg->cache_size, lfs->cfg->block_size/8))) {
         // inline file doesn't fit anymore
-        int err = lfs_file_outline(lfs, file);
+        file->off = file->pos;
+        lfs_alloc_ack(lfs);
+        int err = lfs_file_relocate(lfs, file);
         if (err) {
             file->flags |= LFS_F_ERRED;
             return err;
@@ -3348,14 +3335,6 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) {
         if (err) {
             goto cleanup;
         }
-
-        // force compaction to prevent accidentally mounting any
-        // older version of littlefs that may live on disk
-        root.erased = false;
-        err = lfs_dir_commit(lfs, &root, NULL, 0);
-        if (err) {
-            goto cleanup;
-        }
     }
 
 cleanup:
@@ -3922,7 +3901,7 @@ typedef struct lfs1_superblock {
 
 
 /// Low-level wrappers v1->v2 ///
-static void lfs1_crc(uint32_t *crc, const void *buffer, size_t size) {
+void lfs1_crc(uint32_t *crc, const void *buffer, size_t size) {
     *crc = lfs_crc(*crc, buffer, size);
 }
 
@@ -4417,11 +4396,6 @@ int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg) {
                     goto cleanup;
                 }
             }
-
-            err = lfs_bd_flush(lfs, &lfs->pcache, &lfs->rcache, true);
-            if (err) {
-                goto cleanup;
-            }
         }
 
         // Create new superblock. This marks a successful migration!
@@ -4465,13 +4439,6 @@ int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg) {
         if (err) {
             goto cleanup;
         }
-
-        // force compaction to prevent accidentally mounting v1
-        dir2.erased = false;
-        err = lfs_dir_commit(lfs, &dir2, NULL, 0);
-        if (err) {
-            goto cleanup;
-        }
     }
 
 cleanup:

lfs.h

@@ -122,13 +122,13 @@ enum lfs_type {
 // File open flags
 enum lfs_open_flags {
     // open flags
-    LFS_O_RDONLY  = 1,        // Open a file as read only
-    LFS_O_WRONLY  = 2,        // Open a file as write only
-    LFS_O_RDWR    = 3,        // Open a file as read and write
-    LFS_O_CREAT   = 0x0100,   // Create a file if it does not exist
-    LFS_O_EXCL    = 0x0200,   // Fail if a file already exists
-    LFS_O_TRUNC   = 0x0400,   // Truncate the existing file to zero size
-    LFS_O_APPEND  = 0x0800,   // Move to end of file on every write
+    LFS_O_RDONLY = 1,         // Open a file as read only
+    LFS_O_WRONLY = 2,         // Open a file as write only
+    LFS_O_RDWR   = 3,         // Open a file as read and write
+    LFS_O_CREAT  = 0x0100,    // Create a file if it does not exist
+    LFS_O_EXCL   = 0x0200,    // Fail if a file already exists
+    LFS_O_TRUNC  = 0x0400,    // Truncate the existing file to zero size
+    LFS_O_APPEND = 0x0800,    // Move to end of file on every write
 
     // internally used flags
     LFS_F_DIRTY   = 0x010000, // File does not match storage
@@ -136,7 +136,6 @@ enum lfs_open_flags {
     LFS_F_READING = 0x040000, // File has been read since last flush
     LFS_F_ERRED   = 0x080000, // An error occured during write
     LFS_F_INLINE  = 0x100000, // Currently inlined in directory entry
-    LFS_F_OUTLINE = 0x200000, // Need to delete inlined directory entry
 };
 
 // File seek flags

lfs_util.h

@@ -143,14 +143,14 @@ static inline int lfs_scmp(uint32_t a, uint32_t b) {
 // Convert between 32-bit little-endian and native order
 static inline uint32_t lfs_fromle32(uint32_t a) {
 #if !defined(LFS_NO_INTRINSICS) && ( \
-    (defined(  BYTE_ORDER  ) && defined(  ORDER_LITTLE_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_LITTLE_ENDIAN  ) || \
-    (defined(__BYTE_ORDER  ) && defined(__ORDER_LITTLE_ENDIAN  ) && __BYTE_ORDER   == __ORDER_LITTLE_ENDIAN  ) || \
-    (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
+    (defined(  BYTE_ORDER  ) &&   BYTE_ORDER   ==   ORDER_LITTLE_ENDIAN  ) || \
+    (defined(__BYTE_ORDER  ) && __BYTE_ORDER   == __ORDER_LITTLE_ENDIAN  ) || \
+    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
     return a;
 #elif !defined(LFS_NO_INTRINSICS) && ( \
-    (defined(  BYTE_ORDER  ) && defined(  ORDER_BIG_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
-    (defined(__BYTE_ORDER  ) && defined(__ORDER_BIG_ENDIAN  ) && __BYTE_ORDER   == __ORDER_BIG_ENDIAN  ) || \
-    (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
+    (defined(  BYTE_ORDER  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
+    (defined(__BYTE_ORDER  ) && __BYTE_ORDER   == __ORDER_BIG_ENDIAN  ) || \
+    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
     return __builtin_bswap32(a);
 #else
     return (((uint8_t*)&a)[0] <<  0) |
@@ -167,14 +167,14 @@ static inline uint32_t lfs_tole32(uint32_t a) {
 // Convert between 32-bit big-endian and native order
 static inline uint32_t lfs_frombe32(uint32_t a) {
 #if !defined(LFS_NO_INTRINSICS) && ( \
-    (defined(  BYTE_ORDER  ) && defined(  ORDER_LITTLE_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_LITTLE_ENDIAN  ) || \
-    (defined(__BYTE_ORDER  ) && defined(__ORDER_LITTLE_ENDIAN  ) && __BYTE_ORDER   == __ORDER_LITTLE_ENDIAN  ) || \
-    (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
+    (defined(  BYTE_ORDER  ) &&   BYTE_ORDER   ==   ORDER_LITTLE_ENDIAN  ) || \
+    (defined(__BYTE_ORDER  ) && __BYTE_ORDER   == __ORDER_LITTLE_ENDIAN  ) || \
+    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
     return __builtin_bswap32(a);
 #elif !defined(LFS_NO_INTRINSICS) && ( \
-    (defined(  BYTE_ORDER  ) && defined(  ORDER_BIG_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
-    (defined(__BYTE_ORDER  ) && defined(__ORDER_BIG_ENDIAN  ) && __BYTE_ORDER   == __ORDER_BIG_ENDIAN  ) || \
-    (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
+    (defined(  BYTE_ORDER  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
+    (defined(__BYTE_ORDER  ) && __BYTE_ORDER   == __ORDER_BIG_ENDIAN  ) || \
+    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
     return a;
 #else
     return (((uint8_t*)&a)[0] << 24) |

scripts/prefix.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 
 # This script replaces prefixes of files, and symbols in that file.
 # Useful for creating different versions of the codebase that don't

tests/corrupt.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 
 import struct
 import sys

tests/stats.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 
 import struct
 import sys

tests/test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 
 import re
 import sys

tests/test_seek.sh

@@ -357,73 +357,5 @@ tests/test.py << TEST
     lfs_unmount(&lfs) => 0;
 TEST
 
-echo "--- Inline write and seek ---"
-for SIZE in $SMALLSIZE $MEDIUMSIZE $LARGESIZE
-do
-tests/test.py << TEST
-    lfs_mount(&lfs, &cfg) => 0;
-    lfs_file_open(&lfs, &file[0], "hello/tinykitty$SIZE",
-            LFS_O_RDWR | LFS_O_CREAT) => 0;
-    int j = 0;
-    int k = 0;
-
-    memcpy(buffer, "abcdefghijklmnopqrstuvwxyz", 26);
-    for (unsigned i = 0; i < $SIZE; i++) {
-        lfs_file_write(&lfs, &file[0], &buffer[j++ % 26], 1) => 1;
-        lfs_file_tell(&lfs, &file[0]) => i+1;
-        lfs_file_size(&lfs, &file[0]) => i+1;
-    }
-
-    lfs_file_seek(&lfs, &file[0], 0, LFS_SEEK_SET) => 0;
-    lfs_file_tell(&lfs, &file[0]) => 0;
-    lfs_file_size(&lfs, &file[0]) => $SIZE;
-    for (unsigned i = 0; i < $SIZE; i++) {
-        uint8_t c;
-        lfs_file_read(&lfs, &file[0], &c, 1) => 1;
-        c => buffer[k++ % 26];
-    }
-
-    lfs_file_sync(&lfs, &file[0]) => 0;
-    lfs_file_tell(&lfs, &file[0]) => $SIZE;
-    lfs_file_size(&lfs, &file[0]) => $SIZE;
-
-    lfs_file_seek(&lfs, &file[0], 0, LFS_SEEK_SET) => 0;
-    for (unsigned i = 0; i < $SIZE; i++) {
-        lfs_file_write(&lfs, &file[0], &buffer[j++ % 26], 1) => 1;
-        lfs_file_tell(&lfs, &file[0]) => i+1;
-        lfs_file_size(&lfs, &file[0]) => $SIZE;
-        lfs_file_sync(&lfs, &file[0]) => 0;
-        lfs_file_tell(&lfs, &file[0]) => i+1;
-        lfs_file_size(&lfs, &file[0]) => $SIZE;
-        if (i < $SIZE-2) {
-            uint8_t c[3];
-            lfs_file_seek(&lfs, &file[0], -1, LFS_SEEK_CUR) => i;
-            lfs_file_read(&lfs, &file[0], &c, 3) => 3;
-            lfs_file_tell(&lfs, &file[0]) => i+3;
-            lfs_file_size(&lfs, &file[0]) => $SIZE;
-            lfs_file_seek(&lfs, &file[0], i+1, LFS_SEEK_SET) => i+1;
-            lfs_file_tell(&lfs, &file[0]) => i+1;
-            lfs_file_size(&lfs, &file[0]) => $SIZE;
-        }
-    }
-
-    lfs_file_seek(&lfs, &file[0], 0, LFS_SEEK_SET) => 0;
-    lfs_file_tell(&lfs, &file[0]) => 0;
-    lfs_file_size(&lfs, &file[0]) => $SIZE;
-    for (unsigned i = 0; i < $SIZE; i++) {
-        uint8_t c;
-        lfs_file_read(&lfs, &file[0], &c, 1) => 1;
-        c => buffer[k++ % 26];
-    }
-
-    lfs_file_sync(&lfs, &file[0]) => 0;
-    lfs_file_tell(&lfs, &file[0]) => $SIZE;
-    lfs_file_size(&lfs, &file[0]) => $SIZE;
-
-    lfs_file_close(&lfs, &file[0]) => 0;
-    lfs_unmount(&lfs) => 0;
-TEST
-done
-
 echo "--- Results ---"
 tests/stats.py