Fixed 32-bit/64-bit Ubuntu multilib issue in Travis

Kind of a two-fold issue. One, the programming to the middle of inline
files was causing the cache to get updated to a half programmed state.
While fine, as all programs do occur in order in a block, this is less
efficient when writing to inline files as it would cause the inline file
to need to be reread even if it fits in the cache.

Two, the rereading of the inline file was broken and passed the file's
tag all the way to where a user would expect an error. This was easy to
fix but adds to the reasons we should have test coverage information.

Found by ebinans

.travis.yml

@@ -66,7 +66,10 @@ jobs:
         - CC="arm-linux-gnueabi-gcc --static -mthumb"
         - EXEC="qemu-arm"
       install:
-        - sudo apt-get install gcc-arm-linux-gnueabi qemu-user
+        - sudo apt-get install
+              gcc-arm-linux-gnueabi
+              libc6-dev-armel-cross
+              qemu-user
         - arm-linux-gnueabi-gcc --version
         - qemu-arm -version
 
@@ -78,7 +81,10 @@ jobs:
         - CC="powerpc-linux-gnu-gcc --static"
         - EXEC="qemu-ppc"
       install:
-        - sudo apt-get install gcc-powerpc-linux-gnu qemu-user
+        - sudo apt-get install
+              gcc-powerpc-linux-gnu
+              libc6-dev-powerpc-cross
+              qemu-user
         - powerpc-linux-gnu-gcc --version
         - qemu-ppc -version
 
@@ -90,9 +96,10 @@ jobs:
         - CC="mips-linux-gnu-gcc --static"
         - EXEC="qemu-mips"
       install:
-        - sudo add-apt-repository -y "deb http://archive.ubuntu.com/ubuntu/ xenial main universe"
-        - sudo apt-get -qq update
-        - sudo apt-get install gcc-mips-linux-gnu qemu-user
+        - sudo apt-get install
+              gcc-mips-linux-gnu
+              libc6-dev-mips-cross
+              qemu-user
         - mips-linux-gnu-gcc --version
         - qemu-mips -version
 
@@ -136,6 +143,7 @@ jobs:
       env:
         - STAGE=test
         - NAME=littlefs-migration
+      if: branch !~ -prefix$
       install:
         - sudo apt-get install libfuse-dev
         - git clone --depth 1 https://github.com/geky/littlefs-fuse -b v2-alpha v2

lfs.c

@@ -80,28 +80,16 @@ static int lfs_bd_read(lfs_t *lfs,
             diff = lfs_min(diff, rcache->off-off);
         }
 
-        if (size >= hint && off % lfs->cfg->read_size == 0 &&
-                size >= lfs->cfg->read_size) {
-            // bypass cache?
-            diff = lfs_aligndown(diff, lfs->cfg->read_size);
-            int err = lfs->cfg->read(lfs->cfg, block, off, data, diff);
-            if (err) {
-                return err;
-            }
-
-            data += diff;
-            off += diff;
-            size -= diff;
-            continue;
-        }
-
         // load to cache, first condition can no longer fail
         LFS_ASSERT(block < lfs->cfg->block_count);
         rcache->block = block;
         rcache->off = lfs_aligndown(off, lfs->cfg->read_size);
-        rcache->size = lfs_min(lfs_alignup(off+hint, lfs->cfg->read_size),
-                lfs_min(lfs->cfg->block_size - rcache->off,
-                    lfs->cfg->cache_size));
+        rcache->size = lfs_min(
+                lfs_min(
+                    lfs_alignup(off+hint, lfs->cfg->read_size),
+                    lfs->cfg->block_size)
+                - rcache->off,
+                lfs->cfg->cache_size);
         int err = lfs->cfg->read(lfs->cfg, rcache->block,
                 rcache->off, rcache->buffer, rcache->size);
         if (err) {
@@ -206,7 +194,7 @@ static int lfs_bd_prog(lfs_t *lfs,
             off += diff;
             size -= diff;
 
-            pcache->size = off - pcache->off;
+            pcache->size = lfs_max(pcache->size, off - pcache->off);
             if (pcache->size == lfs->cfg->cache_size) {
                 // eagerly flush out pcache if we fill up
                 int err = lfs_bd_flush(lfs, pcache, rcache, validate);
@@ -629,7 +617,7 @@ static int lfs_dir_getread(lfs_t *lfs, const lfs_mdir_t *dir,
                 lfs->cfg->cache_size);
         int err = lfs_dir_getslice(lfs, dir, gmask, gtag,
                 rcache->off, rcache->buffer, rcache->size);
-        if (err) {
+        if (err < 0) {
             return err;
         }
     }
@@ -2560,7 +2548,7 @@ relocate:
                 }
             }
         } else {
-            file->ctz.size = lfs_max(file->pos, file->ctz.size);
+            file->pos = lfs_max(file->pos, file->ctz.size);
         }
 
         // actual file updates
@@ -2749,7 +2737,7 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file,
 
     if ((file->flags & LFS_F_INLINE) &&
             lfs_max(file->pos+nsize, file->ctz.size) >
-            lfs_min(LFS_ATTR_MAX, lfs_min(
+            lfs_min(0x3fe, lfs_min(
                 lfs->cfg->cache_size, lfs->cfg->block_size/8))) {
         // inline file doesn't fit anymore
         file->off = file->pos;
@@ -2879,13 +2867,14 @@ int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) {
         // lookup new head in ctz skip list
         err = lfs_ctz_find(lfs, NULL, &file->cache,
                 file->ctz.head, file->ctz.size,
-                size, &file->ctz.head, &(lfs_off_t){0});
+                size, &file->block, &file->off);
         if (err) {
             return err;
         }
 
+        file->ctz.head = file->block;
         file->ctz.size = size;
-        file->flags |= LFS_F_DIRTY;
+        file->flags |= LFS_F_DIRTY | LFS_F_READING;
     } else if (size > oldsize) {
         lfs_off_t pos = file->pos;
 
@@ -3349,6 +3338,14 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) {
         if (err) {
             goto cleanup;
         }
+
+        // force compaction to prevent accidentally mounting any
+        // older version of littlefs that may live on disk
+        root.erased = false;
+        err = lfs_dir_commit(lfs, &root, NULL, 0);
+        if (err) {
+            goto cleanup;
+        }
     }
 
 cleanup:
@@ -3915,7 +3912,7 @@ typedef struct lfs1_superblock {
 
 
 /// Low-level wrappers v1->v2 ///
-void lfs1_crc(uint32_t *crc, const void *buffer, size_t size) {
+static void lfs1_crc(uint32_t *crc, const void *buffer, size_t size) {
     *crc = lfs_crc(*crc, buffer, size);
 }
 
@@ -4410,6 +4407,11 @@ int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg) {
                     goto cleanup;
                 }
             }
+
+            err = lfs_bd_flush(lfs, &lfs->pcache, &lfs->rcache, true);
+            if (err) {
+                goto cleanup;
+            }
         }
 
         // Create new superblock. This marks a successful migration!
@@ -4453,6 +4455,13 @@ int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg) {
         if (err) {
             goto cleanup;
         }
+
+        // force compaction to prevent accidentally mounting v1
+        dir2.erased = false;
+        err = lfs_dir_commit(lfs, &dir2, NULL, 0);
+        if (err) {
+            goto cleanup;
+        }
     }
 
 cleanup:

lfs_util.h

@@ -143,14 +143,14 @@ static inline int lfs_scmp(uint32_t a, uint32_t b) {
 // Convert between 32-bit little-endian and native order
 static inline uint32_t lfs_fromle32(uint32_t a) {
 #if !defined(LFS_NO_INTRINSICS) && ( \
-    (defined(  BYTE_ORDER  ) &&   BYTE_ORDER   ==   ORDER_LITTLE_ENDIAN  ) || \
-    (defined(__BYTE_ORDER  ) && __BYTE_ORDER   == __ORDER_LITTLE_ENDIAN  ) || \
-    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
+    (defined(  BYTE_ORDER  ) && defined(  ORDER_LITTLE_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_LITTLE_ENDIAN  ) || \
+    (defined(__BYTE_ORDER  ) && defined(__ORDER_LITTLE_ENDIAN  ) && __BYTE_ORDER   == __ORDER_LITTLE_ENDIAN  ) || \
+    (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
     return a;
 #elif !defined(LFS_NO_INTRINSICS) && ( \
-    (defined(  BYTE_ORDER  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
-    (defined(__BYTE_ORDER  ) && __BYTE_ORDER   == __ORDER_BIG_ENDIAN  ) || \
-    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
+    (defined(  BYTE_ORDER  ) && defined(  ORDER_BIG_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
+    (defined(__BYTE_ORDER  ) && defined(__ORDER_BIG_ENDIAN  ) && __BYTE_ORDER   == __ORDER_BIG_ENDIAN  ) || \
+    (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
     return __builtin_bswap32(a);
 #else
     return (((uint8_t*)&a)[0] <<  0) |
@@ -167,14 +167,14 @@ static inline uint32_t lfs_tole32(uint32_t a) {
 // Convert between 32-bit big-endian and native order
 static inline uint32_t lfs_frombe32(uint32_t a) {
 #if !defined(LFS_NO_INTRINSICS) && ( \
-    (defined(  BYTE_ORDER  ) &&   BYTE_ORDER   ==   ORDER_LITTLE_ENDIAN  ) || \
-    (defined(__BYTE_ORDER  ) && __BYTE_ORDER   == __ORDER_LITTLE_ENDIAN  ) || \
-    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
+    (defined(  BYTE_ORDER  ) && defined(  ORDER_LITTLE_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_LITTLE_ENDIAN  ) || \
+    (defined(__BYTE_ORDER  ) && defined(__ORDER_LITTLE_ENDIAN  ) && __BYTE_ORDER   == __ORDER_LITTLE_ENDIAN  ) || \
+    (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
     return __builtin_bswap32(a);
 #elif !defined(LFS_NO_INTRINSICS) && ( \
-    (defined(  BYTE_ORDER  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
-    (defined(__BYTE_ORDER  ) && __BYTE_ORDER   == __ORDER_BIG_ENDIAN  ) || \
-    (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
+    (defined(  BYTE_ORDER  ) && defined(  ORDER_BIG_ENDIAN  ) &&   BYTE_ORDER   ==   ORDER_BIG_ENDIAN  ) || \
+    (defined(__BYTE_ORDER  ) && defined(__ORDER_BIG_ENDIAN  ) && __BYTE_ORDER   == __ORDER_BIG_ENDIAN  ) || \
+    (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
     return a;
 #else
     return (((uint8_t*)&a)[0] << 24) |

scripts/prefix.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 # This script replaces prefixes of files, and symbols in that file.
 # Useful for creating different versions of the codebase that don't

tests/corrupt.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 import struct
 import sys

tests/stats.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 import struct
 import sys

tests/test.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 import re
 import sys

tests/test_seek.sh

@@ -357,5 +357,73 @@ tests/test.py << TEST
     lfs_unmount(&lfs) => 0;
 TEST
 
+echo "--- Inline write and seek ---"
+for SIZE in $SMALLSIZE $MEDIUMSIZE $LARGESIZE
+do
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "hello/tinykitty$SIZE",
+            LFS_O_RDWR | LFS_O_CREAT) => 0;
+    int j = 0;
+    int k = 0;
+
+    memcpy(buffer, "abcdefghijklmnopqrstuvwxyz", 26);
+    for (unsigned i = 0; i < $SIZE; i++) {
+        lfs_file_write(&lfs, &file[0], &buffer[j++ % 26], 1) => 1;
+        lfs_file_tell(&lfs, &file[0]) => i+1;
+        lfs_file_size(&lfs, &file[0]) => i+1;
+    }
+
+    lfs_file_seek(&lfs, &file[0], 0, LFS_SEEK_SET) => 0;
+    lfs_file_tell(&lfs, &file[0]) => 0;
+    lfs_file_size(&lfs, &file[0]) => $SIZE;
+    for (unsigned i = 0; i < $SIZE; i++) {
+        uint8_t c;
+        lfs_file_read(&lfs, &file[0], &c, 1) => 1;
+        c => buffer[k++ % 26];
+    }
+
+    lfs_file_sync(&lfs, &file[0]) => 0;
+    lfs_file_tell(&lfs, &file[0]) => $SIZE;
+    lfs_file_size(&lfs, &file[0]) => $SIZE;
+
+    lfs_file_seek(&lfs, &file[0], 0, LFS_SEEK_SET) => 0;
+    for (unsigned i = 0; i < $SIZE; i++) {
+        lfs_file_write(&lfs, &file[0], &buffer[j++ % 26], 1) => 1;
+        lfs_file_tell(&lfs, &file[0]) => i+1;
+        lfs_file_size(&lfs, &file[0]) => $SIZE;
+        lfs_file_sync(&lfs, &file[0]) => 0;
+        lfs_file_tell(&lfs, &file[0]) => i+1;
+        lfs_file_size(&lfs, &file[0]) => $SIZE;
+        if (i < $SIZE-2) {
+            uint8_t c[3];
+            lfs_file_seek(&lfs, &file[0], -1, LFS_SEEK_CUR) => i;
+            lfs_file_read(&lfs, &file[0], &c, 3) => 3;
+            lfs_file_tell(&lfs, &file[0]) => i+3;
+            lfs_file_size(&lfs, &file[0]) => $SIZE;
+            lfs_file_seek(&lfs, &file[0], i+1, LFS_SEEK_SET) => i+1;
+            lfs_file_tell(&lfs, &file[0]) => i+1;
+            lfs_file_size(&lfs, &file[0]) => $SIZE;
+        }
+    }
+
+    lfs_file_seek(&lfs, &file[0], 0, LFS_SEEK_SET) => 0;
+    lfs_file_tell(&lfs, &file[0]) => 0;
+    lfs_file_size(&lfs, &file[0]) => $SIZE;
+    for (unsigned i = 0; i < $SIZE; i++) {
+        uint8_t c;
+        lfs_file_read(&lfs, &file[0], &c, 1) => 1;
+        c => buffer[k++ % 26];
+    }
+
+    lfs_file_sync(&lfs, &file[0]) => 0;
+    lfs_file_tell(&lfs, &file[0]) => $SIZE;
+    lfs_file_size(&lfs, &file[0]) => $SIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+done
+
 echo "--- Results ---"
 tests/stats.py

tests/test_truncate.sh

@@ -11,6 +11,150 @@ tests/test.py << TEST
     lfs_format(&lfs, &cfg) => 0;
 TEST
 
+echo "--- Simple truncate ---"
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldynoop",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+
+    strcpy((char*)buffer, "hair");
+    size = strlen((char*)buffer);
+    for (lfs_off_t j = 0; j < $LARGESIZE; j += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldynoop", LFS_O_RDWR) => 0;
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_truncate(&lfs, &file[0], $MEDIUMSIZE) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldynoop", LFS_O_RDONLY) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    size = strlen("hair");
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_read(&lfs, &file[0], buffer, size) => size;
+        memcmp(buffer, "hair", size) => 0;
+    }
+    lfs_file_read(&lfs, &file[0], buffer, size) => 0;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+
+echo "--- Truncate and read ---"
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldyread",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+
+    strcpy((char*)buffer, "hair");
+    size = strlen((char*)buffer);
+    for (lfs_off_t j = 0; j < $LARGESIZE; j += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldyread", LFS_O_RDWR) => 0;
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_truncate(&lfs, &file[0], $MEDIUMSIZE) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    size = strlen("hair");
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_read(&lfs, &file[0], buffer, size) => size;
+        memcmp(buffer, "hair", size) => 0;
+    }
+    lfs_file_read(&lfs, &file[0], buffer, size) => 0;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldyread", LFS_O_RDONLY) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    size = strlen("hair");
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_read(&lfs, &file[0], buffer, size) => size;
+        memcmp(buffer, "hair", size) => 0;
+    }
+    lfs_file_read(&lfs, &file[0], buffer, size) => 0;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+
+echo "--- Truncate and write ---"
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldywrite",
+            LFS_O_WRONLY | LFS_O_CREAT) => 0;
+
+    strcpy((char*)buffer, "hair");
+    size = strlen((char*)buffer);
+    for (lfs_off_t j = 0; j < $LARGESIZE; j += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldywrite", LFS_O_RDWR) => 0;
+    lfs_file_size(&lfs, &file[0]) => $LARGESIZE;
+
+    lfs_file_truncate(&lfs, &file[0], $MEDIUMSIZE) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    strcpy((char*)buffer, "bald");
+    size = strlen((char*)buffer);
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_write(&lfs, &file[0], buffer, size) => size;
+    }
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    lfs_mount(&lfs, &cfg) => 0;
+    lfs_file_open(&lfs, &file[0], "baldywrite", LFS_O_RDONLY) => 0;
+    lfs_file_size(&lfs, &file[0]) => $MEDIUMSIZE;
+
+    size = strlen("bald");
+    for (lfs_off_t j = 0; j < $MEDIUMSIZE; j += size) {
+        lfs_file_read(&lfs, &file[0], buffer, size) => size;
+        memcmp(buffer, "bald", size) => 0;
+    }
+    lfs_file_read(&lfs, &file[0], buffer, size) => 0;
+
+    lfs_file_close(&lfs, &file[0]) => 0;
+    lfs_unmount(&lfs) => 0;
+TEST
+
+# More aggressive general truncation tests
 truncate_test() {
 STARTSIZES="$1"
 STARTSEEKS="$2"