Added conversion to/from little-endian on disk

Required to support big-endian processors, with the most notable being
the PowerPC architecture.

On little-endian architectures, these conversions can be optimized out
and have no code impact.

Initial patch provided by gmouchard

Makefile

@@ -33,7 +33,7 @@ size: $(OBJ)
 	$(SIZE) -t $^
 
 .SUFFIXES:
-test: test_format test_dirs test_files test_seek test_parallel \
+test: test_format test_dirs test_files test_seek test_truncate test_parallel \
 	test_alloc test_paths test_orphan test_move test_corrupt
 test_%: tests/test_%.sh
 ifdef QUIET

lfs.c

@@ -326,6 +326,48 @@ static void lfs_alloc_ack(lfs_t *lfs) {
 }
 
 
+/// Endian swapping functions ///
+static void lfs_dir_fromle32(struct lfs_disk_dir *d) {
+    d->rev     = lfs_fromle32(d->rev);
+    d->size    = lfs_fromle32(d->size);
+    d->tail[0] = lfs_fromle32(d->tail[0]);
+    d->tail[1] = lfs_fromle32(d->tail[1]);
+}
+
+static void lfs_dir_tole32(struct lfs_disk_dir *d) {
+    d->rev     = lfs_tole32(d->rev);
+    d->size    = lfs_tole32(d->size);
+    d->tail[0] = lfs_tole32(d->tail[0]);
+    d->tail[1] = lfs_tole32(d->tail[1]);
+}
+
+static void lfs_entry_fromle32(struct lfs_disk_entry *d) {
+    d->u.dir[0] = lfs_fromle32(d->u.dir[0]);
+    d->u.dir[1] = lfs_fromle32(d->u.dir[1]);
+}
+
+static void lfs_entry_tole32(struct lfs_disk_entry *d) {
+    d->u.dir[0] = lfs_tole32(d->u.dir[0]);
+    d->u.dir[1] = lfs_tole32(d->u.dir[1]);
+}
+
+static void lfs_superblock_fromle32(struct lfs_disk_superblock *d) {
+    d->root[0]     = lfs_fromle32(d->root[0]);
+    d->root[1]     = lfs_fromle32(d->root[1]);
+    d->block_size  = lfs_fromle32(d->block_size);
+    d->block_count = lfs_fromle32(d->block_count);
+    d->version     = lfs_fromle32(d->version);
+}
+
+static void lfs_superblock_tole32(struct lfs_disk_superblock *d) {
+    d->root[0]     = lfs_tole32(d->root[0]);
+    d->root[1]     = lfs_tole32(d->root[1]);
+    d->block_size  = lfs_tole32(d->block_size);
+    d->block_count = lfs_tole32(d->block_count);
+    d->version     = lfs_tole32(d->version);
+}
+
+
 /// Metadata pair and directory operations ///
 static inline void lfs_pairswap(lfs_block_t pair[2]) {
     lfs_block_t t = pair[0];
@@ -367,6 +409,7 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_dir_t *dir) {
     // rather than clobbering one of the blocks we just pretend
     // the revision may be valid
     int err = lfs_bd_read(lfs, dir->pair[0], 0, &dir->d.rev, 4);
+    dir->d.rev = lfs_fromle32(dir->d.rev);
     if (err) {
         return err;
     }
@@ -392,6 +435,7 @@ static int lfs_dir_fetch(lfs_t *lfs,
     for (int i = 0; i < 2; i++) {
         struct lfs_disk_dir test;
         int err = lfs_bd_read(lfs, tpair[i], 0, &test, sizeof(test));
+        lfs_dir_fromle32(&test);
         if (err) {
             return err;
         }
@@ -406,7 +450,9 @@ static int lfs_dir_fetch(lfs_t *lfs,
         }
 
         uint32_t crc = 0xffffffff;
+        lfs_dir_tole32(&test);
         lfs_crc(&crc, &test, sizeof(test));
+        lfs_dir_fromle32(&test);
         err = lfs_bd_crc(lfs, tpair[i], sizeof(test),
                 (0x7fffffff & test.size) - sizeof(test), &crc);
         if (err) {
@@ -466,8 +512,10 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir,
             }
 
             uint32_t crc = 0xffffffff;
+            lfs_dir_tole32(&dir->d);
             lfs_crc(&crc, &dir->d, sizeof(dir->d));
             err = lfs_bd_prog(lfs, dir->pair[0], 0, &dir->d, sizeof(dir->d));
+            lfs_dir_fromle32(&dir->d);
             if (err) {
                 if (err == LFS_ERR_CORRUPT) {
                     goto relocate;
@@ -514,7 +562,9 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir,
                 }
             }
 
+            crc = lfs_tole32(crc);
             err = lfs_bd_prog(lfs, dir->pair[0], newoff, &crc, 4);
+            crc = lfs_fromle32(crc);
             if (err) {
                 if (err == LFS_ERR_CORRUPT) {
                     goto relocate;
@@ -587,11 +637,14 @@ relocate:
 }
 
 static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir,
-        const lfs_entry_t *entry, const void *data) {
-    return lfs_dir_commit(lfs, dir, (struct lfs_region[]){
+        lfs_entry_t *entry, const void *data) {
+    lfs_entry_tole32(&entry->d);
+    int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){
             {entry->off, sizeof(entry->d), &entry->d, sizeof(entry->d)},
             {entry->off+sizeof(entry->d), entry->d.nlen, data, entry->d.nlen}
         }, data ? 2 : 1);
+    lfs_entry_fromle32(&entry->d);
+    return err;
 }
 
 static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir,
@@ -600,10 +653,14 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir,
     while (true) {
         if (dir->d.size + lfs_entry_size(entry) <= lfs->cfg->block_size) {
             entry->off = dir->d.size - 4;
-            return lfs_dir_commit(lfs, dir, (struct lfs_region[]){
+
+            lfs_entry_tole32(&entry->d);
+            int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){
                     {entry->off, 0, &entry->d, sizeof(entry->d)},
                     {entry->off, 0, data, entry->d.nlen}
                 }, 2);
+            lfs_entry_fromle32(&entry->d);
+            return err;
         }
 
         // we need to allocate a new dir block
@@ -617,10 +674,12 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir,
             newdir.d.tail[0] = dir->d.tail[0];
             newdir.d.tail[1] = dir->d.tail[1];
             entry->off = newdir.d.size - 4;
+            lfs_entry_tole32(&entry->d);
             err = lfs_dir_commit(lfs, &newdir, (struct lfs_region[]){
                     {entry->off, 0, &entry->d, sizeof(entry->d)},
                     {entry->off, 0, data, entry->d.nlen}
                 }, 2);
+            lfs_entry_fromle32(&entry->d);
             if (err) {
                 return err;
             }
@@ -706,6 +765,7 @@ static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) {
 
     int err = lfs_bd_read(lfs, dir->pair[0], dir->off,
             &entry->d, sizeof(entry->d));
+    lfs_entry_fromle32(&entry->d);
     if (err) {
         return err;
     }
@@ -1067,6 +1127,7 @@ static int lfs_ctz_find(lfs_t *lfs,
                 lfs_ctz(current));
 
         int err = lfs_cache_read(lfs, rcache, pcache, head, 4*skip, &head, 4);
+        head = lfs_fromle32(head);
         if (err) {
             return err;
         }
@@ -1142,8 +1203,10 @@ static int lfs_ctz_extend(lfs_t *lfs,
             lfs_size_t skips = lfs_ctz(index) + 1;
 
             for (lfs_off_t i = 0; i < skips; i++) {
+                head = lfs_tole32(head);
                 int err = lfs_cache_prog(lfs, pcache, rcache,
                         nblock, 4*i, &head, 4);
+                head = lfs_fromle32(head);
                 if (err) {
                     if (err == LFS_ERR_CORRUPT) {
                         goto relocate;
@@ -1154,6 +1217,7 @@ static int lfs_ctz_extend(lfs_t *lfs,
                 if (i != skips-1) {
                     err = lfs_cache_read(lfs, rcache, NULL,
                             head, 4*i, &head, 4);
+                    head = lfs_fromle32(head);
                     if (err) {
                         return err;
                     }
@@ -1198,6 +1262,8 @@ static int lfs_ctz_traverse(lfs_t *lfs,
         lfs_block_t heads[2];
         int count = 2 - (index & 1);
         err = lfs_cache_read(lfs, rcache, pcache, head, 0, &heads, count*4);
+        heads[0] = lfs_fromle32(heads[0]);
+        heads[1] = lfs_fromle32(heads[1]);
         if (err) {
             return err;
         }
@@ -1458,6 +1524,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) {
         lfs_entry_t entry = {.off = file->poff};
         err = lfs_bd_read(lfs, cwd.pair[0], entry.off,
                 &entry.d, sizeof(entry.d));
+        lfs_entry_fromle32(&entry.d);
         if (err) {
             return err;
         }
@@ -1664,6 +1731,57 @@ lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file,
     return file->pos;
 }
 
+int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) {
+    if ((file->flags & 3) == LFS_O_RDONLY) {
+        return LFS_ERR_INVAL;
+    }
+
+    if (size < lfs_file_size(lfs, file)) {
+        // need to flush since directly changing metadata
+        int err = lfs_file_flush(lfs, file);
+        if (err) {
+            return err;
+        }
+
+        // lookup new head in ctz skip list
+        err = lfs_ctz_find(lfs, &file->cache, NULL,
+                file->head, file->size,
+                size, &file->head, &(lfs_off_t){0});
+        if (err) {
+            return err;
+        }
+
+        file->size = size;
+        file->flags |= LFS_F_DIRTY;
+    } else if (size > lfs_file_size(lfs, file)) {
+        lfs_off_t pos = file->pos;
+
+        // flush+seek if not already at end
+        if (file->pos != lfs_file_size(lfs, file)) {
+            int err = lfs_file_seek(lfs, file, 0, SEEK_END);
+            if (err) {
+                return err;
+            }
+        }
+
+        // fill with zeros
+        while (file->pos < size) {
+            lfs_ssize_t res = lfs_file_write(lfs, file, &(uint8_t){0}, 1);
+            if (res < 0) {
+                return res;
+            }
+        }
+
+        // restore pos
+        int err = lfs_file_seek(lfs, file, pos, LFS_SEEK_SET);
+        if (err < 0) {
+            return err;
+        }
+    }
+
+    return 0;
+}
+
 lfs_soff_t lfs_file_tell(lfs_t *lfs, lfs_file_t *file) {
     return file->pos;
 }
@@ -1678,7 +1796,11 @@ int lfs_file_rewind(lfs_t *lfs, lfs_file_t *file) {
 }
 
 lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file) {
-    return lfs_max(file->pos, file->size);
+    if (file->flags & LFS_F_WRITING) {
+        return lfs_max(file->pos, file->size);
+    } else {
+        return file->size;
+    }
 }
 
 
@@ -2023,6 +2145,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) {
     superdir.d.size = sizeof(superdir.d) + sizeof(superblock.d) + 4;
 
     // write both pairs to be safe
+    lfs_superblock_tole32(&superblock.d);
     bool valid = false;
     for (int i = 0; i < 2; i++) {
         int err = lfs_dir_commit(lfs, &superdir, (struct lfs_region[]){
@@ -2072,6 +2195,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) {
     if (!err) {
         int err = lfs_bd_read(lfs, dir.pair[0], sizeof(dir.d),
                 &superblock.d, sizeof(superblock.d));
+        lfs_superblock_fromle32(&superblock.d);
         if (err) {
             return err;
         }
@@ -2128,6 +2252,7 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) {
         while (dir.off + sizeof(entry.d) <= (0x7fffffff & dir.d.size)-4) {
             int err = lfs_bd_read(lfs, dir.pair[0], dir.off,
                     &entry.d, sizeof(entry.d));
+            lfs_entry_fromle32(&entry.d);
             if (err) {
                 return err;
             }
@@ -2168,7 +2293,7 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) {
             }
         }
     }
-    
+
     return 0;
 }
 
@@ -2202,7 +2327,7 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2],
     if (lfs_pairisnull(lfs->root)) {
         return 0;
     }
-    
+
     parent->d.tail[0] = 0;
     parent->d.tail[1] = 1;
 
@@ -2263,7 +2388,7 @@ static int lfs_moved(lfs_t *lfs, const void *e) {
                 break;
             }
 
-            if (!(0x80 & entry.d.type) && 
+            if (!(0x80 & entry.d.type) &&
                  memcmp(&entry.d.u, e, sizeof(entry.d.u)) == 0) {
                 return true;
             }

lfs.h

@@ -364,6 +364,11 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file,
 lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file,
         lfs_soff_t off, int whence);
 
+// Truncates the size of the file to the specified size
+//
+// Returns a negative error code on failure.
+int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size);
+
 // Return the position of the file
 //
 // Equivalent to lfs_file_seek(lfs, file, 0, LFS_SEEK_CUR)

lfs_util.h

@@ -49,6 +49,23 @@ static inline int lfs_scmp(uint32_t a, uint32_t b) {
     return (int)(unsigned)(a - b);
 }
 
+static inline uint32_t lfs_fromle32(uint32_t a) {
+#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+    return a;
+#elif defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+    return __builtin_bswap32(a);
+#else
+    return (((uint8_t*)&a)[0] <<  0) |
+           (((uint8_t*)&a)[1] <<  8) |
+           (((uint8_t*)&a)[2] << 16) |
+           (((uint8_t*)&a)[3] << 24);
+#endif
+}
+
+static inline uint32_t lfs_tole32(uint32_t a) {
+    return lfs_fromle32(a);
+}
+
 // CRC-32 with polynomial = 0x04c11db7
 void lfs_crc(uint32_t *crc, const void *buffer, size_t size);
 

tests/test_truncate.sh

@@ -0,0 +1,133 @@
+#!/bin/bash
+set -eu
+
+SMALLSIZE=32
+MEDIUMSIZE=2048
+LARGESIZE=8192
+
+echo "=== Truncate tests ==="
+rm -rf blocks
+tests/test.py << TEST
+    lfs_format(&lfs, &cfg) => 0;
+TEST
+
+truncate_test() {
+STARTSIZES="$1"
+HOTSIZES="$2"
+COLDSIZES="$3"
+tests/test.py << TEST
+    static const lfs_off_t startsizes[] = {$STARTSIZES};
+    static const lfs_off_t hotsizes[]   = {$HOTSIZES};
+
+    lfs_mount(&lfs, &cfg) => 0;
+
+    for (int i = 0; i < sizeof(startsizes)/sizeof(startsizes[0]); i++) {
+        sprintf((char*)buffer, "hairyhead%d", i);
+        lfs_file_open(&lfs, &file[0], (const char*)buffer,
+                LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0;
+
+        strcpy((char*)buffer, "hair");
+        size = strlen((char*)buffer);
+        for (int j = 0; j < startsizes[i]; j += size) {
+            lfs_file_write(&lfs, &file[0], buffer, size) => size;
+        }
+        lfs_file_size(&lfs, &file[0]) => startsizes[i];
+
+        lfs_file_truncate(&lfs, &file[0], hotsizes[i]) => 0;
+        lfs_file_size(&lfs, &file[0]) => hotsizes[i];
+
+        lfs_file_close(&lfs, &file[0]) => 0;
+    }
+
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    static const lfs_off_t startsizes[] = {$STARTSIZES};
+    static const lfs_off_t hotsizes[]   = {$HOTSIZES};
+    static const lfs_off_t coldsizes[]  = {$COLDSIZES};
+
+    lfs_mount(&lfs, &cfg) => 0;
+
+    for (int i = 0; i < sizeof(startsizes)/sizeof(startsizes[0]); i++) {
+        sprintf((char*)buffer, "hairyhead%d", i);
+        lfs_file_open(&lfs, &file[0], (const char*)buffer, LFS_O_RDWR) => 0;
+        lfs_file_size(&lfs, &file[0]) => hotsizes[i];
+
+        size = strlen("hair");
+        int j = 0;
+        for (; j < startsizes[i] && j < hotsizes[i]; j += size) {
+            lfs_file_read(&lfs, &file[0], buffer, size) => size;
+            memcmp(buffer, "hair", size) => 0;
+        }
+
+        for (; j < hotsizes[i]; j += size) {
+            lfs_file_read(&lfs, &file[0], buffer, size) => size;
+            memcmp(buffer, "\0\0\0\0", size) => 0;
+        }
+
+        lfs_file_truncate(&lfs, &file[0], coldsizes[i]) => 0;
+        lfs_file_size(&lfs, &file[0]) => coldsizes[i];
+
+        lfs_file_close(&lfs, &file[0]) => 0;
+    }
+
+    lfs_unmount(&lfs) => 0;
+TEST
+tests/test.py << TEST
+    static const lfs_off_t startsizes[] = {$STARTSIZES};
+    static const lfs_off_t hotsizes[]   = {$HOTSIZES};
+    static const lfs_off_t coldsizes[]  = {$COLDSIZES};
+
+    lfs_mount(&lfs, &cfg) => 0;
+
+    for (int i = 0; i < sizeof(startsizes)/sizeof(startsizes[0]); i++) {
+        sprintf((char*)buffer, "hairyhead%d", i);
+        lfs_file_open(&lfs, &file[0], (const char*)buffer, LFS_O_RDONLY) => 0;
+        lfs_file_size(&lfs, &file[0]) => coldsizes[i];
+
+        size = strlen("hair");
+        int j = 0;
+        for (; j < startsizes[i] && j < hotsizes[i] && j < coldsizes[i];
+                j += size) {
+            lfs_file_read(&lfs, &file[0], buffer, size) => size;
+            memcmp(buffer, "hair", size) => 0;
+        }
+
+        for (; j < coldsizes[i]; j += size) {
+            lfs_file_read(&lfs, &file[0], buffer, size) => size;
+            memcmp(buffer, "\0\0\0\0", size) => 0;
+        }
+
+        lfs_file_close(&lfs, &file[0]) => 0;
+    }
+
+    lfs_unmount(&lfs) => 0;
+TEST
+}
+
+echo "--- Cold shrinking truncate ---"
+truncate_test \
+    "2*$LARGESIZE, 2*$LARGESIZE, 2*$LARGESIZE, 2*$LARGESIZE, 2*$LARGESIZE" \
+    "2*$LARGESIZE, 2*$LARGESIZE, 2*$LARGESIZE, 2*$LARGESIZE, 2*$LARGESIZE" \
+    "           0,   $SMALLSIZE,  $MEDIUMSIZE,   $LARGESIZE, 2*$LARGESIZE"
+
+echo "--- Cold expanding truncate ---"
+truncate_test \
+    "           0,   $SMALLSIZE,  $MEDIUMSIZE,   $LARGESIZE, 2*$LARGESIZE" \
+    "           0,   $SMALLSIZE,  $MEDIUMSIZE,   $LARGESIZE, 2*$LARGESIZE" \
+    "2*$LARGESIZE, 2*$LARGESIZE, 2*$LARGESIZE, 2*$LARGESIZE, 2*$LARGESIZE"
+
+echo "--- Warm shrinking truncate ---"
+truncate_test \
+    "2*$LARGESIZE, 2*$LARGESIZE, 2*$LARGESIZE, 2*$LARGESIZE, 2*$LARGESIZE" \
+    "           0,   $SMALLSIZE,  $MEDIUMSIZE,   $LARGESIZE, 2*$LARGESIZE" \
+    "           0,            0,            0,            0,            0"
+
+echo "--- Warm expanding truncate ---"
+truncate_test \
+    "           0,   $SMALLSIZE,  $MEDIUMSIZE,   $LARGESIZE, 2*$LARGESIZE" \
+    "2*$LARGESIZE, 2*$LARGESIZE, 2*$LARGESIZE, 2*$LARGESIZE, 2*$LARGESIZE" \
+    "2*$LARGESIZE, 2*$LARGESIZE, 2*$LARGESIZE, 2*$LARGESIZE, 2*$LARGESIZE"
+
+echo "--- Results ---"
+tests/stats.py