Merge pull request #669 from littlefs-project/devel

Minor release: v2.5

LICENSE.md

@@ -1,3 +1,4 @@
+Copyright (c) 2022, The littlefs authors.  
 Copyright (c) 2017, Arm Limited. All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,

SPEC.md

@@ -233,19 +233,19 @@ Metadata tag fields:
    into a 3-bit abstract type and an 8-bit chunk field. Note that the value
    `0x000` is invalid and not assigned a type.
 
-3. **Type1 (3-bits)** - Abstract type of the tag. Groups the tags into
+    1. **Type1 (3-bits)** - Abstract type of the tag. Groups the tags into
-   8 categories that facilitate bitmasked lookups.
+       8 categories that facilitate bitmasked lookups.
 
-4. **Chunk (8-bits)** - Chunk field used for various purposes by the different
+    2. **Chunk (8-bits)** - Chunk field used for various purposes by the different
-   abstract types.  type1+chunk+id form a unique identifier for each tag in the
+       abstract types.  type1+chunk+id form a unique identifier for each tag in the
-   metadata block.
+       metadata block.
 
-5. **Id (10-bits)** - File id associated with the tag. Each file in a metadata
+3. **Id (10-bits)** - File id associated with the tag. Each file in a metadata
    block gets a unique id which is used to associate tags with that file. The
    special value `0x3ff` is used for any tags that are not associated with a
    file, such as directory and global metadata.
 
-6. **Length (10-bits)** - Length of the data in bytes. The special value
+4. **Length (10-bits)** - Length of the data in bytes. The special value
    `0x3ff` indicates that this tag has been deleted.
 
 ## Metadata types

bd/lfs_filebd.c

@@ -1,6 +1,7 @@
 /*
  * Block device emulated in a file
  *
+ * Copyright (c) 2022, The littlefs authors.
  * Copyright (c) 2017, Arm Limited. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  */
@@ -10,6 +11,10 @@
 #include <unistd.h>
 #include <errno.h>
 
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
 int lfs_filebd_createcfg(const struct lfs_config *cfg, const char *path,
         const struct lfs_filebd_config *bdcfg) {
     LFS_FILEBD_TRACE("lfs_filebd_createcfg(%p {.context=%p, "
@@ -27,7 +32,12 @@ int lfs_filebd_createcfg(const struct lfs_config *cfg, const char *path,
     bd->cfg = bdcfg;
 
     // open file
+    #ifdef _WIN32
+    bd->fd = open(path, O_RDWR | O_CREAT | O_BINARY, 0666);
+    #else
     bd->fd = open(path, O_RDWR | O_CREAT, 0666);
+    #endif
+
     if (bd->fd < 0) {
         int err = -errno;
         LFS_FILEBD_TRACE("lfs_filebd_createcfg -> %d", err);
@@ -80,7 +90,7 @@ int lfs_filebd_read(const struct lfs_config *cfg, lfs_block_t block,
     LFS_ASSERT(size % cfg->read_size == 0);
     LFS_ASSERT(block < cfg->block_count);
 
-    // zero for reproducability (in case file is truncated)
+    // zero for reproducibility (in case file is truncated)
     if (bd->cfg->erase_value != -1) {
         memset(buffer, bd->cfg->erase_value, size);
     }
@@ -193,7 +203,11 @@ int lfs_filebd_sync(const struct lfs_config *cfg) {
     LFS_FILEBD_TRACE("lfs_filebd_sync(%p)", (void*)cfg);
     // file sync
     lfs_filebd_t *bd = cfg->context;
+    #ifdef _WIN32
+    int err = FlushFileBuffers((HANDLE) _get_osfhandle(fd)) ? 0 : -1;
+    #else
     int err = fsync(bd->fd);
+    #endif
     if (err) {
         err = -errno;
         LFS_FILEBD_TRACE("lfs_filebd_sync -> %d", 0);

bd/lfs_filebd.h

@@ -1,6 +1,7 @@
 /*
  * Block device emulated in a file
  *
+ * Copyright (c) 2022, The littlefs authors.
  * Copyright (c) 2017, Arm Limited. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  */

bd/lfs_rambd.c

@@ -1,6 +1,7 @@
 /*
  * Block device emulated in RAM
  *
+ * Copyright (c) 2022, The littlefs authors.
  * Copyright (c) 2017, Arm Limited. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  */
@@ -32,10 +33,12 @@ int lfs_rambd_createcfg(const struct lfs_config *cfg,
         }
     }
 
-    // zero for reproducability?
+    // zero for reproducibility?
     if (bd->cfg->erase_value != -1) {
         memset(bd->buffer, bd->cfg->erase_value,
                 cfg->block_size * cfg->block_count);
+    } else {
+        memset(bd->buffer, 0, cfg->block_size * cfg->block_count);
     }
 
     LFS_RAMBD_TRACE("lfs_rambd_createcfg -> %d", 0);

bd/lfs_rambd.h

@@ -1,6 +1,7 @@
 /*
  * Block device emulated in RAM
  *
+ * Copyright (c) 2022, The littlefs authors.
  * Copyright (c) 2017, Arm Limited. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  */

bd/lfs_testbd.c

@@ -2,6 +2,7 @@
  * Testing block device, wraps filebd and rambd while providing a bunch
  * of hooks for testing littlefs in various conditions.
  *
+ * Copyright (c) 2022, The littlefs authors.
  * Copyright (c) 2017, Arm Limited. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  */

bd/lfs_testbd.h

@@ -2,6 +2,7 @@
  * Testing block device, wraps filebd and rambd while providing a bunch
  * of hooks for testing littlefs in various conditions.
  *
+ * Copyright (c) 2022, The littlefs authors.
  * Copyright (c) 2017, Arm Limited. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  */

lfs.c

@@ -1,6 +1,7 @@
 /*
  * The little filesystem
  *
+ * Copyright (c) 2022, The littlefs authors.
  * Copyright (c) 2017, Arm Limited. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  */
@@ -26,6 +27,7 @@ enum {
 
 
 /// Caching block device operations ///
+
 static inline void lfs_cache_drop(lfs_t *lfs, lfs_cache_t *rcache) {
     // do not zero, cheaper if cache is readonly or only going to be
     // written with identical data (during relocates)
@@ -277,22 +279,26 @@ static inline int lfs_pair_cmp(
              paira[0] == pairb[1] || paira[1] == pairb[0]);
 }
 
+#ifndef LFS_READONLY
 static inline bool lfs_pair_sync(
         const lfs_block_t paira[2],
         const lfs_block_t pairb[2]) {
     return (paira[0] == pairb[0] && paira[1] == pairb[1]) ||
            (paira[0] == pairb[1] && paira[1] == pairb[0]);
 }
+#endif
 
 static inline void lfs_pair_fromle32(lfs_block_t pair[2]) {
     pair[0] = lfs_fromle32(pair[0]);
     pair[1] = lfs_fromle32(pair[1]);
 }
 
+#ifndef LFS_READONLY
 static inline void lfs_pair_tole32(lfs_block_t pair[2]) {
     pair[0] = lfs_tole32(pair[0]);
     pair[1] = lfs_tole32(pair[1]);
 }
+#endif
 
 // operations on 32-bit entry tags
 typedef uint32_t lfs_tag_t;
@@ -374,6 +380,7 @@ static inline bool lfs_gstate_iszero(const lfs_gstate_t *a) {
     return true;
 }
 
+#ifndef LFS_READONLY
 static inline bool lfs_gstate_hasorphans(const lfs_gstate_t *a) {
     return lfs_tag_size(a->tag);
 }
@@ -385,6 +392,7 @@ static inline uint8_t lfs_gstate_getorphans(const lfs_gstate_t *a) {
 static inline bool lfs_gstate_hasmove(const lfs_gstate_t *a) {
     return lfs_tag_type1(a->tag);
 }
+#endif
 
 static inline bool lfs_gstate_hasmovehere(const lfs_gstate_t *a,
         const lfs_block_t *pair) {
@@ -397,11 +405,13 @@ static inline void lfs_gstate_fromle32(lfs_gstate_t *a) {
     a->pair[1] = lfs_fromle32(a->pair[1]);
 }
 
+#ifndef LFS_READONLY
 static inline void lfs_gstate_tole32(lfs_gstate_t *a) {
     a->tag     = lfs_tole32(a->tag);
     a->pair[0] = lfs_tole32(a->pair[0]);
     a->pair[1] = lfs_tole32(a->pair[1]);
 }
+#endif
 
 // other endianness operations
 static void lfs_ctz_fromle32(struct lfs_ctz *ctz) {
@@ -425,6 +435,7 @@ static inline void lfs_superblock_fromle32(lfs_superblock_t *superblock) {
     superblock->attr_max    = lfs_fromle32(superblock->attr_max);
 }
 
+#ifndef LFS_READONLY
 static inline void lfs_superblock_tole32(lfs_superblock_t *superblock) {
     superblock->version     = lfs_tole32(superblock->version);
     superblock->block_size  = lfs_tole32(superblock->block_size);
@@ -433,6 +444,7 @@ static inline void lfs_superblock_tole32(lfs_superblock_t *superblock) {
     superblock->file_max    = lfs_tole32(superblock->file_max);
     superblock->attr_max    = lfs_tole32(superblock->attr_max);
 }
+#endif
 
 #ifndef LFS_NO_ASSERT
 static bool lfs_mlist_isopen(struct lfs_mlist *head,
@@ -755,11 +767,10 @@ static int lfs_dir_traverse_filter(void *p,
 // maximum recursive depth of lfs_dir_traverse, the deepest call:
 //
 // traverse with commit
-// '-> traverse with filter
+// '-> traverse with move
-//     '-> traverse with move
+//     '-> traverse with filter
-//         ' traverse with filter
 //
-#define LFS_DIR_TRAVERSE_DEPTH 4
+#define LFS_DIR_TRAVERSE_DEPTH 3
 
 struct lfs_dir_traverse {
     const lfs_mdir_t *dir;
@@ -881,6 +892,26 @@ popped:
         if (lfs_tag_type3(tag) == LFS_FROM_NOOP) {
             // do nothing
         } else if (lfs_tag_type3(tag) == LFS_FROM_MOVE) {
+            // Without this condition, lfs_dir_traverse can exhibit an
+            // extremely expensive O(n^3) of nested loops when renaming.
+            // This happens because lfs_dir_traverse tries to filter tags by
+            // the tags in the source directory, triggering a second
+            // lfs_dir_traverse with its own filter operation.
+            //
+            // traverse with commit
+            // '-> traverse with filter
+            //     '-> traverse with move
+            //         '-> traverse with filter
+            //
+            // However we don't actually care about filtering the second set of
+            // tags, since duplicate tags have no effect when filtering.
+            //
+            // This check skips this unnecessary recursive filtering explicitly,
+            // reducing this runtime from O(n^3) to O(n^2).
+            if (cb == lfs_dir_traverse_filter) {
+                continue;
+            }
+
             // recurse into move
             stack[sp] = (struct lfs_dir_traverse){
                 .dir        = dir,
@@ -1576,7 +1607,7 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_mdir_t *dir) {
         }
     }
 
-    // zero for reproducability in case initial block is unreadable
+    // zero for reproducibility in case initial block is unreadable
     dir->rev = 0;
 
     // rather than clobbering one of the blocks we just pretend
@@ -1636,7 +1667,6 @@ static int lfs_dir_split(lfs_t *lfs,
         lfs_mdir_t *dir, const struct lfs_mattr *attrs, int attrcount,
         lfs_mdir_t *source, uint16_t split, uint16_t end) {
     // create tail metadata pair
-    lfs_alloc_ack(lfs);
     lfs_mdir_t tail;
     int err = lfs_dir_alloc(lfs, &tail);
     if (err) {
@@ -1934,6 +1964,7 @@ static int lfs_dir_splittingcompact(lfs_t *lfs, lfs_mdir_t *dir,
             // performance
             LFS_WARN("Unable to split {0x%"PRIx32", 0x%"PRIx32"}",
                     dir->pair[0], dir->pair[1]);
+            break;
         } else {
             end = split;
         }
@@ -2858,8 +2889,11 @@ static int lfs_file_rawopencfg(lfs_t *lfs, lfs_file_t *file,
                 {LFS_MKTAG(LFS_TYPE_CREATE, file->id, 0), NULL},
                 {LFS_MKTAG(LFS_TYPE_REG, file->id, nlen), path},
                 {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, file->id, 0), NULL}));
+
+        // it may happen that the file name doesn't fit in the metadata blocks, e.g., a 256 byte file name will
+        // not fit in a 128 byte block.
+        err = (err == LFS_ERR_NOSPC) ? LFS_ERR_NAMETOOLONG : err;
         if (err) {
-            err = LFS_ERR_NAMETOOLONG;
             goto cleanup;
         }
 
@@ -3075,7 +3109,6 @@ static int lfs_file_outline(lfs_t *lfs, lfs_file_t *file) {
 }
 #endif
 
-#ifndef LFS_READONLY
 static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) {
     if (file->flags & LFS_F_READING) {
         if (!(file->flags & LFS_F_INLINE)) {
@@ -3084,6 +3117,7 @@ static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) {
         file->flags &= ~LFS_F_READING;
     }
 
+#ifndef LFS_READONLY
     if (file->flags & LFS_F_WRITING) {
         lfs_off_t pos = file->pos;
 
@@ -3150,10 +3184,10 @@ relocate:
 
         file->pos = pos;
     }
+#endif
 
     return 0;
 }
-#endif
 
 #ifndef LFS_READONLY
 static int lfs_file_rawsync(lfs_t *lfs, lfs_file_t *file) {
@@ -3427,9 +3461,18 @@ static lfs_soff_t lfs_file_rawseek(lfs_t *lfs, lfs_file_t *file,
     if (whence == LFS_SEEK_SET) {
         npos = off;
     } else if (whence == LFS_SEEK_CUR) {
-        npos = file->pos + off;
+        if ((lfs_soff_t)file->pos + off < 0) {
+            return LFS_ERR_INVAL;
+        } else {
+            npos = file->pos + off;
+        }
     } else if (whence == LFS_SEEK_END) {
-        npos = lfs_file_rawsize(lfs, file) + off;
+        lfs_soff_t res = lfs_file_rawsize(lfs, file) + off;
+        if (res < 0) {
+            return LFS_ERR_INVAL;
+        } else {
+            npos = res;
+        }
     }
 
     if (npos > lfs->file_max) {
@@ -3442,13 +3485,32 @@ static lfs_soff_t lfs_file_rawseek(lfs_t *lfs, lfs_file_t *file,
         return npos;
     }
 
+    // if we're only reading and our new offset is still in the file's cache
+    // we can avoid flushing and needing to reread the data
+    if (
 #ifndef LFS_READONLY
+        !(file->flags & LFS_F_WRITING)
+#else
+        true
+#endif
+            ) {
+        int oindex = lfs_ctz_index(lfs, &(lfs_off_t){file->pos});
+        lfs_off_t noff = npos;
+        int nindex = lfs_ctz_index(lfs, &noff);
+        if (oindex == nindex
+                && noff >= file->cache.off
+                && noff < file->cache.off + file->cache.size) {
+            file->pos = npos;
+            file->off = noff;
+            return npos;
+        }
+    }
+
     // write out everything beforehand, may be noop if rdonly
     int err = lfs_file_flush(lfs, file);
     if (err) {
         return err;
     }
-#endif
 
     // update pos
     file->pos = npos;
@@ -4123,6 +4185,20 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) {
 
                 lfs->attr_max = superblock.attr_max;
             }
+
+            if (superblock.block_count != lfs->cfg->block_count) {
+                LFS_ERROR("Invalid block count (%"PRIu32" != %"PRIu32")",
+                        superblock.block_count, lfs->cfg->block_count);
+                err = LFS_ERR_INVAL;
+                goto cleanup;
+            }
+
+            if (superblock.block_size != lfs->cfg->block_size) {
+                LFS_ERROR("Invalid block size (%"PRIu32" != %"PRIu32")",
+                        superblock.block_count, lfs->cfg->block_count);
+                err = LFS_ERR_INVAL;
+                goto cleanup;
+            }
         }
 
         // has gstate?
@@ -5393,6 +5469,7 @@ int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type) {
 }
 #endif
 
+#ifndef LFS_NO_MALLOC
 int lfs_file_open(lfs_t *lfs, lfs_file_t *file, const char *path, int flags) {
     int err = LFS_LOCK(lfs->cfg);
     if (err) {
@@ -5408,6 +5485,7 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, const char *path, int flags) {
     LFS_UNLOCK(lfs->cfg);
     return err;
 }
+#endif
 
 int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file,
         const char *path, int flags,

lfs.h

@@ -1,6 +1,7 @@
 /*
  * The little filesystem
  *
+ * Copyright (c) 2022, The littlefs authors.
  * Copyright (c) 2017, Arm Limited. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  */
@@ -22,7 +23,7 @@ extern "C"
 // Software library version
 // Major (top-nibble), incremented on backwards incompatible changes
 // Minor (bottom-nibble), incremented on feature additions
-#define LFS_VERSION 0x00020004
+#define LFS_VERSION 0x00020005
 #define LFS_VERSION_MAJOR (0xffff & (LFS_VERSION >> 16))
 #define LFS_VERSION_MINOR (0xffff & (LFS_VERSION >>  0))
 
@@ -159,49 +160,49 @@ struct lfs_config {
     // information to the block device operations
     void *context;
 
-    // Read a region in a block. Negative error codes are propogated
+    // Read a region in a block. Negative error codes are propagated
     // to the user.
     int (*read)(const struct lfs_config *c, lfs_block_t block,
             lfs_off_t off, void *buffer, lfs_size_t size);
 
     // Program a region in a block. The block must have previously
-    // been erased. Negative error codes are propogated to the user.
+    // been erased. Negative error codes are propagated to the user.
     // May return LFS_ERR_CORRUPT if the block should be considered bad.
     int (*prog)(const struct lfs_config *c, lfs_block_t block,
             lfs_off_t off, const void *buffer, lfs_size_t size);
 
     // Erase a block. A block must be erased before being programmed.
     // The state of an erased block is undefined. Negative error codes
-    // are propogated to the user.
+    // are propagated to the user.
     // May return LFS_ERR_CORRUPT if the block should be considered bad.
     int (*erase)(const struct lfs_config *c, lfs_block_t block);
 
     // Sync the state of the underlying block device. Negative error codes
-    // are propogated to the user.
+    // are propagated to the user.
     int (*sync)(const struct lfs_config *c);
 
 #ifdef LFS_THREADSAFE
     // Lock the underlying block device. Negative error codes
-    // are propogated to the user.
+    // are propagated to the user.
     int (*lock)(const struct lfs_config *c);
 
     // Unlock the underlying block device. Negative error codes
-    // are propogated to the user.
+    // are propagated to the user.
     int (*unlock)(const struct lfs_config *c);
 #endif
 
-    // Minimum size of a block read. All read operations will be a
+    // Minimum size of a block read in bytes. All read operations will be a
     // multiple of this value.
     lfs_size_t read_size;
 
-    // Minimum size of a block program. All program operations will be a
+    // Minimum size of a block program in bytes. All program operations will be
-    // multiple of this value.
+    // a multiple of this value.
     lfs_size_t prog_size;
 
-    // Size of an erasable block. This does not impact ram consumption and
+    // Size of an erasable block in bytes. This does not impact ram consumption
-    // may be larger than the physical erase size. However, non-inlined files
+    // and may be larger than the physical erase size. However, non-inlined
-    // take up at minimum one block. Must be a multiple of the read
+    // files take up at minimum one block. Must be a multiple of the read and
-    // and program sizes.
+    // program sizes.
     lfs_size_t block_size;
 
     // Number of erasable blocks on the device.
@@ -215,11 +216,11 @@ struct lfs_config {
     // Set to -1 to disable block-level wear-leveling.
     int32_t block_cycles;
 
-    // Size of block caches. Each cache buffers a portion of a block in RAM.
+    // Size of block caches in bytes. Each cache buffers a portion of a block in
-    // The littlefs needs a read cache, a program cache, and one additional
+    // RAM. The littlefs needs a read cache, a program cache, and one additional
     // cache per file. Larger caches can improve performance by storing more
-    // data and reducing the number of disk accesses. Must be a multiple of
+    // data and reducing the number of disk accesses. Must be a multiple of the
-    // the read and program sizes, and a factor of the block size.
+    // read and program sizes, and a factor of the block size.
     lfs_size_t cache_size;
 
     // Size of the lookahead buffer in bytes. A larger lookahead buffer
@@ -485,7 +486,7 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info);
 // Returns the size of the attribute, or a negative error code on failure.
 // Note, the returned size is the size of the attribute on disk, irrespective
 // of the size of the buffer. This can be used to dynamically allocate a buffer
-// or check for existance.
+// or check for existence.
 lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path,
         uint8_t type, void *buffer, lfs_size_t size);
 
@@ -513,6 +514,7 @@ int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type);
 
 /// File operations ///
 
+#ifndef LFS_NO_MALLOC
 // Open a file
 //
 // The mode that the file is opened in is determined by the flags, which
@@ -522,6 +524,10 @@ int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type);
 int lfs_file_open(lfs_t *lfs, lfs_file_t *file,
         const char *path, int flags);
 
+// if LFS_NO_MALLOC is defined, lfs_file_open() will fail with LFS_ERR_NOMEM
+// thus use lfs_file_opencfg() with config.buffer set.
+#endif
+
 // Open a file with extra configuration
 //
 // The mode that the file is opened in is determined by the flags, which

lfs_util.c

@@ -1,6 +1,7 @@
 /*
  * lfs util functions
  *
+ * Copyright (c) 2022, The littlefs authors.
  * Copyright (c) 2017, Arm Limited. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  */

lfs_util.h

@@ -1,6 +1,7 @@
 /*
  * lfs utility functions
  *
+ * Copyright (c) 2022, The littlefs authors.
  * Copyright (c) 2017, Arm Limited. All rights reserved.
  * SPDX-License-Identifier: BSD-3-Clause
  */

scripts/test.py

@@ -565,7 +565,7 @@ class TestSuite:
                     path=self.path))
                 mk.write('\n')
 
-            # add truely global defines globally
+            # add truly global defines globally
             for k, v in sorted(self.defines.items()):
                 mk.write('%s.test: override CFLAGS += -D%s=%r\n'
                     % (self.path, k, v))
@@ -656,7 +656,7 @@ def main(**args):
         for path in glob.glob(testpath):
             suites.append(TestSuite(path, classes, defines, filter, **args))
 
-    # sort for reproducability
+    # sort for reproducibility
     suites = sorted(suites)
 
     # generate permutations