From 3419284689ead61c6905d61359a2b62a8c09c75d Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 21 Aug 2018 18:15:40 -0500 Subject: [PATCH 01/11] Fixed issue with corruption due to different cache sizes The lfs_cache_zero function that was recently added assumed a single cache size, which is incorrect. This would cause a buffer overflow if read_size != prog_size. Since lfs_cache_zero is only used for scrubbing prog caches, the fix here is to use lfs_cache_drop instead on read caches. Info in read caches should never make its way to disk. Found by nstcl --- lfs.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lfs.c b/lfs.c index c6b5870..6b86536 100644 --- a/lfs.c +++ b/lfs.c @@ -1373,7 +1373,10 @@ int lfs_file_opencfg(lfs_t *lfs, lfs_file_t *file, } // zero to avoid information leak - lfs_cache_zero(lfs, &file->cache); + lfs_cache_drop(lfs, &file->cache); + if ((file->flags & 3) != LFS_O_RDONLY) { + lfs_cache_zero(lfs, &file->cache); + } // add to list of files file->next = lfs->files; @@ -2055,8 +2058,8 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { } // zero to avoid information leaks - lfs_cache_zero(lfs, &lfs->rcache); lfs_cache_zero(lfs, &lfs->pcache); + lfs_cache_drop(lfs, &lfs->rcache); // setup lookahead, round down to nearest 32-bits LFS_ASSERT(lfs->cfg->lookahead % 32 == 0); From 6ad544f3f3dde82e0faf1abacf50facc37086d88 Mon Sep 17 00:00:00 2001 From: Chris Date: Wed, 26 Sep 2018 14:04:38 +0200 Subject: [PATCH 02/11] If stats file doesn't exist lfs_emubd_create will fail. This will create default stats file if it doesn't exist. --- emubd/lfs_emubd.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/emubd/lfs_emubd.c b/emubd/lfs_emubd.c index 682ad92..e44602c 100644 --- a/emubd/lfs_emubd.c +++ b/emubd/lfs_emubd.c @@ -47,19 +47,24 @@ int lfs_emubd_create(const struct lfs_config *cfg, const char *path) { // Load stats to continue incrementing snprintf(emu->child, LFS_NAME_MAX, "stats"); + FILE *f = fopen(emu->path, "r"); - if (!f) { + if (!f && errno != ENOENT) { return -errno; } - size_t res = fread(&emu->stats, sizeof(emu->stats), 1, f); - if (res < 1) { - return -errno; - } + if (errno == ENOENT) { + memset(&emu->stats, 0x0, sizeof(emu->stats)); + } else { + size_t res = fread(&emu->stats, sizeof(emu->stats), 1, f); + if (res < 1) { + return -errno; + } - err = fclose(f); - if (err) { - return -errno; + err = fclose(f); + if (err) { + return -errno; + } } return 0; From e5a6938faf75c18a1acc161f2bd785eb57309cf1 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 19 Sep 2018 18:05:39 -0500 Subject: [PATCH 03/11] Fixed possible infinite loop in deorphan step Normally, the linked-list of directory pairs should terminate at a null pointer. However, it is possible if the filesystem is corrupted, that that this linked-list forms a cycle. This should never happen with littlefs's power resilience, but if it does we should recover appropriately. Modified lfs_deorphan to notice if we have a cycle and return LFS_ERR_CORRUPT in that situation. Found by kneko715 --- lfs.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lfs.c b/lfs.c index 6b86536..75f8a91 100644 --- a/lfs.c +++ b/lfs.c @@ -2475,7 +2475,11 @@ int lfs_deorphan(lfs_t *lfs) { lfs_dir_t cwd = {.d.tail[0] = 0, .d.tail[1] = 1}; // iterate over all directory directory entries - while (!lfs_pairisnull(cwd.d.tail)) { + for (int i = 0; i < lfs->cfg->block_count; i++) { + if (lfs_pairisnull(cwd.d.tail)) { + return 0; + } + int err = lfs_dir_fetch(lfs, &cwd, cwd.d.tail); if (err) { return err; @@ -2504,7 +2508,7 @@ int lfs_deorphan(lfs_t *lfs) { return err; } - break; + return 0; } if (!lfs_pairsync(entry.d.u.dir, pdir.d.tail)) { @@ -2520,7 +2524,7 @@ int lfs_deorphan(lfs_t *lfs) { return err; } - break; + return 0; } } @@ -2565,5 +2569,7 @@ int lfs_deorphan(lfs_t *lfs) { memcpy(&pdir, &cwd, sizeof(pdir)); } - return 0; + // If we reached here, we have more directory pairs than blocks in the + // filesystem... So something must be horribly wrong + return LFS_ERR_CORRUPT; } From 646b1b5a6ce432aa0c5c0ecbd9f1d1ee330c2f03 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 26 Sep 2018 10:11:40 -0500 Subject: [PATCH 04/11] Added -Wjump-misses-init and fixed uninitialized warnings --- Makefile | 3 +- lfs.c | 224 ++++++++++++++++++++++++++++--------------------------- 2 files changed, 117 insertions(+), 110 deletions(-) diff --git a/Makefile b/Makefile index 99a3c0c..020942f 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,8 @@ ifdef WORD override CFLAGS += -m$(WORD) endif override CFLAGS += -I. -override CFLAGS += -std=c99 -Wall -pedantic -Wshadow -Wunused-parameter +override CFLAGS += -std=c99 -Wall -pedantic +override CFLAGS += -Wshadow -Wunused-parameter -Wjump-misses-init all: $(TARGET) diff --git a/lfs.c b/lfs.c index 75f8a91..852250f 100644 --- a/lfs.c +++ b/lfs.c @@ -2096,83 +2096,86 @@ cleanup: } int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { - int err = lfs_init(lfs, cfg); - if (err) { - return err; - } + int err = 0; + if (true) { + err = lfs_init(lfs, cfg); + if (err) { + return err; + } - // create free lookahead - memset(lfs->free.buffer, 0, lfs->cfg->lookahead/8); - lfs->free.off = 0; - lfs->free.size = lfs_min(lfs->cfg->lookahead, lfs->cfg->block_count); - lfs->free.i = 0; - lfs_alloc_ack(lfs); + // create free lookahead + memset(lfs->free.buffer, 0, lfs->cfg->lookahead/8); + lfs->free.off = 0; + lfs->free.size = lfs_min(lfs->cfg->lookahead, lfs->cfg->block_count); + lfs->free.i = 0; + lfs_alloc_ack(lfs); - // create superblock dir - lfs_dir_t superdir; - err = lfs_dir_alloc(lfs, &superdir); - if (err) { - goto cleanup; - } - - // write root directory - lfs_dir_t root; - err = lfs_dir_alloc(lfs, &root); - if (err) { - goto cleanup; - } - - err = lfs_dir_commit(lfs, &root, NULL, 0); - if (err) { - goto cleanup; - } - - lfs->root[0] = root.pair[0]; - lfs->root[1] = root.pair[1]; - - // write superblocks - lfs_superblock_t superblock = { - .off = sizeof(superdir.d), - .d.type = LFS_TYPE_SUPERBLOCK, - .d.elen = sizeof(superblock.d) - sizeof(superblock.d.magic) - 4, - .d.nlen = sizeof(superblock.d.magic), - .d.version = LFS_DISK_VERSION, - .d.magic = {"littlefs"}, - .d.block_size = lfs->cfg->block_size, - .d.block_count = lfs->cfg->block_count, - .d.root = {lfs->root[0], lfs->root[1]}, - }; - superdir.d.tail[0] = root.pair[0]; - superdir.d.tail[1] = root.pair[1]; - superdir.d.size = sizeof(superdir.d) + sizeof(superblock.d) + 4; - - // write both pairs to be safe - lfs_superblock_tole32(&superblock.d); - bool valid = false; - for (int i = 0; i < 2; i++) { - err = lfs_dir_commit(lfs, &superdir, (struct lfs_region[]){ - {sizeof(superdir.d), sizeof(superblock.d), - &superblock.d, sizeof(superblock.d)} - }, 1); - if (err && err != LFS_ERR_CORRUPT) { + // create superblock dir + lfs_dir_t superdir; + err = lfs_dir_alloc(lfs, &superdir); + if (err) { goto cleanup; } - valid = valid || !err; - } + // write root directory + lfs_dir_t root; + err = lfs_dir_alloc(lfs, &root); + if (err) { + goto cleanup; + } - if (!valid) { - err = LFS_ERR_CORRUPT; - goto cleanup; - } + err = lfs_dir_commit(lfs, &root, NULL, 0); + if (err) { + goto cleanup; + } - // sanity check that fetch works - err = lfs_dir_fetch(lfs, &superdir, (const lfs_block_t[2]){0, 1}); - if (err) { - goto cleanup; - } + lfs->root[0] = root.pair[0]; + lfs->root[1] = root.pair[1]; - lfs_alloc_ack(lfs); + // write superblocks + lfs_superblock_t superblock = { + .off = sizeof(superdir.d), + .d.type = LFS_TYPE_SUPERBLOCK, + .d.elen = sizeof(superblock.d) - sizeof(superblock.d.magic) - 4, + .d.nlen = sizeof(superblock.d.magic), + .d.version = LFS_DISK_VERSION, + .d.magic = {"littlefs"}, + .d.block_size = lfs->cfg->block_size, + .d.block_count = lfs->cfg->block_count, + .d.root = {lfs->root[0], lfs->root[1]}, + }; + superdir.d.tail[0] = root.pair[0]; + superdir.d.tail[1] = root.pair[1]; + superdir.d.size = sizeof(superdir.d) + sizeof(superblock.d) + 4; + + // write both pairs to be safe + lfs_superblock_tole32(&superblock.d); + bool valid = false; + for (int i = 0; i < 2; i++) { + err = lfs_dir_commit(lfs, &superdir, (struct lfs_region[]){ + {sizeof(superdir.d), sizeof(superblock.d), + &superblock.d, sizeof(superblock.d)} + }, 1); + if (err && err != LFS_ERR_CORRUPT) { + goto cleanup; + } + + valid = valid || !err; + } + + if (!valid) { + err = LFS_ERR_CORRUPT; + goto cleanup; + } + + // sanity check that fetch works + err = lfs_dir_fetch(lfs, &superdir, (const lfs_block_t[2]){0, 1}); + if (err) { + goto cleanup; + } + + lfs_alloc_ack(lfs); + } cleanup: lfs_deinit(lfs); @@ -2180,53 +2183,56 @@ cleanup: } int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { - int err = lfs_init(lfs, cfg); - if (err) { - return err; - } - - // setup free lookahead - lfs->free.off = 0; - lfs->free.size = 0; - lfs->free.i = 0; - lfs_alloc_ack(lfs); - - // load superblock - lfs_dir_t dir; - lfs_superblock_t superblock; - err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); - if (err && err != LFS_ERR_CORRUPT) { - goto cleanup; - } - - if (!err) { - err = lfs_bd_read(lfs, dir.pair[0], sizeof(dir.d), - &superblock.d, sizeof(superblock.d)); - lfs_superblock_fromle32(&superblock.d); + int err = 0; + if (true) { + err = lfs_init(lfs, cfg); if (err) { + return err; + } + + // setup free lookahead + lfs->free.off = 0; + lfs->free.size = 0; + lfs->free.i = 0; + lfs_alloc_ack(lfs); + + // load superblock + lfs_dir_t dir; + lfs_superblock_t superblock; + err = lfs_dir_fetch(lfs, &dir, (const lfs_block_t[2]){0, 1}); + if (err && err != LFS_ERR_CORRUPT) { goto cleanup; } - lfs->root[0] = superblock.d.root[0]; - lfs->root[1] = superblock.d.root[1]; - } + if (!err) { + err = lfs_bd_read(lfs, dir.pair[0], sizeof(dir.d), + &superblock.d, sizeof(superblock.d)); + lfs_superblock_fromle32(&superblock.d); + if (err) { + goto cleanup; + } - if (err || memcmp(superblock.d.magic, "littlefs", 8) != 0) { - LFS_ERROR("Invalid superblock at %d %d", 0, 1); - err = LFS_ERR_CORRUPT; - goto cleanup; - } + lfs->root[0] = superblock.d.root[0]; + lfs->root[1] = superblock.d.root[1]; + } - uint16_t major_version = (0xffff & (superblock.d.version >> 16)); - uint16_t minor_version = (0xffff & (superblock.d.version >> 0)); - if ((major_version != LFS_DISK_VERSION_MAJOR || - minor_version > LFS_DISK_VERSION_MINOR)) { - LFS_ERROR("Invalid version %d.%d", major_version, minor_version); - err = LFS_ERR_INVAL; - goto cleanup; - } + if (err || memcmp(superblock.d.magic, "littlefs", 8) != 0) { + LFS_ERROR("Invalid superblock at %d %d", 0, 1); + err = LFS_ERR_CORRUPT; + goto cleanup; + } - return 0; + uint16_t major_version = (0xffff & (superblock.d.version >> 16)); + uint16_t minor_version = (0xffff & (superblock.d.version >> 0)); + if ((major_version != LFS_DISK_VERSION_MAJOR || + minor_version > LFS_DISK_VERSION_MINOR)) { + LFS_ERROR("Invalid version %d.%d", major_version, minor_version); + err = LFS_ERR_INVAL; + goto cleanup; + } + + return 0; + } cleanup: From cb62bf2188854c5b7c44383571ebb19a414e6137 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 27 Sep 2018 12:04:13 -0500 Subject: [PATCH 05/11] Fixed release script issue with fetching recent tags Fetching all tags was triggering the pagination system inside the github API. This prevent version tags from being found. Modified to use the version tag prefix in the ref lookup, however this still may cause an issue if there are still enough patch releases to trigger pagination. Simpleish solution is to grab the link header to jump to the last page, since pagination results appear to be in sorted order. --- .travis.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3714f7c..6062c42 100644 --- a/.travis.yml +++ b/.travis.yml @@ -138,11 +138,13 @@ jobs: - LFS_VERSION=$(grep -ox '#define LFS_VERSION .*' lfs.h | cut -d ' ' -f3) - LFS_VERSION_MAJOR=$((0xffff & ($LFS_VERSION >> 16))) - LFS_VERSION_MINOR=$((0xffff & ($LFS_VERSION >> 0))) - # Grab latests patch from repo tags, default to 0 - - LFS_VERSION_PATCH=$(curl -f -u "$GEKY_BOT_RELEASES" - https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs - | jq 'map(.ref | match( - "refs/tags/v'"$LFS_VERSION_MAJOR"'\\.'"$LFS_VERSION_MINOR"'\\.(.*)$") + # Grab latests patch from repo tags, default to 0, needs finagling to get past github's pagination api + - PREV_URL=https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs/tags/v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR. + - PREV_URL=$(curl -f -u "$GEKY_BOT_RELEASES" "$PREV_URL" -I + | sed -n '/^Link/{s/.*<\(.*\)>; rel="last"/\1/;p;q0};$q1' + || echo $PREV_URL) + - LFS_VERSION_PATCH=$(curl -f -u "$GEKY_BOT_RELEASES" "$PREV_URL" + | jq 'map(.ref | match("\\bv.*\\..*\\.(.*)$";"g") .captures[].string | tonumber + 1) | max // 0') # We have our new version - LFS_VERSION="v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.$LFS_VERSION_PATCH" From 28d2d96a83dd238acc43523d2fba1223ad0c5ac2 Mon Sep 17 00:00:00 2001 From: Vincent Dupont Date: Fri, 28 Sep 2018 14:12:15 +0200 Subject: [PATCH 06/11] Fix -Wsign-compare error --- Makefile | 2 +- lfs.c | 2 +- tests/test_alloc.sh | 6 +++--- tests/test_seek.sh | 2 +- tests/test_truncate.sh | 12 ++++++------ 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 020942f..17d3616 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ override CFLAGS += -m$(WORD) endif override CFLAGS += -I. override CFLAGS += -std=c99 -Wall -pedantic -override CFLAGS += -Wshadow -Wunused-parameter -Wjump-misses-init +override CFLAGS += -Wshadow -Wunused-parameter -Wjump-misses-init -Wsign-compare all: $(TARGET) diff --git a/lfs.c b/lfs.c index 852250f..d6b45f1 100644 --- a/lfs.c +++ b/lfs.c @@ -2481,7 +2481,7 @@ int lfs_deorphan(lfs_t *lfs) { lfs_dir_t cwd = {.d.tail[0] = 0, .d.tail[1] = 1}; // iterate over all directory directory entries - for (int i = 0; i < lfs->cfg->block_count; i++) { + for (lfs_size_t i = 0; i < lfs->cfg->block_count; i++) { if (lfs_pairisnull(cwd.d.tail)) { return 0; } diff --git a/tests/test_alloc.sh b/tests/test_alloc.sh index 8c81490..6b3b181 100755 --- a/tests/test_alloc.sh +++ b/tests/test_alloc.sh @@ -32,18 +32,18 @@ lfs_alloc_singleproc() { tests/test.py << TEST const char *names[] = {"bacon", "eggs", "pancakes"}; lfs_mount(&lfs, &cfg) => 0; - for (int n = 0; n < sizeof(names)/sizeof(names[0]); n++) { + for (unsigned n = 0; n < sizeof(names)/sizeof(names[0]); n++) { sprintf((char*)buffer, "$1/%s", names[n]); lfs_file_open(&lfs, &file[n], (char*)buffer, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_APPEND) => 0; } - for (int n = 0; n < sizeof(names)/sizeof(names[0]); n++) { + for (unsigned n = 0; n < sizeof(names)/sizeof(names[0]); n++) { size = strlen(names[n]); for (int i = 0; i < $SIZE; i++) { lfs_file_write(&lfs, &file[n], names[n], size) => size; } } - for (int n = 0; n < sizeof(names)/sizeof(names[0]); n++) { + for (unsigned n = 0; n < sizeof(names)/sizeof(names[0]); n++) { lfs_file_close(&lfs, &file[n]) => 0; } lfs_unmount(&lfs) => 0; diff --git a/tests/test_seek.sh b/tests/test_seek.sh index 0084d42..aa8e643 100755 --- a/tests/test_seek.sh +++ b/tests/test_seek.sh @@ -301,7 +301,7 @@ tests/test.py << TEST size = strlen("hedgehoghog"); const lfs_soff_t offsets[] = {512, 1020, 513, 1021, 511, 1019}; - for (int i = 0; i < sizeof(offsets) / sizeof(offsets[0]); i++) { + for (unsigned i = 0; i < sizeof(offsets) / sizeof(offsets[0]); i++) { lfs_soff_t off = offsets[i]; memcpy(buffer, "hedgehoghog", size); lfs_file_seek(&lfs, &file[0], off, LFS_SEEK_SET) => off; diff --git a/tests/test_truncate.sh b/tests/test_truncate.sh index da5ccaf..053b2e0 100755 --- a/tests/test_truncate.sh +++ b/tests/test_truncate.sh @@ -23,14 +23,14 @@ tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - for (int i = 0; i < sizeof(startsizes)/sizeof(startsizes[0]); i++) { + for (unsigned i = 0; i < sizeof(startsizes)/sizeof(startsizes[0]); i++) { sprintf((char*)buffer, "hairyhead%d", i); lfs_file_open(&lfs, &file[0], (const char*)buffer, LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; strcpy((char*)buffer, "hair"); size = strlen((char*)buffer); - for (int j = 0; j < startsizes[i]; j += size) { + for (lfs_off_t j = 0; j < startsizes[i]; j += size) { lfs_file_write(&lfs, &file[0], buffer, size) => size; } lfs_file_size(&lfs, &file[0]) => startsizes[i]; @@ -55,13 +55,13 @@ tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - for (int i = 0; i < sizeof(startsizes)/sizeof(startsizes[0]); i++) { + for (unsigned i = 0; i < sizeof(startsizes)/sizeof(startsizes[0]); i++) { sprintf((char*)buffer, "hairyhead%d", i); lfs_file_open(&lfs, &file[0], (const char*)buffer, LFS_O_RDWR) => 0; lfs_file_size(&lfs, &file[0]) => hotsizes[i]; size = strlen("hair"); - int j = 0; + lfs_off_t j = 0; for (; j < startsizes[i] && j < hotsizes[i]; j += size) { lfs_file_read(&lfs, &file[0], buffer, size) => size; memcmp(buffer, "hair", size) => 0; @@ -87,13 +87,13 @@ tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; - for (int i = 0; i < sizeof(startsizes)/sizeof(startsizes[0]); i++) { + for (unsigned i = 0; i < sizeof(startsizes)/sizeof(startsizes[0]); i++) { sprintf((char*)buffer, "hairyhead%d", i); lfs_file_open(&lfs, &file[0], (const char*)buffer, LFS_O_RDONLY) => 0; lfs_file_size(&lfs, &file[0]) => coldsizes[i]; size = strlen("hair"); - int j = 0; + lfs_off_t j = 0; for (; j < startsizes[i] && j < hotsizes[i] && j < coldsizes[i]; j += size) { lfs_file_read(&lfs, &file[0], buffer, size) => size; From 0bb1f7af17755bd792f0c4966877fb1886dfc802 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 29 Sep 2018 11:18:53 -0500 Subject: [PATCH 07/11] Modified release script to create notes only on minor releases Before, release notes with a list of changes were created every patch release. Unfortunately, it looks like this will create a lot of noise on github, with a notification every patch release, which may be as often as every time a PR is merged. Rather than creating all of this noise for relatively uninteresting changes, the script will now stick to simple tags, and create the release notes only on minor releases. I think this is what several of you were originally suggesting, sorry about the journey, at least I learned a lot. --- .travis.yml | 47 +++++++++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6062c42..e4ea93d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -140,12 +140,13 @@ jobs: - LFS_VERSION_MINOR=$((0xffff & ($LFS_VERSION >> 0))) # Grab latests patch from repo tags, default to 0, needs finagling to get past github's pagination api - PREV_URL=https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs/tags/v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR. - - PREV_URL=$(curl -f -u "$GEKY_BOT_RELEASES" "$PREV_URL" -I + - PREV_URL=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL" -I | sed -n '/^Link/{s/.*<\(.*\)>; rel="last"/\1/;p;q0};$q1' || echo $PREV_URL) - - LFS_VERSION_PATCH=$(curl -f -u "$GEKY_BOT_RELEASES" "$PREV_URL" + - LFS_VERSION_PATCH=$(curl -u "$GEKY_BOT_RELEASES" "$PREV_URL" | jq 'map(.ref | match("\\bv.*\\..*\\.(.*)$";"g") - .captures[].string | tonumber + 1) | max // 0') + .captures[].string | tonumber) | max + 1' + || echo 0) # We have our new version - LFS_VERSION="v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR.$LFS_VERSION_PATCH" - echo "VERSION $LFS_VERSION" @@ -156,24 +157,34 @@ jobs: | jq -re '.sha') if [ "$TRAVIS_COMMIT" == "$CURRENT_COMMIT" ] then - # Build release notes - PREV=$(git tag --sort=-v:refname -l "v*" | head -1) - if [ ! -z "$PREV" ] - then - echo "PREV $PREV" - CHANGES=$'### Changes\n\n'$( \ - git log --oneline $PREV.. --grep='^Merge' --invert-grep) - printf "CHANGES\n%s\n\n" "$CHANGES" - fi - # Create the release + # Create a simple tag curl -f -u "$GEKY_BOT_RELEASES" -X POST \ - https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases \ + https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs \ -d "{ - \"tag_name\": \"$LFS_VERSION\", - \"target_commitish\": \"$TRAVIS_COMMIT\", - \"name\": \"${LFS_VERSION%.0}\", - \"body\": $(jq -sR '.' <<< "$CHANGES") + \"ref\": \"refs/tags/$LFS_VERSION\", + \"sha\": \"$TRAVIS_COMMIT\" }" + # Minor release? + if [[ "$LFS_VERSION" == *.0 ]] + then + # Build release notes + PREV=$(git tag --sort=-v:refname -l "v*.0" | head -1) + if [ ! -z "$PREV" ] + then + echo "PREV $PREV" + CHANGES=$'### Changes\n\n'$( \ + git log --oneline $PREV.. --grep='^Merge' --invert-grep) + printf "CHANGES\n%s\n\n" "$CHANGES" + fi + # Create the release + curl -f -u "$GEKY_BOT_RELEASES" -X POST \ + https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases \ + -d "{ + \"tag_name\": \"$LFS_VERSION\", + \"name\": \"${LFS_VERSION%.0}\", + \"body\": $(jq -sR '.' <<< "$CHANGES") + }" + fi fi # Manage statuses From 97d8d5e96a7781596708664f18f2ea6c3a179330 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 7 Oct 2018 13:47:29 -0500 Subject: [PATCH 08/11] Fixed issue where a rename causes a split and pushes dir out of sync The issue happens when a rename causes a split in the destination pair. If the destination pair is the same as the source pair, this triggers the logic to keep both pairs in sync. Unfortunately, this logic didn't work, because the source entry still resides in the old source pair, unlike the destination pair, which is now in the new pair created by the split. The best fix for now is to refetch the source pair after the changes to the destination pair. This isn't the most efficient solution, but fortunately this bug has already been fixed in the revamped move logic in littlefs v2 (currently in progress). Found by ohoc --- lfs.c | 39 +++++++++++++--------------- lfs.h | 1 + tests/test_dirs.sh | 63 ++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 80 insertions(+), 23 deletions(-) diff --git a/lfs.c b/lfs.c index d6b45f1..438647e 100644 --- a/lfs.c +++ b/lfs.c @@ -888,7 +888,7 @@ nextname: } // check that entry has not been moved - if (entry->d.type & 0x80) { + if (!lfs->moving && entry->d.type & 0x80) { int moved = lfs_moved(lfs, &entry->d.u); if (moved < 0 || moved) { return (moved < 0) ? moved : LFS_ERR_NOENT; @@ -1922,7 +1922,14 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { // find old entry lfs_dir_t oldcwd; lfs_entry_t oldentry; - int err = lfs_dir_find(lfs, &oldcwd, &oldentry, &oldpath); + int err = lfs_dir_find(lfs, &oldcwd, &oldentry, &(const char *){oldpath}); + if (err) { + return err; + } + + // mark as moving + oldentry.d.type |= 0x80; + err = lfs_dir_update(lfs, &oldcwd, &oldentry, NULL); if (err) { return err; } @@ -1935,11 +1942,9 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return err; } - bool prevexists = (err != LFS_ERR_NOENT); - bool samepair = (lfs_paircmp(oldcwd.pair, newcwd.pair) == 0); - // must have same type - if (prevexists && preventry.d.type != oldentry.d.type) { + bool prevexists = (err != LFS_ERR_NOENT); + if (prevexists && preventry.d.type != (0x7f & oldentry.d.type)) { return LFS_ERR_ISDIR; } @@ -1956,18 +1961,6 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } } - // mark as moving - oldentry.d.type |= 0x80; - err = lfs_dir_update(lfs, &oldcwd, &oldentry, NULL); - if (err) { - return err; - } - - // update pair if newcwd == oldcwd - if (samepair) { - newcwd = oldcwd; - } - // move to new location lfs_entry_t newentry = preventry; newentry.d = oldentry.d; @@ -1986,10 +1979,13 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { } } - // update pair if newcwd == oldcwd - if (samepair) { - oldcwd = newcwd; + // fetch old pair again in case dir block changed + lfs->moving = true; + err = lfs_dir_find(lfs, &oldcwd, &oldentry, &oldpath); + if (err) { + return err; } + lfs->moving = false; // remove old entry err = lfs_dir_remove(lfs, &oldcwd, &oldentry); @@ -2087,6 +2083,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->files = NULL; lfs->dirs = NULL; lfs->deorphaned = false; + lfs->moving = false; return 0; diff --git a/lfs.h b/lfs.h index 7dd3604..4b687f4 100644 --- a/lfs.h +++ b/lfs.h @@ -280,6 +280,7 @@ typedef struct lfs { lfs_free_t free; bool deorphaned; + bool moving; } lfs_t; diff --git a/tests/test_dirs.sh b/tests/test_dirs.sh index 53d76f7..874808d 100755 --- a/tests/test_dirs.sh +++ b/tests/test_dirs.sh @@ -326,13 +326,42 @@ tests/test.py << TEST lfs_unmount(&lfs) => 0; TEST +echo "--- Multi-block rename ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + for (int i = 0; i < $LARGESIZE; i++) { + sprintf((char*)buffer, "cactus/test%d", i); + sprintf((char*)wbuffer, "cactus/tedd%d", i); + lfs_rename(&lfs, (char*)buffer, (char*)wbuffer) => 0; + } + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_dir_open(&lfs, &dir[0], "cactus") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + info.type => LFS_TYPE_DIR; + for (int i = 0; i < $LARGESIZE; i++) { + sprintf((char*)buffer, "tedd%d", i); + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, (char*)buffer) => 0; + info.type => LFS_TYPE_DIR; + } + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_unmount(&lfs) => 0; +TEST + echo "--- Multi-block remove ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_remove(&lfs, "cactus") => LFS_ERR_NOTEMPTY; for (int i = 0; i < $LARGESIZE; i++) { - sprintf((char*)buffer, "cactus/test%d", i); + sprintf((char*)buffer, "cactus/tedd%d", i); lfs_remove(&lfs, (char*)buffer) => 0; } @@ -391,13 +420,43 @@ tests/test.py << TEST lfs_unmount(&lfs) => 0; TEST +echo "--- Multi-block rename with files ---" +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + for (int i = 0; i < $LARGESIZE; i++) { + sprintf((char*)buffer, "prickly-pear/test%d", i); + sprintf((char*)wbuffer, "prickly-pear/tedd%d", i); + lfs_rename(&lfs, (char*)buffer, (char*)wbuffer) => 0; + } + lfs_unmount(&lfs) => 0; +TEST +tests/test.py << TEST + lfs_mount(&lfs, &cfg) => 0; + lfs_dir_open(&lfs, &dir[0], "prickly-pear") => 0; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, ".") => 0; + info.type => LFS_TYPE_DIR; + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, "..") => 0; + info.type => LFS_TYPE_DIR; + for (int i = 0; i < $LARGESIZE; i++) { + sprintf((char*)buffer, "tedd%d", i); + lfs_dir_read(&lfs, &dir[0], &info) => 1; + strcmp(info.name, (char*)buffer) => 0; + info.type => LFS_TYPE_REG; + info.size => 6; + } + lfs_dir_read(&lfs, &dir[0], &info) => 0; + lfs_unmount(&lfs) => 0; +TEST + echo "--- Multi-block remove with files ---" tests/test.py << TEST lfs_mount(&lfs, &cfg) => 0; lfs_remove(&lfs, "prickly-pear") => LFS_ERR_NOTEMPTY; for (int i = 0; i < $LARGESIZE; i++) { - sprintf((char*)buffer, "prickly-pear/test%d", i); + sprintf((char*)buffer, "prickly-pear/tedd%d", i); lfs_remove(&lfs, (char*)buffer) => 0; } From 195075819e05a9ce8568d3d98363f2a6f19ed436 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 8 Oct 2018 14:12:20 -0500 Subject: [PATCH 09/11] Added 2GiB file size limit and EFBIG reporting On disk, littlefs uses 32-bit integers to track file size. This sets a theoretical limit of 4GiB for files. However, the API passes file sizes around as signed numbers, with negative values representing error codes. This means that not all of the APIs will work with file sizes > 2GiB. Because of related complications over in FUSE land, I've added the LFS_FILE_MAX constant and proper error reporting if file writes/seeks exceed the 2GiB limit. In v2 this will join the other constants that get stored in the superblock to help portability. Since littlefs is targeting microcontrollers, it's likely this will be a sufficient solution. Note that it's still possible to enable partial-support for 4GiB files by defining LFS_FILE_MAX during compilation. This will work for most of the APIs, except lfs_file_seek, lfs_file_tell, and lfs_file_size. We can also consider improving support for 4GiB files, by making seek a bit more complicated and adding a lfs_file_stat function. I'll leave this for a future improvement if there's interest. Found by cgrozemuller --- lfs.c | 31 ++++++++++++++++++------------- lfs.h | 8 +++++++- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/lfs.c b/lfs.c index 438647e..ed7f687 100644 --- a/lfs.c +++ b/lfs.c @@ -1644,6 +1644,11 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, file->pos = file->size; } + if (file->pos + size > LFS_FILE_MAX) { + // larger than file limit? + return LFS_ERR_FBIG; + } + if (!(file->flags & LFS_F_WRITING) && file->pos > file->size) { // fill with zeros lfs_off_t pos = file->pos; @@ -1730,24 +1735,24 @@ lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file, return err; } - // update pos + // find new pos + lfs_soff_t npos = file->pos; if (whence == LFS_SEEK_SET) { - file->pos = off; + npos = off; } else if (whence == LFS_SEEK_CUR) { - if (off < 0 && (lfs_off_t)-off > file->pos) { - return LFS_ERR_INVAL; - } - - file->pos = file->pos + off; + npos = file->pos + off; } else if (whence == LFS_SEEK_END) { - if (off < 0 && (lfs_off_t)-off > file->size) { - return LFS_ERR_INVAL; - } - - file->pos = file->size + off; + npos = file->size + off; } - return file->pos; + if (npos < 0 || npos > LFS_FILE_MAX) { + // file position out of range + return LFS_ERR_INVAL; + } + + // update pos + file->pos = npos; + return npos; } int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) { diff --git a/lfs.h b/lfs.h index 4b687f4..9c3174e 100644 --- a/lfs.h +++ b/lfs.h @@ -21,7 +21,7 @@ extern "C" // Software library version // Major (top-nibble), incremented on backwards incompatible changes // Minor (bottom-nibble), incremented on feature additions -#define LFS_VERSION 0x00010006 +#define LFS_VERSION 0x00010007 #define LFS_VERSION_MAJOR (0xffff & (LFS_VERSION >> 16)) #define LFS_VERSION_MINOR (0xffff & (LFS_VERSION >> 0)) @@ -49,6 +49,11 @@ typedef uint32_t lfs_block_t; #define LFS_NAME_MAX 255 #endif +// Max file size in bytes +#ifndef LFS_FILE_MAX +#define LFS_FILE_MAX 2147483647 +#endif + // Possible error codes, these are negative to allow // valid positive return values enum lfs_error { @@ -61,6 +66,7 @@ enum lfs_error { LFS_ERR_ISDIR = -21, // Entry is a dir LFS_ERR_NOTEMPTY = -39, // Dir is not empty LFS_ERR_BADF = -9, // Bad file number + LFS_ERR_FBIG = -27, // File too large LFS_ERR_INVAL = -22, // Invalid parameter LFS_ERR_NOSPC = -28, // No space left on device LFS_ERR_NOMEM = -12, // No more memory available From c7894a61e1986f3feca80bf2c0e24c33290063a1 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 14 Oct 2018 11:38:51 -0500 Subject: [PATCH 10/11] Added a handful of links to related projects Interesting open-source projects that I've ran into around embedded storage. May be interesting to others in the embedded space. Added mklfs, SPIFFS, and Dhara. Also a thanks to jolivepetrus for posting the mklfs tool he put together. --- README.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/README.md b/README.md index 623ba0a..82efc54 100644 --- a/README.md +++ b/README.md @@ -175,3 +175,18 @@ handy. [littlefs-js](https://github.com/geky/littlefs-js) - A javascript wrapper for littlefs. I'm not sure why you would want this, but it is handy for demos. You can see it in action [here](http://littlefs.geky.net/demo.html). + +[mklfs](https://github.com/whitecatboard/Lua-RTOS-ESP32/tree/master/components/mklfs/src) - +A command line tool built by the [Lua RTOS](https://github.com/whitecatboard/Lua-RTOS-ESP32) +guys for making littlefs images from a host PC. Supports Windows, Mac OS, +and Linux. + +[SPIFFS](https://github.com/pellepl/spiffs) - Another excellent embedded +filesystem for NOR flash. As a more traditional logging filesystem with full +static wear-leveling, SPIFFS will likely outperform littlefs on small +memories such as the internal flash on microcontrollers. + +[Dhara](https://github.com/dlbeer/dhara) - An interesting NAND flash +translation layer designed for small MCUs. It offers static wear-leveling and +power-resilience with only a fixed O(|address|) pointer structure stored on +each block and in RAM. From ec4d8b68add6a7de021dc09ef08123ab323cbc38 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 20 Oct 2018 12:32:41 -0500 Subject: [PATCH 11/11] Changed release script to generate drafts --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e4ea93d..b28ec1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -182,6 +182,7 @@ jobs: -d "{ \"tag_name\": \"$LFS_VERSION\", \"name\": \"${LFS_VERSION%.0}\", + \"draft\": true, \"body\": $(jq -sR '.' <<< "$CHANGES") }" fi