From 4f08424b51c6e7fda140b91fd551a35531ca0fc3 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 28 Jan 2018 09:56:06 -0600 Subject: [PATCH 1/4] Added software implementations of bitwise instructions This helps significantly with supporting different compilers. Intrinsics for different compilers can be added as they are found. Note that for ARMCC, __builtin_ctz is not used. This was the result of a strange issue where ARMCC only emits __builtin_ctz when passed the --gnu flag, but __builtin_clz and __builtin_popcount are always emitted. This isn't a big problem since the ARM instruction set doesn't have a ctz instruction, and the npw2 based implementation is one of the most efficient. Also note that for littefs's purposes, we consider ctz(0) to be undefined. This lets us save a branch in the software lfs_ctz implementation. --- lfs_util.h | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/lfs_util.h b/lfs_util.h index 9d23ab4..586a32e 100644 --- a/lfs_util.h +++ b/lfs_util.h @@ -33,16 +33,37 @@ static inline uint32_t lfs_min(uint32_t a, uint32_t b) { return (a < b) ? a : b; } -static inline uint32_t lfs_ctz(uint32_t a) { - return __builtin_ctz(a); +static inline uint32_t lfs_npw2(uint32_t a) { +#if defined(__GNUC__) || defined(__CC_ARM) + return 32 - __builtin_clz(a-1); +#else + uint32_t r = 0; + uint32_t s; + a -= 1; + s = (a > 0xffff) << 4; a >>= s; r |= s; + s = (a > 0xff ) << 3; a >>= s; r |= s; + s = (a > 0xf ) << 2; a >>= s; r |= s; + s = (a > 0x3 ) << 1; a >>= s; r |= s; + return (r | (a >> 1)) + 1; +#endif } -static inline uint32_t lfs_npw2(uint32_t a) { - return 32 - __builtin_clz(a-1); +static inline uint32_t lfs_ctz(uint32_t a) { +#if defined(__GNUC__) + return __builtin_ctz(a); +#else + return lfs_npw2((a & -a) + 1) - 1; +#endif } static inline uint32_t lfs_popc(uint32_t a) { +#if defined(__GNUC__) || defined(__CC_ARM) return __builtin_popcount(a); +#else + a = a - ((a >> 1) & 0x55555555); + a = (a & 0x33333333) + ((a >> 2) & 0x33333333); + return (((a + (a >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24; +#endif } static inline int lfs_scmp(uint32_t a, uint32_t b) { From a0a55fb9e5942b51331947b855756abcac42a3c5 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 2 Feb 2018 05:58:43 -0600 Subject: [PATCH 2/4] Added conversion to/from little-endian on disk Required to support big-endian processors, with the most notable being the PowerPC architecture. On little-endian architectures, these conversions can be optimized out and have no code impact. Initial patch provided by gmouchard --- lfs.c | 82 ++++++++++++++++++++++++++++++++++++++++++++++++++---- lfs_util.h | 23 +++++++++++++++ 2 files changed, 99 insertions(+), 6 deletions(-) diff --git a/lfs.c b/lfs.c index b7cc675..0374853 100644 --- a/lfs.c +++ b/lfs.c @@ -323,6 +323,48 @@ static void lfs_alloc_ack(lfs_t *lfs) { } +/// Endian swapping functions /// +static void lfs_dir_fromle32(struct lfs_disk_dir *d) { + d->rev = lfs_fromle32(d->rev); + d->size = lfs_fromle32(d->size); + d->tail[0] = lfs_fromle32(d->tail[0]); + d->tail[1] = lfs_fromle32(d->tail[1]); +} + +static void lfs_dir_tole32(struct lfs_disk_dir *d) { + d->rev = lfs_tole32(d->rev); + d->size = lfs_tole32(d->size); + d->tail[0] = lfs_tole32(d->tail[0]); + d->tail[1] = lfs_tole32(d->tail[1]); +} + +static void lfs_entry_fromle32(struct lfs_disk_entry *d) { + d->u.dir[0] = lfs_fromle32(d->u.dir[0]); + d->u.dir[1] = lfs_fromle32(d->u.dir[1]); +} + +static void lfs_entry_tole32(struct lfs_disk_entry *d) { + d->u.dir[0] = lfs_tole32(d->u.dir[0]); + d->u.dir[1] = lfs_tole32(d->u.dir[1]); +} + +static void lfs_superblock_fromle32(struct lfs_disk_superblock *d) { + d->root[0] = lfs_fromle32(d->root[0]); + d->root[1] = lfs_fromle32(d->root[1]); + d->block_size = lfs_fromle32(d->block_size); + d->block_count = lfs_fromle32(d->block_count); + d->version = lfs_fromle32(d->version); +} + +static void lfs_superblock_tole32(struct lfs_disk_superblock *d) { + d->root[0] = lfs_tole32(d->root[0]); + d->root[1] = lfs_tole32(d->root[1]); + d->block_size = lfs_tole32(d->block_size); + d->block_count = lfs_tole32(d->block_count); + d->version = lfs_tole32(d->version); +} + + /// Metadata pair and directory operations /// static inline void lfs_pairswap(lfs_block_t pair[2]) { lfs_block_t t = pair[0]; @@ -364,6 +406,7 @@ static int lfs_dir_alloc(lfs_t *lfs, lfs_dir_t *dir) { // rather than clobbering one of the blocks we just pretend // the revision may be valid int err = lfs_bd_read(lfs, dir->pair[0], 0, &dir->d.rev, 4); + dir->d.rev = lfs_fromle32(dir->d.rev); if (err) { return err; } @@ -389,6 +432,7 @@ static int lfs_dir_fetch(lfs_t *lfs, for (int i = 0; i < 2; i++) { struct lfs_disk_dir test; int err = lfs_bd_read(lfs, tpair[i], 0, &test, sizeof(test)); + lfs_dir_fromle32(&test); if (err) { return err; } @@ -403,7 +447,9 @@ static int lfs_dir_fetch(lfs_t *lfs, } uint32_t crc = 0xffffffff; + lfs_dir_tole32(&test); lfs_crc(&crc, &test, sizeof(test)); + lfs_dir_fromle32(&test); err = lfs_bd_crc(lfs, tpair[i], sizeof(test), (0x7fffffff & test.size) - sizeof(test), &crc); if (err) { @@ -463,8 +509,10 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, } uint32_t crc = 0xffffffff; + lfs_dir_tole32(&dir->d); lfs_crc(&crc, &dir->d, sizeof(dir->d)); err = lfs_bd_prog(lfs, dir->pair[0], 0, &dir->d, sizeof(dir->d)); + lfs_dir_fromle32(&dir->d); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -511,7 +559,9 @@ static int lfs_dir_commit(lfs_t *lfs, lfs_dir_t *dir, } } + crc = lfs_tole32(crc); err = lfs_bd_prog(lfs, dir->pair[0], newoff, &crc, 4); + crc = lfs_fromle32(crc); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -584,11 +634,14 @@ relocate: } static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, - const lfs_entry_t *entry, const void *data) { - return lfs_dir_commit(lfs, dir, (struct lfs_region[]){ + lfs_entry_t *entry, const void *data) { + lfs_entry_tole32(&entry->d); + int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ {entry->off, sizeof(entry->d), &entry->d, sizeof(entry->d)}, {entry->off+sizeof(entry->d), entry->d.nlen, data, entry->d.nlen} }, data ? 2 : 1); + lfs_entry_fromle32(&entry->d); + return err; } static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, @@ -597,10 +650,14 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, while (true) { if (dir->d.size + lfs_entry_size(entry) <= lfs->cfg->block_size) { entry->off = dir->d.size - 4; - return lfs_dir_commit(lfs, dir, (struct lfs_region[]){ + + lfs_entry_tole32(&entry->d); + int err = lfs_dir_commit(lfs, dir, (struct lfs_region[]){ {entry->off, 0, &entry->d, sizeof(entry->d)}, {entry->off, 0, data, entry->d.nlen} }, 2); + lfs_entry_fromle32(&entry->d); + return err; } // we need to allocate a new dir block @@ -614,10 +671,12 @@ static int lfs_dir_append(lfs_t *lfs, lfs_dir_t *dir, newdir.d.tail[0] = dir->d.tail[0]; newdir.d.tail[1] = dir->d.tail[1]; entry->off = newdir.d.size - 4; + lfs_entry_tole32(&entry->d); err = lfs_dir_commit(lfs, &newdir, (struct lfs_region[]){ {entry->off, 0, &entry->d, sizeof(entry->d)}, {entry->off, 0, data, entry->d.nlen} }, 2); + lfs_entry_fromle32(&entry->d); if (err) { return err; } @@ -703,6 +762,7 @@ static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { int err = lfs_bd_read(lfs, dir->pair[0], dir->off, &entry->d, sizeof(entry->d)); + lfs_entry_fromle32(&entry->d); if (err) { return err; } @@ -1065,6 +1125,7 @@ static int lfs_ctz_find(lfs_t *lfs, lfs_ctz(current)); int err = lfs_cache_read(lfs, rcache, pcache, head, 4*skip, &head, 4); + head = lfs_fromle32(head); if (err) { return err; } @@ -1140,8 +1201,10 @@ static int lfs_ctz_extend(lfs_t *lfs, lfs_size_t skips = lfs_ctz(index) + 1; for (lfs_off_t i = 0; i < skips; i++) { + head = lfs_tole32(head); err = lfs_cache_prog(lfs, pcache, rcache, nblock, 4*i, &head, 4); + head = lfs_fromle32(head); if (err) { if (err == LFS_ERR_CORRUPT) { goto relocate; @@ -1152,6 +1215,7 @@ static int lfs_ctz_extend(lfs_t *lfs, if (i != skips-1) { err = lfs_cache_read(lfs, rcache, NULL, head, 4*i, &head, 4); + head = lfs_fromle32(head); if (err) { return err; } @@ -1196,6 +1260,8 @@ static int lfs_ctz_traverse(lfs_t *lfs, lfs_block_t heads[2]; int count = 2 - (index & 1); err = lfs_cache_read(lfs, rcache, pcache, head, 0, &heads, count*4); + heads[0] = lfs_fromle32(heads[0]); + heads[1] = lfs_fromle32(heads[1]); if (err) { return err; } @@ -1456,6 +1522,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { lfs_entry_t entry = {.off = file->poff}; err = lfs_bd_read(lfs, cwd.pair[0], entry.off, &entry.d, sizeof(entry.d)); + lfs_entry_fromle32(&entry.d); if (err) { return err; } @@ -2075,6 +2142,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { superdir.d.size = sizeof(superdir.d) + sizeof(superblock.d) + 4; // write both pairs to be safe + lfs_superblock_tole32(&superblock.d); bool valid = false; for (int i = 0; i < 2; i++) { err = lfs_dir_commit(lfs, &superdir, (struct lfs_region[]){ @@ -2125,6 +2193,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { if (!err) { err = lfs_bd_read(lfs, dir.pair[0], sizeof(dir.d), &superblock.d, sizeof(superblock.d)); + lfs_superblock_fromle32(&superblock.d); if (err) { return err; } @@ -2182,6 +2251,7 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { while (dir.off + sizeof(entry.d) <= (0x7fffffff & dir.d.size)-4) { err = lfs_bd_read(lfs, dir.pair[0], dir.off, &entry.d, sizeof(entry.d)); + lfs_entry_fromle32(&entry.d); if (err) { return err; } @@ -2222,7 +2292,7 @@ int lfs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data) { } } } - + return 0; } @@ -2256,7 +2326,7 @@ static int lfs_parent(lfs_t *lfs, const lfs_block_t dir[2], if (lfs_pairisnull(lfs->root)) { return 0; } - + parent->d.tail[0] = 0; parent->d.tail[1] = 1; @@ -2317,7 +2387,7 @@ static int lfs_moved(lfs_t *lfs, const void *e) { break; } - if (!(0x80 & entry.d.type) && + if (!(0x80 & entry.d.type) && memcmp(&entry.d.u, e, sizeof(entry.d.u)) == 0) { return true; } diff --git a/lfs_util.h b/lfs_util.h index 586a32e..9fe77de 100644 --- a/lfs_util.h +++ b/lfs_util.h @@ -70,6 +70,29 @@ static inline int lfs_scmp(uint32_t a, uint32_t b) { return (int)(unsigned)(a - b); } +static inline uint32_t lfs_fromle32(uint32_t a) { +#if ( \ + (defined( BYTE_ORDER ) && BYTE_ORDER == ORDER_LITTLE_ENDIAN ) || \ + (defined(__BYTE_ORDER ) && __BYTE_ORDER == __ORDER_LITTLE_ENDIAN ) || \ + (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) + return a; +#elif ( \ + (defined( BYTE_ORDER ) && BYTE_ORDER == ORDER_BIG_ENDIAN ) || \ + (defined(__BYTE_ORDER ) && __BYTE_ORDER == __ORDER_BIG_ENDIAN ) || \ + (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) + return __builtin_bswap32(a); +#else + return (((uint8_t*)&a)[0] << 0) | + (((uint8_t*)&a)[1] << 8) | + (((uint8_t*)&a)[2] << 16) | + (((uint8_t*)&a)[3] << 24); +#endif +} + +static inline uint32_t lfs_tole32(uint32_t a) { + return lfs_fromle32(a); +} + // CRC-32 with polynomial = 0x04c11db7 void lfs_crc(uint32_t *crc, const void *buffer, size_t size); From a3fd2d4d6d5e125f878177931a8bd52e5d114f7a Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 29 Jan 2018 15:20:12 -0600 Subject: [PATCH 3/4] Added more configurable utils Note: It's still expected to modify lfs_utils.h when porting littlefs to a new target/system. There's just too much room for system-specific improvements, such as taking advantage of CRC hardware. Rather, encouraging modification of lfs_util.h and making it easy to modify and debug should result in better integration with the consuming systems. This just adds a bunch of quality-of-life improvements that should help development and integration in littlefs. - Macros that require no side-effects are all-caps - System includes are only brought in when needed - Malloc/free wrappers - LFS_NO_* checks for quickly disabling things at the command line - At least a little-bit more docs --- lfs.c | 52 ++++++++++++++---------------- lfs.h | 2 +- lfs_util.h | 92 +++++++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 103 insertions(+), 43 deletions(-) diff --git a/lfs.c b/lfs.c index 0374853..a0e0b0f 100644 --- a/lfs.c +++ b/lfs.c @@ -18,17 +18,13 @@ #include "lfs.h" #include "lfs_util.h" -#include -#include -#include - /// Caching block device operations /// static int lfs_cache_read(lfs_t *lfs, lfs_cache_t *rcache, const lfs_cache_t *pcache, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size) { uint8_t *data = buffer; - assert(block < lfs->cfg->block_count); + LFS_ASSERT(block < lfs->cfg->block_count); while (size > 0) { if (pcache && block == pcache->block && off >= pcache->off && @@ -153,7 +149,7 @@ static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_block_t block, lfs_off_t off, const void *buffer, lfs_size_t size) { const uint8_t *data = buffer; - assert(block < lfs->cfg->block_count); + LFS_ASSERT(block < lfs->cfg->block_count); while (size > 0) { if (block == pcache->block && off >= pcache->off && @@ -180,7 +176,7 @@ static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache, // pcache must have been flushed, either by programming and // entire block or manually flushing the pcache - assert(pcache->block == 0xffffffff); + LFS_ASSERT(pcache->block == 0xffffffff); if (off % lfs->cfg->prog_size == 0 && size >= lfs->cfg->prog_size) { @@ -1130,7 +1126,7 @@ static int lfs_ctz_find(lfs_t *lfs, return err; } - assert(head >= 2 && head <= lfs->cfg->block_count); + LFS_ASSERT(head >= 2 && head <= lfs->cfg->block_count); current -= 1 << skip; } @@ -1150,7 +1146,7 @@ static int lfs_ctz_extend(lfs_t *lfs, if (err) { return err; } - assert(nblock >= 2 && nblock <= lfs->cfg->block_count); + LFS_ASSERT(nblock >= 2 && nblock <= lfs->cfg->block_count); if (true) { err = lfs_bd_erase(lfs, nblock); @@ -1221,7 +1217,7 @@ static int lfs_ctz_extend(lfs_t *lfs, } } - assert(head >= 2 && head <= lfs->cfg->block_count); + LFS_ASSERT(head >= 2 && head <= lfs->cfg->block_count); } *block = nblock; @@ -1347,12 +1343,12 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, if (lfs->cfg->file_buffer) { file->cache.buffer = lfs->cfg->file_buffer; } else if ((file->flags & 3) == LFS_O_RDONLY) { - file->cache.buffer = malloc(lfs->cfg->read_size); + file->cache.buffer = lfs_malloc(lfs->cfg->read_size); if (!file->cache.buffer) { return LFS_ERR_NOMEM; } } else { - file->cache.buffer = malloc(lfs->cfg->prog_size); + file->cache.buffer = lfs_malloc(lfs->cfg->prog_size); if (!file->cache.buffer) { return LFS_ERR_NOMEM; } @@ -1378,7 +1374,7 @@ int lfs_file_close(lfs_t *lfs, lfs_file_t *file) { // clean up memory if (!lfs->cfg->file_buffer) { - free(file->cache.buffer); + lfs_free(file->cache.buffer); } return err; @@ -1527,7 +1523,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } - assert(entry.d.type == LFS_TYPE_REG); + LFS_ASSERT(entry.d.type == LFS_TYPE_REG); entry.d.u.file.head = file->head; entry.d.u.file.size = file->size; @@ -1753,7 +1749,7 @@ int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size) { // flush+seek if not already at end if (file->pos != oldsize) { - int err = lfs_file_seek(lfs, file, 0, SEEK_END); + int err = lfs_file_seek(lfs, file, 0, LFS_SEEK_END); if (err < 0) { return err; } @@ -1886,7 +1882,7 @@ int lfs_remove(lfs_t *lfs, const char *path) { return res; } - assert(res); // must have pred + LFS_ASSERT(res); // must have pred cwd.d.tail[0] = dir.d.tail[0]; cwd.d.tail[1] = dir.d.tail[1]; @@ -2003,7 +1999,7 @@ int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath) { return res; } - assert(res); // must have pred + LFS_ASSERT(res); // must have pred newcwd.d.tail[0] = dir.d.tail[0]; newcwd.d.tail[1] = dir.d.tail[1]; @@ -2026,7 +2022,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { if (lfs->cfg->read_buffer) { lfs->rcache.buffer = lfs->cfg->read_buffer; } else { - lfs->rcache.buffer = malloc(lfs->cfg->read_size); + lfs->rcache.buffer = lfs_malloc(lfs->cfg->read_size); if (!lfs->rcache.buffer) { return LFS_ERR_NOMEM; } @@ -2037,30 +2033,30 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { if (lfs->cfg->prog_buffer) { lfs->pcache.buffer = lfs->cfg->prog_buffer; } else { - lfs->pcache.buffer = malloc(lfs->cfg->prog_size); + lfs->pcache.buffer = lfs_malloc(lfs->cfg->prog_size); if (!lfs->pcache.buffer) { return LFS_ERR_NOMEM; } } // setup lookahead, round down to nearest 32-bits - assert(lfs->cfg->lookahead % 32 == 0); - assert(lfs->cfg->lookahead > 0); + LFS_ASSERT(lfs->cfg->lookahead % 32 == 0); + LFS_ASSERT(lfs->cfg->lookahead > 0); if (lfs->cfg->lookahead_buffer) { lfs->free.buffer = lfs->cfg->lookahead_buffer; } else { - lfs->free.buffer = malloc(lfs->cfg->lookahead/8); + lfs->free.buffer = lfs_malloc(lfs->cfg->lookahead/8); if (!lfs->free.buffer) { return LFS_ERR_NOMEM; } } // check that program and read sizes are multiples of the block size - assert(lfs->cfg->prog_size % lfs->cfg->read_size == 0); - assert(lfs->cfg->block_size % lfs->cfg->prog_size == 0); + LFS_ASSERT(lfs->cfg->prog_size % lfs->cfg->read_size == 0); + LFS_ASSERT(lfs->cfg->block_size % lfs->cfg->prog_size == 0); // check that the block size is large enough to fit ctz pointers - assert(4*lfs_npw2(0xffffffff / (lfs->cfg->block_size-2*4)) + LFS_ASSERT(4*lfs_npw2(0xffffffff / (lfs->cfg->block_size-2*4)) <= lfs->cfg->block_size); // setup default state @@ -2076,15 +2072,15 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { static int lfs_deinit(lfs_t *lfs) { // free allocated memory if (!lfs->cfg->read_buffer) { - free(lfs->rcache.buffer); + lfs_free(lfs->rcache.buffer); } if (!lfs->cfg->prog_buffer) { - free(lfs->pcache.buffer); + lfs_free(lfs->pcache.buffer); } if (!lfs->cfg->lookahead_buffer) { - free(lfs->free.buffer); + lfs_free(lfs->free.buffer); } return 0; diff --git a/lfs.h b/lfs.h index e24c308..ad37985 100644 --- a/lfs.h +++ b/lfs.h @@ -27,7 +27,7 @@ // Software library version // Major (top-nibble), incremented on backwards incompatible changes // Minor (bottom-nibble), incremented on feature additions -#define LFS_VERSION 0x00010002 +#define LFS_VERSION 0x00010003 #define LFS_VERSION_MAJOR (0xffff & (LFS_VERSION >> 16)) #define LFS_VERSION_MINOR (0xffff & (LFS_VERSION >> 0)) diff --git a/lfs_util.h b/lfs_util.h index 9fe77de..0b7ccae 100644 --- a/lfs_util.h +++ b/lfs_util.h @@ -18,13 +18,60 @@ #ifndef LFS_UTIL_H #define LFS_UTIL_H -#include #include +#include +#include + +#ifndef LFS_NO_MALLOC +#include +#endif +#ifndef LFS_NO_ASSERT +#include +#endif +#if !defined(LFS_NO_DEBUG) || !defined(LFS_NO_WARN) || !defined(LFS_NO_ERROR) #include +#endif -// Builtin functions, these may be replaced by more -// efficient implementations in the system +// Macros, may be replaced by system specific wrappers. Arguments to these +// macros must not have side-effects as the macros can be removed for a smaller +// code footprint + +// Logging functions +#ifndef LFS_NO_DEBUG +#define LFS_DEBUG(fmt, ...) \ + printf("lfs debug:%d: " fmt "\n", __LINE__, __VA_ARGS__) +#else +#define LFS_DEBUG(fmt, ...) +#endif + +#ifndef LFS_NO_WARN +#define LFS_WARN(fmt, ...) \ + printf("lfs warn:%d: " fmt "\n", __LINE__, __VA_ARGS__) +#else +#define LFS_WARN(fmt, ...) +#endif + +#ifndef LFS_NO_ERROR +#define LFS_ERROR(fmt, ...) \ + printf("lfs error:%d: " fmt "\n", __LINE__, __VA_ARGS__) +#else +#define LFS_ERROR(fmt, ...) +#endif + +// Runtime assertions +#ifndef LFS_NO_ASSERT +#define LFS_ASSERT(test) assert(test) +#else +#define LFS_ASSERT(test) +#endif + + +// Builtin functions, these may be replaced by more efficient +// toolchain-specific implementations. LFS_NO_INTRINSICS falls back to a more +// expensive basic C implementation for debugging purposes + +// Min/max functions for unsigned 32-bit numbers static inline uint32_t lfs_max(uint32_t a, uint32_t b) { return (a > b) ? a : b; } @@ -33,8 +80,9 @@ static inline uint32_t lfs_min(uint32_t a, uint32_t b) { return (a < b) ? a : b; } +// Find the next smallest power of 2 less than or equal to a static inline uint32_t lfs_npw2(uint32_t a) { -#if defined(__GNUC__) || defined(__CC_ARM) +#if !defined(LFS_NO_INTRINSICS) && (defined(__GNUC__) || defined(__CC_ARM)) return 32 - __builtin_clz(a-1); #else uint32_t r = 0; @@ -48,16 +96,19 @@ static inline uint32_t lfs_npw2(uint32_t a) { #endif } +// Count the number of trailing binary zeros in a +// lfs_ctz(0) may be undefined static inline uint32_t lfs_ctz(uint32_t a) { -#if defined(__GNUC__) +#if !defined(LFS_NO_INTRINSICS) && defined(__GNUC__) return __builtin_ctz(a); #else return lfs_npw2((a & -a) + 1) - 1; #endif } +// Count the number of binary ones in a static inline uint32_t lfs_popc(uint32_t a) { -#if defined(__GNUC__) || defined(__CC_ARM) +#if !defined(LFS_NO_INTRINSICS) && (defined(__GNUC__) || defined(__CC_ARM)) return __builtin_popcount(a); #else a = a - ((a >> 1) & 0x55555555); @@ -66,17 +117,20 @@ static inline uint32_t lfs_popc(uint32_t a) { #endif } +// Find the sequence comparison of a and b, this is the distance +// between a and b ignoring overflow static inline int lfs_scmp(uint32_t a, uint32_t b) { return (int)(unsigned)(a - b); } +// Convert from 32-bit little-endian to native order static inline uint32_t lfs_fromle32(uint32_t a) { -#if ( \ +#if !defined(LFS_NO_INTRINSICS) && ( \ (defined( BYTE_ORDER ) && BYTE_ORDER == ORDER_LITTLE_ENDIAN ) || \ (defined(__BYTE_ORDER ) && __BYTE_ORDER == __ORDER_LITTLE_ENDIAN ) || \ (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) return a; -#elif ( \ +#elif !defined(LFS_NO_INTRINSICS) && ( \ (defined( BYTE_ORDER ) && BYTE_ORDER == ORDER_BIG_ENDIAN ) || \ (defined(__BYTE_ORDER ) && __BYTE_ORDER == __ORDER_BIG_ENDIAN ) || \ (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) @@ -89,19 +143,29 @@ static inline uint32_t lfs_fromle32(uint32_t a) { #endif } +// Convert to 32-bit little-endian from native order static inline uint32_t lfs_tole32(uint32_t a) { return lfs_fromle32(a); } -// CRC-32 with polynomial = 0x04c11db7 +// Calculate CRC-32 with polynomial = 0x04c11db7 void lfs_crc(uint32_t *crc, const void *buffer, size_t size); +// Allocate memory, only used if buffers are not provided to littlefs +static inline void *lfs_malloc(size_t size) { +#ifndef LFS_NO_MALLOC + return malloc(size); +#else + return NULL; +#endif +} -// Logging functions, these may be replaced by system-specific -// logging functions -#define LFS_DEBUG(fmt, ...) printf("lfs debug: " fmt "\n", __VA_ARGS__) -#define LFS_WARN(fmt, ...) printf("lfs warn: " fmt "\n", __VA_ARGS__) -#define LFS_ERROR(fmt, ...) printf("lfs error: " fmt "\n", __VA_ARGS__) +// Deallocate memory, only used if buffers are not provided to littlefs +static inline void lfs_free(void *p) { +#ifndef LFS_NO_MALLOC + free(p); +#endif +} #endif From 67daf9e2c5f98e1cd820bc8fd5fd9e725a2a9390 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 18 Feb 2018 14:22:24 -0600 Subject: [PATCH 4/4] Added cross-compile targets for testing Using gcc cross compilers and qemu: - make test CC="arm-linux-gnueabi-gcc --static -mthumb" EXEC="qemu-arm" - make test CC="powerpc-linux-gnu-gcc --static" EXEC="qemu-ppc" - make test CC="mips-linux-gnu-gcc --static" EXEC="qemu-mips" Also separated out Travis jobs and added some size reporting --- .travis.yml | 271 ++++++++++++++++++++++++++++++++++---------------- Makefile | 6 +- tests/test.py | 9 +- 3 files changed, 197 insertions(+), 89 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8538492..14f91da 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,107 +1,210 @@ +# Environment variables env: + global: - CFLAGS=-Werror +# Common test script script: - # make sure example can at least compile - - sed -n '/``` c/,/```/{/```/d; p;}' README.md > test.c && - make all size CFLAGS+=" + # make sure example can at least compile + - sed -n '/``` c/,/```/{/```/d; p;}' README.md > test.c && + make all CFLAGS+=" -Duser_provided_block_device_read=NULL -Duser_provided_block_device_prog=NULL -Duser_provided_block_device_erase=NULL -Duser_provided_block_device_sync=NULL -include stdio.h" - # run tests - - make test QUIET=1 + # run tests + - make test QUIET=1 - # run tests with a few different configurations - - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=1 -DLFS_PROG_SIZE=1" - - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=512 -DLFS_PROG_SIZE=512" - - make test QUIET=1 CFLAGS+="-DLFS_BLOCK_COUNT=1023 -DLFS_LOOKAHEAD=2048" + # run tests with a few different configurations + - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=1 -DLFS_PROG_SIZE=1" + - make test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=512 -DLFS_PROG_SIZE=512" + - make test QUIET=1 CFLAGS+="-DLFS_BLOCK_COUNT=1023 -DLFS_LOOKAHEAD=2048" + + - make clean test QUIET=1 CFLAGS+="-DLFS_NO_INTRINSICS" + + # compile and find the code size with the smalles configuration + - make clean size + OBJ="$(ls lfs*.o | tr '\n' ' ')" + CFLAGS+="-DLFS_NO{ASSERT,DEBUG,WARN,ERROR}" + | tee sizes + + # update status if we succeeded, compare with master if possible + - | + if [ "$TRAVIS_TEST_RESULT" -eq 0 ] + then + CURR=$(tail -n1 sizes | awk '{print $1}') + STATUS="Passed, code size is ${CURR}B" + + PREV=$(curl https://api.github.com/repos/$TRAVIS_REPO_SLUG/status/master \ + | jq -r ".statuses[] | select(.context == \"ci/$NAME\").description" \ + | sed 's/.*code size is \([0-9]*\).*/\1/' \ + || echo 0) + if [ "$PREV" -ne 0 ] + then + STATUS="$STATUS ($(python -c "print '%+.2f' % (100*($CURR-$PREV)/$PREV.0)")%)" + fi + fi + +# CI matrix +matrix: + include: + # native testing + - env: + - NAME=littlefs-x86 + + # cross-compile with ARM (thumb mode) + - env: + - NAME=littlefs-arm + - CC="arm-linux-gnueabi-gcc --static -mthumb" + - EXEC="qemu-arm" + install: + - sudo apt-get install gcc-arm-linux-gnueabi qemu-user + - arm-linux-gnueabi-gcc --version + - qemu-arm -version + + # cross-compile with PowerPC + - env: + - NAME=littlefs-powerpc + - CC="powerpc-linux-gnu-gcc --static" + - EXEC="qemu-ppc" + install: + - sudo apt-get install gcc-powerpc-linux-gnu qemu-user + - powerpc-linux-gnu-gcc --version + - qemu-ppc -version + + # cross-compile with MIPS + - env: + - NAME=littlefs-mips + - CC="mips-linux-gnu-gcc --static" + - EXEC="qemu-mips" + install: + - sudo add-apt-repository -y "deb http://archive.ubuntu.com/ubuntu/ xenial main universe" + - sudo apt-get -qq update + - sudo apt-get install gcc-mips-linux-gnu qemu-user + - mips-linux-gnu-gcc --version + - qemu-mips -version # self-host with littlefs-fuse for fuzz test - - make -C littlefs-fuse + - env: + - NAME=littlefs-fuse + install: + - sudo apt-get install libfuse-dev + - git clone --depth 1 https://github.com/geky/littlefs-fuse + - fusermount -V + - gcc --version + before_script: + # setup disk for littlefs-fuse + - rm -rf littlefs-fuse/littlefs/* + - cp -r $(git ls-tree --name-only HEAD) littlefs-fuse/littlefs - - littlefs-fuse/lfs --format /dev/loop0 - - littlefs-fuse/lfs /dev/loop0 mount + - mkdir mount + - sudo chmod a+rw /dev/loop0 + - dd if=/dev/zero bs=512 count=2048 of=disk + - losetup /dev/loop0 disk + script: + # self-host test + - make -C littlefs-fuse - - ls mount - - mkdir mount/littlefs - - cp -r $(git ls-tree --name-only HEAD) mount/littlefs - - cd mount/littlefs - - ls - - make -B test_dirs test_files QUIET=1 + - littlefs-fuse/lfs --format /dev/loop0 + - littlefs-fuse/lfs /dev/loop0 mount + - ls mount + - mkdir mount/littlefs + - cp -r $(git ls-tree --name-only HEAD) mount/littlefs + - cd mount/littlefs + - ls + - make -B test_dirs test_files QUIET=1 + +# Manage statuses before_install: - - fusermount -V - - gcc --version + - | + curl -u $GEKY_BOT_STATUSES -X POST \ + https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \ + -d "{ + \"context\": \"ci/$NAME\", + \"state\": \"pending\", + \"description\": \"${STATUS:-In progress}\", + \"target_url\": \"https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$TRAVIS_JOB_ID\" + }" -install: - - sudo apt-get install libfuse-dev - - git clone --depth 1 https://github.com/geky/littlefs-fuse +after_failure: + - | + curl -u $GEKY_BOT_STATUSES -X POST \ + https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \ + -d "{ + \"context\": \"ci/$NAME\", + \"state\": \"failure\", + \"description\": \"${STATUS:-Failed}\", + \"target_url\": \"https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$TRAVIS_JOB_ID\" + }" -before_script: - - rm -rf littlefs-fuse/littlefs/* - - cp -r $(git ls-tree --name-only HEAD) littlefs-fuse/littlefs - - - mkdir mount - - sudo chmod a+rw /dev/loop0 - - dd if=/dev/zero bs=512 count=2048 of=disk - - losetup /dev/loop0 disk +after_success: + - | + curl -u $GEKY_BOT_STATUSES -X POST \ + https://api.github.com/repos/$TRAVIS_REPO_SLUG/statuses/${TRAVIS_PULL_REQUEST_SHA:-$TRAVIS_COMMIT} \ + -d "{ + \"context\": \"ci/$NAME\", + \"state\": \"success\", + \"description\": \"${STATUS:-Passed}\", + \"target_url\": \"https://travis-ci.org/$TRAVIS_REPO_SLUG/jobs/$TRAVIS_JOB_ID\" + }" +# Automatically update releases deploy: - # Let before_deploy take over - provider: script - script: 'true' - on: - branch: master + # Let before_deploy take over + provider: script + script: 'true' + on: + branch: master before_deploy: - - cd $TRAVIS_BUILD_DIR - # Update tag for version defined in lfs.h - - LFS_VERSION=$(grep -ox '#define LFS_VERSION .*' lfs.h | cut -d ' ' -f3) - - LFS_VERSION_MAJOR=$((0xffff & ($LFS_VERSION >> 16))) - - LFS_VERSION_MINOR=$((0xffff & ($LFS_VERSION >> 0))) - - LFS_VERSION="v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR" - - echo "littlefs version $LFS_VERSION" - - | - curl -u $GEKY_BOT -X POST \ - https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs \ - -d "{ - \"ref\": \"refs/tags/$LFS_VERSION\", - \"sha\": \"$TRAVIS_COMMIT\" - }" - - | - curl -f -u $GEKY_BOT -X PATCH \ - https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs/tags/$LFS_VERSION \ - -d "{ - \"sha\": \"$TRAVIS_COMMIT\" - }" - # Create release notes from commits - - LFS_PREV_VERSION="v$LFS_VERSION_MAJOR.$(($LFS_VERSION_MINOR-1))" - - | - if [ $(git tag -l "$LFS_PREV_VERSION") ] - then - curl -u $GEKY_BOT -X POST \ - https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases \ - -d "{ - \"tag_name\": \"$LFS_VERSION\", - \"name\": \"$LFS_VERSION\" - }" - RELEASE=$( - curl -f https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases/tags/$LFS_VERSION - ) - CHANGES=$( - git log --oneline $LFS_PREV_VERSION.. --grep='^Merge' --invert-grep - ) - curl -f -u $GEKY_BOT -X PATCH \ - https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases/$( - jq -r '.id' <<< "$RELEASE" - ) \ - -d "$( - jq -s '{ - "body": ((.[0] // "" | sub("(?<=\n)#+ Changes.*"; ""; "mi")) - + "### Changes\n\n" + .[1]) - }' <(jq '.body' <<< "$RELEASE") <(jq -sR '.' <<< "$CHANGES") - )" - fi + - cd $TRAVIS_BUILD_DIR + # Update tag for version defined in lfs.h + - LFS_VERSION=$(grep -ox '#define LFS_VERSION .*' lfs.h | cut -d ' ' -f3) + - LFS_VERSION_MAJOR=$((0xffff & ($LFS_VERSION >> 16))) + - LFS_VERSION_MINOR=$((0xffff & ($LFS_VERSION >> 0))) + - LFS_VERSION="v$LFS_VERSION_MAJOR.$LFS_VERSION_MINOR" + - echo "littlefs version $LFS_VERSION" + - | + curl -u $GEKY_BOT_RELEASES -X POST \ + https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs \ + -d "{ + \"ref\": \"refs/tags/$LFS_VERSION\", + \"sha\": \"$TRAVIS_COMMIT\" + }" + - | + curl -f -u $GEKY_BOT_RELEASES -X PATCH \ + https://api.github.com/repos/$TRAVIS_REPO_SLUG/git/refs/tags/$LFS_VERSION \ + -d "{ + \"sha\": \"$TRAVIS_COMMIT\" + }" + # Create release notes from commits + - LFS_PREV_VERSION="v$LFS_VERSION_MAJOR.$(($LFS_VERSION_MINOR-1))" + - | + if [ $(git tag -l "$LFS_PREV_VERSION") ] + then + curl -u $GEKY_BOT_RELEASES -X POST \ + https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases \ + -d "{ + \"tag_name\": \"$LFS_VERSION\", + \"name\": \"$LFS_VERSION\" + }" + RELEASE=$( + curl -f https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases/tags/$LFS_VERSION + ) + CHANGES=$( + git log --oneline $LFS_PREV_VERSION.. --grep='^Merge' --invert-grep + ) + curl -f -u $GEKY_BOT_RELEASES -X PATCH \ + https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases/$( + jq -r '.id' <<< "$RELEASE" + ) \ + -d "$( + jq -s '{ + "body": ((.[0] // "" | sub("(?<=\n)#+ Changes.*"; ""; "mi")) + + "### Changes\n\n" + .[1]) + }' <(jq '.body' <<< "$RELEASE") <(jq -sR '.' <<< "$CHANGES") + )" + fi diff --git a/Makefile b/Makefile index 259ed0e..ee71793 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ TARGET = lfs -CC = gcc -AR = ar -SIZE = size +CC ?= gcc +AR ?= ar +SIZE ?= size SRC += $(wildcard *.c emubd/*.c) OBJ := $(SRC:.c=.o) diff --git a/tests/test.py b/tests/test.py index e74bc3c..24b0d1a 100755 --- a/tests/test.py +++ b/tests/test.py @@ -33,10 +33,15 @@ def generate(test): pass def compile(): - subprocess.check_call(['make', '--no-print-directory', '-s']) + subprocess.check_call([ + os.environ.get('MAKE', 'make'), + '--no-print-directory', '-s']) def execute(): - subprocess.check_call(["./lfs"]) + if 'EXEC' in os.environ: + subprocess.check_call([os.environ['EXEC'], "./lfs"]) + else: + subprocess.check_call(["./lfs"]) def main(test=None): if test and not test.startswith('-'):