github-mirrors/thirdparty-littlefs
1
0
Fork 0
mirror of https://github.com/eledio-devices/thirdparty-littlefs.git synced 2025-10-31 16:14:16 +01:00
Code Issues Packages Projects Releases Wiki Activity

Shoehorned in hacky implementation of inline files

Browse Source 
Proof-of-concept implementation of inline files that stores the file's
content directly in its parent's directory pair.

Inline files are indicated by a different type stored in an entry's
struct field, and take advantage of resizable entries. Where a normal
file's entry would normally hold the reference to the CTZ skip-list, an
inline file's entry contains the contents of the actual file.

Unfortunately, storing the inline file on disk is the easy part. We also
need to manage inline files in the internals of littlefs and provide the
same operations that we do on normal files, all while reusing as much
code as possible to avoid a significant increase in code cost.

There is a relatively simple, though maybe a bit hacky, solution here. If a
file fits entirely in a cache line, the file logic never actually has to go to
disk. This means we can just give the file a "pretend" block (hopefully
one that would assert if ever written to), and carry out file operations
as normal, as long as we catch the file before it exceeds the cache line
and write out the file to an actual disk.
This commit is contained in:
Christopher Haster
2018-03-17 10:28:14 -05:00
parent fb23044872
commit 836e23895a
3 changed files with 133 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

163
lfs.c
View File

@@ -24,7 +24,7 @@ static int lfs_cache_read(lfs_t *lfs, lfs_cache_t *rcache,
const lfs_cache_t *pcache, lfs_block_t block, const lfs_cache_t *pcache, lfs_block_t block,
lfs_off_t off, void *buffer, lfs_size_t size) { lfs_off_t off, void *buffer, lfs_size_t size) {
uint8_t *data = buffer; uint8_t *data = buffer;
LFS_ASSERT(block < lfs->cfg->block_count); LFS_ASSERT(block != 0xffffffff);
while (size > 0) { while (size > 0) {
if (pcache && block == pcache->block && off >= pcache->off && if (pcache && block == pcache->block && off >= pcache->off &&
@@ -68,6 +68,7 @@ static int lfs_cache_read(lfs_t *lfs, lfs_cache_t *rcache,
} }
// load to cache, first condition can no longer fail // load to cache, first condition can no longer fail
LFS_ASSERT(block < lfs->cfg->block_count);
rcache->block = block; rcache->block = block;
rcache->off = off - (off % lfs->cfg->read_size); rcache->off = off - (off % lfs->cfg->read_size);
int err = lfs->cfg->read(lfs->cfg, rcache->block, int err = lfs->cfg->read(lfs->cfg, rcache->block,
@@ -121,6 +122,7 @@ static int lfs_cache_crc(lfs_t *lfs, lfs_cache_t *rcache,
static int lfs_cache_flush(lfs_t *lfs, static int lfs_cache_flush(lfs_t *lfs,
lfs_cache_t *pcache, lfs_cache_t *rcache) { lfs_cache_t *pcache, lfs_cache_t *rcache) {
if (pcache->block != 0xffffffff) { if (pcache->block != 0xffffffff) {
LFS_ASSERT(pcache->block < lfs->cfg->block_count);
int err = lfs->cfg->prog(lfs->cfg, pcache->block, int err = lfs->cfg->prog(lfs->cfg, pcache->block,
pcache->off, pcache->buffer, lfs->cfg->prog_size); pcache->off, pcache->buffer, lfs->cfg->prog_size);
if (err) { if (err) {
@@ -149,7 +151,7 @@ static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache,
lfs_cache_t *rcache, lfs_block_t block, lfs_cache_t *rcache, lfs_block_t block,
lfs_off_t off, const void *buffer, lfs_size_t size) { lfs_off_t off, const void *buffer, lfs_size_t size) {
const uint8_t *data = buffer; const uint8_t *data = buffer;
LFS_ASSERT(block < lfs->cfg->block_count); LFS_ASSERT(block != 0xffffffff);
while (size > 0) { while (size > 0) {
if (block == pcache->block && off >= pcache->off && if (block == pcache->block && off >= pcache->off &&
@@ -181,6 +183,7 @@ static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache,
if (off % lfs->cfg->prog_size == 0 && if (off % lfs->cfg->prog_size == 0 &&
size >= lfs->cfg->prog_size) { size >= lfs->cfg->prog_size) {
// bypass pcache? // bypass pcache?
LFS_ASSERT(block < lfs->cfg->block_count);
lfs_size_t diff = size - (size % lfs->cfg->prog_size); lfs_size_t diff = size - (size % lfs->cfg->prog_size);
int err = lfs->cfg->prog(lfs->cfg, block, off, data, diff); int err = lfs->cfg->prog(lfs->cfg, block, off, data, diff);
if (err) { if (err) {
@@ -240,6 +243,7 @@ static int lfs_bd_crc(lfs_t *lfs, lfs_block_t block,
} }
static int lfs_bd_erase(lfs_t *lfs, lfs_block_t block) { static int lfs_bd_erase(lfs_t *lfs, lfs_block_t block) {
LFS_ASSERT(block < lfs->cfg->block_count);
return lfs->cfg->erase(lfs->cfg, block); return lfs->cfg->erase(lfs->cfg, block);
} }
@@ -851,7 +855,7 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir,
} }
static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) {
while (dir->off + sizeof(entry->d) > (0x7fffffff & dir->d.size)-4) { while (dir->off >= (0x7fffffff & dir->d.size)-4) {
if (!(0x80000000 & dir->d.size)) { if (!(0x80000000 & dir->d.size)) {
entry->off = dir->off; entry->off = dir->off;
return LFS_ERR_NOENT; return LFS_ERR_NOENT;
@@ -942,8 +946,8 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir,
return err; return err;
} }
if (((0x7f & entry->d.type) != (LFS_STRUCT_CTZ | LFS_TYPE_REG) && if (((0xf & entry->d.type) != LFS_TYPE_REG &&
(0x7f & entry->d.type) != (LFS_STRUCT_DIR | LFS_TYPE_DIR)) || (0xf & entry->d.type) != LFS_TYPE_DIR) ||
entry->d.nlen != pathlen) { entry->d.nlen != pathlen) {
continue; continue;
} }
@@ -1126,8 +1130,8 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) {
return (err == LFS_ERR_NOENT) ? 0 : err; return (err == LFS_ERR_NOENT) ? 0 : err;
} }
if ((0x7f & entry.d.type) != (LFS_STRUCT_CTZ | LFS_TYPE_REG) && if ((0xf & entry.d.type) != LFS_TYPE_REG &&
(0x7f & entry.d.type) != (LFS_STRUCT_DIR | LFS_TYPE_DIR)) { (0xf & entry.d.type) != LFS_TYPE_DIR) {
continue; continue;
} }
@@ -1149,8 +1153,10 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) {
} }
info->type = 0xf & entry.d.type; info->type = 0xf & entry.d.type;
if (info->type == LFS_TYPE_REG) { if (entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) {
info->size = entry.d.u.file.size; info->size = entry.d.u.file.size;
} else if (entry.d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) {
info->size = entry.d.elen;
} }
int err = lfs_bd_read(lfs, dir->pair[0], int err = lfs_bd_read(lfs, dir->pair[0],
@@ -1424,18 +1430,16 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file,
} }
// create entry to remember name // create entry to remember name
entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG; entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG;
entry.d.elen = sizeof(entry.d) - 4; entry.d.elen = 0;
entry.d.alen = 0; entry.d.alen = 0;
entry.d.nlen = strlen(path); entry.d.nlen = strlen(path);
entry.d.u.file.head = 0xffffffff;
entry.d.u.file.size = 0;
entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen;
err = lfs_dir_append(lfs, &cwd, &entry, err = lfs_dir_append(lfs, &cwd, &entry,
&(struct lfs_region){ &(struct lfs_region){
0, +sizeof(entry.d), 0, +4,
lfs_commit_mem, &entry.d, sizeof(entry.d), lfs_commit_mem, &entry.d, 4,
&(struct lfs_region){ &(struct lfs_region){
0, +entry.d.nlen, 0, +entry.d.nlen,
lfs_commit_mem, path, entry.d.nlen}}); lfs_commit_mem, path, entry.d.nlen}});
@@ -1481,6 +1485,22 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file,
} }
} }
// load inline files
if ((0x70 & entry.d.type) == LFS_STRUCT_INLINE) {
file->head = 0xfffffffe;
file->size = entry.d.elen;
file->flags |= LFS_F_INLINE;
file->cache.block = file->head;
file->cache.off = 0;
err = lfs_bd_read(lfs, cwd.pair[0],
entry.off + 4,
file->cache.buffer, file->size);
if (err) {
lfs_free(file->cache.buffer);
return err;
}
}
// add to list of files // add to list of files
file->next = lfs->files; file->next = lfs->files;
lfs->files = file; lfs->files = file;
@@ -1556,6 +1576,20 @@ relocate:
} }
static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) {
if (file->flags & LFS_F_INLINE) {
// do nothing since we won't need the cache for anything else
if (file->flags & LFS_F_READING) {
file->flags &= ~LFS_F_READING;
}
if (file->flags & LFS_F_WRITING) {
file->flags &= ~LFS_F_WRITING;
file->flags |= LFS_F_DIRTY;
}
return 0;
}
if (file->flags & LFS_F_READING) { if (file->flags & LFS_F_READING) {
// just drop read cache // just drop read cache
file->cache.block = 0xffffffff; file->cache.block = 0xffffffff;
@@ -1642,6 +1676,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) {
return err; return err;
} }
// TODO entry read?
lfs_entry_t entry = {.off = file->poff}; lfs_entry_t entry = {.off = file->poff};
err = lfs_bd_read(lfs, cwd.pair[0], entry.off, err = lfs_bd_read(lfs, cwd.pair[0], entry.off,
&entry.d, sizeof(entry.d)); &entry.d, sizeof(entry.d));
@@ -1649,19 +1684,35 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) {
if (err) { if (err) {
return err; return err;
} }
LFS_ASSERT((0xf & entry.d.type) == LFS_TYPE_REG);
entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen;
LFS_ASSERT(entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)); if (file->flags & LFS_F_INLINE) {
entry.d.u.file.head = file->head; file->size = lfs_max(file->pos, file->size);
entry.d.u.file.size = file->size; lfs_ssize_t diff = file->size - entry.d.elen;
entry.d.elen = file->size;
lfs_entry_tole32(&entry.d); err = lfs_dir_update(lfs, &cwd, &entry,
err = lfs_dir_update(lfs, &cwd, &entry, &(struct lfs_region){
&(struct lfs_region){ 0, 0,
0, 0, lfs_commit_mem, &entry.d, 4,
lfs_commit_mem, &entry.d, sizeof(entry.d)}); &(struct lfs_region){
lfs_entry_fromle32(&entry.d); 4, diff,
if (err) { lfs_commit_mem, file->cache.buffer, file->size}});
return err; if (err) {
return err;
}
} else {
entry.d.u.file.head = file->head;
entry.d.u.file.size = file->size;
err = lfs_dir_update(lfs, &cwd, &entry,
&(struct lfs_region){
0, 0,
lfs_commit_mem, &entry.d, sizeof(entry.d)});
if (err) {
return err;
}
} }
file->flags &= ~LFS_F_DIRTY; file->flags &= ~LFS_F_DIRTY;
@@ -1699,11 +1750,16 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file,
// check if we need a new block // check if we need a new block
if (!(file->flags & LFS_F_READING) || if (!(file->flags & LFS_F_READING) ||
file->off == lfs->cfg->block_size) { file->off == lfs->cfg->block_size) {
int err = lfs_ctz_find(lfs, &file->cache, NULL, if (file->flags & LFS_F_INLINE) {
file->head, file->size, file->block = 0xfffffffe;
file->pos, &file->block, &file->off); file->off = 0;
if (err) { } else {
return err; int err = lfs_ctz_find(lfs, &file->cache, NULL,
file->head, file->size,
file->pos, &file->block, &file->off);
if (err) {
return err;
}
} }
file->flags |= LFS_F_READING; file->flags |= LFS_F_READING;
@@ -1761,31 +1817,42 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file,
} }
while (nsize > 0) { while (nsize > 0) {
//printf("pos %d\n", file->pos + nsize);
// TODO combine with block allocation?
if (file->pos + nsize >= LFS_INLINE_MAX) {
file->flags &= ~LFS_F_INLINE;
}
// check if we need a new block // check if we need a new block
if (!(file->flags & LFS_F_WRITING) || if (!(file->flags & LFS_F_WRITING) ||
file->off == lfs->cfg->block_size) { file->off == lfs->cfg->block_size) {
if (!(file->flags & LFS_F_WRITING) && file->pos > 0) { if (file->flags & LFS_F_INLINE) {
// find out which block we're extending from file->block = 0xfffffffe;
int err = lfs_ctz_find(lfs, &file->cache, NULL, file->off = 0;
file->head, file->size, } else {
file->pos-1, &file->block, &file->off); if (!(file->flags & LFS_F_WRITING) && file->pos > 0) {
// find out which block we're extending from
int err = lfs_ctz_find(lfs, &file->cache, NULL,
file->head, file->size,
file->pos-1, &file->block, &file->off);
if (err) {
file->flags |= LFS_F_ERRED;
return err;
}
// mark cache as dirty since we may have read data into it
file->cache.block = 0xffffffff;
}
// extend file with new blocks
lfs_alloc_ack(lfs);
int err = lfs_ctz_extend(lfs, &lfs->rcache, &file->cache,
file->block, file->pos,
&file->block, &file->off);
if (err) { if (err) {
file->flags |= LFS_F_ERRED; file->flags |= LFS_F_ERRED;
return err; return err;
} }
// mark cache as dirty since we may have read data into it
file->cache.block = 0xffffffff;
}
// extend file with new blocks
lfs_alloc_ack(lfs);
int err = lfs_ctz_extend(lfs, &lfs->rcache, &file->cache,
file->block, file->pos,
&file->block, &file->off);
if (err) {
file->flags |= LFS_F_ERRED;
return err;
} }
file->flags |= LFS_F_WRITING; file->flags |= LFS_F_WRITING;

28
lfs.h
View File

@@ -55,6 +55,10 @@ typedef uint32_t lfs_block_t;
#define LFS_NAME_MAX 255 #define LFS_NAME_MAX 255
#endif #endif
#ifndef LFS_INLINE_MAX
#define LFS_INLINE_MAX 255
#endif
// Possible error codes, these are negative to allow // Possible error codes, these are negative to allow
// valid positive return values // valid positive return values
enum lfs_error { enum lfs_error {
@@ -82,25 +86,27 @@ enum lfs_type {
// on disk structure // on disk structure
LFS_STRUCT_CTZ = 0x10, LFS_STRUCT_CTZ = 0x10,
LFS_STRUCT_DIR = 0x20, LFS_STRUCT_DIR = 0x20,
LFS_STRUCT_INLINE = 0x30,
LFS_STRUCT_MOVED = 0x80, LFS_STRUCT_MOVED = 0x80,
}; };
// File open flags // File open flags
enum lfs_open_flags { enum lfs_open_flags {
// open flags // open flags
LFS_O_RDONLY = 1, // Open a file as read only LFS_O_RDONLY = 1, // Open a file as read only
LFS_O_WRONLY = 2, // Open a file as write only LFS_O_WRONLY = 2, // Open a file as write only
LFS_O_RDWR = 3, // Open a file as read and write LFS_O_RDWR = 3, // Open a file as read and write
LFS_O_CREAT = 0x0100, // Create a file if it does not exist LFS_O_CREAT = 0x0100, // Create a file if it does not exist
LFS_O_EXCL = 0x0200, // Fail if a file already exists LFS_O_EXCL = 0x0200, // Fail if a file already exists
LFS_O_TRUNC = 0x0400, // Truncate the existing file to zero size LFS_O_TRUNC = 0x0400, // Truncate the existing file to zero size
LFS_O_APPEND = 0x0800, // Move to end of file on every write LFS_O_APPEND = 0x0800, // Move to end of file on every write
// internally used flags // internally used flags
LFS_F_DIRTY = 0x10000, // File does not match storage LFS_F_DIRTY = 0x10000, // File does not match storage
LFS_F_WRITING = 0x20000, // File has been written since last flush LFS_F_WRITING = 0x20000, // File has been written since last flush
LFS_F_READING = 0x40000, // File has been read since last flush LFS_F_READING = 0x40000, // File has been read since last flush
LFS_F_ERRED = 0x80000, // An error occured during write LFS_F_ERRED = 0x80000, // An error occured during write
LFS_F_INLINE = 0x100000, // Currently inlined in directory entry
}; };
// File seek flags // File seek flags

2
tests/test_dirs.sh
View File

@@ -357,7 +357,7 @@ tests/test.py << TEST
TEST TEST
echo "--- Multi-block directory with files ---" echo "--- Multi-block directory with files ---"
tests/test.py << TEST tests/test.py -s << TEST
lfs_mount(&lfs, &cfg) => 0; lfs_mount(&lfs, &cfg) => 0;
lfs_mkdir(&lfs, "prickly-pear") => 0; lfs_mkdir(&lfs, "prickly-pear") => 0;
for (int i = 0; i < $LARGESIZE; i++) { for (int i = 0; i < $LARGESIZE; i++) {