Added thread-safe build+size reporting to CI

This removes quite a bit of extra code needed to entertwine the
LFS_TRACE calls into the original funcions.

Also changed temporary return type to match API declaration where
necessary.

.travis.yml

@@ -208,22 +208,6 @@ jobs:
     script:
       - make test TFLAGS+="-k --valgrind"
 
-  # test compilation in read-only mode
-  - stage: test
-    env:
-      - NAME=littlefs-readonly
-      - CC="arm-linux-gnueabi-gcc --static -mthumb"
-      - CFLAGS="-Werror -DLFS_READONLY"
-    if: branch !~ -prefix$
-    install:
-      - *install-common
-      - sudo apt-get install
-            gcc-arm-linux-gnueabi
-            libc6-dev-armel-cross
-      - arm-linux-gnueabi-gcc --version
-    # report-size will compile littlefs and report the size
-    script: [*report-size]
-
   # test compilation in thread-safe mode
   - stage: test
     env:

Makefile

@@ -6,7 +6,6 @@ endif
 CC ?= gcc
 AR ?= ar
 SIZE ?= size
-NM ?= nm
 
 SRC += $(wildcard *.c bd/*.c)
 OBJ := $(SRC:.c=.o)
@@ -30,7 +29,6 @@ override CFLAGS += -Wextra -Wshadow -Wjump-misses-init -Wundef
 
 ifdef VERBOSE
 override TFLAGS += -v
-override SFLAGS += -v
 endif
 
 
@@ -41,9 +39,6 @@ asm: $(ASM)
 size: $(OBJ)
 	$(SIZE) -t $^
 
-code_size:
-	./scripts/code_size.py $(SFLAGS)
-
 test:
 	./scripts/test.py $(TFLAGS)
 .SECONDEXPANSION:
@@ -70,4 +65,3 @@ clean:
 	rm -f $(DEP)
 	rm -f $(ASM)
 	rm -f tests/*.toml.*
-	rm -f sizes/*

README.md

@@ -221,11 +221,6 @@ License Identifiers that are here available: http://spdx.org/licenses/
 - [littlefs-js] - A javascript wrapper for littlefs. I'm not sure why you would
   want this, but it is handy for demos.  You can see it in action
   [here][littlefs-js-demo].
-  
-- [littlefs-python] - A Python wrapper for littlefs. The project allows you
-  to create images of the filesystem on your PC. Check if littlefs will fit
-  your needs, create images for a later download to the target memory or
-  inspect the content of a binary image of the target memory.
 
 - [mklfs] - A command line tool built by the [Lua RTOS] guys for making
   littlefs images from a host PC. Supports Windows, Mac OS, and Linux.
@@ -255,4 +250,3 @@ License Identifiers that are here available: http://spdx.org/licenses/
 [LittleFileSystem]: https://os.mbed.com/docs/mbed-os/v5.12/apis/littlefilesystem.html
 [SPIFFS]: https://github.com/pellepl/spiffs
 [Dhara]: https://github.com/dlbeer/dhara
-[littlefs-python]: https://pypi.org/project/littlefs-python/

bd/lfs_testbd.c

@@ -207,7 +207,7 @@ int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block,
         bd->power_cycles -= 1;
         if (bd->power_cycles == 0) {
             // sync to make sure we persist the last changes
-            LFS_ASSERT(lfs_testbd_rawsync(cfg) == 0);
+            assert(lfs_testbd_rawsync(cfg) == 0);
             // simulate power loss
             exit(33);
         }
@@ -254,7 +254,7 @@ int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) {
         bd->power_cycles -= 1;
         if (bd->power_cycles == 0) {
             // sync to make sure we persist the last changes
-            LFS_ASSERT(lfs_testbd_rawsync(cfg) == 0);
+            assert(lfs_testbd_rawsync(cfg) == 0);
             // simulate power loss
             exit(33);
         }

lfs.h

@@ -22,7 +22,7 @@ extern "C"
 // Software library version
 // Major (top-nibble), incremented on backwards incompatible changes
 // Minor (bottom-nibble), incremented on feature additions
-#define LFS_VERSION 0x00020003
+#define LFS_VERSION 0x00020002
 #define LFS_VERSION_MAJOR (0xffff & (LFS_VERSION >> 16))
 #define LFS_VERSION_MINOR (0xffff & (LFS_VERSION >>  0))
 
@@ -124,25 +124,20 @@ enum lfs_type {
 enum lfs_open_flags {
     // open flags
     LFS_O_RDONLY = 1,         // Open a file as read only
-#ifndef LFS_READONLY
     LFS_O_WRONLY = 2,         // Open a file as write only
     LFS_O_RDWR   = 3,         // Open a file as read and write
     LFS_O_CREAT  = 0x0100,    // Create a file if it does not exist
     LFS_O_EXCL   = 0x0200,    // Fail if a file already exists
     LFS_O_TRUNC  = 0x0400,    // Truncate the existing file to zero size
     LFS_O_APPEND = 0x0800,    // Move to end of file on every write
-#endif
 
     // internally used flags
-#ifndef LFS_READONLY
     LFS_F_DIRTY   = 0x010000, // File does not match storage
     LFS_F_WRITING = 0x020000, // File has been written since last flush
-#endif
     LFS_F_READING = 0x040000, // File has been read since last flush
-#ifndef LFS_READONLY
-    LFS_F_ERRED   = 0x080000, // An error occurred during write
-#endif
+    LFS_F_ERRED   = 0x080000, // An error occured during write
     LFS_F_INLINE  = 0x100000, // Currently inlined in directory entry
+    LFS_F_OPENED  = 0x200000, // File has been opened
 };
 
 // File seek flags
@@ -415,7 +410,6 @@ typedef struct lfs {
 
 /// Filesystem functions ///
 
-#ifndef LFS_READONLY
 // Format a block device with the littlefs
 //
 // Requires a littlefs object and config struct. This clobbers the littlefs
@@ -424,7 +418,6 @@ typedef struct lfs {
 //
 // Returns a negative error code on failure.
 int lfs_format(lfs_t *lfs, const struct lfs_config *config);
-#endif
 
 // Mounts a littlefs
 //
@@ -444,15 +437,12 @@ int lfs_unmount(lfs_t *lfs);
 
 /// General operations ///
 
-#ifndef LFS_READONLY
 // Removes a file or directory
 //
 // If removing a directory, the directory must be empty.
 // Returns a negative error code on failure.
 int lfs_remove(lfs_t *lfs, const char *path);
-#endif
 
-#ifndef LFS_READONLY
 // Rename or move a file or directory
 //
 // If the destination exists, it must match the source in type.
@@ -460,7 +450,6 @@ int lfs_remove(lfs_t *lfs, const char *path);
 //
 // Returns a negative error code on failure.
 int lfs_rename(lfs_t *lfs, const char *oldpath, const char *newpath);
-#endif
 
 // Find info about a file or directory
 //
@@ -483,7 +472,6 @@ int lfs_stat(lfs_t *lfs, const char *path, struct lfs_info *info);
 lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path,
         uint8_t type, void *buffer, lfs_size_t size);
 
-#ifndef LFS_READONLY
 // Set custom attributes
 //
 // Custom attributes are uniquely identified by an 8-bit type and limited
@@ -493,16 +481,13 @@ lfs_ssize_t lfs_getattr(lfs_t *lfs, const char *path,
 // Returns a negative error code on failure.
 int lfs_setattr(lfs_t *lfs, const char *path,
         uint8_t type, const void *buffer, lfs_size_t size);
-#endif
 
-#ifndef LFS_READONLY
 // Removes a custom attribute
 //
 // If an attribute is not found, nothing happens.
 //
 // Returns a negative error code on failure.
 int lfs_removeattr(lfs_t *lfs, const char *path, uint8_t type);
-#endif
 
 
 /// File operations ///
@@ -551,7 +536,6 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file);
 lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file,
         void *buffer, lfs_size_t size);
 
-#ifndef LFS_READONLY
 // Write data to file
 //
 // Takes a buffer and size indicating the data to write. The file will not
@@ -560,7 +544,6 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file,
 // Returns the number of bytes written, or a negative error code on failure.
 lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file,
         const void *buffer, lfs_size_t size);
-#endif
 
 // Change the position of the file
 //
@@ -569,12 +552,10 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file,
 lfs_soff_t lfs_file_seek(lfs_t *lfs, lfs_file_t *file,
         lfs_soff_t off, int whence);
 
-#ifndef LFS_READONLY
 // Truncates the size of the file to the specified size
 //
 // Returns a negative error code on failure.
 int lfs_file_truncate(lfs_t *lfs, lfs_file_t *file, lfs_off_t size);
-#endif
 
 // Return the position of the file
 //
@@ -597,12 +578,10 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file);
 
 /// Directory operations ///
 
-#ifndef LFS_READONLY
 // Create a directory
 //
 // Returns a negative error code on failure.
 int lfs_mkdir(lfs_t *lfs, const char *path);
-#endif
 
 // Open a directory
 //
@@ -664,7 +643,6 @@ lfs_ssize_t lfs_fs_size(lfs_t *lfs);
 // Returns a negative error code on failure.
 int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data);
 
-#ifndef LFS_READONLY
 #ifdef LFS_MIGRATE
 // Attempts to migrate a previous version of littlefs
 //
@@ -679,7 +657,6 @@ int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data);
 // Returns a negative error code on failure.
 int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg);
 #endif
-#endif
 
 
 #ifdef __cplusplus

scripts/code_size.py

@@ -1,328 +0,0 @@
-#!/usr/bin/env python3
-#
-# This script finds the code size at the function level, with/without
-# static functions, and has some conveniences for comparing different
-# versions. It's basically one big wrapper around nm, and may or may
-# not have been written out of jealousy of Linux's Bloat-O-Meter.
-#
-# Here's a useful bash script to use while developing:
-# ./scripts/code_size.py -qo old.csv
-# while true ; do ./code_scripts/size.py -d old.csv ; inotifywait -rqe modify * ; done
-#
-# Or even better, to automatically update results on commit:
-# ./scripts/code_size.py -qo commit.csv
-# while true ; do ./scripts/code_size.py -d commit.csv -o current.csv ; git diff --exit-code --quiet && cp current.csv commit.csv ; inotifywait -rqe modify * ; done
-#
-# Or my personal favorite:
-# ./scripts/code_size.py -qo master.csv && cp master.csv commit.csv
-# while true ; do ( ./scripts/code_size.py -i commit.csv -d master.csv -s ; ./scripts/code_size.py -i current.csv -d master.csv -s ; ./scripts/code_size.py -d master.csv -o current.csv -s ) | awk 'BEGIN {printf "%-16s %7s %7s %7s\n","","old","new","diff"} (NR==2 && $1="commit") || (NR==4 && $1="prev") || (NR==6 && $1="current") {printf "%-16s %7s %7s %7s %s\n",$1,$2,$3,$5,$6}' ; git diff --exit-code --quiet && cp current.csv commit.csv ; inotifywait -rqe modify * ; done
-#
-
-import os
-import itertools as it
-import subprocess as sp
-import shlex
-import re
-import csv
-import collections as co
-
-SIZEDIR = 'sizes'
-RULES = """
-define FLATTEN
-%(sizedir)s/%(build)s.$(subst /,.,$(target)): $(target)
-    ( echo "#line 1 \\"$$<\\"" ; %(cat)s $$< ) > $$@
-%(sizedir)s/%(build)s.$(subst /,.,$(target:.c=.size)): \\
-        %(sizedir)s/%(build)s.$(subst /,.,$(target:.c=.o))
-    $(NM) --size-sort $$^ | sed 's/^/$(subst /,\\/,$(target:.c=.o)):/' > $$@
-endef
-$(foreach target,$(SRC),$(eval $(FLATTEN)))
-
--include %(sizedir)s/*.d
-.SECONDARY:
-
-%%.size: $(foreach t,$(subst /,.,$(SRC:.c=.size)),%%.$t)
-    cat $^ > $@
-"""
-CATS = {
-    'code': 'cat',
-    'code_inlined': 'sed \'s/^static\( inline\)\?//\'',
-}
-
-def build(**args):
-    # mkdir -p sizedir
-    os.makedirs(args['sizedir'], exist_ok=True)
-
-    if args.get('inlined', False):
-        builds = ['code', 'code_inlined']
-    else:
-        builds = ['code']
-
-    # write makefiles for the different types of builds
-    makefiles = []
-    targets = []
-    for build in builds:
-        path = args['sizedir'] + '/' + build
-        with open(path + '.mk', 'w') as mk:
-            mk.write(RULES.replace(4*' ', '\t') % dict(
-                sizedir=args['sizedir'],
-                build=build,
-                cat=CATS[build]))
-            mk.write('\n')
-
-            # pass on defines
-            for d in args['D']:
-                mk.write('%s: override CFLAGS += -D%s\n' % (
-                    path+'.size', d))
-
-        makefiles.append(path + '.mk')
-        targets.append(path + '.size')
-
-    # build in parallel
-    cmd = (['make', '-f', 'Makefile'] +
-        list(it.chain.from_iterable(['-f', m] for m in makefiles)) +
-        [target for target in targets])
-    if args.get('verbose', False):
-        print(' '.join(shlex.quote(c) for c in cmd))
-    proc = sp.Popen(cmd,
-        stdout=sp.DEVNULL if not args.get('verbose', False) else None)
-    proc.wait()
-    if proc.returncode != 0:
-        sys.exit(-1)
-
-    # find results
-    build_results = co.defaultdict(lambda: 0)
-    # notes
-    # - filters type
-    # - discards internal/debug functions (leading __)
-    pattern = re.compile(
-        '^(?P<file>[^:]+)' +
-        ':(?P<size>[0-9a-fA-F]+)' +
-        ' (?P<type>[%s])' % re.escape(args['type']) +
-        ' (?!__)(?P<name>.+?)$')
-    for build in builds:
-        path = args['sizedir'] + '/' + build
-        with open(path + '.size') as size:
-            for line in size:
-                match = pattern.match(line)
-                if match:
-                    file = match.group('file')
-                    # discard .8449 suffixes created by optimizer
-                    name = re.sub('\.[0-9]+', '', match.group('name'))
-                    size = int(match.group('size'), 16)
-                    build_results[(build, file, name)] += size
-
-    results = []
-    for (build, file, name), size in build_results.items():
-        if build == 'code':
-            results.append((file, name, size, False))
-        elif (build == 'code_inlined' and
-                ('inlined', file, name) not in results):
-            results.append((file, name, size, True))
-
-    return results
-
-def main(**args):
-    # find results
-    if not args.get('input', None):
-        results = build(**args)
-    else:
-        with open(args['input']) as f:
-            r = csv.DictReader(f)
-            results = [
-                (   result['file'],
-                    result['name'],
-                    int(result['size']),
-                    bool(int(result.get('inlined', 0))))
-                for result in r
-                if (not bool(int(result.get('inlined', 0))) or
-                    args.get('inlined', False))]
-
-    total = 0
-    for _, _, size, inlined in results:
-        if not inlined:
-            total += size
-
-    # find previous results?
-    if args.get('diff', None):
-        with open(args['diff']) as f:
-            r = csv.DictReader(f)
-            prev_results = [
-                (   result['file'],
-                    result['name'],
-                    int(result['size']),
-                    bool(int(result.get('inlined', 0))))
-                for result in r
-                if (not bool(int(result.get('inlined', 0))) or
-                    args.get('inlined', False))]
-
-        prev_total = 0
-        for _, _, size, inlined in prev_results:
-            if not inlined:
-                prev_total += size
-
-    # write results to CSV
-    if args.get('output', None):
-        results.sort(key=lambda x: (-x[2], x))
-        with open(args['output'], 'w') as f:
-            w = csv.writer(f)
-            if args.get('inlined', False):
-                w.writerow(['file', 'name', 'size', 'inlined'])
-                for file, name, size, inlined in results:
-                    w.writerow((file, name, size, int(inlined)))
-            else:
-                w.writerow(['file', 'name', 'size'])
-                for file, name, size, inlined in results:
-                    w.writerow((file, name, size))
-
-    # print results
-    def dedup_functions(results):
-        functions = co.defaultdict(lambda: (0, True))
-        for _, name, size, inlined in results:
-            if not inlined:
-                functions[name] = (functions[name][0] + size, False)
-        for _, name, size, inlined in results:
-            if inlined and functions[name][1]:
-                functions[name] = (functions[name][0] + size, True)
-        return functions
-
-    def dedup_files(results):
-        files = co.defaultdict(lambda: 0)
-        for file, _, size, inlined in results:
-            if not inlined:
-                files[file] += size
-        return files
-
-    def diff_sizes(olds, news):
-        diff = co.defaultdict(lambda: (None, None, None))
-        for name, new in news.items():
-            diff[name] = (None, new, new)
-        for name, old in olds.items():
-            new = diff[name][1] or 0
-            diff[name] = (old, new, new-old)
-        return diff
-
-    def print_header(name=''):
-        if not args.get('diff', False):
-            print('%-40s %7s' % (name, 'size'))
-        else:
-            print('%-40s %7s %7s %7s' % (name, 'old', 'new', 'diff'))
-
-    def print_functions():
-        functions = dedup_functions(results)
-        functions = {
-            name+' (inlined)' if inlined else name: size
-            for name, (size, inlined) in functions.items()}
-
-        if not args.get('diff', None):
-            print_header('function')
-            for name, size in sorted(functions.items(),
-                    key=lambda x: (-x[1], x)):
-                print("%-40s %7d" % (name, size))
-        else:
-            prev_functions = dedup_functions(prev_results)
-            prev_functions = {
-                name+' (inlined)' if inlined else name: size
-                for name, (size, inlined) in prev_functions.items()}
-            diff = diff_sizes(functions, prev_functions)
-            print_header('function (%d added, %d removed)' % (
-                sum(1 for old, _, _ in diff.values() if not old),
-                sum(1 for _, new, _ in diff.values() if not new)))
-            for name, (old, new, diff) in sorted(diff.items(),
-                    key=lambda x: (-(x[1][2] or 0), x)):
-                if diff or args.get('all', False):
-                    print("%-40s %7s %7s %+7d%s" % (
-                        name, old or "-", new or "-", diff,
-                        ' (%+.2f%%)' % (100*((new-old)/old))
-                        if old and new else
-                        ''))
-
-    def print_files():
-        files = dedup_files(results)
-
-        if not args.get('diff', None):
-            print_header('file')
-            for file, size in sorted(files.items(),
-                    key=lambda x: (-x[1], x)):
-                print("%-40s %7d" % (file, size))
-        else:
-            prev_files = dedup_files(prev_results)
-            diff = diff_sizes(files, prev_files)
-            print_header('file (%d added, %d removed)' % (
-                sum(1 for old, _, _ in diff.values() if not old),
-                sum(1 for _, new, _ in diff.values() if not new)))
-            for name, (old, new, diff) in sorted(diff.items(),
-                    key=lambda x: (-(x[1][2] or 0), x)):
-                if diff or args.get('all', False):
-                    print("%-40s %7s %7s %+7d%s" % (
-                        name, old or "-", new or "-", diff,
-                        ' (%+.2f%%)' % (100*((new-old)/old))
-                        if old and new else
-                        ''))
-
-    def print_totals():
-        if not args.get('diff', None):
-            print("%-40s %7d" % ('TOTALS', total))
-        else:
-            print("%-40s %7s %7s %+7d%s" % (
-                'TOTALS', prev_total, total, total-prev_total,
-                ' (%+.2f%%)' % (100*((total-prev_total)/total))
-                if prev_total and total else
-                ''))
-
-    def print_status():
-        if not args.get('diff', None):
-            print(total)
-        else:
-            print("%d (%+.2f%%)" % (total, 100*((total-prev_total)/total)))
-
-    if args.get('quiet', False):
-        pass
-    elif args.get('status', False):
-        print_status()
-    elif args.get('summary', False):
-        print_header()
-        print_totals()
-    elif args.get('files', False):
-        print_files()
-        print_totals()
-    else:
-        print_functions()
-        print_totals()
-
-if __name__ == "__main__":
-    import argparse
-    import sys
-    parser = argparse.ArgumentParser(
-        description="Find code size at the function level.")
-    parser.add_argument('sizedir', nargs='?', default=SIZEDIR,
-        help="Directory to store intermediary results. Defaults "
-            "to \"%s\"." % SIZEDIR)
-    parser.add_argument('-D', action='append', default=[],
-        help="Specify compile-time define.")
-    parser.add_argument('-v', '--verbose', action='store_true',
-        help="Output commands that run behind the scenes.")
-    parser.add_argument('-i', '--input',
-        help="Don't compile and find code sizes, instead use this CSV file.")
-    parser.add_argument('-o', '--output',
-        help="Specify CSV file to store results.")
-    parser.add_argument('-d', '--diff',
-        help="Specify CSV file to diff code size against.")
-    parser.add_argument('-a', '--all', action='store_true',
-        help="Show all functions, not just the ones that changed.")
-    parser.add_argument('--inlined', action='store_true',
-        help="Run a second compilation to find the sizes of functions normally "
-            "removed by optimizations. These will be shown as \"*.inlined\" "
-            "functions, and will not be included in the total.")
-    parser.add_argument('--files', action='store_true',
-        help="Show file-level code sizes. Note this does not include padding! "
-            "So sizes may differ from other tools.")
-    parser.add_argument('-s', '--summary', action='store_true',
-        help="Only show the total code size.")
-    parser.add_argument('-S', '--status', action='store_true',
-        help="Show minimum info useful for a single-line status.")
-    parser.add_argument('-q', '--quiet', action='store_true',
-        help="Don't show anything, useful with -o.")
-    parser.add_argument('--type', default='tTrRdDbB',
-        help="Type of symbols to report, this uses the same single-character "
-            "type-names emitted by nm. Defaults to %(default)r.")
-    sys.exit(main(**vars(parser.parse_args())))

scripts/readtree.py

@@ -106,7 +106,7 @@ def main(args):
             struct.unpack('<HH', superblock[1].data[0:4].ljust(4, b'\xff'))))
     print("%-47s%s" % ("littlefs v%s.%s" % version,
         "data (truncated, if it fits)"
-        if not any([args.no_truncate, args.log, args.all]) else ""))
+        if not any([args.no_truncate, args.tags, args.log, args.all]) else ""))
 
     # print gstate
     print("gstate 0x%s" % ''.join('%02x' % c for c in gstate))