github-mirrors/thirdparty-littlefs
1
0
Fork 0
mirror of https://github.com/eledio-devices/thirdparty-littlefs.git synced 2025-10-31 00:32:38 +01:00
Code Issues Packages Projects Releases Wiki Activity

Added reentrant and gdb testing mechanisms to test framework

Browse Source 
Aside from reworking the internals of test_.py to work well with
inherited TestCase classes, this also provides the two main features
that were the main reason for revamping the test framework

1. ./scripts/test_.py --reentrant

   Runs reentrant tests (tests with reentrant=true in the .toml
   configuration) under gdb such that the program is killed on every
   call to lfs_emubd_prog or lfs_emubd_erase.

   Currently this just increments a number of prog/erases to skip, which
   means it doesn't necessarily check every possible branch of the test,
   but this should still provide a good coverage of power-loss tests.

2. ./scripts/test_.py --gdb

   Run the tests and if a failure is hit, drop into GDB. In theory this
   will be very useful for reproducing and debugging test failures.

   Note this can be combined with --reentrant to drop into GDB on the
   exact cycle of power-loss where the tests fail.
This commit is contained in:
Christopher Haster
2019-12-31 11:51:52 -06:00
parent ed8341ec4c
commit 53d2b02f2a
3 changed files with 165 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
scripts/explode_asserts.py
View File

@@ -16,7 +16,8 @@ ASSERT_TESTS = {
printf("%s:%d:assert: " printf("%s:%d:assert: "
"assert failed with %"PRIiMAX", expected {comp} %"PRIiMAX"\\n", "assert failed with %"PRIiMAX", expected {comp} %"PRIiMAX"\\n",
{file}, {line}, (intmax_t)_lh, (intmax_t)_rh); {file}, {line}, (intmax_t)_lh, (intmax_t)_rh);
exit(-2); fflush(NULL);
raise(SIGABRT);
}} }}
""", """,
'str': """ 'str': """
@@ -26,7 +27,8 @@ ASSERT_TESTS = {
printf("%s:%d:assert: " printf("%s:%d:assert: "
"assert failed with \\\"%s\\\", expected {comp} \\\"%s\\\"\\n", "assert failed with \\\"%s\\\", expected {comp} \\\"%s\\\"\\n",
{file}, {line}, _lh, _rh); {file}, {line}, _lh, _rh);
exit(-2); fflush(NULL);
raise(SIGABRT);
}} }}
""", """,
'bool': """ 'bool': """
@@ -36,7 +38,8 @@ ASSERT_TESTS = {
printf("%s:%d:assert: " printf("%s:%d:assert: "
"assert failed with %s, expected {comp} %s\\n", "assert failed with %s, expected {comp} %s\\n",
{file}, {line}, _lh ? "true" : "false", _rh ? "true" : "false"); {file}, {line}, _lh ? "true" : "false", _rh ? "true" : "false");
exit(-2); fflush(NULL);
raise(SIGABRT);
}} }}
""", """,
} }
@@ -180,6 +183,7 @@ def main(args):
outf.write("#include <stdbool.h>\n") outf.write("#include <stdbool.h>\n")
outf.write("#include <stdint.h>\n") outf.write("#include <stdint.h>\n")
outf.write("#include <inttypes.h>\n") outf.write("#include <inttypes.h>\n")
outf.write("#include <signal.h>\n")
outf.write(mkdecl('int', 'eq', '==')) outf.write(mkdecl('int', 'eq', '=='))
outf.write(mkdecl('int', 'ne', '!=')) outf.write(mkdecl('int', 'ne', '!='))
outf.write(mkdecl('int', 'lt', '<')) outf.write(mkdecl('int', 'lt', '<'))

214
scripts/test_.py
View File

@@ -16,9 +16,9 @@ import base64
import sys import sys
import copy import copy
import shutil import shutil
import shlex
TEST_DIR = 'tests_' TESTDIR = 'tests_'
RULES = """ RULES = """
define FLATTEN define FLATTEN
%$(subst /,.,$(target:.c=.t.c)): $(target) %$(subst /,.,$(target:.c=.t.c)): $(target)
@@ -28,9 +28,12 @@ $(foreach target,$(SRC),$(eval $(FLATTEN)))
-include tests_/*.d -include tests_/*.d
.SECONDARY:
%.c: %.t.c %.c: %.t.c
./scripts/explode_asserts.py $< -o $@ ./scripts/explode_asserts.py $< -o $@
%.test: override CFLAGS += -fdiagnostics-color=always
%.test: override CFLAGS += -ggdb
%.test: %.test.o $(foreach f,$(subst /,.,$(SRC:.c=.o)),%.test.$f) %.test: %.test.o $(foreach f,$(subst /,.,$(SRC:.c=.o)),%.test.$f)
$(CC) $(CFLAGS) $^ $(LFLAGS) -o $@ $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@
""" """
@@ -60,18 +63,18 @@ PROLOGUE = """
__attribute__((unused)) char path[1024]; __attribute__((unused)) char path[1024];
__attribute__((unused)) const struct lfs_config cfg = { __attribute__((unused)) const struct lfs_config cfg = {
.context = &bd, .context = &bd,
.read = &lfs_emubd_read, .read = &lfs_emubd_read,
.prog = &lfs_emubd_prog, .prog = &lfs_emubd_prog,
.erase = &lfs_emubd_erase, .erase = &lfs_emubd_erase,
.sync = &lfs_emubd_sync, .sync = &lfs_emubd_sync,
.read_size = LFS_READ_SIZE, .read_size = LFS_READ_SIZE,
.prog_size = LFS_PROG_SIZE, .prog_size = LFS_PROG_SIZE,
.block_size = LFS_BLOCK_SIZE, .block_size = LFS_BLOCK_SIZE,
.block_count = LFS_BLOCK_COUNT, .block_count = LFS_BLOCK_COUNT,
.block_cycles = LFS_BLOCK_CYCLES, .block_cycles = LFS_BLOCK_CYCLES,
.cache_size = LFS_CACHE_SIZE, .cache_size = LFS_CACHE_SIZE,
.lookahead_size = LFS_LOOKAHEAD_SIZE, .lookahead_size = LFS_LOOKAHEAD_SIZE,
}; };
@@ -85,13 +88,14 @@ PASS = '\033[32m✓\033[0m'
FAIL = '\033[31m✗\033[0m' FAIL = '\033[31m✗\033[0m'
class TestFailure(Exception): class TestFailure(Exception):
def __init__(self, case, stdout=None, assert_=None): def __init__(self, case, returncode=None, stdout=None, assert_=None):
self.case = case self.case = case
self.returncode = returncode
self.stdout = stdout self.stdout = stdout
self.assert_ = assert_ self.assert_ = assert_
class TestCase: class TestCase:
def __init__(self, suite, config, caseno=None, lineno=None, **_): def __init__(self, config, suite=None, caseno=None, lineno=None, **_):
self.suite = suite self.suite = suite
self.caseno = caseno self.caseno = caseno
self.lineno = lineno self.lineno = lineno
@@ -148,25 +152,29 @@ class TestCase:
f.write('}\n') f.write('}\n')
def test(self, **args): def test(self, exec=[], persist=False, gdb=False, failure=None, **args):
# clear disk first # clear disk first
shutil.rmtree('blocks') if not persist:
shutil.rmtree('blocks', True)
# build command # build command
cmd = ['./%s.test' % self.suite.path, cmd = exec + ['./%s.test' % self.suite.path,
repr(self.caseno), repr(self.permno)] repr(self.caseno), repr(self.permno)]
# run in valgrind? # failed? drop into debugger?
if args.get('valgrind', False) and not self.leaky: if gdb and failure:
cmd = ['valgrind', cmd = (['gdb', '-ex', 'r'
'--leak-check=full', ] + (['-ex', 'up'] if failure.assert_ else []) + [
'--error-exitcode=4', '--args'] + cmd)
'-q'] + cmd if args.get('verbose', False):
print(' '.join(shlex.quote(c) for c in cmd))
sys.exit(sp.call(cmd))
# run test case! # run test case!
stdout = [] stdout = []
assert_ = None
if args.get('verbose', False): if args.get('verbose', False):
print(' '.join(cmd)) print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd, proc = sp.Popen(cmd,
universal_newlines=True, universal_newlines=True,
bufsize=1, bufsize=1,
@@ -176,33 +184,84 @@ class TestCase:
stdout.append(line) stdout.append(line)
if args.get('verbose', False): if args.get('verbose', False):
sys.stdout.write(line) sys.stdout.write(line)
proc.wait() # intercept asserts
m = re.match('^([^:]+):([0-9]+):(assert): (.*)$', line)
if proc.returncode != 0: if m and assert_ is None:
# failed, try to parse assert?
assert_ = None
for line in stdout:
try: try:
m = re.match('^([^:\\n]+):([0-9]+):assert: (.*)$', line)
# found an assert, print info from file
with open(m.group(1)) as f: with open(m.group(1)) as f:
lineno = int(m.group(2)) lineno = int(m.group(2))
line = next(it.islice(f, lineno-1, None)).strip('\n') line = next(it.islice(f, lineno-1, None)).strip('\n')
assert_ = { assert_ = {
'path': m.group(1), 'path': m.group(1),
'lineno': lineno, 'line': line,
'line': line, 'lineno': lineno,
'message': m.group(3), 'message': m.group(4)}
}
except: except:
pass pass
proc.wait()
self.result = TestFailure(self, stdout, assert_) # did we pass?
raise self.result if proc.returncode != 0:
raise TestFailure(self, proc.returncode, stdout, assert_)
else: else:
self.result = PASS return PASS
return self.result
class ValgrindTestCase(TestCase):
def __init__(self, config, **args):
self.leaky = config.get('leaky', False)
super().__init__(config, **args)
def test(self, exec=[], **args):
if self.leaky:
return
exec = exec + [
'valgrind',
'--leak-check=full',
'--error-exitcode=4',
'-q']
return super().test(exec=exec, **args)
class ReentrantTestCase(TestCase):
def __init__(self, config, **args):
self.reentrant = config.get('reentrant', False)
super().__init__(config, **args)
def test(self, exec=[], persist=False, gdb=False, failure=None, **args):
if not self.reentrant:
return
for cycles in it.count(1):
npersist = persist or cycles > 1
# exact cycle we should drop into debugger?
if gdb and failure and failure.cycleno == cycles:
return super().test(exec=exec, persist=npersist,
gdb=gdb, failure=failure, **args)
# run tests, but kill the program after lfs_emubd_prog/erase has
# been hit n cycles. We exit with a special return code if the
# program has not finished, since this isn't a test failure.
nexec = exec + [
'gdb', '-batch-silent',
'-ex', 'handle all nostop',
'-ex', 'b lfs_emubd_prog',
'-ex', 'b lfs_emubd_erase',
'-ex', 'r',
] + cycles*['-ex', 'c'] + [
'-ex', 'q '
'!$_isvoid($_exitsignal) ? $_exitsignal : '
'!$_isvoid($_exitcode) ? $_exitcode : '
'33',
'--args']
try:
return super().test(exec=nexec, persist=npersist, **args)
except TestFailure as nfailure:
if nfailure.returncode == 33:
continue
else:
nfailure.cycleno = cycles
raise
class TestSuite: class TestSuite:
def __init__(self, path, TestCase=TestCase, **args): def __init__(self, path, TestCase=TestCase, **args):
@@ -229,8 +288,8 @@ class TestSuite:
# create initial test cases # create initial test cases
self.cases = [] self.cases = []
for i, (case, lineno) in enumerate(zip(config['case'], linenos)): for i, (case, lineno) in enumerate(zip(config['case'], linenos)):
self.cases.append(self.TestCase( self.cases.append(self.TestCase(case,
self, case, caseno=i, lineno=lineno, **args)) suite=self, caseno=i, lineno=lineno, **args))
def __str__(self): def __str__(self):
return self.name return self.name
@@ -343,7 +402,7 @@ class TestSuite:
# write test.c in base64 so make can decide when to rebuild # write test.c in base64 so make can decide when to rebuild
mk.write('%s: %s\n' % (self.path+'.test.t.c', self.path)) mk.write('%s: %s\n' % (self.path+'.test.t.c', self.path))
mk.write('\tbase64 -d <<< ') mk.write('\t@base64 -d <<< ')
mk.write(base64.b64encode( mk.write(base64.b64encode(
f.getvalue().encode('utf8')).decode('utf8')) f.getvalue().encode('utf8')).decode('utf8'))
mk.write(' > $@\n') mk.write(' > $@\n')
@@ -364,8 +423,9 @@ class TestSuite:
continue continue
try: try:
perm.test(**args) result = perm.test(**args)
except TestFailure as failure: except TestFailure as failure:
perm.result = failure
if not args.get('verbose', True): if not args.get('verbose', True):
sys.stdout.write(FAIL) sys.stdout.write(FAIL)
sys.stdout.flush() sys.stdout.flush()
@@ -374,9 +434,11 @@ class TestSuite:
sys.stdout.write('\n') sys.stdout.write('\n')
raise raise
else: else:
if not args.get('verbose', True): if result == PASS:
sys.stdout.write(PASS) perm.result = PASS
sys.stdout.flush() if not args.get('verbose', True):
sys.stdout.write(PASS)
sys.stdout.flush()
if not args.get('verbose', True): if not args.get('verbose', True):
sys.stdout.write('\n') sys.stdout.write('\n')
@@ -400,14 +462,19 @@ def main(**args):
elif os.path.isfile(testpath): elif os.path.isfile(testpath):
testpath = testpath testpath = testpath
elif testpath.endswith('.toml'): elif testpath.endswith('.toml'):
testpath = TEST_DIR + '/' + testpath testpath = TESTDIR + '/' + testpath
else: else:
testpath = TEST_DIR + '/' + testpath + '.toml' testpath = TESTDIR + '/' + testpath + '.toml'
# find tests # find tests
suites = [] suites = []
for path in glob.glob(testpath): for path in glob.glob(testpath):
suites.append(TestSuite(path, **args)) if args.get('valgrind', False):
suites.append(TestSuite(path, TestCase=ValgrindTestCase, **args))
elif args.get('reentrant', False):
suites.append(TestSuite(path, TestCase=ReentrantTestCase, **args))
else:
suites.append(TestSuite(path, **args))
# sort for reproducability # sort for reproducability
suites = sorted(suites) suites = sorted(suites)
@@ -432,11 +499,10 @@ def main(**args):
cmd = (['make', '-f', 'Makefile'] + cmd = (['make', '-f', 'Makefile'] +
list(it.chain.from_iterable(['-f', m] for m in makefiles)) + list(it.chain.from_iterable(['-f', m] for m in makefiles)) +
['CFLAGS+=-fdiagnostics-color=always'] +
[target for target in targets]) [target for target in targets])
stdout = [] stdout = []
if args.get('verbose', False): if args.get('verbose', False):
print(' '.join(cmd)) print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd, proc = sp.Popen(cmd,
universal_newlines=True, universal_newlines=True,
bufsize=1, bufsize=1,
@@ -466,6 +532,18 @@ def main(**args):
except TestFailure: except TestFailure:
pass pass
if args.get('gdb', False):
failure = None
for suite in suites:
for perm in suite.perms:
if getattr(perm, 'result', PASS) != PASS:
failure = perm.result
if failure is not None:
print('======= gdb ======')
# drop into gdb
failure.case.test(failure=failure, **args)
sys.exit(0)
print('====== results ======') print('====== results ======')
passed = 0 passed = 0
failed = 0 failed = 0
@@ -498,26 +576,26 @@ if __name__ == "__main__":
import argparse import argparse
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Run parameterized tests in various configurations.") description="Run parameterized tests in various configurations.")
parser.add_argument('testpath', nargs='?', default=TEST_DIR, parser.add_argument('testpath', nargs='?', default=TESTDIR,
help="Description of test(s) to run. By default, this is all tests \ help="Description of test(s) to run. By default, this is all tests \
found in the \"{0}\" directory. Here, you can specify a different \ found in the \"{0}\" directory. Here, you can specify a different \
directory of tests, a specific file, a suite by name, and even a \ directory of tests, a specific file, a suite by name, and even a \
specific test case by adding brackets. For example \ specific test case by adding brackets. For example \
\"test_dirs[0]\" or \"{0}/test_dirs.toml[0]\".".format(TEST_DIR)) \"test_dirs[0]\" or \"{0}/test_dirs.toml[0]\".".format(TESTDIR))
parser.add_argument('-D', action='append', default=[], parser.add_argument('-D', action='append', default=[],
help="Overriding parameter definitions.") help="Overriding parameter definitions.")
parser.add_argument('-v', '--verbose', action='store_true', parser.add_argument('-v', '--verbose', action='store_true',
help="Output everything that is happening.") help="Output everything that is happening.")
parser.add_argument('-t', '--trace', action='store_true',
help="Normally trace output is captured for internal usage, this \
enables forwarding trace output which is usually too verbose to \
be useful.")
parser.add_argument('-k', '--keep-going', action='store_true', parser.add_argument('-k', '--keep-going', action='store_true',
help="Run all tests instead of stopping on first error. Useful for CI.") help="Run all tests instead of stopping on first error. Useful for CI.")
# TODO parser.add_argument('-p', '--persist', action='store_true',
# parser.add_argument('--gdb', action='store_true', help="Don't reset the tests disk before each test.")
# help="Run tests under gdb. Useful for debugging failures.") parser.add_argument('-g', '--gdb', action='store_true',
help="Drop into gdb on failure.")
parser.add_argument('--valgrind', action='store_true', parser.add_argument('--valgrind', action='store_true',
help="Run non-leaky tests under valgrind to check for memory leaks. \ help="Run non-leaky tests under valgrind to check for memory leaks.")
Tests marked as \"leaky = true\" run normally.") parser.add_argument('--reentrant', action='store_true',
help="Run reentrant tests with simulated power-loss.")
parser.add_argument('-e', '--exec', default=[], type=lambda e: e.split(' '),
help="Run tests with another executable prefixed on the command line.")
main(**vars(parser.parse_args())) main(**vars(parser.parse_args()))

13
tests_/test_dirs.toml
View File

@@ -10,6 +10,17 @@ code = """
lfs_unmount(&lfs) => 0; lfs_unmount(&lfs) => 0;
""" """
[[case]] # reentrant format
code = """
int err = lfs_mount(&lfs, &cfg);
if (err) {
lfs_format(&lfs, &cfg) => 0;
lfs_mount(&lfs, &cfg) => 0;
}
lfs_unmount(&lfs) => 0;
"""
reentrant = true
[[case]] # root [[case]] # root
code = """ code = """
lfs_format(&lfs, &cfg) => 0; lfs_format(&lfs, &cfg) => 0;
@@ -53,7 +64,7 @@ code = """
} }
lfs_dir_read(&lfs, &dir, &info) => 0; lfs_dir_read(&lfs, &dir, &info) => 0;
lfs_dir_close(&lfs, &dir) => 0; lfs_dir_close(&lfs, &dir) => 0;
lfs_unmount(&lfs); lfs_unmount(&lfs) => 0;
""" """
define.N = 'range(0, 100, 3)' define.N = 'range(0, 100, 3)'