Added reentrant and gdb testing mechanisms to test framework

Aside from reworking the internals of test_.py to work well with
inherited TestCase classes, this also provides the two main features
that were the main reason for revamping the test framework

1. ./scripts/test_.py --reentrant

   Runs reentrant tests (tests with reentrant=true in the .toml
   configuration) under gdb such that the program is killed on every
   call to lfs_emubd_prog or lfs_emubd_erase.

   Currently this just increments a number of prog/erases to skip, which
   means it doesn't necessarily check every possible branch of the test,
   but this should still provide a good coverage of power-loss tests.

2. ./scripts/test_.py --gdb

   Run the tests and if a failure is hit, drop into GDB. In theory this
   will be very useful for reproducing and debugging test failures.

   Note this can be combined with --reentrant to drop into GDB on the
   exact cycle of power-loss where the tests fail.

scripts/explode_asserts.py

@@ -16,7 +16,8 @@ ASSERT_TESTS = {
             printf("%s:%d:assert: "
                 "assert failed with %"PRIiMAX", expected {comp} %"PRIiMAX"\\n",
                 {file}, {line}, (intmax_t)_lh, (intmax_t)_rh);
-            exit(-2);
+            fflush(NULL);
+            raise(SIGABRT);
         }}
     """,
     'str': """
@@ -26,7 +27,8 @@ ASSERT_TESTS = {
             printf("%s:%d:assert: "
                 "assert failed with \\\"%s\\\", expected {comp} \\\"%s\\\"\\n",
                 {file}, {line}, _lh, _rh);
-            exit(-2);
+            fflush(NULL);
+            raise(SIGABRT);
         }}
     """,
     'bool': """
@@ -36,7 +38,8 @@ ASSERT_TESTS = {
             printf("%s:%d:assert: "
                 "assert failed with %s, expected {comp} %s\\n",
                 {file}, {line}, _lh ? "true" : "false", _rh ? "true" : "false");
-            exit(-2);
+            fflush(NULL);
+            raise(SIGABRT);
         }}
     """,
 }
@@ -180,6 +183,7 @@ def main(args):
     outf.write("#include <stdbool.h>\n")
     outf.write("#include <stdint.h>\n")
     outf.write("#include <inttypes.h>\n")
+    outf.write("#include <signal.h>\n")
     outf.write(mkdecl('int',  'eq', '=='))
     outf.write(mkdecl('int',  'ne', '!='))
     outf.write(mkdecl('int',  'lt', '<'))

scripts/test_.py

@@ -16,9 +16,9 @@ import base64
 import sys
 import copy
 import shutil
+import shlex
 
-TEST_DIR = 'tests_'
+TESTDIR = 'tests_'
-
 RULES = """
 define FLATTEN
 %$(subst /,.,$(target:.c=.t.c)): $(target)
@@ -28,9 +28,12 @@ $(foreach target,$(SRC),$(eval $(FLATTEN)))
 
 -include tests_/*.d
 
+.SECONDARY:
 %.c: %.t.c
     ./scripts/explode_asserts.py $< -o $@
 
+%.test: override CFLAGS += -fdiagnostics-color=always
+%.test: override CFLAGS += -ggdb
 %.test: %.test.o $(foreach f,$(subst /,.,$(SRC:.c=.o)),%.test.$f)
     $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@
 """
@@ -85,13 +88,14 @@ PASS = '\033[32m✓\033[0m'
 FAIL = '\033[31m✗\033[0m'
 
 class TestFailure(Exception):
-    def __init__(self, case, stdout=None, assert_=None):
+    def __init__(self, case, returncode=None, stdout=None, assert_=None):
         self.case = case
+        self.returncode = returncode
         self.stdout = stdout
         self.assert_ = assert_
 
 class TestCase:
-    def __init__(self, suite, config, caseno=None, lineno=None, **_):
+    def __init__(self, config, suite=None, caseno=None, lineno=None, **_):
         self.suite = suite
         self.caseno = caseno
         self.lineno = lineno
@@ -148,25 +152,29 @@ class TestCase:
 
         f.write('}\n')
 
-    def test(self, **args):
+    def test(self, exec=[], persist=False, gdb=False, failure=None, **args):
         # clear disk first
-        shutil.rmtree('blocks')
+        if not persist:
+            shutil.rmtree('blocks', True)
 
         # build command
-        cmd = ['./%s.test' % self.suite.path,
+        cmd = exec + ['./%s.test' % self.suite.path,
             repr(self.caseno), repr(self.permno)]
 
-        # run in valgrind?
+        # failed? drop into debugger?
-        if args.get('valgrind', False) and not self.leaky:
+        if gdb and failure:
-            cmd = ['valgrind',
+            cmd = (['gdb', '-ex', 'r'
-                '--leak-check=full',
+                ] + (['-ex', 'up'] if failure.assert_ else []) + [
-                '--error-exitcode=4',
+                '--args'] + cmd)
-                '-q'] + cmd
+            if args.get('verbose', False):
+                print(' '.join(shlex.quote(c) for c in cmd))
+            sys.exit(sp.call(cmd))
 
         # run test case!
         stdout = []
+        assert_ = None
         if args.get('verbose', False):
-            print(' '.join(cmd))
+            print(' '.join(shlex.quote(c) for c in cmd))
         proc = sp.Popen(cmd,
             universal_newlines=True,
             bufsize=1,
@@ -176,33 +184,84 @@ class TestCase:
             stdout.append(line)
             if args.get('verbose', False):
                 sys.stdout.write(line)
-        proc.wait()
+            # intercept asserts
-
+            m = re.match('^([^:]+):([0-9]+):(assert): (.*)$', line)
-        if proc.returncode != 0:
+            if m and assert_ is None:
-            # failed, try to parse assert?
-            assert_ = None
-            for line in stdout:
                 try:
-                    m = re.match('^([^:\\n]+):([0-9]+):assert: (.*)$', line)
-                    # found an assert, print info from file
                     with open(m.group(1)) as f:
                         lineno = int(m.group(2))
                         line = next(it.islice(f, lineno-1, None)).strip('\n')
                     assert_ = {
                         'path': m.group(1),
-                            'lineno': lineno,
                         'line': line,
-                            'message': m.group(3),
+                        'lineno': lineno,
-                        }
+                        'message': m.group(4)}
                 except:
                     pass
+        proc.wait()
 
-            self.result = TestFailure(self, stdout, assert_)
+        # did we pass?
-            raise self.result
+        if proc.returncode != 0:
-
+            raise TestFailure(self, proc.returncode, stdout, assert_)
         else:
-            self.result = PASS
+            return PASS
-            return self.result
+
+class ValgrindTestCase(TestCase):
+    def __init__(self, config, **args):
+        self.leaky = config.get('leaky', False)
+        super().__init__(config, **args)
+
+    def test(self, exec=[], **args):
+        if self.leaky:
+            return
+
+        exec = exec + [
+            'valgrind',
+            '--leak-check=full',
+            '--error-exitcode=4',
+            '-q']
+        return super().test(exec=exec, **args)
+
+class ReentrantTestCase(TestCase):
+    def __init__(self, config, **args):
+        self.reentrant = config.get('reentrant', False)
+        super().__init__(config, **args)
+
+    def test(self, exec=[], persist=False, gdb=False, failure=None, **args):
+        if not self.reentrant:
+            return
+
+        for cycles in it.count(1):
+            npersist = persist or cycles > 1
+
+            # exact cycle we should drop into debugger?
+            if gdb and failure and failure.cycleno == cycles:
+                return super().test(exec=exec, persist=npersist,
+                    gdb=gdb, failure=failure, **args)
+
+            # run tests, but kill the program after lfs_emubd_prog/erase has
+            # been hit n cycles. We exit with a special return code if the
+            # program has not finished, since this isn't a test failure.
+            nexec = exec + [
+                'gdb', '-batch-silent',
+                '-ex', 'handle all nostop',
+                '-ex', 'b lfs_emubd_prog',
+                '-ex', 'b lfs_emubd_erase',
+                '-ex', 'r',
+                ] + cycles*['-ex', 'c'] + [
+                '-ex', 'q '
+                    '!$_isvoid($_exitsignal) ? $_exitsignal : '
+                    '!$_isvoid($_exitcode) ? $_exitcode : '
+                    '33',
+                '--args']
+            try:
+                return super().test(exec=nexec, persist=npersist, **args)
+            except TestFailure as nfailure:
+                if nfailure.returncode == 33:
+                    continue
+                else:
+                    nfailure.cycleno = cycles
+                    raise
 
 class TestSuite:
     def __init__(self, path, TestCase=TestCase, **args):
@@ -229,8 +288,8 @@ class TestSuite:
         # create initial test cases
         self.cases = []
         for i, (case, lineno) in enumerate(zip(config['case'], linenos)):
-            self.cases.append(self.TestCase(
+            self.cases.append(self.TestCase(case,
-                self, case, caseno=i, lineno=lineno, **args))
+                suite=self, caseno=i, lineno=lineno, **args))
 
     def __str__(self):
         return self.name
@@ -343,7 +402,7 @@ class TestSuite:
 
             # write test.c in base64 so make can decide when to rebuild
             mk.write('%s: %s\n' % (self.path+'.test.t.c', self.path))
-            mk.write('\tbase64 -d <<< ')
+            mk.write('\t@base64 -d <<< ')
             mk.write(base64.b64encode(
                 f.getvalue().encode('utf8')).decode('utf8'))
             mk.write(' > $@\n')
@@ -364,8 +423,9 @@ class TestSuite:
                 continue
 
             try:
-                perm.test(**args)
+                result = perm.test(**args)
             except TestFailure as failure:
+                perm.result = failure
                 if not args.get('verbose', True):
                     sys.stdout.write(FAIL)
                     sys.stdout.flush()
@@ -374,6 +434,8 @@ class TestSuite:
                         sys.stdout.write('\n')
                     raise
             else:
+                if result == PASS:
+                    perm.result = PASS
                     if not args.get('verbose', True):
                         sys.stdout.write(PASS)
                         sys.stdout.flush()
@@ -400,13 +462,18 @@ def main(**args):
     elif os.path.isfile(testpath):
         testpath = testpath
     elif testpath.endswith('.toml'):
-        testpath = TEST_DIR + '/' + testpath
+        testpath = TESTDIR + '/' + testpath
     else:
-        testpath = TEST_DIR + '/' + testpath + '.toml'
+        testpath = TESTDIR + '/' + testpath + '.toml'
 
     # find tests
     suites = []
     for path in glob.glob(testpath):
+        if args.get('valgrind', False):
+            suites.append(TestSuite(path, TestCase=ValgrindTestCase, **args))
+        elif args.get('reentrant', False):
+            suites.append(TestSuite(path, TestCase=ReentrantTestCase, **args))
+        else:
             suites.append(TestSuite(path, **args))
 
     # sort for reproducability
@@ -432,11 +499,10 @@ def main(**args):
 
     cmd = (['make', '-f', 'Makefile'] +
         list(it.chain.from_iterable(['-f', m] for m in makefiles)) +
-        ['CFLAGS+=-fdiagnostics-color=always'] +
         [target for target in targets])
     stdout = []
     if args.get('verbose', False):
-        print(' '.join(cmd))
+        print(' '.join(shlex.quote(c) for c in cmd))
     proc = sp.Popen(cmd,
         universal_newlines=True,
         bufsize=1,
@@ -466,6 +532,18 @@ def main(**args):
     except TestFailure:
         pass
 
+    if args.get('gdb', False):
+        failure = None
+        for suite in suites:
+            for perm in suite.perms:
+                if getattr(perm, 'result', PASS) != PASS:
+                    failure = perm.result
+        if failure is not None:
+            print('======= gdb ======')
+            # drop into gdb
+            failure.case.test(failure=failure, **args)
+            sys.exit(0)
+
     print('====== results ======')
     passed = 0
     failed = 0
@@ -498,26 +576,26 @@ if __name__ == "__main__":
     import argparse
     parser = argparse.ArgumentParser(
         description="Run parameterized tests in various configurations.")
-    parser.add_argument('testpath', nargs='?', default=TEST_DIR,
+    parser.add_argument('testpath', nargs='?', default=TESTDIR,
         help="Description of test(s) to run. By default, this is all tests \
             found in the \"{0}\" directory. Here, you can specify a different \
             directory of tests, a specific file, a suite by name, and even a \
             specific test case by adding brackets. For example \
-            \"test_dirs[0]\" or \"{0}/test_dirs.toml[0]\".".format(TEST_DIR))
+            \"test_dirs[0]\" or \"{0}/test_dirs.toml[0]\".".format(TESTDIR))
     parser.add_argument('-D', action='append', default=[],
         help="Overriding parameter definitions.")
     parser.add_argument('-v', '--verbose', action='store_true',
         help="Output everything that is happening.")
-    parser.add_argument('-t', '--trace', action='store_true',
-        help="Normally trace output is captured for internal usage, this \
-            enables forwarding trace output which is usually too verbose to \
-            be useful.")
     parser.add_argument('-k', '--keep-going', action='store_true',
         help="Run all tests instead of stopping on first error. Useful for CI.")
-# TODO
+    parser.add_argument('-p', '--persist', action='store_true',
-#    parser.add_argument('--gdb', action='store_true',
+        help="Don't reset the tests disk before each test.")
-#        help="Run tests under gdb. Useful for debugging failures.")
+    parser.add_argument('-g', '--gdb', action='store_true',
+        help="Drop into gdb on failure.")
     parser.add_argument('--valgrind', action='store_true',
-        help="Run non-leaky tests under valgrind to check for memory leaks. \
+        help="Run non-leaky tests under valgrind to check for memory leaks.")
-            Tests marked as \"leaky = true\" run normally.")
+    parser.add_argument('--reentrant', action='store_true',
+        help="Run reentrant tests with simulated power-loss.")
+    parser.add_argument('-e', '--exec', default=[], type=lambda e: e.split(' '),
+        help="Run tests with another executable prefixed on the command line.")
     main(**vars(parser.parse_args()))

tests_/test_dirs.toml

@@ -10,6 +10,17 @@ code = """
     lfs_unmount(&lfs) => 0;
 """
 
+[[case]] # reentrant format
+code = """
+    int err = lfs_mount(&lfs, &cfg);
+    if (err) {
+        lfs_format(&lfs, &cfg) => 0;
+        lfs_mount(&lfs, &cfg) => 0;
+    }
+    lfs_unmount(&lfs) => 0;
+"""
+reentrant = true
+
 [[case]] # root
 code = """
     lfs_format(&lfs, &cfg) => 0;
@@ -53,7 +64,7 @@ code = """
     }
     lfs_dir_read(&lfs, &dir, &info) => 0;
     lfs_dir_close(&lfs, &dir) => 0;
-    lfs_unmount(&lfs);
+    lfs_unmount(&lfs) => 0;
 """
 define.N = 'range(0, 100, 3)'