mirror of
				https://github.com/eledio-devices/thirdparty-littlefs.git
				synced 2025-10-31 16:14:16 +01:00 
			
		
		
		
	This is the start of reworking littlefs's testing framework based on
lessons learned from the initial testing framework.
1. The testing framework needs to be _flexible_. It was hacky, which by
   itself isn't a downside, but it wasn't _flexible_. This limited what
   could be done with the tests and there ended up being many
   workarounds just to reproduce bugs.
   The idea behind this revamped framework is to separate the
   description of tests (tests/test_dirs.toml) and the running of tests
   (scripts/test.py).
   Now, with the logic moved entirely to python, it's possible to run
   the test under varying environments. In addition to the "just don't
   assert" run, I'm also looking to run the tests in valgrind for memory
   checking, and an environment with simulated power-loss.
   The test description can also contain abstract attributes that help
   control how tests can be ran, such as "leaky" to identify tests where
   memory leaks are expected. This keeps test limitations at a minimum
   without limiting how the tests can be ran.
2. Multi-stage-process tests didn't really add value and limited what
   the testing environment.
   Unmounting + mounting can be done in a single process to test the
   same logic. It would be really difficult to make this fail only
   when memory is zeroed, though that can still be caught by
   power-resilient tests.
   Requiring every test to be a single process adds several options
   for test execution, such as using a RAM-backed block device for
   speed, or even running the tests on a device.
3. Added fancy assert interception. This wasn't really a requirement,
   but something I've been wanting to experiment with for a while.
   During testing, scripts/explode_asserts.py is added to the build
   process. This is a custom C-preprocessor that parses out assert
   statements and replaces them with _very_ verbose asserts that
   wouldn't normally be possible with just C macros.
   It even goes as far as to report the arguments to strcmp, since the
   lack of visibility here was very annoying.
   tests_/test_dirs.toml:186:assert: assert failed with "..", expected eq "..."
       assert(strcmp(info.name, "...") == 0);
   One downside is that simply parsing C in python is slower than the
   entire rest of the compilation, but fortunately this can be
   alleviated by parallelizing the test builds through make.
Other neat bits:
- All generated files are a suffix of the test description, this helps
  cleanup and means it's (theoretically) possible to parallelize the
  tests.
- The generated test.c is shoved base64 into an ad-hoc Makefile, this
  means it doesn't force a rebuild of tests all the time.
- Test parameterizing is now easier.
- Hopefully this framework can be repurposed also for benchmarks in the
  future.
		
	
		
			
				
	
	
		
			212 lines
		
	
	
		
			6.2 KiB
		
	
	
	
		
			Python
		
	
	
		
			Executable File
		
	
	
	
	
			
		
		
	
	
			212 lines
		
	
	
		
			6.2 KiB
		
	
	
	
		
			Python
		
	
	
		
			Executable File
		
	
	
	
	
| #!/usr/bin/env python3
 | |
| 
 | |
| import parsy as p
 | |
| import re
 | |
| import io
 | |
| import sys
 | |
| 
 | |
| ASSERT_PATTERN = p.string('LFS_ASSERT') | p.string('assert')
 | |
| ASSERT_CHARS = 'La'
 | |
| ASSERT_TARGET = '__LFS_ASSERT_{TYPE}_{COMP}'
 | |
| ASSERT_TESTS = {
 | |
|     'int': """
 | |
|         __typeof__({lh}) _lh = {lh};
 | |
|         __typeof__({lh}) _rh = (__typeof__({lh})){rh};
 | |
|         if (!(_lh {op} _rh)) {{
 | |
|             printf("%s:%d:assert: "
 | |
|                 "assert failed with %"PRIiMAX", expected {comp} %"PRIiMAX"\\n",
 | |
|                 {file}, {line}, (intmax_t)_lh, (intmax_t)_rh);
 | |
|             exit(-2);
 | |
|         }}
 | |
|     """,
 | |
|     'str': """
 | |
|         const char *_lh = {lh};
 | |
|         const char *_rh = {rh};
 | |
|         if (!(strcmp(_lh, _rh) {op} 0)) {{
 | |
|             printf("%s:%d:assert: "
 | |
|                 "assert failed with \\\"%s\\\", expected {comp} \\\"%s\\\"\\n",
 | |
|                 {file}, {line}, _lh, _rh);
 | |
|             exit(-2);
 | |
|         }}
 | |
|     """,
 | |
|     'bool': """
 | |
|         bool _lh = !!({lh});
 | |
|         bool _rh = !!({rh});
 | |
|         if (!(_lh {op} _rh)) {{
 | |
|             printf("%s:%d:assert: "
 | |
|                 "assert failed with %s, expected {comp} %s\\n",
 | |
|                 {file}, {line}, _lh ? "true" : "false", _rh ? "true" : "false");
 | |
|             exit(-2);
 | |
|         }}
 | |
|     """,
 | |
| }
 | |
| 
 | |
| def mkassert(lh, rh='true', type='bool', comp='eq'):
 | |
|     return ((ASSERT_TARGET + "({lh}, {rh}, __FILE__, __LINE__, __func__)")
 | |
|         .format(
 | |
|             type=type, TYPE=type.upper(),
 | |
|             comp=comp, COMP=comp.upper(),
 | |
|             lh=lh.strip(' '),
 | |
|             rh=rh.strip(' ')))
 | |
| 
 | |
| def mkdecl(type, comp, op):
 | |
|     return ((
 | |
|         "#define "+ASSERT_TARGET+"(lh, rh, file, line, func)"
 | |
|         "   do {{"+re.sub('\s+', ' ', ASSERT_TESTS[type])+"}} while (0)\n")
 | |
|         .format(
 | |
|             type=type, TYPE=type.upper(),
 | |
|             comp=comp, COMP=comp.upper(),
 | |
|             lh='lh', rh='rh', op=op,
 | |
|             file='file', line='line', func='func'))
 | |
| 
 | |
| # add custom until combinator
 | |
| def until(self, end):
 | |
|     return end.should_fail('should fail').then(self).many()
 | |
| p.Parser.until = until
 | |
| 
 | |
| pcomp = (
 | |
|     p.string('==').tag('eq') |
 | |
|     p.string('!=').tag('ne') |
 | |
|     p.string('<=').tag('le') |
 | |
|     p.string('>=').tag('ge') |
 | |
|     p.string('<').tag('lt') |
 | |
|     p.string('>').tag('gt'));
 | |
| 
 | |
| plogic = p.string('&&') | p.string('||')
 | |
| 
 | |
| @p.generate
 | |
| def pstrassert():
 | |
|     yield ASSERT_PATTERN + p.regex('\s*') + p.string('(') + p.regex('\s*')
 | |
|     yield p.string('strcmp') + p.regex('\s*') + p.string('(') + p.regex('\s*')
 | |
|     lh = yield pexpr.until(p.string(',') | p.string(')') | plogic)
 | |
|     yield p.string(',') + p.regex('\s*')
 | |
|     rh = yield pexpr.until(p.string(')') | plogic)
 | |
|     yield p.string(')') + p.regex('\s*')
 | |
|     op = yield pcomp
 | |
|     yield p.regex('\s*') + p.string('0') + p.regex('\s*') + p.string(')')
 | |
|     return mkassert(''.join(lh), ''.join(rh), 'str', op[0])
 | |
| 
 | |
| @p.generate
 | |
| def pintassert():
 | |
|     yield ASSERT_PATTERN + p.regex('\s*') + p.string('(') + p.regex('\s*')
 | |
|     lh = yield pexpr.until(pcomp | p.string(')') | plogic)
 | |
|     op = yield pcomp
 | |
|     rh = yield pexpr.until(p.string(')') | plogic)
 | |
|     yield p.string(')')
 | |
|     return mkassert(''.join(lh), ''.join(rh), 'int', op[0])
 | |
| 
 | |
| @p.generate
 | |
| def pboolassert():
 | |
|     yield ASSERT_PATTERN + p.regex('\s*') + p.string('(') + p.regex('\s*')
 | |
|     expr = yield pexpr.until(p.string(')'))
 | |
|     yield p.string(')')
 | |
|     return mkassert(''.join(expr), 'true', 'bool', 'eq')
 | |
| 
 | |
| passert = p.peek(ASSERT_PATTERN) >> (pstrassert | pintassert | pboolassert)
 | |
| 
 | |
| @p.generate
 | |
| def pcomment1():
 | |
|     yield p.string('//')
 | |
|     s = yield p.regex('[^\\n]*')
 | |
|     yield p.string('\n')
 | |
|     return '//' + s + '\n'
 | |
| 
 | |
| @p.generate
 | |
| def pcomment2():
 | |
|     yield p.string('/*')
 | |
|     s = yield p.regex('((?!\*/).)*')
 | |
|     yield p.string('*/')
 | |
|     return '/*' + ''.join(s) + '*/'
 | |
| 
 | |
| @p.generate
 | |
| def pcomment3():
 | |
|     yield p.string('#')
 | |
|     s = yield p.regex('[^\\n]*')
 | |
|     yield p.string('\n')
 | |
|     return '#' + s + '\n'
 | |
| 
 | |
| pws = p.regex('\s+') | pcomment1 | pcomment2 | pcomment3
 | |
| 
 | |
| @p.generate
 | |
| def pstring():
 | |
|     q = yield p.regex('["\']')
 | |
|     s = yield (p.string('\\%s' % q) | p.regex('[^%s]' % q)).many()
 | |
|     yield p.string(q)
 | |
|     return q + ''.join(s) + q
 | |
| 
 | |
| @p.generate
 | |
| def pnested():
 | |
|     l = yield p.string('(')
 | |
|     n = yield pexpr.until(p.string(')'))
 | |
|     r = yield p.string(')')
 | |
|     return l + ''.join(n) + r
 | |
| 
 | |
| pexpr = (
 | |
|     # shortcut for a bit better performance
 | |
|     p.regex('[^%s/#\'"();{}=><,&|-]+' % ASSERT_CHARS) |
 | |
|     pws |
 | |
|     passert |
 | |
|     pstring |
 | |
|     pnested |
 | |
|     p.string('->') |
 | |
|     p.regex('.', re.DOTALL))
 | |
| 
 | |
| @p.generate
 | |
| def pstmt():
 | |
|     ws = yield pws.many()
 | |
|     lh = yield pexpr.until(p.string('=>') | p.regex('[;{}]'))
 | |
|     op = yield p.string('=>').optional()
 | |
|     if op == '=>':
 | |
|         rh = yield pstmt
 | |
|         return ''.join(ws) + mkassert(''.join(lh), rh, 'int', 'eq')
 | |
|     else:
 | |
|         return ''.join(ws) + ''.join(lh)
 | |
| 
 | |
| @p.generate
 | |
| def pstmts():
 | |
|     a = yield pstmt
 | |
|     b = yield (p.regex('[;{}]') + pstmt).many()
 | |
|     return [a] + b
 | |
| 
 | |
| def main(args):
 | |
|     inf = open(args.input, 'r') if args.input else sys.stdin
 | |
|     outf = open(args.output, 'w') if args.output else sys.stdout
 | |
| 
 | |
|     # parse C code
 | |
|     input = inf.read()
 | |
|     stmts = pstmts.parse(input)
 | |
| 
 | |
|     # write extra verbose asserts
 | |
|     outf.write("#include <stdbool.h>\n")
 | |
|     outf.write("#include <stdint.h>\n")
 | |
|     outf.write("#include <inttypes.h>\n")
 | |
|     outf.write(mkdecl('int',  'eq', '=='))
 | |
|     outf.write(mkdecl('int',  'ne', '!='))
 | |
|     outf.write(mkdecl('int',  'lt', '<'))
 | |
|     outf.write(mkdecl('int',  'gt', '>'))
 | |
|     outf.write(mkdecl('int',  'le', '<='))
 | |
|     outf.write(mkdecl('int',  'ge', '>='))
 | |
|     outf.write(mkdecl('str',  'eq', '=='))
 | |
|     outf.write(mkdecl('str',  'ne', '!='))
 | |
|     outf.write(mkdecl('str',  'lt', '<'))
 | |
|     outf.write(mkdecl('str',  'gt', '>'))
 | |
|     outf.write(mkdecl('str',  'le', '<='))
 | |
|     outf.write(mkdecl('str',  'ge', '>='))
 | |
|     outf.write(mkdecl('bool', 'eq', '=='))
 | |
|     if args.input:
 | |
|         outf.write("#line %d \"%s\"\n" % (1, args.input))
 | |
| 
 | |
|     # write parsed statements
 | |
|     for stmt in stmts:
 | |
|         outf.write(stmt)
 | |
| 
 | |
| if __name__ == "__main__":
 | |
|     import argparse
 | |
|     parser = argparse.ArgumentParser(
 | |
|         description="Cpp step that increases assert verbosity")
 | |
|     parser.add_argument('input', nargs='?',
 | |
|         help="Input C file after cpp.")
 | |
|     parser.add_argument('-o', '--output',
 | |
|         help="Output C file.")
 | |
|     main(parser.parse_args())
 |