WIP crazy new idea work in progress

passing non-reentrant tests already!
2025-10-31 16:14:16 +01:00 · 2020-03-06 20:14:27 -06:00
parent 3ee291de59
commit eecb06a9dc
5 changed files with 329 additions and 104 deletions
--- a/scripts/readmdir.py
+++ b/scripts/readmdir.py
@@ -326,7 +326,7 @@ def main(args):
        mdir.rev,
        ' (was %s)' % ', '.join('%d' % m.rev for m in mdir.pair[1:])
        if len(mdir.pair) > 1 else '',
-        ' (corrupted)' if not mdir else ''))
+        ' (corrupted!)' if not mdir else ''))
    if args.all:
        mdir.dump_all(truncate=not args.no_truncate)
    elif args.log:
--- a/scripts/readtree.py
+++ b/scripts/readtree.py
@@ -5,6 +5,7 @@ import sys
 import json
 import io
 import itertools as it
+import collections as c
 from readmdir import Tag, MetadataPair

 def popc(x):
@@ -13,7 +14,7 @@ def popc(x):
 def ctz(x):
    return len(bin(x)) - len(bin(x).rstrip('0'))

-def dumpentries(args, mdir, f):
+def dumpentries(args, mdir, mdirs, f):
    for k, id_ in enumerate(mdir.ids):
        name = mdir[Tag('name', id_, 0)]
        struct_ = mdir[Tag('struct', id_, 0)]
@@ -22,8 +23,10 @@ def dumpentries(args, mdir, f):
            id_, name.typerepr(),
            json.dumps(name.data.decode('utf8')))
        if struct_.is_('dirstruct'):
-            desc += " dir {%#x, %#x}" % struct.unpack(
-                '<II', struct_.data[:8].ljust(8, b'\xff'))
+            pair = struct.unpack('<II', struct_.data[:8].ljust(8, b'\xff'))
+            desc += " dir {%#x, %#x}%s" % (
+                pair[0], pair[1],
+                '?' if frozenset(pair) not in mdirs else '')
        if struct_.is_('ctzstruct'):
            desc += " ctz {%#x} size %d" % struct.unpack(
                '<II', struct_.data[:8].ljust(8, b'\xff'))
@@ -93,19 +96,15 @@ def dumpentries(args, mdir, f):

 def main(args):
    with open(args.disk, 'rb') as f:
-        dirs = []
        superblock = None
-        gstate = b''
-        mdirs = []
+        gstate = b'\0\0\0\0\0\0\0\0\0\0\0\0'
+        mdirs = c.OrderedDict()
        cycle = False
        tail = (args.block1, args.block2)
-        hard = False
-        while True:
-            for m in it.chain((m for d in dirs for m in d), mdirs):
-                if set(m.blocks) == set(tail):
-                    # cycle detected
-                    cycle = m.blocks
-            if cycle:
+        while tail:
+            if frozenset(tail) in mdirs:
+                # cycle detected
+                cycle = tail
                break

            # load mdir
@@ -128,6 +127,13 @@ def main(args):
            except KeyError:
                mdir.tail = None

+            try:
+                mdir.branch = mdir[Tag('branch', 0, 0)]
+                if mdir.branch.size != 8 or mdir.branch.data == 8*b'\xff':
+                    mdir.branch = None
+            except KeyError:
+                mdir.branch = None
+
            # have superblock?
            try:
                nsuperblock = mdir[
@@ -144,41 +150,55 @@ def main(args):
            except KeyError:
                pass

-            # add to directories
-            mdirs.append(mdir)
-            if mdir.tail is None or not mdir.tail.is_('hardtail'):
-                dirs.append(mdirs)
-                mdirs = []
+            # add to metadata-pairs
+            mdirs[frozenset(mdir.blocks)] = mdir
+            tail = (struct.unpack('<II', mdir.tail.data)
+                if mdir.tail else None)

-            if mdir.tail is None:
-                break
-
-            tail = struct.unpack('<II', mdir.tail.data)
-            hard = mdir.tail.is_('hardtail')
-
-    # find paths
-    dirtable = {}
-    for dir in dirs:
-        dirtable[frozenset(dir[0].blocks)] = dir
-
-    pending = [("/", dirs[0])]
+    # derive paths and build directories
+    dirs = {}
+    rogue = {}
+    pending = [('/', (args.block1, args.block2))]
    while pending:
-        path, dir = pending.pop(0)
-        for mdir in dir:
+        path, branch = pending.pop(0)
+        dir = []
+        while branch and frozenset(branch) in mdirs:
+            mdir = mdirs[frozenset(branch)]
+            dir.append(mdir)
+
            for tag in mdir.tags:
                if tag.is_('dir'):
                    try:
-                        npath = tag.data.decode('utf8')
+                        npath = path + '/' + tag.data.decode('utf8')
+                        npath = npath.replace('//', '/')
                        dirstruct = mdir[Tag('dirstruct', tag.id, 0)]
-                        nblocks = struct.unpack('<II', dirstruct.data)
-                        nmdir = dirtable[frozenset(nblocks)]
-                        pending.append(((path + '/' + npath), nmdir))
+                        npair = struct.unpack('<II', dirstruct.data)
+                        pending.append((npath, npair))
                    except KeyError:
                        pass

-        dir[0].path = path.replace('//', '/')
+            branch = (struct.unpack('<II', mdir.branch.data)
+                if mdir.branch else None)

-    # dump tree
+        if not dir:
+            rogue[path] = branch
+        else:
+            dirs[path] = dir
+
+    # also find orphans
+    not_orphans = {frozenset(mdir.blocks)
+        for dir in dirs.values()
+        for mdir in dir}
+    orphans = []
+    for pair, mdir in mdirs.items():
+        if pair not in not_orphans:
+            if len(orphans) > 0 and (pair == frozenset(
+                    struct.unpack('<II', orphans[-1][-1].tail.data))):
+                orphans[-1].append(mdir)
+            else:
+                orphans.append([mdir])
+
+    # print littlefs + version info
    version = ('?', '?')
    if superblock:
        version = tuple(reversed(
@@ -187,24 +207,30 @@ def main(args):
        "data (truncated, if it fits)"
        if not any([args.no_truncate, args.tags, args.log, args.all]) else ""))

-    if gstate:
-        print("gstate 0x%s" % ''.join('%02x' % c for c in gstate))
-        tag = Tag(struct.unpack('<I', gstate[0:4].ljust(4, b'\xff'))[0])
-        blocks = struct.unpack('<II', gstate[4:4+8].ljust(8, b'\xff'))
-        if tag.size or not tag.isvalid:
-            print("  orphans >=%d" % max(tag.size, 1))
-        if tag.type:
-            print("  move dir {%#x, %#x} id %d" % (
-                blocks[0], blocks[1], tag.id))
+    # print gstate
+    print("gstate 0x%s" % ''.join('%02x' % c for c in gstate))
+    tag = Tag(struct.unpack('<I', gstate[0:4].ljust(4, b'\xff'))[0])
+    blocks = struct.unpack('<II', gstate[4:4+8].ljust(8, b'\xff'))
+    if tag.size or not tag.isvalid:
+        print("  orphans >=%d" % max(tag.size, 1))
+    if tag.type:
+        print("  move dir {%#x, %#x}%s id %d" % (
+            blocks[0], blocks[1],
+            '?' if frozenset(blocks) not in mdirs else '',
+            tag.id))

-    for i, dir in enumerate(dirs):
-        print("dir %s" % (json.dumps(dir[0].path)
-            if hasattr(dir[0], 'path') else '(orphan)'))
+    # print dir info
+    for path, dir in it.chain(
+            sorted(dirs.items()),
+            zip(it.repeat(None), orphans)):
+        print("dir %s" % json.dumps(path) if path else "orphaned")

        for j, mdir in enumerate(dir):
-            print("mdir {%#x, %#x} rev %d%s" % (
-                mdir.blocks[0], mdir.blocks[1], mdir.rev,
-                ' (corrupted)' if not mdir else ''))
+            print("mdir {%#x, %#x} rev %d (was %d)%s%s" % (
+                mdir.blocks[0], mdir.blocks[1], mdir.rev, mdir.pair[1].rev,
+                ' (corrupted!)' if not mdir else '',
+                ' -> {%#x, %#x}' % struct.unpack('<II', mdir.tail.data)
+                if mdir.tail else ''))

            f = io.StringIO()
            if args.tags:
@@ -214,21 +240,27 @@ def main(args):
            elif args.all:
                mdir.dump_all(f, truncate=not args.no_truncate)
            else:
-                dumpentries(args, mdir, f)
+                dumpentries(args, mdir, mdirs, f)

            lines = list(filter(None, f.getvalue().split('\n')))
            for k, line in enumerate(lines):
                print("%s %s" % (
-                    ' ' if i == len(dirs)-1 and j == len(dir)-1 else
+                    ' ' if j == len(dir)-1 else
                    'v' if k == len(lines)-1 else
-                    '.' if j == len(dir)-1 else
-                    '|',
+                    '|' if path else '.',
                    line))

-    if cycle:
-        print("*** cycle detected! -> {%#x, %#x} ***" % (cycle[0], cycle[1]))
+    for path, pair in rogue.items():
+        print("*** couldn't find dir %s {%#x, %#x}! ***" % (
+            json.dumps(path), pair[0], pair[1]))

    if cycle:
+        print("*** cycle detected {%#x, %#x}! ***" % (
+            cycle[0], cycle[1]))
+
+    if cycle:
+        return 3
+    elif rogue:
        return 2
    elif not all(mdir for dir in dirs for mdir in dir):
        return 1
--- a/scripts/test.py
+++ b/scripts/test.py
@@ -231,7 +231,7 @@ class TestCase:
                ncmd.extend(['-ex', 'r'])
                if failure.assert_:
                    ncmd.extend(['-ex', 'up 2'])
-            elif gdb == 'start':
+            elif gdb == 'start' or isinstance(gdb, int):
                ncmd.extend([
                    '-ex', 'b %s:%d' % (self.suite.path, self.code_lineno),
                    '-ex', 'r'])
@@ -329,7 +329,9 @@ class ReentrantTestCase(TestCase):
                persist = 'noerase'

            # exact cycle we should drop into debugger?
-            if gdb and failure and failure.cycleno == cycles:
+            if gdb and failure and (
+                    failure.cycleno == cycles or
+                    (isinstance(gdb, int) and gdb == cycles)):
                return super().test(gdb=gdb, persist=persist, cycles=cycles,
                    failure=failure, **args)

@@ -760,7 +762,8 @@ if __name__ == "__main__":
        help="Store disk image in a file.")
    parser.add_argument('-b', '--build', action='store_true',
        help="Only build the tests, do not execute.")
-    parser.add_argument('-g', '--gdb', choices=['init', 'start', 'assert'],
+    parser.add_argument('-g', '--gdb', metavar='{init,start,assert},CYCLE',
+        type=lambda n: n if n in {'init', 'start', 'assert'} else int(n, 0),
        nargs='?', const='assert',
        help="Drop into gdb on test failure.")
    parser.add_argument('--no-internal', action='store_true',