Some improvements to size scripts

- Added -L/--depth argument to show dependencies for scripts/stack.py,
  this replaces calls.py
- Additional internal restructuring to avoid repeated code
- Removed incorrect diff percentage when there is no actual size
- Consistent percentage rendering in test.py
This commit is contained in:
Christopher Haster
2022-03-05 14:16:56 -06:00
parent 50ad2adc96
commit eb8be9f351
7 changed files with 202 additions and 309 deletions

View File

@@ -1,170 +0,0 @@
#!/usr/bin/env python3
#
# Script to show the callgraph in a human readable manner. Basically just a
# wrapper aroung GCC's -fcallgraph-info flag.
#
import os
import glob
import itertools as it
import re
import csv
import collections as co
CI_PATHS = ['*.ci']
def collect(paths, **args):
# parse the vcg format
k_pattern = re.compile('([a-z]+)\s*:', re.DOTALL)
v_pattern = re.compile('(?:"(.*?)"|([a-z]+))', re.DOTALL)
def parse_vcg(rest):
def parse_vcg(rest):
node = []
while True:
rest = rest.lstrip()
m = k_pattern.match(rest)
if not m:
return (node, rest)
k, rest = m.group(1), rest[m.end(0):]
rest = rest.lstrip()
if rest.startswith('{'):
v, rest = parse_vcg(rest[1:])
assert rest[0] == '}', "unexpected %r" % rest[0:1]
rest = rest[1:]
node.append((k, v))
else:
m = v_pattern.match(rest)
assert m, "unexpected %r" % rest[0:1]
v, rest = m.group(1) or m.group(2), rest[m.end(0):]
node.append((k, v))
node, rest = parse_vcg(rest)
assert rest == '', "unexpected %r" % rest[0:1]
return node
# collect into functions
results = co.defaultdict(lambda: (None, None, set()))
f_pattern = re.compile(r'([^\\]*)\\n([^:]*)')
for path in paths:
with open(path) as f:
vcg = parse_vcg(f.read())
for k, graph in vcg:
if k != 'graph':
continue
for k, info in graph:
if k == 'node':
info = dict(info)
m = f_pattern.match(info['label'])
if m:
function, file = m.groups()
_, _, targets = results[info['title']]
results[info['title']] = (file, function, targets)
elif k == 'edge':
info = dict(info)
_, _, targets = results[info['sourcename']]
targets.add(info['targetname'])
else:
continue
if not args.get('everything'):
for source, (s_file, s_function, _) in list(results.items()):
# discard internal functions
if s_file.startswith('<') or s_file.startswith('/usr/include'):
del results[source]
# flatten into a list
flat_results = []
for _, (s_file, s_function, targets) in results.items():
for target in targets:
if target not in results:
continue
t_file, t_function, _ = results[target]
flat_results.append((s_file, s_function, t_file, t_function))
return flat_results
def main(**args):
# find sizes
if not args.get('use', None):
# find .ci files
paths = []
for path in args['ci_paths']:
if os.path.isdir(path):
path = path + '/*.ci'
for path in glob.glob(path):
paths.append(path)
if not paths:
print('no .ci files found in %r?' % args['ci_paths'])
sys.exit(-1)
results = collect(paths, **args)
else:
with open(args['use']) as f:
r = csv.DictReader(f)
results = [
( result['file'],
result['function'],
result['callee_file'],
result['callee_function'])
for result in r]
# write results to CSV
if args.get('output'):
with open(args['output'], 'w') as f:
w = csv.writer(f)
w.writerow(['file', 'function', 'callee_file', 'callee_function'])
for file, func, c_file, c_func in sorted(results):
w.writerow((file, func, c_file, c_func))
# print results
def dedup_entries(results, by='function'):
entries = co.defaultdict(lambda: set())
for file, func, c_file, c_func in results:
entry = (file if by == 'file' else func)
entries[entry].add(c_file if by == 'file' else c_func)
return entries
def print_entries(by='function'):
entries = dedup_entries(results, by=by)
for name, callees in sorted(entries.items()):
print(name)
for i, c_name in enumerate(sorted(callees)):
print(" -> %s" % c_name)
if args.get('quiet'):
pass
elif args.get('files'):
print_entries(by='file')
else:
print_entries(by='function')
if __name__ == "__main__":
import argparse
import sys
parser = argparse.ArgumentParser(
description="Find and show callgraph.")
parser.add_argument('ci_paths', nargs='*', default=CI_PATHS,
help="Description of where to find *.ci files. May be a directory \
or a list of paths. Defaults to %r." % CI_PATHS)
parser.add_argument('-v', '--verbose', action='store_true',
help="Output commands that run behind the scenes.")
parser.add_argument('-o', '--output',
help="Specify CSV file to store results.")
parser.add_argument('-u', '--use',
help="Don't parse callgraph files, instead use this CSV file.")
parser.add_argument('-A', '--everything', action='store_true',
help="Include builtin and libc specific symbols.")
parser.add_argument('--files', action='store_true',
help="Show file-level calls.")
parser.add_argument('-q', '--quiet', action='store_true',
help="Don't show anything, useful with -o.")
parser.add_argument('--build-dir',
help="Specify the relative build directory. Used to map object files \
to the correct source files.")
sys.exit(main(**vars(parser.parse_args())))

View File

@@ -152,13 +152,23 @@ def main(**args):
else:
print('%-36s %7s %7s %7s' % (by, 'old', 'new', 'diff'))
def print_entry(name, size):
print("%-36s %7d" % (name, size))
def print_diff_entry(name, old, new, diff, ratio):
print("%-36s %7s %7s %+7d%s" % (name,
old or "-",
new or "-",
diff,
' (%+.1f%%)' % (100*ratio) if ratio else ''))
def print_entries(by='function'):
entries = dedup_entries(results, by=by)
if not args.get('diff'):
print_header(by=by)
for name, size in sorted_entries(entries.items()):
print("%-36s %7d" % (name, size))
print_entry(name, size)
else:
prev_entries = dedup_entries(prev_results, by=by)
diff = diff_entries(prev_entries, entries)
@@ -168,23 +178,19 @@ def main(**args):
for name, (old, new, diff, ratio) in sorted_diff_entries(
diff.items()):
if ratio or args.get('all'):
print("%-36s %7s %7s %+7d%s" % (name,
old or "-",
new or "-",
diff,
' (%+.1f%%)' % (100*ratio) if ratio else ''))
print_diff_entry(name, old, new, diff, ratio)
def print_totals():
if not args.get('diff'):
print("%-36s %7d" % ('TOTAL', total))
print_entry('TOTAL', total)
else:
ratio = (total-prev_total)/prev_total if prev_total else 1.0
print("%-36s %7s %7s %+7d%s" % (
'TOTAL',
prev_total if prev_total else '-',
total if total else '-',
ratio = (0.0 if not prev_total and not total
else 1.0 if not prev_total
else (total-prev_total)/prev_total)
print_diff_entry('TOTAL',
prev_total, total,
total-prev_total,
' (%+.1f%%)' % (100*ratio) if ratio else ''))
ratio)
if args.get('quiet'):
pass

View File

@@ -173,17 +173,37 @@ def main(**args):
else:
print('%-36s %19s %19s %11s' % (by, 'old', 'new', 'diff'))
def print_entry(name, hits, count):
print("%-36s %11s %7s" % (name,
'%d/%d' % (hits, count)
if count else '-',
'%.1f%%' % (100*hits/count)
if count else '-'))
def print_diff_entry(name,
old_hits, old_count,
new_hits, new_count,
diff_hits, diff_count,
ratio):
print("%-36s %11s %7s %11s %7s %11s%s" % (name,
'%d/%d' % (old_hits, old_count)
if old_count else '-',
'%.1f%%' % (100*old_hits/old_count)
if old_count else '-',
'%d/%d' % (new_hits, new_count)
if new_count else '-',
'%.1f%%' % (100*new_hits/new_count)
if new_count else '-',
'%+d/%+d' % (diff_hits, diff_count),
' (%+.1f%%)' % (100*ratio) if ratio else ''))
def print_entries(by='function'):
entries = dedup_entries(results, by=by)
if not args.get('diff'):
print_header(by=by)
for name, (hits, count) in sorted_entries(entries.items()):
print("%-36s %11s %7s" % (name,
'%d/%d' % (hits, count)
if count else '-',
'%.1f%%' % (100*hits/count)
if count else '-'))
print_entry(name, hits, count)
else:
prev_entries = dedup_entries(prev_results, by=by)
diff = diff_entries(prev_entries, entries)
@@ -196,42 +216,25 @@ def main(**args):
diff_hits, diff_count, ratio) in sorted_diff_entries(
diff.items()):
if ratio or args.get('all'):
print("%-36s %11s %7s %11s %7s %11s%s" % (name,
'%d/%d' % (old_hits, old_count)
if old_count else '-',
'%.1f%%' % (100*old_hits/old_count)
if old_count else '-',
'%d/%d' % (new_hits, new_count)
if new_count else '-',
'%.1f%%' % (100*new_hits/new_count)
if new_count else '-',
'%+d/%+d' % (diff_hits, diff_count),
' (%+.1f%%)' % (100*ratio) if ratio else ''))
print_diff_entry(name,
old_hits, old_count,
new_hits, new_count,
diff_hits, diff_count,
ratio)
def print_totals():
if not args.get('diff'):
print("%-36s %11s %7s" % ('TOTAL',
'%d/%d' % (total_hits, total_count)
if total_count else '-',
'%.1f%%' % (100*total_hits/total_count)
if total_count else '-'))
print_entry('TOTAL', total_hits, total_count)
else:
ratio = ((total_hits/total_count
if total_count else 1.0)
- (prev_total_hits/prev_total_count
if prev_total_count else 1.0))
print("%-36s %11s %7s %11s %7s %11s%s" % ('TOTAL',
'%d/%d' % (prev_total_hits, prev_total_count)
if prev_total_count else '-',
'%.1f%%' % (100*prev_total_hits/prev_total_count)
if prev_total_count else '-',
'%d/%d' % (total_hits, total_count)
if total_count else '-',
'%.1f%%' % (100*total_hits/total_count)
if total_count else '-',
'%+d/%+d' % (total_hits-prev_total_hits,
total_count-prev_total_count),
' (%+.1f%%)' % (100*ratio) if ratio else ''))
print_diff_entry('TOTAL',
prev_total_hits, prev_total_count,
total_hits, total_count,
total_hits-prev_total_hits, total_count-prev_total_count,
ratio)
if args.get('quiet'):
pass

View File

@@ -152,13 +152,23 @@ def main(**args):
else:
print('%-36s %7s %7s %7s' % (by, 'old', 'new', 'diff'))
def print_entry(name, size):
print("%-36s %7d" % (name, size))
def print_diff_entry(name, old, new, diff, ratio):
print("%-36s %7s %7s %+7d%s" % (name,
old or "-",
new or "-",
diff,
' (%+.1f%%)' % (100*ratio) if ratio else ''))
def print_entries(by='function'):
entries = dedup_entries(results, by=by)
if not args.get('diff'):
print_header(by=by)
for name, size in sorted_entries(entries.items()):
print("%-36s %7d" % (name, size))
print_entry(name, size)
else:
prev_entries = dedup_entries(prev_results, by=by)
diff = diff_entries(prev_entries, entries)
@@ -168,23 +178,19 @@ def main(**args):
for name, (old, new, diff, ratio) in sorted_diff_entries(
diff.items()):
if ratio or args.get('all'):
print("%-36s %7s %7s %+7d%s" % (name,
old or "-",
new or "-",
diff,
' (%+.1f%%)' % (100*ratio) if ratio else ''))
print_diff_entry(name, old, new, diff, ratio)
def print_totals():
if not args.get('diff'):
print("%-36s %7d" % ('TOTAL', total))
print_entry('TOTAL', total)
else:
ratio = (total-prev_total)/prev_total if prev_total else 1.0
print("%-36s %7s %7s %+7d%s" % (
'TOTAL',
prev_total if prev_total else '-',
total if total else '-',
ratio = (0.0 if not prev_total and not total
else 1.0 if not prev_total
else (total-prev_total)/prev_total)
print_diff_entry('TOTAL',
prev_total, total,
total-prev_total,
' (%+.1f%%)' % (100*ratio) if ratio else ''))
ratio)
if args.get('quiet'):
pass

View File

@@ -82,7 +82,7 @@ def collect(paths, **args):
# find maximum stack size recursively, this requires also detecting cycles
# (in case of recursion)
def stack_limit(source, seen=None):
def find_limit(source, seen=None):
seen = seen or set()
if source not in results:
return 0
@@ -93,16 +93,25 @@ def collect(paths, **args):
if target in seen:
# found a cycle
return float('inf')
limit_ = stack_limit(target, seen | {target})
limit_ = find_limit(target, seen | {target})
limit = max(limit, limit_)
return frame + limit
def find_deps(targets):
deps = set()
for target in targets:
if target in results:
t_file, t_function, _, _ = results[target]
deps.add((t_file, t_function))
return deps
# flatten into a list
flat_results = []
for source, (s_file, s_function, frame, targets) in results.items():
limit = stack_limit(source)
flat_results.append((s_file, s_function, frame, limit))
limit = find_limit(source)
deps = find_deps(targets)
flat_results.append((s_file, s_function, frame, limit, deps))
return flat_results
@@ -130,12 +139,13 @@ def main(**args):
( result['file'],
result['function'],
int(result['stack_frame']),
float(result['stack_limit'])) # note limit can be inf
float(result['stack_limit']), # note limit can be inf
set())
for result in r]
total_frame = 0
total_limit = 0
for _, _, frame, limit in results:
for _, _, frame, limit, _ in results:
total_frame += frame
total_limit = max(total_limit, limit)
@@ -148,14 +158,15 @@ def main(**args):
( result['file'],
result['function'],
int(result['stack_frame']),
float(result['stack_limit']))
float(result['stack_limit']),
set())
for result in r]
except FileNotFoundError:
prev_results = []
prev_total_frame = 0
prev_total_limit = 0
for _, _, frame, limit in prev_results:
for _, _, frame, limit, _ in prev_results:
prev_total_frame += frame
prev_total_limit = max(prev_total_limit, limit)
@@ -164,28 +175,33 @@ def main(**args):
with open(args['output'], 'w') as f:
w = csv.writer(f)
w.writerow(['file', 'function', 'stack_frame', 'stack_limit'])
for file, func, frame, limit in sorted(results):
for file, func, frame, limit, _ in sorted(results):
w.writerow((file, func, frame, limit))
# print results
def dedup_entries(results, by='function'):
entries = co.defaultdict(lambda: (0, 0))
for file, func, frame, limit in results:
entries = co.defaultdict(lambda: (0, 0, set()))
for file, func, frame, limit, deps in results:
entry = (file if by == 'file' else func)
entry_frame, entry_limit = entries[entry]
entries[entry] = (entry_frame + frame, max(entry_limit, limit))
entry_frame, entry_limit, entry_deps = entries[entry]
entries[entry] = (
entry_frame + frame,
max(entry_limit, limit),
entry_deps | {file if by == 'file' else func
for file, func in deps})
return entries
def diff_entries(olds, news):
diff = co.defaultdict(lambda: (None, None, None, None, 0, 0, 0))
for name, (new_frame, new_limit) in news.items():
diff = co.defaultdict(lambda: (None, None, None, None, 0, 0, 0, set()))
for name, (new_frame, new_limit, deps) in news.items():
diff[name] = (
None, None,
new_frame, new_limit,
new_frame, new_limit,
1.0)
for name, (old_frame, old_limit) in olds.items():
_, _, new_frame, new_limit, _, _, _ = diff[name]
1.0,
deps)
for name, (old_frame, old_limit, _) in olds.items():
_, _, new_frame, new_limit, _, _, _, deps = diff[name]
diff[name] = (
old_frame, old_limit,
new_frame, new_limit,
@@ -197,7 +213,8 @@ def main(**args):
else -float('inf') if m.isinf(old_limit or 0)
else +0.0 if not old_limit and not new_limit
else +1.0 if not old_limit
else ((new_limit or 0) - (old_limit or 0))/(old_limit or 0))
else ((new_limit or 0) - (old_limit or 0))/(old_limit or 0),
deps)
return diff
def sorted_entries(entries):
@@ -230,46 +247,78 @@ def main(**args):
else:
print('%-36s %15s %15s %15s' % (by, 'old', 'new', 'diff'))
def print_entry(name, frame, limit):
print("%-36s %7d %7s" % (name,
frame, '' if m.isinf(limit) else int(limit)))
def print_diff_entry(name,
old_frame, old_limit,
new_frame, new_limit,
diff_frame, diff_limit,
ratio):
print('%-36s %7s %7s %7s %7s %+7d %7s%s' % (name,
old_frame if old_frame is not None else "-",
('' if m.isinf(old_limit) else int(old_limit))
if old_limit is not None else "-",
new_frame if new_frame is not None else "-",
('' if m.isinf(new_limit) else int(new_limit))
if new_limit is not None else "-",
diff_frame,
('+∞' if diff_limit > 0 and m.isinf(diff_limit)
else '-∞' if diff_limit < 0 and m.isinf(diff_limit)
else '%+d' % diff_limit),
'' if not ratio
else ' (+∞%)' if ratio > 0 and m.isinf(ratio)
else ' (-∞%)' if ratio < 0 and m.isinf(ratio)
else ' (%+.1f%%)' % (100*ratio)))
def print_entries(by='function'):
# build optional tree of dependencies
def print_deps(entries, depth, print,
filter=lambda _: True,
prefixes=('', '', '', '')):
entries = entries if isinstance(entries, list) else list(entries)
filtered_entries = [(name, entry)
for name, entry in entries
if filter(name)]
for i, (name, entry) in enumerate(filtered_entries):
last = (i == len(filtered_entries)-1)
print(prefixes[0+last] + name, entry)
if depth > 0:
deps = entry[-1]
print_deps(entries, depth-1, print,
lambda name: name in deps,
( prefixes[2+last] + "|-> ",
prefixes[2+last] + "'-> ",
prefixes[2+last] + "| ",
prefixes[2+last] + " "))
entries = dedup_entries(results, by=by)
if not args.get('diff'):
print_header(by=by)
for name, (frame, limit) in sorted_entries(entries.items()):
print("%-36s %7d %7s" % (name,
frame, '' if m.isinf(limit) else int(limit)))
print_deps(
sorted_entries(entries.items()),
args.get('depth') or 0,
lambda name, entry: print_entry(name, *entry[:-1]))
else:
prev_entries = dedup_entries(prev_results, by=by)
diff = diff_entries(prev_entries, entries)
print_header(by='%s (%d added, %d removed)' % (by,
sum(1 for _, old, _, _, _, _, _ in diff.values() if old is None),
sum(1 for _, _, _, new, _, _, _ in diff.values() if new is None)))
for name, (
old_frame, old_limit,
new_frame, new_limit,
diff_frame, diff_limit, ratio) in sorted_diff_entries(
diff.items()):
if ratio or args.get('all'):
print("%-36s %7s %7s %7s %7s %+7d %7s%s" % (name,
old_frame if old_frame is not None else "-",
('' if m.isinf(old_limit) else int(old_limit))
if old_limit is not None else "-",
new_frame if new_frame is not None else "-",
('' if m.isinf(new_limit) else int(new_limit))
if new_limit is not None else "-",
diff_frame,
('+∞' if diff_limit > 0 and m.isinf(diff_limit)
else '-∞' if diff_limit < 0 and m.isinf(diff_limit)
else '%+d' % diff_limit),
'' if not ratio
else ' (+∞%)' if ratio > 0 and m.isinf(ratio)
else ' (-∞%)' if ratio < 0 and m.isinf(ratio)
else ' (%+.1f%%)' % (100*ratio)))
sum(1 for _, old, _, _, _, _, _, _ in diff.values() if old is None),
sum(1 for _, _, _, new, _, _, _, _ in diff.values() if new is None)))
print_deps(
filter(
lambda x: x[1][6] or args.get('all'),
sorted_diff_entries(diff.items())),
args.get('depth') or 0,
lambda name, entry: print_diff_entry(name, *entry[:-1]))
def print_totals():
if not args.get('diff'):
print("%-36s %7d %7s" % ('TOTAL',
total_frame, '' if m.isinf(total_limit) else int(total_limit)))
print_entry('TOTAL', total_frame, total_limit)
else:
diff_frame = total_frame - prev_total_frame
diff_limit = (
@@ -279,25 +328,14 @@ def main(**args):
0.0 if m.isinf(total_limit or 0) and m.isinf(prev_total_limit or 0)
else +float('inf') if m.isinf(total_limit or 0)
else -float('inf') if m.isinf(prev_total_limit or 0)
else +0.0 if not prev_total_limit and not total_limit
else +1.0 if not prev_total_limit
else 0.0 if not prev_total_limit and not total_limit
else 1.0 if not prev_total_limit
else ((total_limit or 0) - (prev_total_limit or 0))/(prev_total_limit or 0))
print("%-36s %7s %7s %7s %7s %+7d %7s%s" % ('TOTAL',
prev_total_frame if prev_total_frame is not None else '-',
('' if m.isinf(prev_total_limit) else int(prev_total_limit))
if prev_total_limit is not None else '-',
total_frame if total_frame is not None else '-',
('' if m.isinf(total_limit) else int(total_limit))
if total_limit is not None else '-',
diff_frame,
('+∞' if diff_limit > 0 and m.isinf(diff_limit)
else '-∞' if diff_limit < 0 and m.isinf(diff_limit)
else '%+d' % diff_limit),
'' if not ratio
else ' (+∞%)' if ratio > 0 and m.isinf(ratio)
else ' (-∞%)' if ratio < 0 and m.isinf(ratio)
else ' (%+.1f%%)' % (100*ratio)))
print_diff_entry('TOTAL',
prev_total_frame, prev_total_limit,
total_frame, total_limit,
diff_frame, diff_limit,
ratio)
if args.get('quiet'):
pass
@@ -311,6 +349,7 @@ def main(**args):
print_entries(by='function')
print_totals()
if __name__ == "__main__":
import argparse
import sys
@@ -339,6 +378,9 @@ if __name__ == "__main__":
help="Sort by stack frame size.")
parser.add_argument('-F', '--reverse-frame-sort', action='store_true',
help="Sort by stack frame size, but backwards.")
parser.add_argument('-L', '--depth', default=0, type=lambda x: int(x, 0),
nargs='?', const=float('inf'),
help="Depth of dependencies to show.")
parser.add_argument('--files', action='store_true',
help="Show file-level calls.")
parser.add_argument('--summary', action='store_true',

View File

@@ -160,13 +160,23 @@ def main(**args):
else:
print('%-36s %7s %7s %7s' % (by, 'old', 'new', 'diff'))
def print_entry(name, size):
print("%-36s %7d" % (name, size))
def print_diff_entry(name, old, new, diff, ratio):
print("%-36s %7s %7s %+7d%s" % (name,
old or "-",
new or "-",
diff,
' (%+.1f%%)' % (100*ratio) if ratio else ''))
def print_entries(by='struct'):
entries = dedup_entries(results, by=by)
if not args.get('diff'):
print_header(by=by)
for name, size in sorted_entries(entries.items()):
print("%-36s %7d" % (name, size))
print_entry(name, size)
else:
prev_entries = dedup_entries(prev_results, by=by)
diff = diff_entries(prev_entries, entries)
@@ -176,23 +186,19 @@ def main(**args):
for name, (old, new, diff, ratio) in sorted_diff_entries(
diff.items()):
if ratio or args.get('all'):
print("%-36s %7s %7s %+7d%s" % (name,
old or "-",
new or "-",
diff,
' (%+.1f%%)' % (100*ratio) if ratio else ''))
print_diff_entry(name, old, new, diff, ratio)
def print_totals():
if not args.get('diff'):
print("%-36s %7d" % ('TOTAL', total))
print_entry('TOTAL', total)
else:
ratio = (total-prev_total)/prev_total if prev_total else 1.0
print("%-36s %7s %7s %+7d%s" % (
'TOTAL',
prev_total if prev_total else '-',
total if total else '-',
ratio = (0.0 if not prev_total and not total
else 1.0 if not prev_total
else (total-prev_total)/prev_total)
print_diff_entry('TOTAL',
prev_total, total,
total-prev_total,
' (%+.1f%%)' % (100*ratio) if ratio else ''))
ratio)
if args.get('quiet'):
pass

View File

@@ -803,9 +803,9 @@ def main(**args):
failure.case.test(failure=failure, **args)
sys.exit(0)
print('tests passed %d/%d (%.2f%%)' % (passed, total,
print('tests passed %d/%d (%.1f%%)' % (passed, total,
100*(passed/total if total else 1.0)))
print('tests failed %d/%d (%.2f%%)' % (failed, total,
print('tests failed %d/%d (%.1f%%)' % (failed, total,
100*(failed/total if total else 1.0)))
return 1 if failed > 0 else 0