Added ./script/summary.py

A full summary of static measurements (code size, stack usage, etc) can now
be found with:

    make summary

This is done through the combination of a new ./scripts/summary.py
script and the ability of existing scripts to merge into existing csv
files, allowing multiple results to be merged either in a pipeline, or
in parallel with a single ./script/summary.py call.

The ./scripts/summary.py script can also be used to quickly compare
different builds or configurations. This is a proper implementation
of a similar but hacky shell script that has already been very useful
for making optimization decisions:

    $ ./scripts/structs.py new.csv -d old.csv --summary
    name (2 added, 0 removed)               code             stack            structs
    TOTAL                                  28648 (-2.7%)      2448               1012

Also some other small tweaks to scripts:

- Removed state saving diff rules. This isn't the most useful way to
  handle comparing changes.

- Added short flags for --summary (-Y) and --files (-F), since these
  are quite often used.

scripts/code.py

@@ -48,17 +48,30 @@ def collect(paths, **args):
         # map to source files
         if args.get('build_dir'):
             file = re.sub('%s/*' % re.escape(args['build_dir']), '', file)
+        # replace .o with .c, different scripts report .o/.c, we need to
+        # choose one if we want to deduplicate csv files
+        file = re.sub('\.o$', '.c', file)
         # discard internal functions
         if not args.get('everything'):
             if func.startswith('__'):
                 continue
         # discard .8449 suffixes created by optimizer
         func = re.sub('\.[0-9]+', '', func)
+
         flat_results.append((file, func, size))
 
     return flat_results
 
 def main(**args):
+    def openio(path, mode='r'):
+        if path == '-':
+            if 'r' in mode:
+                return os.fdopen(os.dup(sys.stdin.fileno()), 'r')
+            else:
+                return os.fdopen(os.dup(sys.stdout.fileno()), 'w')
+        else:
+            return open(path, mode)
+
     # find sizes
     if not args.get('use', None):
         # find .o files
@@ -76,13 +89,14 @@ def main(**args):
 
         results = collect(paths, **args)
     else:
-        with open(args['use']) as f:
+        with openio(args['use']) as f:
             r = csv.DictReader(f)
             results = [
                 (   result['file'],
-                    result['function'],
+                    result['name'],
                     int(result['code_size']))
-                for result in r]
+                for result in r
+                if result.get('code_size') not in {None, ''}]
 
     total = 0
     for _, _, size in results:
@@ -91,13 +105,14 @@ def main(**args):
     # find previous results?
     if args.get('diff'):
         try:
-            with open(args['diff']) as f:
+            with openio(args['diff']) as f:
                 r = csv.DictReader(f)
                 prev_results = [
                     (   result['file'],
-                        result['function'],
+                        result['name'],
                         int(result['code_size']))
-                    for result in r]
+                    for result in r
+                    if result.get('code_size') not in {None, ''}]
         except FileNotFoundError:
             prev_results = []
 
@@ -107,14 +122,34 @@ def main(**args):
 
     # write results to CSV
     if args.get('output'):
-        with open(args['output'], 'w') as f:
-            w = csv.writer(f)
-            w.writerow(['file', 'function', 'code_size'])
-            for file, func, size in sorted(results):
-                w.writerow((file, func, size))
+        merged_results = co.defaultdict(lambda: {})
+        other_fields = []
+
+        # merge?
+        if args.get('merge'):
+            try:
+                with openio(args['merge']) as f:
+                    r = csv.DictReader(f)
+                    for result in r:
+                        file = result.pop('file', '')
+                        func = result.pop('name', '')
+                        result.pop('code_size', None)
+                        merged_results[(file, func)] = result
+                        other_fields = result.keys()
+            except FileNotFoundError:
+                pass
+
+        for file, func, size in results:
+            merged_results[(file, func)]['code_size'] = size
+
+        with openio(args['output'], 'w') as f:
+            w = csv.DictWriter(f, ['file', 'name', *other_fields, 'code_size'])
+            w.writeheader()
+            for (file, func), result in sorted(merged_results.items()):
+                w.writerow({'file': file, 'name': func, **result})
 
     # print results
-    def dedup_entries(results, by='function'):
+    def dedup_entries(results, by='name'):
         entries = co.defaultdict(lambda: 0)
         for file, func, size in results:
             entry = (file if by == 'file' else func)
@@ -162,7 +197,7 @@ def main(**args):
             diff,
             ' (%+.1f%%)' % (100*ratio) if ratio else ''))
 
-    def print_entries(by='function'):
+    def print_entries(by='name'):
         entries = dedup_entries(results, by=by)
 
         if not args.get('diff'):
@@ -201,7 +236,7 @@ def main(**args):
         print_entries(by='file')
         print_totals()
     else:
-        print_entries(by='function')
+        print_entries(by='name')
         print_totals()
 
 if __name__ == "__main__":
@@ -214,12 +249,16 @@ if __name__ == "__main__":
             or a list of paths. Defaults to %r." % OBJ_PATHS)
     parser.add_argument('-v', '--verbose', action='store_true',
         help="Output commands that run behind the scenes.")
+    parser.add_argument('-q', '--quiet', action='store_true',
+        help="Don't show anything, useful with -o.")
     parser.add_argument('-o', '--output',
         help="Specify CSV file to store results.")
     parser.add_argument('-u', '--use',
         help="Don't compile and find code sizes, instead use this CSV file.")
     parser.add_argument('-d', '--diff',
         help="Specify CSV file to diff code size against.")
+    parser.add_argument('-m', '--merge',
+        help="Merge with an existing CSV file when writing to output.")
     parser.add_argument('-a', '--all', action='store_true',
         help="Show all functions, not just the ones that changed.")
     parser.add_argument('-A', '--everything', action='store_true',
@@ -228,13 +267,11 @@ if __name__ == "__main__":
         help="Sort by size.")
     parser.add_argument('-S', '--reverse-size-sort', action='store_true',
         help="Sort by size, but backwards.")
-    parser.add_argument('--files', action='store_true',
+    parser.add_argument('-F', '--files', action='store_true',
         help="Show file-level code sizes. Note this does not include padding! "
             "So sizes may differ from other tools.")
-    parser.add_argument('--summary', action='store_true',
+    parser.add_argument('-Y', '--summary', action='store_true',
         help="Only show the total code size.")
-    parser.add_argument('-q', '--quiet', action='store_true',
-        help="Don't show anything, useful with -o.")
     parser.add_argument('--type', default='tTrRdD',
         help="Type of symbols to report, this uses the same single-character "
             "type-names emitted by nm. Defaults to %(default)r.")