Changed scripts to not infer field purposes from CSV values

Note there's a bit of subtlety here, field _types_ are still infered,
but the intention of the fields, i.e. if the field contains data vs
row name/other properties, must be unambiguous in the scripts.

There is still a _tiny_ bit of inference. For most scripts only one
of --by or --fields is strictly needed, since this makes the purpose of
the other fields unambiguous.

The reason for this change is so the scripts are a bit more reliable,
but also because this simplifies the data parsing/inference a bit.

Oh, and this also changes field inference to use the csv.DictReader's
fieldnames field instead of only inspecting the returned dicts. This
should also save a bit of O(n) overhead when parsing CSV files.

scripts/amor.py

@@ -46,11 +46,15 @@ def dat(x):
 
 def collect(csv_paths, renames=[], defines=[]):
     # collect results from CSV files
+    fields = []
     results = []
     for path in csv_paths:
         try:
             with openio(path) as f:
                 reader = csv.DictReader(f, restval='')
+                fields.extend(
+                    k for k in reader.fieldnames
+                    if k not in fields)
                 for r in reader:
                     # apply any renames
                     if renames:
@@ -69,15 +73,15 @@ def collect(csv_paths, renames=[], defines=[]):
         except FileNotFoundError:
             pass
 
-    return results
+    return fields, results
 
 def main(csv_paths, output, *,
         amor=False,
         per=False,
+        by=None,
         meas=None,
         iter=None,
         size=None,
-        by=None,
         fields=None,
         defines=[]):
     # default to amortizing and per-byte results if size is present
@@ -95,40 +99,43 @@ def main(csv_paths, output, *,
     if fields is not None:
         fields = [k for k, _ in fields]
 
+    if by is None and fields is None:
+        print("error: needs --by or --fields to figure out fields")
+        sys.exit(-1)
+
     # collect results from csv files
-    results = collect(csv_paths, renames, defines)
+    fields_, results = collect(csv_paths, renames, defines)
 
-    # if fields not specified, try to guess from data
+    # if by not specified, guess it's anything not in
-    if fields is None:
+    # iter/size/fields/renames/defines
-        fields = co.OrderedDict()
-        for r in results:
-            for k, v in r.items():
-                if k not in (by or []) and k != iter and v.strip():
-                    try:
-                        dat(v)
-                        fields[k] = True
-                    except ValueError:
-                        fields[k] = False
-        fields = list(k for k,v in fields.items() if v)
-
-    # if by not specified, guess it's anything not in iter/fields and not a
-    # source of a rename
     if by is None:
-        by = co.OrderedDict()
+        by = [
-        for r in results:
+            k for k in fields_
-            # also ignore None keys, these are introduced by csv.DictReader
+            if k != iter
-            # when header + row mismatch
+                and k != size
-            by.update((k, True) for k in r.keys()
+                and k not in (fields or [])
-                if k is not None
+                and not any(k == old_k for _, old_k in renames)
-                    and k != iter
+                and not any(k == k_ for k_, _ in defines)]
-                    and k not in fields
+
-                    and not any(k == old_k for _, old_k in renames))
+    # if fields not specified, guess it's anything not in
-        by = list(by.keys())
+    # by/iter/size/renames/defines
+    if fields is None:
+        fields = [
+            k for k in fields_
+            if k not in (by or [])
+                and k != iter
+                and k != size
+                and not any(k == old_k for _, old_k in renames)
+                and not any(k == k_ for k_, _ in defines)]
+
+    # add meas to by if it isn't already present
+    if meas is not None and meas not in by:
+        by.append(meas)
 
     # convert iter/fields to ints/floats
     for r in results:
-        for k in {iter} | set(fields) | ({size} if size is not None else {}):
+        for k in it.chain([iter], [size] if size is not None else [], fields):
-            if k in r:
+            if k in r and isinstance(r[k], str):
                 r[k] = dat(r[k]) if r[k].strip() else 0
 
     # organize by 'by' values
@@ -141,7 +148,7 @@ def main(csv_paths, output, *,
     # for each key compute the amortized results
     amors = []
     for key, rs in results.items():
-        # keep a running sum for each fied
+        # keep a running sum for each field
         sums = {f: 0 for f in fields}
         size_ = 0
         for j, (i, r) in enumerate(sorted(
@@ -171,7 +178,7 @@ def main(csv_paths, output, *,
     # write results to CSV
     with openio(output, 'w') as f:
         writer = csv.DictWriter(f,
-            by + ([meas] if meas not in by else []) + [iter] + fields)
+            by + [iter] + ([size] if size is not None else []) + fields)
         writer.writeheader()
         for r in amors:
             writer.writerow(r)
@@ -199,6 +206,16 @@ if __name__ == "__main__":
         '--per',
         action='store_true',
         help="Compute per-byte results.")
+    parser.add_argument(
+        '-b', '--by',
+        action='append',
+        type=lambda x: (
+            lambda k, vs=None: (
+                k.strip(),
+                tuple(v.strip() for v in vs.split(','))
+                    if vs is not None else ())
+            )(*x.split('=', 1)),
+        help="Group by this field. Can rename fields with new_name=old_name.")
     parser.add_argument(
         '-m', '--meas',
         help="Optional name of measurement name field. If provided, the name "
@@ -210,16 +227,6 @@ if __name__ == "__main__":
     parser.add_argument(
         '-n', '--size',
         help="Optional name of size field.")
-    parser.add_argument(
-        '-b', '--by',
-        action='append',
-        type=lambda x: (
-            lambda k, vs=None: (
-                k.strip(),
-                tuple(v.strip() for v in vs.split(','))
-                    if vs is not None else ())
-            )(*x.split('=', 1)),
-        help="Group by this field. Can rename fields with new_name=old_name.")
     parser.add_argument(
         '-f', '--field',
         dest='fields',

scripts/avg.py

@@ -46,11 +46,15 @@ def dat(x):
 
 def collect(csv_paths, renames=[], defines=[]):
     # collect results from CSV files
+    fields = []
     results = []
     for path in csv_paths:
         try:
             with openio(path) as f:
                 reader = csv.DictReader(f, restval='')
+                fields.extend(
+                    k for k in reader.fieldnames
+                    if k not in fields)
                 for r in reader:
                     # apply any renames
                     if renames:
@@ -69,7 +73,7 @@ def collect(csv_paths, renames=[], defines=[]):
         except FileNotFoundError:
             pass
 
-    return results
+    return fields, results
 
 def main(csv_paths, output, *,
         sum=False,
@@ -81,8 +85,8 @@ def main(csv_paths, output, *,
         stddev=False,
         gmean=False,
         gstddev=False,
-        meas=None,
         by=None,
+        meas=None,
         seeds=None,
         fields=None,
         defines=[]):
@@ -113,40 +117,41 @@ def main(csv_paths, output, *,
     if fields is not None:
         fields = [k for k, _ in fields]
 
+    if by is None and fields is None:
+        print("error: needs --by or --fields to figure out fields")
+        sys.exit(-1)
+
     # collect results from csv files
-    results = collect(csv_paths, renames, defines)
+    fields_, results = collect(csv_paths, renames, defines)
 
-    # if fields not specified, try to guess from data
+    # if by not specified, guess it's anything not in
-    if fields is None:
+    # seeds/fields/renames/defines
-        fields = co.OrderedDict()
-        for r in results:
-            for k, v in r.items():
-                if k not in (by or []) and k not in (seeds or []) and v.strip():
-                    try:
-                        dat(v)
-                        fields[k] = True
-                    except ValueError:
-                        fields[k] = False
-        fields = list(k for k,v in fields.items() if v)
-
-    # if by not specified, guess it's anything not in seeds/fields and not a
-    # source of a rename
     if by is None:
-        by = co.OrderedDict()
+        by = [
-        for r in results:
+            k for k in fields_
-            # also ignore None keys, these are introduced by csv.DictReader
+            if k not in (seeds or [])
-            # when header + row mismatch
+                and k not in (fields or [])
-            by.update((k, True) for k in r.keys()
+                and not any(k == old_k for _, old_k in renames)
-                if k is not None
+                and not any(k == k_ for k_, _ in defines)]
-                    and k not in (seeds or [])
+
-                    and k not in fields
+    # if fields not specified, guess it's anything not in
-                    and not any(k == old_k for _, old_k in renames))
+    # by/seeds/renames/defines
-        by = list(by.keys())
+    if fields is None:
+        fields = [
+            k for k in fields_
+            if k not in (by or [])
+                and k not in (seeds or [])
+                and not any(k == old_k for _, old_k in renames)
+                and not any(k == k_ for k_, _ in defines)]
+
+    # add meas to by if it isn't already present
+    if meas is not None and meas not in by:
+        by.append(meas)
 
     # convert fields to ints/floats
     for r in results:
         for k in fields:
-            if k in r:
+            if k in r and isinstance(r[k], str):
                 r[k] = dat(r[k]) if r[k].strip() else 0
 
     # organize by 'by' values
@@ -162,11 +167,10 @@ def main(csv_paths, output, *,
         vs = {f: [] for f in fields}
         meas__ = None
         for r in rs:
-            if all(k in r and r[k] == v for k, v in zip(by, key)):
+            for f in fields:
-                for f in fields:
+                vs[f].append(r.get(f, 0))
-                    vs[f].append(r.get(f, 0))
+            if meas is not None and meas in r:
-                if meas is not None and meas in r:
+                meas__ = r[meas]
-                    meas__ = r[meas]
 
         def append(meas_, f_):
             avgs.append(
@@ -197,8 +201,7 @@ def main(csv_paths, output, *,
 
     # write results to CSVS
     with openio(output, 'w') as f:
-        writer = csv.DictWriter(f,
+        writer = csv.DictWriter(f, by + fields)
-            by + ([meas] if meas not in by else []) + fields)
         writer.writeheader()
         for r in avgs:
             writer.writerow(r)
@@ -254,10 +257,6 @@ if __name__ == "__main__":
         '--gstddev',
         action='store_true',
         help="Compute the geometric standard deviation.")
-    parser.add_argument(
-        '-m', '--meas',
-        help="Optional name of measurement name field. If provided, the name "
-            "will be modified with +amor or +per.")
     parser.add_argument(
         '-b', '--by',
         action='append',
@@ -268,6 +267,10 @@ if __name__ == "__main__":
                     if vs is not None else ())
             )(*x.split('=', 1)),
         help="Group by this field. Can rename fields with new_name=old_name.")
+    parser.add_argument(
+        '-m', '--meas',
+        help="Optional name of measurement name field. If provided, the name "
+            "will be modified with +amor or +per.")
     parser.add_argument(
         '-s', '--seed',
         dest='seeds',

scripts/code.py

@@ -315,10 +315,7 @@ def collect(obj_paths, *,
     return results
 
 
-def fold(Result, results, *,
+def fold(Result, results, by=None, defines=[]):
-        by=None,
-        defines=[],
-        **_):
     if by is None:
         by = Result._by
 

scripts/cov.py

@@ -297,10 +297,7 @@ def collect(gcda_paths, *,
     return results
 
 
-def fold(Result, results, *,
+def fold(Result, results, by=None, defines=[]):
-        by=None,
-        defines=[],
-        **_):
     if by is None:
         by = Result._by
 

scripts/data.py

@@ -315,10 +315,7 @@ def collect(obj_paths, *,
     return results
 
 
-def fold(Result, results, *,
+def fold(Result, results, by=None, defines=[]):
-        by=None,
-        defines=[],
-        **_):
     if by is None:
         by = Result._by
 

scripts/perf.py

@@ -627,10 +627,7 @@ def collect(perf_paths, *,
     return results
 
 
-def fold(Result, results, *,
+def fold(Result, results, by=None, defines=[]):
-        by=None,
-        defines=[],
-        **_):
     if by is None:
         by = Result._by
 

scripts/perfbd.py

@@ -593,10 +593,7 @@ def collect(obj_path, trace_paths, *,
     return results
 
 
-def fold(Result, results, *,
+def fold(Result, results, by=None, defines=[]):
-        by=None,
-        defines=[],
-        **_):
     if by is None:
         by = Result._by
 

scripts/plot.py

@@ -445,11 +445,15 @@ class Plot:
 
 def collect(csv_paths, renames=[], defines=[]):
     # collect results from CSV files
+    fields = []
     results = []
     for path in csv_paths:
         try:
             with openio(path) as f:
                 reader = csv.DictReader(f, restval='')
+                fields.extend(
+                    k for k in reader.fieldnames
+                    if k not in fields)
                 for r in reader:
                     # apply any renames
                     if renames:
@@ -468,7 +472,7 @@ def collect(csv_paths, renames=[], defines=[]):
         except FileNotFoundError:
             pass
 
-    return results
+    return fields, results
 
 def fold(results, by=None, x=None, y=None, defines=[]):
     # filter by matching defines
@@ -479,29 +483,16 @@ def fold(results, by=None, x=None, y=None, defines=[]):
                 results_.append(r)
         results = results_
 
-    # if y not specified, try to guess from data
-    if not y:
-        y = co.OrderedDict()
-        for r in results:
-            for k, v in r.items():
-                if (not by or k not in by) and v.strip():
-                    try:
-                        dat(v)
-                        y[k] = True
-                    except ValueError:
-                        y[k] = False
-        y = list(k for k,v in y.items() if v)
-
     if by:
         # find all 'by' values
-        ks = set()
+        keys = set()
         for r in results:
-            ks.add(tuple(r.get(k, '') for k in by))
+            keys.add(tuple(r.get(k, '') for k in by))
-        ks = sorted(ks)
+        keys = sorted(keys)
 
     # collect all datasets
     datasets = co.OrderedDict()
-    for ks_ in (ks if by else [()]):
+    for key in (keys if by else [()]):
         for x_ in (x if x else [None]):
             for y_ in y:
                 # organize by 'by', x, and y
@@ -511,7 +502,7 @@ def fold(results, by=None, x=None, y=None, defines=[]):
                     # filter by 'by'
                     if by and not all(
                             k in r and r[k] == v
-                            for k, v in zip(by, ks_)):
+                            for k, v in zip(by, key)):
                         continue
 
                     # find xs
@@ -542,8 +533,8 @@ def fold(results, by=None, x=None, y=None, defines=[]):
 
                 # hide x/y if there is only one field
                 k_x = x_ if len(x or []) > 1 else ''
-                k_y = y_ if len(y or []) > 1 or (not ks_ and not k_x) else ''
+                k_y = y_ if len(y or []) > 1 or (not key and not k_x) else ''
-                datasets[ks_ + (k_x, k_y)] = dataset
+                datasets[key + (k_x, k_y)] = dataset
 
     return datasets
 
@@ -904,13 +895,17 @@ def main(csv_paths, *,
     all_defines = sorted(all_defines.items())
 
     # separate out renames
-    renames = list(it.chain.from_iterable(
+    all_renames = list(it.chain.from_iterable(
         ((k, v) for v in vs)
         for k, vs in it.chain(all_by, all_x, all_y)))
     all_by = [k for k, _ in all_by]
     all_x = [k for k, _ in all_x]
     all_y = [k for k, _ in all_y]
 
+    if not all_by and not all_y:
+        print("error: needs --by or -y to figure out fields")
+        sys.exit(-1)
+
     # create a grid of subplots
     grid = Grid.fromargs(**subplot, subplots=subplots)
 
@@ -994,10 +989,19 @@ def main(csv_paths, *,
         f.writeln = writeln
 
         # first collect results from CSV files
-        results = collect(csv_paths, renames, all_defines)
+        fields_, results = collect(csv_paths, all_renames, all_defines)
+
+        # if y not specified, guess it's anything not in by/defines/x/renames
+        all_y_ = all_y
+        if not all_y:
+            all_y_ = [
+                k for k in fields_
+                    if k not in all_by
+                        and not any(k == k_ for k_, _ in all_defines)
+                        and not any(k == old_k for _, old_k in all_renames)]
 
         # then extract the requested datasets
-        datasets_ = fold(results, all_by, all_x, all_y)
+        datasets_ = fold(results, all_by, all_x, all_y_)
 
         # figure out colors/chars here so that subplot defines
         # don't change them later, that'd be bad
@@ -1143,7 +1147,7 @@ def main(csv_paths, *,
 
             # data can be constrained by subplot-specific defines,
             # so re-extract for each plot
-            subdatasets = fold(results, all_by, all_x, all_y, define_)
+            subdatasets = fold(results, all_by, all_x, all_y_, define_)
 
             # filter by subplot x/y
             subdatasets = co.OrderedDict([(name, dataset)

scripts/plotmpl.py

@@ -191,11 +191,15 @@ def dat(x):
 
 def collect(csv_paths, renames=[], defines=[]):
     # collect results from CSV files
+    fields = []
     results = []
     for path in csv_paths:
         try:
             with openio(path) as f:
                 reader = csv.DictReader(f, restval='')
+                fields.extend(
+                    k for k in reader.fieldnames
+                    if k not in fields)
                 for r in reader:
                     # apply any renames
                     if renames:
@@ -214,7 +218,7 @@ def collect(csv_paths, renames=[], defines=[]):
         except FileNotFoundError:
             pass
 
-    return results
+    return fields, results
 
 def fold(results, by=None, x=None, y=None, defines=[]):
     # filter by matching defines
@@ -225,29 +229,16 @@ def fold(results, by=None, x=None, y=None, defines=[]):
                 results_.append(r)
         results = results_
 
-    # if y not specified, try to guess from data
-    if not y:
-        y = co.OrderedDict()
-        for r in results:
-            for k, v in r.items():
-                if (not by or k not in by) and v.strip():
-                    try:
-                        dat(v)
-                        y[k] = True
-                    except ValueError:
-                        y[k] = False
-        y = list(k for k,v in y.items() if v)
-
     if by:
         # find all 'by' values
-        ks = set()
+        keys = set()
         for r in results:
-            ks.add(tuple(r.get(k, '') for k in by))
+            keys.add(tuple(r.get(k, '') for k in by))
-        ks = sorted(ks)
+        keys = sorted(keys)
 
     # collect all datasets
     datasets = co.OrderedDict()
-    for ks_ in (ks if by else [()]):
+    for key in (keys if by else [()]):
         for x_ in (x if x else [None]):
             for y_ in y:
                 # organize by 'by', x, and y
@@ -257,7 +248,7 @@ def fold(results, by=None, x=None, y=None, defines=[]):
                     # filter by 'by'
                     if by and not all(
                             k in r and r[k] == v
-                            for k, v in zip(by, ks_)):
+                            for k, v in zip(by, key)):
                         continue
 
                     # find xs
@@ -288,8 +279,8 @@ def fold(results, by=None, x=None, y=None, defines=[]):
 
                 # hide x/y if there is only one field
                 k_x = x_ if len(x or []) > 1 else ''
-                k_y = y_ if len(y or []) > 1 or (not ks_ and not k_x) else ''
+                k_y = y_ if len(y or []) > 1 or (not key and not k_x) else ''
-                datasets[ks_ + (k_x, k_y)] = dataset
+                datasets[key + (k_x, k_y)] = dataset
 
     return datasets
 
@@ -746,15 +737,27 @@ def main(csv_paths, output, *,
     all_defines = sorted(all_defines.items())
 
     # separate out renames
-    renames = list(it.chain.from_iterable(
+    all_renames = list(it.chain.from_iterable(
         ((k, v) for v in vs)
         for k, vs in it.chain(all_by, all_x, all_y)))
     all_by = [k for k, _ in all_by]
     all_x = [k for k, _ in all_x]
     all_y = [k for k, _ in all_y]
 
+    if not all_by and not all_y:
+        print("error: needs --by or -y to figure out fields")
+        sys.exit(-1)
+
     # first collect results from CSV files
-    results = collect(csv_paths, renames, all_defines)
+    fields_, results = collect(csv_paths, all_renames, all_defines)
+
+    # if y not specified, guess it's anything not in by/defines/x/renames
+    if not all_y:
+        all_y = [
+            k for k in fields_
+                if k not in all_by
+                    and not any(k == k_ for k_, _ in all_defines)
+                    and not any(k == old_k for _, old_k in all_renames)]
 
     # then extract the requested datasets
     datasets_ = fold(results, all_by, all_x, all_y)

scripts/stack.py

@@ -273,10 +273,7 @@ def collect(ci_paths, *,
     return results
 
 
-def fold(Result, results, *,
+def fold(Result, results, by=None, defines=[]):
-        by=None,
-        defines=[],
-        **_):
     if by is None:
         by = Result._by
 

scripts/structs.py

@@ -264,10 +264,7 @@ def collect(obj_paths, *,
     return results
 
 
-def fold(Result, results, *,
+def fold(Result, results, by=None, defines=[]):
-        by=None,
-        defines=[],
-        **_):
     if by is None:
         by = Result._by
 

scripts/summary.py

@@ -251,11 +251,15 @@ def openio(path, mode='r', buffering=-1):
 
 def collect(csv_paths, renames=[], defines=[]):
     # collect results from CSV files
+    fields = []
     results = []
     for path in csv_paths:
         try:
             with openio(path) as f:
                 reader = csv.DictReader(f, restval='')
+                fields.extend(
+                    k for k in reader.fieldnames
+                    if k not in fields)
                 for r in reader:
                     # apply any renames
                     if renames:
@@ -274,49 +278,34 @@ def collect(csv_paths, renames=[], defines=[]):
         except FileNotFoundError:
             pass
 
-    return results
+    return fields, results
 
-def infer(results, *,
+def infer(fields_, results,
         by=None,
         fields=None,
         types={},
         ops={},
         renames=[],
-        **_):
+        defines=[]):
-    # if fields not specified, try to guess from data
+    # if by not specified, guess it's anything not in fields/renames/defines
-    if fields is None:
-        fields = co.OrderedDict()
-        for r in results:
-            for k, v in r.items():
-                if (by is None or k not in by) and v.strip():
-                    types_ = []
-                    for t in fields.get(k, TYPES.values()):
-                        try:
-                            t(v)
-                            types_.append(t)
-                        except ValueError:
-                            pass
-                    fields[k] = types_
-        fields = list(k for k, v in fields.items() if v)
-
-    # deduplicate fields
-    fields = list(co.OrderedDict.fromkeys(fields).keys())
-
-    # if by not specified, guess it's anything not in fields and not a
-    # source of a rename
     if by is None:
-        by = co.OrderedDict()
+        by = [
-        for r in results:
+            k for k in fields_
-            # also ignore None keys, these are introduced by csv.DictReader
+            if k not in (fields or [])
-            # when header + row mismatch
+                and not any(k == old_k for _, old_k in renames)
-            by.update((k, True) for k in r.keys()
+                and not any(k == k_ for k_, _ in defines)]
-                if k is not None
-                    and k not in fields
-                    and not any(k == old_k for _, old_k in renames))
-        by = list(by.keys())
 
-    # deduplicate fields
+    # if fields not specified, guess it's anything not in by/renames/defines
+    if fields is None:
+        fields = [
+            k for k in fields_
+            if k not in (by or [])
+                and not any(k == old_k for _, old_k in renames)
+                and not any(k == k_ for k_, _ in defines)]
+
+    # deduplicate by/fields
     by = list(co.OrderedDict.fromkeys(by).keys())
+    fields = list(co.OrderedDict.fromkeys(fields).keys())
 
     # find best type for all fields
     types_ = {}
@@ -381,10 +370,7 @@ def infer(results, *,
     })
 
 
-def fold(Result, results, *,
+def fold(Result, results, by=None, defines=[]):
-        by=None,
-        defines=[],
-        **_):
     if by is None:
         by = Result._by
 
@@ -634,16 +620,21 @@ def main(csv_paths, *,
                 ops_[new_k] = ops[old_k]
         ops.update(ops_)
 
+    if by is None and fields is None:
+        print("error: needs --by or --fields to figure out fields")
+        sys.exit(-1)
+
     # find CSV files
-    results = collect(csv_paths, renames=renames, defines=defines)
+    fields_, results = collect(csv_paths, renames, defines)
 
     # homogenize
-    Result = infer(results,
+    Result = infer(fields_, results,
         by=by,
         fields=fields,
         types=types,
         ops=ops,
-        renames=renames)
+        renames=renames,
+        defines=defines)
     results_ = []
     for r in results:
         if not any(k in r and r[k].strip()
@@ -682,7 +673,7 @@ def main(csv_paths, *,
 
     # find previous results?
     if args.get('diff'):
-        diff_results = collect([args['diff']], renames=renames, defines=defines)
+        _, diff_results = collect([args['diff']], renames, defines)
         diff_results_ = []
         for r in diff_results:
             if not any(k in r and r[k].strip()