scripts: Adopted globs in all field matchers (-D/--define, -c/--compare)

Globs in CLI attrs (-L'*=bs=%(bs)s' for example), have been remarkably
useful. It makes sense to extend this to the other flags that match
against CSV fields, though this does add complexity to a large number of
smaller scripts.

- -D/--define can now use globs when filtering:

    $ ./scripts/code.py lfs.o -Dfunction='lfsr_file_*'

  -D/--define already accepted a comma-separated list of options, so
  extending this to globs makes sense.

  Note this differs from test.py/bench.py's -D/--define. Globbing in
  test.py/bench.py wouldn't really work since -D/--define is generative,
  not matching. But there's already other differences such as integer
  parsing, range, etc. It's not worth making these perfectly consistent
  as they are really two different tools that just happen to look the
  same.

- -c/--compare now matches with globs when finding the compare entry:

    $ ./scripts/code.py lfs.o -c'lfs*_file_sync'

  This is quite a bit less useful that -D/--define, but makes sense for
  consistency.

  Note -c/--compare just chooses the first match. It doesn't really make
  sense to compare against multiple entries.

This raised the question of globs in the field specifiers themselves
(-f'bench_*' for example), but I'm rejecting this for now as I need to
draw the complexity/scope _somewhere_, and I'm worried it's already way
over on the too-complex side.

So, for now, field names must always be specified explicitly. Globbing
field names would add too much complexity. Especially considering how
many flags accept field names in these scripts.

scripts/code.py

@@ -18,6 +18,7 @@ if __name__ == "__main__":
 
 import collections as co
 import csv
+import fnmatch
 import functools as ft
 import io
 import itertools as it
@@ -543,7 +544,9 @@ def fold(Result, results, *,
     if defines:
         results_ = []
         for r in results:
-            if all(str(getattr(r, k)) in vs for k, vs in defines):
+            if all(any(fnmatch.fnmatchcase(str(getattr(r, k, '')), v)
+                        for v in vs)
+                    for k, vs in defines):
                 results_.append(r)
         results = results_
 
@@ -634,7 +637,13 @@ def table(Result, results, diff_results=None, *,
 
     # find compare entry if there is one
     if compare:
-        compare_r = table.get(','.join(str(k) for k in compare))
+        compare_ = min(
+            (n for n in table.keys()
+                if all(fnmatch.fnmatchcase(k, c)
+                    for k, c in it.zip_longest(n.split(','), compare,
+                        fillvalue=''))),
+            default=compare)
+        compare_r = table.get(compare_)
 
     # build up our lines
     lines = []
@@ -1156,7 +1165,8 @@ if __name__ == "__main__":
                     k.strip(),
                     {v.strip() for v in vs.split(',')})
                 )(*x.split('=', 1)),
-            help="Only include results where this field is this value.")
+            help="Only include results where this field is this value. May "
+                "include comma-separated options and globs.")
     class AppendSort(argparse.Action):
         def __call__(self, parser, namespace, value, option):
             if namespace.sort is None:

scripts/cov.py

@@ -18,6 +18,7 @@ if __name__ == "__main__":
 
 import collections as co
 import csv
+import fnmatch
 import io
 import itertools as it
 import json
@@ -403,7 +404,9 @@ def fold(Result, results, *,
     if defines:
         results_ = []
         for r in results:
-            if all(str(getattr(r, k)) in vs for k, vs in defines):
+            if all(any(fnmatch.fnmatchcase(str(getattr(r, k, '')), v)
+                        for v in vs)
+                    for k, vs in defines):
                 results_.append(r)
         results = results_
 
@@ -494,7 +497,13 @@ def table(Result, results, diff_results=None, *,
 
     # find compare entry if there is one
     if compare:
-        compare_r = table.get(','.join(str(k) for k in compare))
+        compare_ = min(
+            (n for n in table.keys()
+                if all(fnmatch.fnmatchcase(k, c)
+                    for k, c in it.zip_longest(n.split(','), compare,
+                        fillvalue=''))),
+            default=compare)
+        compare_r = table.get(compare_)
 
     # build up our lines
     lines = []
@@ -1126,7 +1135,8 @@ if __name__ == "__main__":
                     k.strip(),
                     {v.strip() for v in vs.split(',')})
                 )(*x.split('=', 1)),
-            help="Only include results where this field is this value.")
+            help="Only include results where this field is this value. May "
+                "include comma-separated options and globs.")
     class AppendSort(argparse.Action):
         def __call__(self, parser, namespace, value, option):
             if namespace.sort is None:

scripts/csv.py

@@ -16,6 +16,7 @@ if __name__ == "__main__":
 
 import collections as co
 import csv
+import fnmatch
 import functools as ft
 import itertools as it
 import math as mt
@@ -1566,7 +1567,9 @@ def homogenize(Result, results, *,
         # evaluation order of exprs/mods/etc, note this isn't really
         # inconsistent with the other scripts, since they don't really
         # evaluate anything
-        if not all(k in r and str(r[k]) in vs for k, vs in defines):
+        if not all(any(fnmatch.fnmatchcase(str(r.get(k, '')), v)
+                    for v in vs)
+                for k, vs in defines):
             continue
 
         # append a result
@@ -1628,7 +1631,9 @@ def fold(Result, results, *,
     if defines:
         results_ = []
         for r in results:
-            if all(str(getattr(r, k)) in vs for k, vs in defines):
+            if all(any(fnmatch.fnmatchcase(str(getattr(r, k, '')), v)
+                        for v in vs)
+                    for k, vs in defines):
                 results_.append(r)
         results = results_
 
@@ -1766,7 +1771,13 @@ def table(Result, results, diff_results=None, *,
 
     # find compare entry if there is one
     if compare:
-        compare_r = table.get(','.join(str(k) for k in compare))
+        compare_ = min(
+            (n for n in table.keys()
+                if all(fnmatch.fnmatchcase(k, c)
+                    for k, c in it.zip_longest(n.split(','), compare,
+                        fillvalue=''))),
+            default=compare)
+        compare_r = table.get(compare_)
 
     # build up our lines
     lines = []
@@ -2496,7 +2507,7 @@ if __name__ == "__main__":
                     {v.strip() for v in vs.split(',')})
                 )(*x.split('=', 1)),
             help="Only include results where this field is this value. May "
-                "include comma-separated options.")
+                "include comma-separated options and globs.")
     class AppendSort(argparse.Action):
         def __call__(self, parser, namespace, value, option):
             if namespace.sort is None:

scripts/ctx.py

@@ -15,6 +15,7 @@ if __name__ == "__main__":
 
 import collections as co
 import csv
+import fnmatch
 import functools as ft
 import io
 import itertools as it
@@ -756,7 +757,9 @@ def fold(Result, results, *,
     if defines:
         results_ = []
         for r in results:
-            if all(str(getattr(r, k)) in vs for k, vs in defines):
+            if all(any(fnmatch.fnmatchcase(str(getattr(r, k, '')), v)
+                        for v in vs)
+                    for k, vs in defines):
                 results_.append(r)
         results = results_
 
@@ -894,7 +897,13 @@ def table(Result, results, diff_results=None, *,
 
     # find compare entry if there is one
     if compare:
-        compare_r = table.get(','.join(str(k) for k in compare))
+        compare_ = min(
+            (n for n in table.keys()
+                if all(fnmatch.fnmatchcase(k, c)
+                    for k, c in it.zip_longest(n.split(','), compare,
+                        fillvalue=''))),
+            default=compare)
+        compare_r = table.get(compare_)
 
     # build up our lines
     lines = []
@@ -1452,7 +1461,8 @@ if __name__ == "__main__":
                     k.strip(),
                     {v.strip() for v in vs.split(',')})
                 )(*x.split('=', 1)),
-            help="Only include results where this field is this value.")
+            help="Only include results where this field is this value. May "
+                "include comma-separated options and globs.")
     class AppendSort(argparse.Action):
         def __call__(self, parser, namespace, value, option):
             if namespace.sort is None:

scripts/data.py

@@ -18,6 +18,7 @@ if __name__ == "__main__":
 
 import collections as co
 import csv
+import fnmatch
 import functools as ft
 import io
 import itertools as it
@@ -543,7 +544,9 @@ def fold(Result, results, *,
     if defines:
         results_ = []
         for r in results:
-            if all(str(getattr(r, k)) in vs for k, vs in defines):
+            if all(any(fnmatch.fnmatchcase(str(getattr(r, k, '')), v)
+                        for v in vs)
+                    for k, vs in defines):
                 results_.append(r)
         results = results_
 
@@ -634,7 +637,13 @@ def table(Result, results, diff_results=None, *,
 
     # find compare entry if there is one
     if compare:
-        compare_r = table.get(','.join(str(k) for k in compare))
+        compare_ = min(
+            (n for n in table.keys()
+                if all(fnmatch.fnmatchcase(k, c)
+                    for k, c in it.zip_longest(n.split(','), compare,
+                        fillvalue=''))),
+            default=compare)
+        compare_r = table.get(compare_)
 
     # build up our lines
     lines = []
@@ -1156,7 +1165,8 @@ if __name__ == "__main__":
                     k.strip(),
                     {v.strip() for v in vs.split(',')})
                 )(*x.split('=', 1)),
-            help="Only include results where this field is this value.")
+            help="Only include results where this field is this value. May "
+                "include comma-separated options and globs.")
     class AppendSort(argparse.Action):
         def __call__(self, parser, namespace, value, option):
             if namespace.sort is None:

scripts/perf.py

@@ -19,6 +19,7 @@ import collections as co
 import csv
 import errno
 import fcntl
+import fnmatch
 import functools as ft
 import io
 import itertools as it
@@ -857,7 +858,9 @@ def fold(Result, results, *,
     if defines:
         results_ = []
         for r in results:
-            if all(str(getattr(r, k)) in vs for k, vs in defines):
+            if all(any(fnmatch.fnmatchcase(str(getattr(r, k, '')), v)
+                        for v in vs)
+                    for k, vs in defines):
                 results_.append(r)
         results = results_
 
@@ -995,7 +998,13 @@ def table(Result, results, diff_results=None, *,
 
     # find compare entry if there is one
     if compare:
-        compare_r = table.get(','.join(str(k) for k in compare))
+        compare_ = min(
+            (n for n in table.keys()
+                if all(fnmatch.fnmatchcase(k, c)
+                    for k, c in it.zip_longest(n.split(','), compare,
+                        fillvalue=''))),
+            default=compare)
+        compare_r = table.get(compare_)
 
     # build up our lines
     lines = []
@@ -1698,7 +1707,8 @@ if __name__ == "__main__":
                     k.strip(),
                     {v.strip() for v in vs.split(',')})
                 )(*x.split('=', 1)),
-            help="Only include results where this field is this value.")
+            help="Only include results where this field is this value. May "
+                "include comma-separated options and globs.")
     class AppendSort(argparse.Action):
         def __call__(self, parser, namespace, value, option):
             if namespace.sort is None:

scripts/perfbd.py

@@ -18,6 +18,7 @@ if __name__ == "__main__":
 import bisect
 import collections as co
 import csv
+import fnmatch
 import functools as ft
 import io
 import itertools as it
@@ -831,7 +832,9 @@ def fold(Result, results, *,
     if defines:
         results_ = []
         for r in results:
-            if all(str(getattr(r, k)) in vs for k, vs in defines):
+            if all(any(fnmatch.fnmatchcase(str(getattr(r, k, '')), v)
+                        for v in vs)
+                    for k, vs in defines):
                 results_.append(r)
         results = results_
 
@@ -969,7 +972,13 @@ def table(Result, results, diff_results=None, *,
 
     # find compare entry if there is one
     if compare:
-        compare_r = table.get(','.join(str(k) for k in compare))
+        compare_ = min(
+            (n for n in table.keys()
+                if all(fnmatch.fnmatchcase(k, c)
+                    for k, c in it.zip_longest(n.split(','), compare,
+                        fillvalue=''))),
+            default=compare)
+        compare_r = table.get(compare_)
 
     # build up our lines
     lines = []
@@ -1699,7 +1708,8 @@ if __name__ == "__main__":
                     k.strip(),
                     {v.strip() for v in vs.split(',')})
                 )(*x.split('=', 1)),
-            help="Only include results where this field is this value.")
+            help="Only include results where this field is this value. May "
+                "include comma-separated options and globs.")
     class AppendSort(argparse.Action):
         def __call__(self, parser, namespace, value, option):
             if namespace.sort is None:

scripts/plot.py

@@ -327,7 +327,9 @@ def collect(csv_paths, defines=[]):
                             if k not in fields)
                 for r in reader:
                     # filter by matching defines
-                    if not all(k in r and r[k] in vs for k, vs in defines):
+                    if not all(any(fnmatch.fnmatchcase(r.get(k, ''), v)
+                                for v in vs)
+                            for k, vs in defines):
                         continue
 
                     results.append(r)
@@ -341,7 +343,9 @@ def fold(results, by=None, x=None, y=None, defines=[]):
     if defines:
         results_ = []
         for r in results:
-            if all(k in r and r[k] in vs for k, vs in defines):
+            if all(any(fnmatch.fnmatchcase(r.get(k, ''), v)
+                        for v in vs)
+                    for k, vs in defines):
                 results_.append(r)
         results = results_
 
@@ -1913,7 +1917,7 @@ if __name__ == "__main__":
                 )(*x.split('=', 1)),
             action='append',
             help="Only include results where this field is this value. May "
-                "include comma-separated options.")
+                "include comma-separated options and globs.")
     parser.add_argument(
             '-L', '--add-label',
             dest='labels',

scripts/plotmpl.py

@@ -211,7 +211,9 @@ def collect(csv_paths, defines=[]):
                             if k not in fields)
                 for r in reader:
                     # filter by matching defines
-                    if not all(k in r and r[k] in vs for k, vs in defines):
+                    if not all(any(fnmatch.fnmatchcase(r.get(k, ''), v)
+                                for v in vs)
+                            for k, vs in defines):
                         continue
 
                     results.append(r)
@@ -225,7 +227,9 @@ def fold(results, by=None, x=None, y=None, defines=[]):
     if defines:
         results_ = []
         for r in results:
-            if all(k in r and r[k] in vs for k, vs in defines):
+            if all(any(fnmatch.fnmatchcase(r.get(k, ''), v)
+                        for v in vs)
+                    for k, vs in defines):
                 results_.append(r)
         results = results_
 
@@ -1367,7 +1371,7 @@ if __name__ == "__main__":
                 )(*x.split('=', 1)),
             action='append',
             help="Only include results where this field is this value. May "
-                "include comma-separated options.")
+                "include comma-separated options and globs.")
     parser.add_argument(
             '-L', '--add-label',
             dest='labels',

scripts/stack.py

@@ -16,6 +16,7 @@ if __name__ == "__main__":
 
 import collections as co
 import csv
+import fnmatch
 import functools as ft
 import io
 import itertools as it
@@ -499,7 +500,9 @@ def fold(Result, results, *,
     if defines:
         results_ = []
         for r in results:
-            if all(str(getattr(r, k)) in vs for k, vs in defines):
+            if all(any(fnmatch.fnmatchcase(str(getattr(r, k, '')), v)
+                        for v in vs)
+                    for k, vs in defines):
                 results_.append(r)
         results = results_
 
@@ -637,7 +640,13 @@ def table(Result, results, diff_results=None, *,
 
     # find compare entry if there is one
     if compare:
-        compare_r = table.get(','.join(str(k) for k in compare))
+        compare_ = min(
+            (n for n in table.keys()
+                if all(fnmatch.fnmatchcase(k, c)
+                    for k, c in it.zip_longest(n.split(','), compare,
+                        fillvalue=''))),
+            default=compare)
+        compare_r = table.get(compare_)
 
     # build up our lines
     lines = []
@@ -1197,7 +1206,8 @@ if __name__ == "__main__":
                     k.strip(),
                     {v.strip() for v in vs.split(',')})
                 )(*x.split('=', 1)),
-            help="Only include results where this field is this value.")
+            help="Only include results where this field is this value. May "
+                "include comma-separated options and globs.")
     class AppendSort(argparse.Action):
         def __call__(self, parser, namespace, value, option):
             if namespace.sort is None:

scripts/structs.py

@@ -15,9 +15,10 @@ if __name__ == "__main__":
 
 import collections as co
 import csv
-import itertools as it
+import fnmatch
-import io
 import functools as ft
+import io
+import itertools as it
 import math as mt
 import os
 import re
@@ -637,7 +638,9 @@ def fold(Result, results, *,
     if defines:
         results_ = []
         for r in results:
-            if all(str(getattr(r, k)) in vs for k, vs in defines):
+            if all(any(fnmatch.fnmatchcase(str(getattr(r, k, '')), v)
+                        for v in vs)
+                    for k, vs in defines):
                 results_.append(r)
         results = results_
 
@@ -775,7 +778,13 @@ def table(Result, results, diff_results=None, *,
 
     # find compare entry if there is one
     if compare:
-        compare_r = table.get(','.join(str(k) for k in compare))
+        compare_ = min(
+            (n for n in table.keys()
+                if all(fnmatch.fnmatchcase(k, c)
+                    for k, c in it.zip_longest(n.split(','), compare,
+                        fillvalue=''))),
+            default=compare)
+        compare_r = table.get(compare_)
 
     # build up our lines
     lines = []
@@ -1333,7 +1342,8 @@ if __name__ == "__main__":
                     k.strip(),
                     {v.strip() for v in vs.split(',')})
                 )(*x.split('=', 1)),
-            help="Only include results where this field is this value.")
+            help="Only include results where this field is this value. May "
+                "include comma-separated options and globs.")
     class AppendSort(argparse.Action):
         def __call__(self, parser, namespace, value, option):
             if namespace.sort is None:

scripts/treemap.py

@@ -236,7 +236,9 @@ def collect(csv_paths, defines=[]):
                             if k not in fields)
                 for r in reader:
                     # filter by matching defines
-                    if not all(k in r and r[k] in vs for k, vs in defines):
+                    if not all(any(fnmatch.fnmatchcase(r.get(k, ''), v)
+                                for v in vs)
+                            for k, vs in defines):
                         continue
 
                     results.append(r)
@@ -250,7 +252,9 @@ def fold(results, by=None, fields=None, defines=[]):
     if defines:
         results_ = []
         for r in results:
-            if all(k in r and r[k] in vs for k, vs in defines):
+            if all(any(fnmatch.fnmatchcase(r.get(k, ''), v)
+                        for v in vs)
+                    for k, vs in defines):
                 results_.append(r)
         results = results_
 
@@ -1339,7 +1343,8 @@ if __name__ == "__main__":
                     k.strip(),
                     {v.strip() for v in vs.split(',')})
                 )(*x.split('=', 1)),
-            help="Only include results where this field is this value.")
+            help="Only include results where this field is this value. May "
+                "include comma-separated options and globs.")
     parser.add_argument(
             '-L', '--add-label',
             dest='labels',

scripts/treemapd3.py

@@ -104,7 +104,9 @@ def collect(csv_paths, defines=[]):
                             if k not in fields)
                 for r in reader:
                     # filter by matching defines
-                    if not all(k in r and r[k] in vs for k, vs in defines):
+                    if not all(any(fnmatch.fnmatchcase(r.get(k, ''), v)
+                                for v in vs)
+                            for k, vs in defines):
                         continue
 
                     results.append(r)
@@ -118,7 +120,9 @@ def fold(results, by=None, fields=None, defines=[]):
     if defines:
         results_ = []
         for r in results:
-            if all(k in r and r[k] in vs for k, vs in defines):
+            if all(any(fnmatch.fnmatchcase(r.get(k, ''), v)
+                        for v in vs)
+                    for k, vs in defines):
                 results_.append(r)
         results = results_
 
@@ -1043,7 +1047,8 @@ if __name__ == "__main__":
                     k.strip(),
                     {v.strip() for v in vs.split(',')})
                 )(*x.split('=', 1)),
-            help="Only include results where this field is this value.")
+            help="Only include results where this field is this value. May "
+                "include comma-separated options and globs.")
     parser.add_argument(
             '-L', '--add-label',
             dest='labels',