From 86f3bad2a46cad62d2b24971c58dd0d37df6b3ae Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 20 Feb 2025 05:08:55 -0600 Subject: [PATCH] scripts: Adopted Attr rework in plot.py/plotmpl.py Unifying these complicated attr-assigning flags across all the scripts is the main benefit of the new internal Attr system. The only tricky bit is we need to somehow keep track of all input fields in case % modifiers reference fields, when we could previously discard non-data fields. Tricky but doable. Updated flags: - -L/--label -> -L/--add-label - --colors -> -C/--add-color - --formats -> -F/--add-format - --chars -> -*/--add-char/--chars - --line-chars -> -_/--add-line-char/--line-chars I've also tweaked Attr to accept glob matches when figuring out group assignments. This is useful for matching slightly different, but similarly named results in our benchmark scripts. There's probably a clever way to do this by injecting new by fields with csv.py, but just adding globbing is simpler and makes attr assignment even more flexible. --- scripts/bench.py | 4 +- scripts/plot.py | 739 +++++++++++++++++++++++++++---------------- scripts/plotmpl.py | 370 +++++++++++++++------- scripts/test.py | 4 +- scripts/treemap.py | 44 ++- scripts/treemapd3.py | 38 ++- 6 files changed, 802 insertions(+), 397 deletions(-) diff --git a/scripts/bench.py b/scripts/bench.py index 3f491e66..e106432b 100755 --- a/scripts/bench.py +++ b/scripts/bench.py @@ -877,11 +877,11 @@ def find_ids(runner, bench_ids=[], **args): if '*' in name: bench_ids__.extend(suite for suite in expected_suite_perms.keys() - if fnmatch.fnmatch(suite, name)) + if fnmatch.fnmatchcase(suite, name)) if not bench_ids__: bench_ids__.extend(case_ for case_ in expected_case_perms.keys() - if fnmatch.fnmatch(case_, name)) + if fnmatch.fnmatchcase(case_, name)) # literal suite elif name in expected_suite_perms: bench_ids__.append(id) diff --git a/scripts/plot.py b/scripts/plot.py index 6c58ea5a..3127ee05 100755 --- a/scripts/plot.py +++ b/scripts/plot.py @@ -16,6 +16,7 @@ if __name__ == "__main__": import bisect import collections as co import csv +import fnmatch import io import itertools as it import math as mt @@ -131,31 +132,6 @@ def si2(x, w=5): s = s.rstrip('.') return '%s%s%s' % ('-' if x < 0 else '', s, SI2_PREFIXES[p]) -# parse %-escaped strings -def unescape(s): - pattern = re.compile( - '%[%=,abfnrtv0]' - '|' '%x..' - '|' '%u....' - '|' '%U........') - def unescape(m): - if m.group()[1] == '%': return '%' - elif m.group()[1] == '=': return '=' - elif m.group()[1] == ',': return ',' - elif m.group()[1] == 'a': return '\a' - elif m.group()[1] == 'b': return '\b' - elif m.group()[1] == 'f': return '\f' - elif m.group()[1] == 'n': return '\n' - elif m.group()[1] == 'r': return '\r' - elif m.group()[1] == 't': return '\t' - elif m.group()[1] == 'v': return '\v' - elif m.group()[1] == '0': return '\0' - elif m.group()[1] == 'x': return chr(int(m.group()[2:], 16)) - elif m.group()[1] == 'u': return chr(int(m.group()[2:], 16)) - elif m.group()[1] == 'U': return chr(int(m.group()[2:], 16)) - else: assert False - return re.sub(pattern, unescape, s) - def openio(path, mode='r', buffering=-1): # allow '-' for stdin/stdout if path == '-': @@ -292,6 +268,249 @@ def dat(x): # else give up raise ValueError("invalid dat %r" % x) +def try_dat(x): + try: + return dat(x) + except ValueError: + return None + +def collect(csv_paths, defines=[]): + # collect results from CSV files + fields = [] + results = [] + for path in csv_paths: + try: + with openio(path) as f: + reader = csv.DictReader(f, restval='') + fields.extend( + k for k in reader.fieldnames + if k not in fields) + for r in reader: + # filter by matching defines + if not all(k in r and r[k] in vs for k, vs in defines): + continue + + results.append(r) + except FileNotFoundError: + pass + + return fields, results + +def fold(results, by=None, x=None, y=None, defines=[]): + # filter by matching defines + if defines: + results_ = [] + for r in results: + if all(k in r and r[k] in vs for k, vs in defines): + results_.append(r) + results = results_ + + if by: + # find all 'by' values + keys = set() + for r in results: + keys.add(tuple(r.get(k, '') for k in by)) + keys = sorted(keys) + + # collect all datasets + datasets = co.OrderedDict() + dataattrs = co.OrderedDict() + for key in (keys if by else [()]): + for x_ in (x if x else [None]): + for y_ in y: + # organize by 'by', x, and y + dataset = [] + dataattr = {} + i = 0 + for r in results: + # filter by 'by' + if by and not all( + k in r and r[k] == v + for k, v in zip(by, key)): + continue + + # find xs + if x_ is not None: + if x_ not in r: + continue + try: + x__ = dat(r[x_]) + except ValueError: + continue + else: + # fallback to enumeration + x__ = i + i += 1 + + # find ys + if y_ is not None: + if y_ not in r: + continue + try: + y__ = dat(r[y_]) + except ValueError: + continue + else: + y__ = None + + # do _not_ sum ys here, it's tempting but risks + # incorrect and misleading results + dataset.append((x__, y__)) + + # include all fields in dataattrs in case we use + # them for % modifiers + dataattr.update(r) + + # hide x/y if there is only one field + key_ = key + if len(x or []) > 1: + key_ += (x_,) + if len(y or []) > 1 or not key_: + key_ += (y_,) + datasets[key_] = dataset + dataattrs[key_] = dataattr + + return datasets, dataattrs + +# a representation of optionally key-mapped attrs +class Attr: + def __init__(self, attrs, *, + defaults=None): + # include defaults? + if (defaults is not None + and not any( + not isinstance(attr, tuple) + or attr[0] in {None, (), ('*',)} + for attr in (attrs or []))): + attrs = defaults + (attrs or []) + + # normalize + self.attrs = [] + self.keyed = co.OrderedDict() + for attr in (attrs or []): + if not isinstance(attr, tuple): + attr = ((), attr) + elif attr[0] in {None, (), ('*',)}: + attr = ((), attr[1]) + + self.attrs.append(attr) + if attr[0] not in self.keyed: + self.keyed[attr[0]] = [] + self.keyed[attr[0]].append(attr[1]) + + def __repr__(self): + return 'Attr(%r)' % [ + (','.join(attr[0]), attr[1]) + for attr in self.attrs] + + def __iter__(self): + return it.cycle(self.keyed[()]) + + def __bool__(self): + return bool(self.attrs) + + def __getitem__(self, key): + if isinstance(key, tuple): + if len(key) > 0 and not isinstance(key[0], str): + i, key = key + else: + i, key = 0, key + else: + i, key = key, () + + # try to lookup by key + best = None + for ks, vs in self.keyed.items(): + prefix = [] + for j, k in enumerate(ks): + if j < len(key) and fnmatch.fnmatchcase(key[j], k): + prefix.append(k) + else: + prefix = None + break + + if prefix is not None and ( + best is None or len(prefix) >= len(best[0])): + best = (prefix, vs) + + if best is not None: + # cycle based on index + return best[1][i % len(best[1])] + + return None + + def __contains__(self, key): + return self.__getitem__(key) is not None + + # a key function for sorting by key order + def key(self, key): + # allow key to be a tuple to make sorting dicts easier + if (isinstance(key, tuple) + and len(key) >= 1 + and isinstance(key[0], tuple)): + key = key[0] + + best = None + for i, ks in enumerate(self.keyed.keys()): + prefix = [] + for j, k in enumerate(ks): + if j < len(key) and (not k or key[j] == k): + prefix.append(k) + else: + prefix = None + break + + if prefix is not None and ( + best is None or len(prefix) >= len(best[0])): + best = (prefix, i) + + if best is not None: + return best[1] + + return len(self.keyed) + +# parse %-escaped strings +def punescape(s, attrs=None): + if attrs is None: + attrs = {} + if isinstance(attrs, dict): + attrs_ = attrs + attrs = lambda k: attrs_[k] + + pattern = re.compile( + '%[%n]' + '|' '%x..' + '|' '%u....' + '|' '%U........' + '|' '%\((?P[^)]*)\)' + '(?P[+\- #0-9\.]*[scdboxXfFeEgG])') + def unescape(m): + if m.group()[1] == '%': return '%' + elif m.group()[1] == 'n': return '\n' + elif m.group()[1] == 'x': return chr(int(m.group()[2:], 16)) + elif m.group()[1] == 'u': return chr(int(m.group()[2:], 16)) + elif m.group()[1] == 'U': return chr(int(m.group()[2:], 16)) + elif m.group()[1] == '(': + try: + v = attrs(m.group('field')) + except KeyError: + return m.group() + if m.group('format')[-1] in 'dboxX': + if isinstance(v, str): + v = try_dat(v) or 0 + v = int(v) + elif m.group('format')[-1] in 'fFeEgG': + if isinstance(v, str): + v = try_dat(v) or 0 + v = float(v) + else: + v = str(v) + # note we need Python's new format syntax for binary + f = '{:%s}' % m.group('format') + return f.format(v) + else: assert False + return re.sub(pattern, unescape, s) + # a hack log that preserves sign, with a linear region between -1 and 1 def symlog(x): @@ -477,110 +696,6 @@ class Plot: return ''.join(row_) -def collect(csv_paths, defines=[]): - # collect results from CSV files - fields = [] - results = [] - for path in csv_paths: - try: - with openio(path) as f: - reader = csv.DictReader(f, restval='') - fields.extend( - k for k in reader.fieldnames - if k not in fields) - for r in reader: - # filter by matching defines - if not all(k in r and r[k] in vs for k, vs in defines): - continue - - results.append(r) - except FileNotFoundError: - pass - - return fields, results - -def fold(results, by=None, x=None, y=None, defines=[], labels=None): - # filter by matching defines - if defines: - results_ = [] - for r in results: - if all(k in r and r[k] in vs for k, vs in defines): - results_.append(r) - results = results_ - - if by: - # find all 'by' values - keys = set() - for r in results: - keys.add(tuple(r.get(k, '') for k in by)) - keys = sorted(keys) - - # collect all datasets - datasets = co.OrderedDict() - for key in (keys if by else [()]): - for x_ in (x if x else [None]): - for y_ in y: - # organize by 'by', x, and y - dataset = [] - i = 0 - for r in results: - # filter by 'by' - if by and not all( - k in r and r[k] == v - for k, v in zip(by, key)): - continue - - # find xs - if x_ is not None: - if x_ not in r: - continue - try: - x__ = dat(r[x_]) - except ValueError: - continue - else: - # fallback to enumeration - x__ = i - i += 1 - - # find ys - if y_ is not None: - if y_ not in r: - continue - try: - y__ = dat(r[y_]) - except ValueError: - continue - else: - y__ = None - - # do _not_ sum ys here, it's tempting but risks - # incorrect and misleading results - dataset.append((x__, y__)) - - # hide x/y if there is only one field - key_ = key - if len(x or []) > 1: - key_ += (x_,) - if len(y or []) > 1 or not key_: - key_ += (y_,) - datasets[key_] = dataset - - # order by labels - if labels: - datasets_ = co.OrderedDict() - for _, key in labels: - if key in datasets: - datasets_[key] = datasets[key] - # include unlabeled data to help with debugging - for key, dataset in datasets.items(): - if key not in datasets_: - datasets_[key] = datasets[key] - datasets = datasets_ - - return datasets - - # some classes for organizing subplots into a grid class Subplot: def __init__(self, **args): @@ -850,12 +965,12 @@ def main(csv_paths, *, x=None, y=None, define=[], - label=None, + labels=[], + chars=[], + line_chars=[], + colors=[], color=False, braille=False, - colors=None, - chars=None, - line_chars=None, points=False, points_and_lines=False, width=None, @@ -891,30 +1006,38 @@ def main(csv_paths, *, else: color = False - # what colors to use? - if colors is not None: - colors_ = colors - else: - colors_ = COLORS + # what chars/colors to use? + chars_ = [] + for char in chars: + if isinstance(char, tuple): + chars_.extend((char[0], c) for c in char[1]) + else: + chars_.extend(char) + chars_ = Attr(chars_, defaults=( + CHARS_POINTS_AND_LINES if points_and_lines + else [True])) - if chars is not None: - chars_ = chars - elif points_and_lines: - chars_ = CHARS_POINTS_AND_LINES - else: - chars_ = [True] + line_chars_ = [] + for line_char in line_chars: + if isinstance(line_char, tuple): + line_chars_.extend((line_char[0], c) for c in line_char[1]) + else: + line_chars_.extend(line_char) + line_chars_ = Attr(line_chars_, defaults=( + [True] if points_and_lines or not points + else [False])) - if line_chars is not None: - line_chars_ = line_chars - elif points_and_lines or not points: - line_chars_ = [True] - else: - line_chars_ = [False] + colors_ = Attr(colors, defaults=COLORS) - # allow %-escaped codes in labels/titles - title = unescape(title).splitlines() if title is not None else [] - xlabel = unescape(xlabel).splitlines() if xlabel is not None else [] - ylabel = unescape(ylabel).splitlines() if ylabel is not None else [] + labels_ = Attr(labels) + + # split %n newlines early + title = (title.replace('%n', '\n').split('\n') + if title is not None else []) + xlabel = (xlabel.replace('%n', '\n').split('\n') + if xlabel is not None else []) + ylabel = (ylabel.replace('%n', '\n').split('\n') + if ylabel is not None else []) # subplot can also contribute to subplots, resolve this here or things # become a mess... @@ -935,9 +1058,6 @@ def main(csv_paths, *, subplots_get('define', **subplot, subplots=subplots)): all_defines[k] |= vs all_defines = sorted(all_defines.items()) - all_labels = [(unescape(k), vs) for k, vs in ( - (label or []) - + subplots_get('label', **subplot, subplots=subplots))] if not all_by and not all_y: print("error: needs --by or -y to figure out fields", @@ -961,23 +1081,21 @@ def main(csv_paths, *, xsublabel = s.args.get('xlabel') ysublabel = s.args.get('ylabel') - # allow escape codes in sublabels/subtitles - subtitle = (unescape(subtitle).splitlines() + # split %n newlines early + subtitle = (subtitle.replace('%n', '\n').split('\n') if subtitle is not None else []) - xsublabel = (unescape(xsublabel).splitlines() + xsublabel = (xsublabel.replace('%n', '\n').split('\n') if xsublabel is not None else []) - ysublabel = (unescape(ysublabel).splitlines() + ysublabel = (ysublabel.replace('%n', '\n').split('\n') if ysublabel is not None else []) # don't allow >2 ticklabels and render single ticklabels only once if xticklabels_ is not None: - xticklabels_ = [unescape(l) for l in xticklabels_] if len(xticklabels_) == 1: xticklabels_ = ["", xticklabels_[0]] elif len(xticklabels_) > 2: xticklabels_ = [xticklabels_[0], xticklabels_[-1]] if yticklabels_ is not None: - yticklabels_ = [unescape(l) for l in yticklabels_] if len(yticklabels_) == 1: yticklabels_ = ["", yticklabels_[0]] elif len(yticklabels_) > 2: @@ -1000,7 +1118,11 @@ def main(csv_paths, *, + (1 if s.x > 0 else 0), # space between ((5 if s.y2 else 4) + len(s.yunits) # fit yticklabels if s.yticklabels is None - else max((len(t) for t in s.yticklabels), default=0)) + else max( + # bit of a hack, we just guess the yticklabel size + # since we don't have the data yet + (len(punescape(l)) for l in s.yticklabels), + default=0)) + (1 if s.yticklabels != [] else 0), ) s.ymargin = ( @@ -1029,7 +1151,7 @@ def main(csv_paths, *, f.writeln = writeln # first collect results from CSV files - fields_, results = collect(csv_paths, all_defines) + fields_, results = collect(csv_paths) # if y not specified, guess it's anything not in by/defines/x all_y_ = all_y @@ -1039,43 +1161,61 @@ def main(csv_paths, *, and not any(k == k_ for k_, _ in all_defines)] # then extract the requested datasets - datasets_ = fold(results, all_by, all_x, all_y_, None, all_labels) + # + # note we don't need to filter by defines again + datasets_, dataattrs_ = fold(results, all_by, all_x, all_y) + + # order by labels + datasets_ = co.OrderedDict(sorted( + datasets_.items(), + key=labels_.key)) + + # and merge dataattrs + mergedattrs_ = {k: v + for dataattr in dataattrs_.values() + for k, v in dataattr.items()} + + # figure out labels/titles now that we have our data + title_ = [punescape(l, mergedattrs_) for l in title] + xlabel_ = [punescape(l, mergedattrs_) for l in xlabel] + ylabel_ = [punescape(l, mergedattrs_) for l in ylabel] # figure out colors/chars here so that subplot defines # don't change them later, that'd be bad - datacolors_ = { - name: colors_[i % len(colors_)] - for i, name in enumerate(datasets_.keys())} - datachars_ = { - name: chars_[i % len(chars_)] - for i, name in enumerate(datasets_.keys())} - dataline_chars_ = { - name: line_chars_[i % len(line_chars_)] - for i, name in enumerate(datasets_.keys())} + datachars_ = {name: chars_[i, name] + for i, name in enumerate(datasets_.keys())} + dataline_chars_ = {name: line_chars_[i, name] + for i, name in enumerate(datasets_.keys())} + datacolors_ = {name: colors_[i, name] + for i, name in enumerate(datasets_.keys())} + datalabels_ = {name: punescape(labels_[i, name], mergedattrs_) + for i, name in enumerate(datasets_.keys()) + if (i, name) in labels_} # build legend? legend_width = 0 if legend_right or legend_above or legend_below: legend_ = [] - if all_labels: - all_labels_ = {key: l for l, key in all_labels} for i, name in enumerate(datasets_.keys()): - if (all_labels - and name in all_labels_ - and not all_labels_[name]): + if name in datalabels_ and not datalabels_[name]: continue label = '%s%s' % ( - '%s ' % datachars_[name] - if chars is not None + '. ' if chars + and isinstance(datachars_[name], bool) + else '%s ' % datachars_[name] + if chars + else '. ' + if line_chars + and isinstance(dataline_chars_[name], bool) else '%s ' % dataline_chars_[name] - if line_chars is not None + if line_chars else '', - all_labels_[name] - if all_labels and name in all_labels_ + datalabels_[name] + if name in datalabels_ else ','.join(name)) if label: - legend_.append((label, colors_[i % len(colors_)])) + legend_.append((label, colors_[i, name])) legend_width = max(legend_width, len(label)+1) # figure out our canvas size @@ -1087,22 +1227,22 @@ def main(csv_paths, *, width_ = shutil.get_terminal_size((80, None))[0] if height is None: - height_ = 17 + len(title) + len(xlabel) + height_ = 17 + len(title_) + len(xlabel_) elif height: height_ = height else: height_ = shutil.get_terminal_size((None, - 17 + len(title) + len(xlabel)))[1] + 17 + len(title_) + len(xlabel_)))[1] # make space for shell prompt if not keep_open: height_ -= 1 # carve out space for the xlabel - height_ -= len(xlabel) + height_ -= len(xlabel_) # carve out space for the ylabel - width_ -= len(ylabel) + (1 if ylabel else 0) + width_ -= len(ylabel_) + (1 if ylabel_ else 0) # carve out space for title - height_ -= len(title) + height_ -= len(title_) # carve out space for the legend if legend_right and legend_: @@ -1156,7 +1296,9 @@ def main(csv_paths, *, 2, 2*((5 if s.x2 else 4)+len(s.xunits)) if s.xticklabels is None - else sum(len(t) for t in s.xticklabels)) + # bit of a hack, we just guess the xticklabel size + # since we don't have the data yet + else sum(len(punescape(l)) for l in s.xticklabels)) # fit yunits minheight = sum(s.ymargin) + 2 @@ -1192,8 +1334,13 @@ def main(csv_paths, *, # data can be constrained by subplot-specific defines, # so re-extract for each plot - subdatasets = fold(results, - all_by, all_x, all_y_, define_, all_labels) + subdatasets, subdataattrs = fold( + results, all_by, all_x, all_y_, define_) + + # order by labels + subdatasets = co.OrderedDict(sorted( + subdatasets.items(), + key=labels_.key)) # filter by subplot x/y subdatasets = co.OrderedDict([(name, dataset) @@ -1202,6 +1349,16 @@ def main(csv_paths, *, or name[-(1 if len(all_y_) <= 1 else 2)] in x_ if len(all_y_) <= 1 or name[-1] in y_]) + subdataattrs = co.OrderedDict([(name, dataattr) + for name, dataattr in subdataattrs.items() + if len(all_x) <= 1 + or name[-(1 if len(all_y) <= 1 else 2)] in x_ + if len(all_y) <= 1 + or name[-1] in y_]) + # and merge dataattrs + submergedattrs = {k: v + for dataattr in subdataattrs.values() + for k, v in dataattr.items()} # find actual xlim/ylim xlim_ = ( @@ -1228,6 +1385,17 @@ def main(csv_paths, *, for _, y in dataset if y is not None)))) + # figure out labels/titles now that we have our data + subtitle = [punescape(l, submergedattrs) for l in s.title] + subxlabel = [punescape(l, submergedattrs) for l in s.xlabel] + subylabel = [punescape(l, submergedattrs) for l in s.ylabel] + subxticklabels = ( + [punescape(l, submergedattrs) for l in s.xticklabels] + if s.xticklabels is not None else None) + subyticklabels = ( + [punescape(l, submergedattrs) for l in s.yticklabels] + if s.yticklabels is not None else None) + # find actual width/height subwidth = sum(widths[s.x:s.x+s.xspan]) - sum(s.xmargin) subheight = sum(heights[s.y:s.y+s.yspan]) - sum(s.ymargin) @@ -1240,28 +1408,33 @@ def main(csv_paths, *, ylim=ylim_, xlog=xlog_, ylog=ylog_, - braille=line_chars is None and braille, - dots=line_chars is None and not braille) + braille=not line_chars and braille, + dots=not line_chars and not braille) for name, dataset in subdatasets.items(): plot.plot( sorted((x,y) for x,y in dataset), - color=datacolors_[name], char=datachars_[name], - line_char=dataline_chars_[name]) + line_char=dataline_chars_[name], + color=datacolors_[name]) - s.plot = plot - s.width = subwidth - s.height = subheight - s.xlim = xlim_ - s.ylim = ylim_ + s.plot_ = plot + s.width_ = subwidth + s.height_ = subheight + s.xlim_ = xlim_ + s.ylim_ = ylim_ + s.title_ = subtitle + s.xlabel_ = subxlabel + s.ylabel_ = subylabel + s.xticklabels_ = subxticklabels + s.yticklabels_ = subyticklabels # now that everything's plotted, let's render things to the terminal # figure out margin xmargin = ( - len(ylabel) + (1 if ylabel else 0), + len(ylabel_) + (1 if ylabel_ else 0), sum(grid[0,0].xmargin[:2]), ) ymargin = ( @@ -1270,7 +1443,7 @@ def main(csv_paths, *, ) # draw title? - for line in title: + for line in title_: f.writeln('%*s%s' % ( sum(xmargin[:2]), '', line.center(width_-xmargin[1]))) @@ -1298,8 +1471,8 @@ def main(csv_paths, *, ymargin[-1], '', line.center(height_-sum(ymargin)), ymargin[0], ''))[row] - for line in ylabel) - if ylabel else '') + for line in ylabel_) + if ylabel_ else '') for x_ in range(grid.width): # figure out the grid x/y position @@ -1315,82 +1488,83 @@ def main(csv_paths, *, # header if subrow < s.ymargin[-1]: # draw subtitle? - if subrow < len(s.title): + if subrow < len(s.title_): f.write('%*s%s' % ( sum(s.xmargin[:2]), '', - s.title[subrow].center(s.width))) + s.title_[subrow].center(s.width_))) else: f.write('%*s%*s' % ( sum(s.xmargin[:2]), '', - s.width, '')) + s.width_, '')) # draw plot? - elif subrow-s.ymargin[-1] < s.height: + elif subrow-s.ymargin[-1] < s.height_: subrow = subrow-s.ymargin[-1] # draw ysublabel? f.write('%-*s' % ( s.xmargin[0], '%s ' % ''.join( - line.center(s.height)[subrow] - for line in s.ylabel) - if s.ylabel else '')) + line.center(s.height_)[subrow] + for line in s.ylabel_) + if s.ylabel_ else '')) # draw yunits? - if subrow == 0 and s.yticklabels != []: + if subrow == 0 and s.yticklabels_ != []: f.write('%*s' % ( s.xmargin[1], - ((si2 if s.y2 else si)(s.ylim[1]) + s.yunits - if s.yticklabels is None - else s.yticklabels[1]) + ((si2 if s.y2 else si)(s.ylim_[1]) + s.yunits + if s.yticklabels_ is None + else s.yticklabels_[1]) + ' ')) - elif subrow == s.height-1 and s.yticklabels != []: + elif subrow == s.height_-1 and s.yticklabels_ != []: f.write('%*s' % ( s.xmargin[1], - ((si2 if s.y2 else si)(s.ylim[0]) + s.yunits - if s.yticklabels is None - else s.yticklabels[0]) + ((si2 if s.y2 else si)(s.ylim_[0]) + s.yunits + if s.yticklabels_ is None + else s.yticklabels_[0]) + ' ')) else: f.write('%*s' % ( s.xmargin[1], '')) # draw plot! - f.write(s.plot.draw(subrow, color=color)) + f.write(s.plot_.draw(subrow, color=color)) # footer else: - subrow = subrow-s.ymargin[-1]-s.height + subrow = subrow-s.ymargin[-1]-s.height_ # draw xunits? - if subrow < (1 if s.xticklabels != [] else 0): + if subrow < (1 if s.xticklabels_ != [] else 0): f.write('%*s%-*s%*s%*s' % ( sum(s.xmargin[:2]), '', (5 if s.x2 else 4) + len(s.xunits) - if s.xticklabels is None - else len(s.xticklabels[0]), - (si2 if s.x2 else si)(s.xlim[0]) + s.xunits - if s.xticklabels is None - else s.xticklabels[0], - s.width - (2*((5 if s.x2 else 4)+len(s.xunits)) - if s.xticklabels is None + if s.xticklabels_ is None + else len(s.xticklabels_[0]), + (si2 if s.x2 else si)(s.xlim_[0]) + s.xunits + if s.xticklabels_ is None + else s.xticklabels_[0], + s.width_ - (2*((5 if s.x2 else 4)+len(s.xunits)) + if s.xticklabels_ is None else sum(len(t) - for t in s.xticklabels)), '', + for t in s.xticklabels_)), '', (5 if s.x2 else 4) + len(s.xunits) - if s.xticklabels is None - else len(s.xticklabels[1]), - (si2 if s.x2 else si)(s.xlim[1]) + s.xunits - if s.xticklabels is None - else s.xticklabels[1])) + if s.xticklabels_ is None + else len(s.xticklabels_[1]), + (si2 if s.x2 else si)(s.xlim_[1]) + s.xunits + if s.xticklabels_ is None + else s.xticklabels_[1])) # draw xsublabel? elif (subrow < s.ymargin[1] - or subrow-s.ymargin[1] >= len(s.xlabel)): + or subrow-s.ymargin[1] >= len(s.xlabel_)): f.write('%*s%*s' % ( sum(s.xmargin[:2]), '', - s.width, '')) + s.width_, '')) else: f.write('%*s%s' % ( sum(s.xmargin[:2]), '', - s.xlabel[subrow-s.ymargin[1]].center(s.width))) + s.xlabel_[subrow-s.ymargin[1]] + .center(s.width_))) # draw legend_right? if (legend_right and legend_ @@ -1405,7 +1579,7 @@ def main(csv_paths, *, f.writeln() # draw xlabel? - for line in xlabel: + for line in xlabel_: f.writeln('%*s%s' % ( sum(xmargin[:2]), '', line.center(width_-xmargin[1]))) @@ -1494,17 +1668,56 @@ if __name__ == "__main__": help="Only include results where this field is this value. May " "include comma-separated options.") parser.add_argument( - '-L', '--label', + '-L', '--add-label', + dest='labels', action='append', type=lambda x: ( - lambda k, vs: ( - k.strip(), - tuple(v.strip() for v in vs.split(','))) - )(*re.split(r'(? 1: @@ -305,20 +293,144 @@ def fold(results, by=None, x=None, y=None, defines=[], labels=None): if len(y or []) > 1 or not key_: key_ += (y_,) datasets[key_] = dataset + dataattrs[key_] = dataattr - # order by labels - if labels: - datasets_ = co.OrderedDict() - for _, key in labels: - if key in datasets: - datasets_[key] = datasets[key] - # include unlabeled data to help with debugging - for key, dataset in datasets.items(): - if key not in datasets_: - datasets_[key] = datasets[key] - datasets = datasets_ + return datasets, dataattrs - return datasets +# a representation of optionally key-mapped attrs +class Attr: + def __init__(self, attrs, *, + defaults=None): + # include defaults? + if (defaults is not None + and not any( + not isinstance(attr, tuple) + or attr[0] in {None, (), ('*',)} + for attr in (attrs or []))): + attrs = defaults + (attrs or []) + + # normalize + self.attrs = [] + self.keyed = co.OrderedDict() + for attr in (attrs or []): + if not isinstance(attr, tuple): + attr = ((), attr) + elif attr[0] in {None, (), ('*',)}: + attr = ((), attr[1]) + + self.attrs.append(attr) + if attr[0] not in self.keyed: + self.keyed[attr[0]] = [] + self.keyed[attr[0]].append(attr[1]) + + def __repr__(self): + return 'Attr(%r)' % [ + (','.join(attr[0]), attr[1]) + for attr in self.attrs] + + def __iter__(self): + return it.cycle(self.keyed[()]) + + def __bool__(self): + return bool(self.attrs) + + def __getitem__(self, key): + if isinstance(key, tuple): + if len(key) > 0 and not isinstance(key[0], str): + i, key = key + else: + i, key = 0, key + else: + i, key = key, () + + # try to lookup by key + best = None + for ks, vs in self.keyed.items(): + prefix = [] + for j, k in enumerate(ks): + if j < len(key) and fnmatch.fnmatchcase(key[j], k): + prefix.append(k) + else: + prefix = None + break + + if prefix is not None and ( + best is None or len(prefix) >= len(best[0])): + best = (prefix, vs) + + if best is not None: + # cycle based on index + return best[1][i % len(best[1])] + + return None + + def __contains__(self, key): + return self.__getitem__(key) is not None + + # a key function for sorting by key order + def key(self, key): + # allow key to be a tuple to make sorting dicts easier + if (isinstance(key, tuple) + and len(key) >= 1 + and isinstance(key[0], tuple)): + key = key[0] + + best = None + for i, ks in enumerate(self.keyed.keys()): + prefix = [] + for j, k in enumerate(ks): + if j < len(key) and (not k or key[j] == k): + prefix.append(k) + else: + prefix = None + break + + if prefix is not None and ( + best is None or len(prefix) >= len(best[0])): + best = (prefix, i) + + if best is not None: + return best[1] + + return len(self.keyed) + +# parse %-escaped strings +def punescape(s, attrs=None): + if attrs is None: + attrs = {} + if isinstance(attrs, dict): + attrs_ = attrs + attrs = lambda k: attrs_[k] + + pattern = re.compile( + '%[%n]' + '|' '%x..' + '|' '%u....' + '|' '%U........' + '|' '%\((?P[^)]*)\)' + '(?P[+\- #0-9\.]*[scdboxXfFeEgG])') + def unescape(m): + if m.group()[1] == '%': return '%' + elif m.group()[1] == 'n': return '\n' + elif m.group()[1] == 'x': return chr(int(m.group()[2:], 16)) + elif m.group()[1] == 'u': return chr(int(m.group()[2:], 16)) + elif m.group()[1] == 'U': return chr(int(m.group()[2:], 16)) + elif m.group()[1] == '(': + try: + v = attrs(m.group('field')) + except KeyError: + return m.group() + if m.group('format')[-1] in 'dboxXfFeEgG': + if isinstance(v, str): + v = try_dat(v) or 0 + else: + if not isinstance(v, str): + v = str(v) + # note we need Python's new format syntax for binary + f = '{:%s}' % m.group('format') + return f.format(v) + else: assert False + return re.sub(pattern, unescape, s) # some classes for organizing subplots into a grid @@ -593,11 +705,11 @@ def main(csv_paths, output, *, x=None, y=None, define=[], - label=None, + labels=[], + colors=[], + formats=[], points=False, points_and_lines=False, - colors=None, - formats=None, width=WIDTH, height=HEIGHT, xlim=(None,None), @@ -637,21 +749,14 @@ def main(csv_paths, output, *, svg = True # what colors/alphas/formats to use? - if colors is not None: - colors_ = colors - elif dark: - colors_ = COLORS_DARK - else: - colors_ = COLORS + colors_ = Attr(colors, defaults=COLORS_DARK if dark else COLORS) - if formats is not None: - formats_ = [unescape(f) for f in formats] - elif points_and_lines: - formats_ = FORMATS_POINTS_AND_LINES - elif points: - formats_ = FORMATS_POINTS - else: - formats_ = FORMATS + formats_ = Attr(formats, defaults=( + FORMATS_POINTS_AND_LINES if points_and_lines + else FORMATS_POINTS if points + else FORMATS)) + + labels_ = Attr(labels) if font_color is not None: font_color_ = font_color @@ -750,9 +855,6 @@ def main(csv_paths, output, *, subplots_get('define', **subplot, subplots=subplots)): all_defines[k] |= vs all_defines = sorted(all_defines.items()) - all_labels = [(unescape(k), vs) for k, vs in ( - (label or []) - + subplots_get('label', **subplot, subplots=subplots))] if not all_by and not all_y: print("error: needs --by or -y to figure out fields", @@ -760,7 +862,7 @@ def main(csv_paths, output, *, sys.exit(-1) # first collect results from CSV files - fields_, results = collect(csv_paths, all_defines) + fields_, results = collect(csv_paths) # if y not specified, guess it's anything not in by/defines/x if not all_y: @@ -771,16 +873,27 @@ def main(csv_paths, output, *, # then extract the requested datasets # # note we don't need to filter by defines again - datasets_ = fold(results, all_by, all_x, all_y, None, all_labels) + datasets_, dataattrs_ = fold(results, all_by, all_x, all_y) + + # order by labels + datasets_ = co.OrderedDict(sorted( + datasets_.items(), + key=labels_.key)) + + # and merge dataattrs + mergedattrs_ = {k: v + for dataattr in dataattrs_.values() + for k, v in dataattr.items()} # figure out formats/colors here so that subplot defines don't change # them later, that'd be bad - dataformats_ = { - name: formats_[i % len(formats_)] - for i, name in enumerate(datasets_.keys())} - datacolors_ = { - name: colors_[i % len(colors_)] - for i, name in enumerate(datasets_.keys())} + dataformats_ = {name: formats_[i, name] + for i, name in enumerate(datasets_.keys())} + datacolors_ = {name: colors_[i, name] + for i, name in enumerate(datasets_.keys())} + datalabels_ = {name: punescape(labels_[i, name], dataattrs_[name]) + for i, name in enumerate(datasets_.keys()) + if (i, name) in labels_} # create a grid of subplots grid = Grid.fromargs(**subplot, subplots=subplots) @@ -846,13 +959,27 @@ def main(csv_paths, output, *, # data can be constrained by subplot-specific defines, # so re-extract for each plot - subdatasets = fold(results, all_by, all_x, all_y, define_, all_labels) + subdatasets, subdataattrs = fold( + results, all_by, all_x, all_y, define_) + + # order by labels + subdatasets = co.OrderedDict(sorted( + subdatasets.items(), + key=labels_.key)) # filter by subplot x/y subdatasets = co.OrderedDict([(name, dataset) for name, dataset in subdatasets.items() if len(all_x) <= 1 or name[-(1 if len(all_y) <= 1 else 2)] in x_ if len(all_y) <= 1 or name[-1] in y_]) + subdataattrs = co.OrderedDict([(name, dataattr) + for name, dataattr in subdataattrs.items() + if len(all_x) <= 1 or name[-(1 if len(all_y) <= 1 else 2)] in x_ + if len(all_y) <= 1 or name[-1] in y_]) + # and merge dataattrs + submergedattrs = {k: v + for dataattr in subdataattrs.values() + for k, v in dataattr.items()} # plot! ax = s.ax @@ -898,7 +1025,8 @@ def main(csv_paths, output, *, ax.xaxis.set_major_formatter(lambda x, pos: si2(x)+(xunits_ if xunits_ else '')) if xticklabels_ is not None: - ax.xaxis.set_ticklabels([unescape(l) for l in xticklabels_]) + ax.xaxis.set_ticklabels([punescape(l, submergedattrs) + for l in xticklabels_]) if xticks_ is None: ax.xaxis.set_major_locator(AutoMultipleLocator(2)) elif isinstance(xticks_, list): @@ -911,7 +1039,8 @@ def main(csv_paths, output, *, ax.xaxis.set_major_formatter(lambda x, pos: si(x)+(xunits_ if xunits_ else '')) if xticklabels_ is not None: - ax.xaxis.set_ticklabels([unescape(l) for l in xticklabels_]) + ax.xaxis.set_ticklabels([punescape(l, submergedattrs) + for l in xticklabels_]) if xticks_ is None: ax.xaxis.set_major_locator(mpl.ticker.AutoLocator()) elif isinstance(xticks_, list): @@ -924,7 +1053,8 @@ def main(csv_paths, output, *, ax.yaxis.set_major_formatter(lambda x, pos: si2(x)+(yunits_ if yunits_ else '')) if yticklabels_ is not None: - ax.yaxis.set_ticklabels([unescape(l) for l in yticklabels_]) + ax.xaxis.set_ticklabels([punescape(l, submergedattrs) + for l in yticklabels_]) if yticks_ is None: ax.yaxis.set_major_locator(AutoMultipleLocator(2)) elif isinstance(yticks_, list): @@ -937,7 +1067,8 @@ def main(csv_paths, output, *, ax.yaxis.set_major_formatter(lambda x, pos: si(x)+(yunits_ if yunits_ else '')) if yticklabels_ is not None: - ax.yaxis.set_ticklabels([unescape(l) for l in yticklabels_]) + ax.xaxis.set_ticklabels([punescape(l, submergedattrs) + for l in yticklabels_]) if yticks_ is None: ax.yaxis.set_major_locator(mpl.ticker.AutoLocator()) elif isinstance(yticks_, list): @@ -951,11 +1082,11 @@ def main(csv_paths, output, *, # axes subplot labels if xsublabel is not None: - ax.set_xlabel(unescape(xsublabel)) + ax.set_xlabel(punescape(xsublabel, submergedattrs)) if ysublabel is not None: - ax.set_ylabel(unescape(ysublabel)) + ax.set_ylabel(punescape(ysublabel, submergedattrs)) if subtitle is not None: - ax.set_title(unescape(subtitle)) + ax.set_title(punescape(subtitle, submergedattrs)) # add a legend? a bit tricky with matplotlib # @@ -968,16 +1099,14 @@ def main(csv_paths, output, *, for s in grid: for h, l in zip(*s.ax.get_legend_handles_labels()): legend[l] = h - if all_labels: - all_labels_ = {key: l for l, key in all_labels} # sort in dataset order legend_ = [] - for name in datasets_.keys(): + for i, name in enumerate(datasets_.keys()): name_ = ','.join(name) if name_ in legend: - if all_labels and name in all_labels_: - if all_labels_[name]: - legend_.append((all_labels_[name], legend[name_])) + if name in datalabels_: + if datalabels_[name]: + legend_.append((datalabels_[name], legend[name_])) else: legend_.append((name_, legend[name_])) legend = legend_ @@ -1026,7 +1155,7 @@ def main(csv_paths, output, *, # big hack to get xlabel above the legend! but hey this # works really well actually if xlabel: - ax.set_title(unescape(xlabel), + ax.set_title(punescape(xlabel, mergedattrs_), size=plt.rcParams['axes.labelsize'], weight=plt.rcParams['axes.labelweight']) @@ -1056,11 +1185,11 @@ def main(csv_paths, output, *, # axes labels, NOTE we reposition these below if xlabel is not None and not legend_below: - fig.supxlabel(unescape(xlabel)) + fig.supxlabel(punescape(xlabel, mergedattrs_)) if ylabel is not None: - fig.supylabel(unescape(ylabel)) + fig.supylabel(punescape(ylabel, mergedattrs_)) if title is not None: - fig.suptitle(unescape(title)) + fig.suptitle(punescape(title, mergedattrs_)) # precompute constrained layout and find midpoints to adjust things # that should be centered so they are actually centered @@ -1069,11 +1198,11 @@ def main(csv_paths, output, *, ymid = (grid[0,0].ax.get_position().y0 + grid[0,-1].ax.get_position().y1)/2 if xlabel is not None and not legend_below: - fig.supxlabel(unescape(xlabel), x=xmid) + fig.supxlabel(punescape(xlabel, mergedattrs_), x=xmid) if ylabel is not None: - fig.supylabel(unescape(ylabel), y=ymid) + fig.supylabel(punescape(ylabel, mergedattrs_), y=ymid) if title is not None: - fig.suptitle(unescape(title), x=xmid) + fig.suptitle(punescape(title, mergedattrs_), x=xmid) # write the figure! @@ -1137,17 +1266,43 @@ if __name__ == "__main__": help="Only include results where this field is this value. May " "include comma-separated options.") parser.add_argument( - '-L', '--label', + '-L', '--add-label', + dest='labels', action='append', type=lambda x: ( - lambda k, vs: ( - k.strip(), - tuple(v.strip() for v in vs.split(','))) - )(*re.split(r'(?= 1 + and isinstance(key[0], tuple)): + key = key[0] + + best = None + for i, ks in enumerate(self.keyed.keys()): + prefix = [] + for j, k in enumerate(ks): + if j < len(key) and (not k or key[j] == k): + prefix.append(k) + else: + prefix = None + break + + if prefix is not None and ( + best is None or len(prefix) >= len(best[0])): + best = (prefix, i) + + if best is not None: + return best[1] + + return len(self.keyed) + # parse %-escaped strings def punescape(s, attrs=None): if attrs is None: @@ -673,9 +701,9 @@ def main(csv_paths, *, by=None, fields=None, defines=[], - labels=None, - chars=None, - colors=None, + labels=[], + chars=[], + colors=[], color=False, dots=False, braille=False, @@ -698,7 +726,7 @@ def main(csv_paths, *, # what chars/colors/labels to use? chars_ = [] - for char in (chars or []): + for char in chars: if isinstance(char, tuple): chars_.extend((char[0], c) for c in char[1]) else: @@ -1004,7 +1032,7 @@ if __name__ == "__main__": "where a group is the comma-separated 'by' fields. Accepts %% " "modifiers.") parser.add_argument( - '-.', '--add-char', '--chars', + '-*', '--add-char', '--chars', dest='chars', action='append', type=lambda x: ( diff --git a/scripts/treemapd3.py b/scripts/treemapd3.py index d694b2bd..a8c498e4 100755 --- a/scripts/treemapd3.py +++ b/scripts/treemapd3.py @@ -11,6 +11,7 @@ if __name__ == "__main__": import bisect import collections as co import csv +import fnmatch import itertools as it import math as mt import re @@ -179,7 +180,7 @@ class Attr: if (defaults is not None and not any( not isinstance(attr, tuple) - or attr[0] in {None, (), ('',)} + or attr[0] in {None, (), ('*',)} for attr in (attrs or []))): attrs = defaults + (attrs or []) @@ -189,7 +190,7 @@ class Attr: for attr in (attrs or []): if not isinstance(attr, tuple): attr = ((), attr) - elif attr[0] in {None, (), ('',)}: + elif attr[0] in {None, (), ('*',)}: attr = ((), attr[1]) self.attrs.append(attr) @@ -222,7 +223,7 @@ class Attr: for ks, vs in self.keyed.items(): prefix = [] for j, k in enumerate(ks): - if j < len(key) and (not k or key[j] == k): + if j < len(key) and fnmatch.fnmatchcase(key[j], k): prefix.append(k) else: prefix = None @@ -241,6 +242,33 @@ class Attr: def __contains__(self, key): return self.__getitem__(key) is not None + # a key function for sorting by key order + def key(self, key): + # allow key to be a tuple to make sorting dicts easier + if (isinstance(key, tuple) + and len(key) >= 1 + and isinstance(key[0], tuple)): + key = key[0] + + best = None + for i, ks in enumerate(self.keyed.keys()): + prefix = [] + for j, k in enumerate(ks): + if j < len(key) and (not k or key[j] == k): + prefix.append(k) + else: + prefix = None + break + + if prefix is not None and ( + best is None or len(prefix) >= len(best[0])): + best = (prefix, i) + + if best is not None: + return best[1] + + return len(self.keyed) + # parse %-escaped strings def punescape(s, attrs=None): if attrs is None: @@ -526,8 +554,8 @@ def main(csv_paths, output, *, by=None, fields=None, defines=[], - labels=None, - colors=None, + labels=[], + colors=[], width=None, height=None, no_header=False,