Github/littlefs
1
Fork 0
Code Actions Activity

scripts: Adopted Attr rework in plot.py/plotmpl.py

Browse Source 
Unifying these complicated attr-assigning flags across all the scripts
is the main benefit of the new internal Attr system.

The only tricky bit is we need to somehow keep track of all input fields
in case % modifiers reference fields, when we could previously discard
non-data fields.

Tricky but doable.

Updated flags:

- -L/--label -> -L/--add-label
- --colors -> -C/--add-color
- --formats -> -F/--add-format
- --chars -> -*/--add-char/--chars
- --line-chars -> -_/--add-line-char/--line-chars

I've also tweaked Attr to accept glob matches when figuring out group
assignments. This is useful for matching slightly different, but
similarly named results in our benchmark scripts.

There's probably a clever way to do this by injecting new by fields with
csv.py, but just adding globbing is simpler and makes attr assignment
even more flexible.
This commit is contained in:
Christopher Haster
2025-02-20 05:08:55 -06:00
parent 8b04e35ea5
commit 86f3bad2a4
6 changed files with 802 additions and 397 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/bench.py
+2 -2
View File
@@ -877,11 +877,11 @@ def find_ids(runner, bench_ids=[], **args):
if '*' in name: if '*' in name:
bench_ids__.extend(suite bench_ids__.extend(suite
for suite in expected_suite_perms.keys() for suite in expected_suite_perms.keys()
if fnmatch.fnmatch(suite, name)) if fnmatch.fnmatchcase(suite, name))
if not bench_ids__: if not bench_ids__:
bench_ids__.extend(case_ bench_ids__.extend(case_
for case_ in expected_case_perms.keys() for case_ in expected_case_perms.keys()
if fnmatch.fnmatch(case_, name)) if fnmatch.fnmatchcase(case_, name))
# literal suite # literal suite
elif name in expected_suite_perms: elif name in expected_suite_perms:
bench_ids__.append(id) bench_ids__.append(id)
scripts/plot.py
+466 -263
View File
@@ -16,6 +16,7 @@ if __name__ == "__main__":
import bisect import bisect
import collections as co import collections as co
import csv import csv
import fnmatch
import io import io
import itertools as it import itertools as it
import math as mt import math as mt
@@ -131,31 +132,6 @@ def si2(x, w=5):
s = s.rstrip('.') s = s.rstrip('.')
return '%s%s%s' % ('-' if x < 0 else '', s, SI2_PREFIXES[p]) return '%s%s%s' % ('-' if x < 0 else '', s, SI2_PREFIXES[p])
# parse %-escaped strings
def unescape(s):
pattern = re.compile(
'%[%=,abfnrtv0]'
'|' '%x..'
'|' '%u....'
'|' '%U........')
def unescape(m):
if m.group()[1] == '%': return '%'
elif m.group()[1] == '=': return '='
elif m.group()[1] == ',': return ','
elif m.group()[1] == 'a': return '\a'
elif m.group()[1] == 'b': return '\b'
elif m.group()[1] == 'f': return '\f'
elif m.group()[1] == 'n': return '\n'
elif m.group()[1] == 'r': return '\r'
elif m.group()[1] == 't': return '\t'
elif m.group()[1] == 'v': return '\v'
elif m.group()[1] == '0': return '\0'
elif m.group()[1] == 'x': return chr(int(m.group()[2:], 16))
elif m.group()[1] == 'u': return chr(int(m.group()[2:], 16))
elif m.group()[1] == 'U': return chr(int(m.group()[2:], 16))
else: assert False
return re.sub(pattern, unescape, s)
def openio(path, mode='r', buffering=-1): def openio(path, mode='r', buffering=-1):
# allow '-' for stdin/stdout # allow '-' for stdin/stdout
if path == '-': if path == '-':
@@ -292,6 +268,249 @@ def dat(x):
# else give up # else give up
raise ValueError("invalid dat %r" % x) raise ValueError("invalid dat %r" % x)
def try_dat(x):
try:
return dat(x)
except ValueError:
return None
def collect(csv_paths, defines=[]):
# collect results from CSV files
fields = []
results = []
for path in csv_paths:
try:
with openio(path) as f:
reader = csv.DictReader(f, restval='')
fields.extend(
k for k in reader.fieldnames
if k not in fields)
for r in reader:
# filter by matching defines
if not all(k in r and r[k] in vs for k, vs in defines):
continue
results.append(r)
except FileNotFoundError:
pass
return fields, results
def fold(results, by=None, x=None, y=None, defines=[]):
# filter by matching defines
if defines:
results_ = []
for r in results:
if all(k in r and r[k] in vs for k, vs in defines):
results_.append(r)
results = results_
if by:
# find all 'by' values
keys = set()
for r in results:
keys.add(tuple(r.get(k, '') for k in by))
keys = sorted(keys)
# collect all datasets
datasets = co.OrderedDict()
dataattrs = co.OrderedDict()
for key in (keys if by else [()]):
for x_ in (x if x else [None]):
for y_ in y:
# organize by 'by', x, and y
dataset = []
dataattr = {}
i = 0
for r in results:
# filter by 'by'
if by and not all(
k in r and r[k] == v
for k, v in zip(by, key)):
continue
# find xs
if x_ is not None:
if x_ not in r:
continue
try:
x__ = dat(r[x_])
except ValueError:
continue
else:
# fallback to enumeration
x__ = i
i += 1
# find ys
if y_ is not None:
if y_ not in r:
continue
try:
y__ = dat(r[y_])
except ValueError:
continue
else:
y__ = None
# do _not_ sum ys here, it's tempting but risks
# incorrect and misleading results
dataset.append((x__, y__))
# include all fields in dataattrs in case we use
# them for % modifiers
dataattr.update(r)
# hide x/y if there is only one field
key_ = key
if len(x or []) > 1:
key_ += (x_,)
if len(y or []) > 1 or not key_:
key_ += (y_,)
datasets[key_] = dataset
dataattrs[key_] = dataattr
return datasets, dataattrs
# a representation of optionally key-mapped attrs
class Attr:
def __init__(self, attrs, *,
defaults=None):
# include defaults?
if (defaults is not None
and not any(
not isinstance(attr, tuple)
or attr[0] in {None, (), ('*',)}
for attr in (attrs or []))):
attrs = defaults + (attrs or [])
# normalize
self.attrs = []
self.keyed = co.OrderedDict()
for attr in (attrs or []):
if not isinstance(attr, tuple):
attr = ((), attr)
elif attr[0] in {None, (), ('*',)}:
attr = ((), attr[1])
self.attrs.append(attr)
if attr[0] not in self.keyed:
self.keyed[attr[0]] = []
self.keyed[attr[0]].append(attr[1])
def __repr__(self):
return 'Attr(%r)' % [
(','.join(attr[0]), attr[1])
for attr in self.attrs]
def __iter__(self):
return it.cycle(self.keyed[()])
def __bool__(self):
return bool(self.attrs)
def __getitem__(self, key):
if isinstance(key, tuple):
if len(key) > 0 and not isinstance(key[0], str):
i, key = key
else:
i, key = 0, key
else:
i, key = key, ()
# try to lookup by key
best = None
for ks, vs in self.keyed.items():
prefix = []
for j, k in enumerate(ks):
if j < len(key) and fnmatch.fnmatchcase(key[j], k):
prefix.append(k)
else:
prefix = None
break
if prefix is not None and (
best is None or len(prefix) >= len(best[0])):
best = (prefix, vs)
if best is not None:
# cycle based on index
return best[1][i % len(best[1])]
return None
def __contains__(self, key):
return self.__getitem__(key) is not None
# a key function for sorting by key order
def key(self, key):
# allow key to be a tuple to make sorting dicts easier
if (isinstance(key, tuple)
and len(key) >= 1
and isinstance(key[0], tuple)):
key = key[0]
best = None
for i, ks in enumerate(self.keyed.keys()):
prefix = []
for j, k in enumerate(ks):
if j < len(key) and (not k or key[j] == k):
prefix.append(k)
else:
prefix = None
break
if prefix is not None and (
best is None or len(prefix) >= len(best[0])):
best = (prefix, i)
if best is not None:
return best[1]
return len(self.keyed)
# parse %-escaped strings
def punescape(s, attrs=None):
if attrs is None:
attrs = {}
if isinstance(attrs, dict):
attrs_ = attrs
attrs = lambda k: attrs_[k]
pattern = re.compile(
'%[%n]'
'|' '%x..'
'|' '%u....'
'|' '%U........'
'|' '%\((?P<field>[^)]*)\)'
'(?P<format>[+\- #0-9\.]*[scdboxXfFeEgG])')
def unescape(m):
if m.group()[1] == '%': return '%'
elif m.group()[1] == 'n': return '\n'
elif m.group()[1] == 'x': return chr(int(m.group()[2:], 16))
elif m.group()[1] == 'u': return chr(int(m.group()[2:], 16))
elif m.group()[1] == 'U': return chr(int(m.group()[2:], 16))
elif m.group()[1] == '(':
try:
v = attrs(m.group('field'))
except KeyError:
return m.group()
if m.group('format')[-1] in 'dboxX':
if isinstance(v, str):
v = try_dat(v) or 0
v = int(v)
elif m.group('format')[-1] in 'fFeEgG':
if isinstance(v, str):
v = try_dat(v) or 0
v = float(v)
else:
v = str(v)
# note we need Python's new format syntax for binary
f = '{:%s}' % m.group('format')
return f.format(v)
else: assert False
return re.sub(pattern, unescape, s)
# a hack log that preserves sign, with a linear region between -1 and 1 # a hack log that preserves sign, with a linear region between -1 and 1
def symlog(x): def symlog(x):
@@ -477,110 +696,6 @@ class Plot:
return ''.join(row_) return ''.join(row_)
def collect(csv_paths, defines=[]):
# collect results from CSV files
fields = []
results = []
for path in csv_paths:
try:
with openio(path) as f:
reader = csv.DictReader(f, restval='')
fields.extend(
k for k in reader.fieldnames
if k not in fields)
for r in reader:
# filter by matching defines
if not all(k in r and r[k] in vs for k, vs in defines):
continue
results.append(r)
except FileNotFoundError:
pass
return fields, results
def fold(results, by=None, x=None, y=None, defines=[], labels=None):
# filter by matching defines
if defines:
results_ = []
for r in results:
if all(k in r and r[k] in vs for k, vs in defines):
results_.append(r)
results = results_
if by:
# find all 'by' values
keys = set()
for r in results:
keys.add(tuple(r.get(k, '') for k in by))
keys = sorted(keys)
# collect all datasets
datasets = co.OrderedDict()
for key in (keys if by else [()]):
for x_ in (x if x else [None]):
for y_ in y:
# organize by 'by', x, and y
dataset = []
i = 0
for r in results:
# filter by 'by'
if by and not all(
k in r and r[k] == v
for k, v in zip(by, key)):
continue
# find xs
if x_ is not None:
if x_ not in r:
continue
try:
x__ = dat(r[x_])
except ValueError:
continue
else:
# fallback to enumeration
x__ = i
i += 1
# find ys
if y_ is not None:
if y_ not in r:
continue
try:
y__ = dat(r[y_])
except ValueError:
continue
else:
y__ = None
# do _not_ sum ys here, it's tempting but risks
# incorrect and misleading results
dataset.append((x__, y__))
# hide x/y if there is only one field
key_ = key
if len(x or []) > 1:
key_ += (x_,)
if len(y or []) > 1 or not key_:
key_ += (y_,)
datasets[key_] = dataset
# order by labels
if labels:
datasets_ = co.OrderedDict()
for _, key in labels:
if key in datasets:
datasets_[key] = datasets[key]
# include unlabeled data to help with debugging
for key, dataset in datasets.items():
if key not in datasets_:
datasets_[key] = datasets[key]
datasets = datasets_
return datasets
# some classes for organizing subplots into a grid # some classes for organizing subplots into a grid
class Subplot: class Subplot:
def __init__(self, **args): def __init__(self, **args):
@@ -850,12 +965,12 @@ def main(csv_paths, *,
x=None, x=None,
y=None, y=None,
define=[], define=[],
label=None, labels=[],
chars=[],
line_chars=[],
colors=[],
color=False, color=False,
braille=False, braille=False,
colors=None,
chars=None,
line_chars=None,
points=False, points=False,
points_and_lines=False, points_and_lines=False,
width=None, width=None,
@@ -891,30 +1006,38 @@ def main(csv_paths, *,
else: else:
color = False color = False
# what colors to use? # what chars/colors to use?
if colors is not None: chars_ = []
colors_ = colors for char in chars:
if isinstance(char, tuple):
chars_.extend((char[0], c) for c in char[1])
else: else:
colors_ = COLORS chars_.extend(char)
chars_ = Attr(chars_, defaults=(
CHARS_POINTS_AND_LINES if points_and_lines
else [True]))
if chars is not None: line_chars_ = []
chars_ = chars for line_char in line_chars:
elif points_and_lines: if isinstance(line_char, tuple):
chars_ = CHARS_POINTS_AND_LINES line_chars_.extend((line_char[0], c) for c in line_char[1])
else: else:
chars_ = [True] line_chars_.extend(line_char)
line_chars_ = Attr(line_chars_, defaults=(
[True] if points_and_lines or not points
else [False]))
if line_chars is not None: colors_ = Attr(colors, defaults=COLORS)
line_chars_ = line_chars
elif points_and_lines or not points:
line_chars_ = [True]
else:
line_chars_ = [False]
# allow %-escaped codes in labels/titles labels_ = Attr(labels)
title = unescape(title).splitlines() if title is not None else []
xlabel = unescape(xlabel).splitlines() if xlabel is not None else [] # split %n newlines early
ylabel = unescape(ylabel).splitlines() if ylabel is not None else [] title = (title.replace('%n', '\n').split('\n')
if title is not None else [])
xlabel = (xlabel.replace('%n', '\n').split('\n')
if xlabel is not None else [])
ylabel = (ylabel.replace('%n', '\n').split('\n')
if ylabel is not None else [])
# subplot can also contribute to subplots, resolve this here or things # subplot can also contribute to subplots, resolve this here or things
# become a mess... # become a mess...
@@ -935,9 +1058,6 @@ def main(csv_paths, *,
subplots_get('define', **subplot, subplots=subplots)): subplots_get('define', **subplot, subplots=subplots)):
all_defines[k] |= vs all_defines[k] |= vs
all_defines = sorted(all_defines.items()) all_defines = sorted(all_defines.items())
all_labels = [(unescape(k), vs) for k, vs in (
(label or [])
+ subplots_get('label', **subplot, subplots=subplots))]
if not all_by and not all_y: if not all_by and not all_y:
print("error: needs --by or -y to figure out fields", print("error: needs --by or -y to figure out fields",
@@ -961,23 +1081,21 @@ def main(csv_paths, *,
xsublabel = s.args.get('xlabel') xsublabel = s.args.get('xlabel')
ysublabel = s.args.get('ylabel') ysublabel = s.args.get('ylabel')
# allow escape codes in sublabels/subtitles # split %n newlines early
subtitle = (unescape(subtitle).splitlines() subtitle = (subtitle.replace('%n', '\n').split('\n')
if subtitle is not None else []) if subtitle is not None else [])
xsublabel = (unescape(xsublabel).splitlines() xsublabel = (xsublabel.replace('%n', '\n').split('\n')
if xsublabel is not None else []) if xsublabel is not None else [])
ysublabel = (unescape(ysublabel).splitlines() ysublabel = (ysublabel.replace('%n', '\n').split('\n')
if ysublabel is not None else []) if ysublabel is not None else [])
# don't allow >2 ticklabels and render single ticklabels only once # don't allow >2 ticklabels and render single ticklabels only once
if xticklabels_ is not None: if xticklabels_ is not None:
xticklabels_ = [unescape(l) for l in xticklabels_]
if len(xticklabels_) == 1: if len(xticklabels_) == 1:
xticklabels_ = ["", xticklabels_[0]] xticklabels_ = ["", xticklabels_[0]]
elif len(xticklabels_) > 2: elif len(xticklabels_) > 2:
xticklabels_ = [xticklabels_[0], xticklabels_[-1]] xticklabels_ = [xticklabels_[0], xticklabels_[-1]]
if yticklabels_ is not None: if yticklabels_ is not None:
yticklabels_ = [unescape(l) for l in yticklabels_]
if len(yticklabels_) == 1: if len(yticklabels_) == 1:
yticklabels_ = ["", yticklabels_[0]] yticklabels_ = ["", yticklabels_[0]]
elif len(yticklabels_) > 2: elif len(yticklabels_) > 2:
@@ -1000,7 +1118,11 @@ def main(csv_paths, *,
+ (1 if s.x > 0 else 0), # space between + (1 if s.x > 0 else 0), # space between
((5 if s.y2 else 4) + len(s.yunits) # fit yticklabels ((5 if s.y2 else 4) + len(s.yunits) # fit yticklabels
if s.yticklabels is None if s.yticklabels is None
else max((len(t) for t in s.yticklabels), default=0)) else max(
# bit of a hack, we just guess the yticklabel size
# since we don't have the data yet
(len(punescape(l)) for l in s.yticklabels),
default=0))
+ (1 if s.yticklabels != [] else 0), + (1 if s.yticklabels != [] else 0),
) )
s.ymargin = ( s.ymargin = (
@@ -1029,7 +1151,7 @@ def main(csv_paths, *,
f.writeln = writeln f.writeln = writeln
# first collect results from CSV files # first collect results from CSV files
fields_, results = collect(csv_paths, all_defines) fields_, results = collect(csv_paths)
# if y not specified, guess it's anything not in by/defines/x # if y not specified, guess it's anything not in by/defines/x
all_y_ = all_y all_y_ = all_y
@@ -1039,43 +1161,61 @@ def main(csv_paths, *,
and not any(k == k_ for k_, _ in all_defines)] and not any(k == k_ for k_, _ in all_defines)]
# then extract the requested datasets # then extract the requested datasets
datasets_ = fold(results, all_by, all_x, all_y_, None, all_labels) #
# note we don't need to filter by defines again
datasets_, dataattrs_ = fold(results, all_by, all_x, all_y)
# order by labels
datasets_ = co.OrderedDict(sorted(
datasets_.items(),
key=labels_.key))
# and merge dataattrs
mergedattrs_ = {k: v
for dataattr in dataattrs_.values()
for k, v in dataattr.items()}
# figure out labels/titles now that we have our data
title_ = [punescape(l, mergedattrs_) for l in title]
xlabel_ = [punescape(l, mergedattrs_) for l in xlabel]
ylabel_ = [punescape(l, mergedattrs_) for l in ylabel]
# figure out colors/chars here so that subplot defines # figure out colors/chars here so that subplot defines
# don't change them later, that'd be bad # don't change them later, that'd be bad
datacolors_ = { datachars_ = {name: chars_[i, name]
name: colors_[i % len(colors_)]
for i, name in enumerate(datasets_.keys())} for i, name in enumerate(datasets_.keys())}
datachars_ = { dataline_chars_ = {name: line_chars_[i, name]
name: chars_[i % len(chars_)]
for i, name in enumerate(datasets_.keys())} for i, name in enumerate(datasets_.keys())}
dataline_chars_ = { datacolors_ = {name: colors_[i, name]
name: line_chars_[i % len(line_chars_)]
for i, name in enumerate(datasets_.keys())} for i, name in enumerate(datasets_.keys())}
datalabels_ = {name: punescape(labels_[i, name], mergedattrs_)
for i, name in enumerate(datasets_.keys())
if (i, name) in labels_}
# build legend? # build legend?
legend_width = 0 legend_width = 0
if legend_right or legend_above or legend_below: if legend_right or legend_above or legend_below:
legend_ = [] legend_ = []
if all_labels:
all_labels_ = {key: l for l, key in all_labels}
for i, name in enumerate(datasets_.keys()): for i, name in enumerate(datasets_.keys()):
if (all_labels if name in datalabels_ and not datalabels_[name]:
and name in all_labels_
and not all_labels_[name]):
continue continue
label = '%s%s' % ( label = '%s%s' % (
'%s ' % datachars_[name] '. ' if chars
if chars is not None and isinstance(datachars_[name], bool)
else '%s ' % datachars_[name]
if chars
else '. '
if line_chars
and isinstance(dataline_chars_[name], bool)
else '%s ' % dataline_chars_[name] else '%s ' % dataline_chars_[name]
if line_chars is not None if line_chars
else '', else '',
all_labels_[name] datalabels_[name]
if all_labels and name in all_labels_ if name in datalabels_
else ','.join(name)) else ','.join(name))
if label: if label:
legend_.append((label, colors_[i % len(colors_)])) legend_.append((label, colors_[i, name]))
legend_width = max(legend_width, len(label)+1) legend_width = max(legend_width, len(label)+1)
# figure out our canvas size # figure out our canvas size
@@ -1087,22 +1227,22 @@ def main(csv_paths, *,
width_ = shutil.get_terminal_size((80, None))[0] width_ = shutil.get_terminal_size((80, None))[0]
if height is None: if height is None:
height_ = 17 + len(title) + len(xlabel) height_ = 17 + len(title_) + len(xlabel_)
elif height: elif height:
height_ = height height_ = height
else: else:
height_ = shutil.get_terminal_size((None, height_ = shutil.get_terminal_size((None,
17 + len(title) + len(xlabel)))[1] 17 + len(title_) + len(xlabel_)))[1]
# make space for shell prompt # make space for shell prompt
if not keep_open: if not keep_open:
height_ -= 1 height_ -= 1
# carve out space for the xlabel # carve out space for the xlabel
height_ -= len(xlabel) height_ -= len(xlabel_)
# carve out space for the ylabel # carve out space for the ylabel
width_ -= len(ylabel) + (1 if ylabel else 0) width_ -= len(ylabel_) + (1 if ylabel_ else 0)
# carve out space for title # carve out space for title
height_ -= len(title) height_ -= len(title_)
# carve out space for the legend # carve out space for the legend
if legend_right and legend_: if legend_right and legend_:
@@ -1156,7 +1296,9 @@ def main(csv_paths, *,
2, 2,
2*((5 if s.x2 else 4)+len(s.xunits)) 2*((5 if s.x2 else 4)+len(s.xunits))
if s.xticklabels is None if s.xticklabels is None
else sum(len(t) for t in s.xticklabels)) # bit of a hack, we just guess the xticklabel size
# since we don't have the data yet
else sum(len(punescape(l)) for l in s.xticklabels))
# fit yunits # fit yunits
minheight = sum(s.ymargin) + 2 minheight = sum(s.ymargin) + 2
@@ -1192,8 +1334,13 @@ def main(csv_paths, *,
# data can be constrained by subplot-specific defines, # data can be constrained by subplot-specific defines,
# so re-extract for each plot # so re-extract for each plot
subdatasets = fold(results, subdatasets, subdataattrs = fold(
all_by, all_x, all_y_, define_, all_labels) results, all_by, all_x, all_y_, define_)
# order by labels
subdatasets = co.OrderedDict(sorted(
subdatasets.items(),
key=labels_.key))
# filter by subplot x/y # filter by subplot x/y
subdatasets = co.OrderedDict([(name, dataset) subdatasets = co.OrderedDict([(name, dataset)
@@ -1202,6 +1349,16 @@ def main(csv_paths, *,
or name[-(1 if len(all_y_) <= 1 else 2)] in x_ or name[-(1 if len(all_y_) <= 1 else 2)] in x_
if len(all_y_) <= 1 if len(all_y_) <= 1
or name[-1] in y_]) or name[-1] in y_])
subdataattrs = co.OrderedDict([(name, dataattr)
for name, dataattr in subdataattrs.items()
if len(all_x) <= 1
or name[-(1 if len(all_y) <= 1 else 2)] in x_
if len(all_y) <= 1
or name[-1] in y_])
# and merge dataattrs
submergedattrs = {k: v
for dataattr in subdataattrs.values()
for k, v in dataattr.items()}
# find actual xlim/ylim # find actual xlim/ylim
xlim_ = ( xlim_ = (
@@ -1228,6 +1385,17 @@ def main(csv_paths, *,
for _, y in dataset for _, y in dataset
if y is not None)))) if y is not None))))
# figure out labels/titles now that we have our data
subtitle = [punescape(l, submergedattrs) for l in s.title]
subxlabel = [punescape(l, submergedattrs) for l in s.xlabel]
subylabel = [punescape(l, submergedattrs) for l in s.ylabel]
subxticklabels = (
[punescape(l, submergedattrs) for l in s.xticklabels]
if s.xticklabels is not None else None)
subyticklabels = (
[punescape(l, submergedattrs) for l in s.yticklabels]
if s.yticklabels is not None else None)
# find actual width/height # find actual width/height
subwidth = sum(widths[s.x:s.x+s.xspan]) - sum(s.xmargin) subwidth = sum(widths[s.x:s.x+s.xspan]) - sum(s.xmargin)
subheight = sum(heights[s.y:s.y+s.yspan]) - sum(s.ymargin) subheight = sum(heights[s.y:s.y+s.yspan]) - sum(s.ymargin)
@@ -1240,28 +1408,33 @@ def main(csv_paths, *,
ylim=ylim_, ylim=ylim_,
xlog=xlog_, xlog=xlog_,
ylog=ylog_, ylog=ylog_,
braille=line_chars is None and braille, braille=not line_chars and braille,
dots=line_chars is None and not braille) dots=not line_chars and not braille)
for name, dataset in subdatasets.items(): for name, dataset in subdatasets.items():
plot.plot( plot.plot(
sorted((x,y) for x,y in dataset), sorted((x,y) for x,y in dataset),
color=datacolors_[name],
char=datachars_[name], char=datachars_[name],
line_char=dataline_chars_[name]) line_char=dataline_chars_[name],
color=datacolors_[name])
s.plot = plot s.plot_ = plot
s.width = subwidth s.width_ = subwidth
s.height = subheight s.height_ = subheight
s.xlim = xlim_ s.xlim_ = xlim_
s.ylim = ylim_ s.ylim_ = ylim_
s.title_ = subtitle
s.xlabel_ = subxlabel
s.ylabel_ = subylabel
s.xticklabels_ = subxticklabels
s.yticklabels_ = subyticklabels
# now that everything's plotted, let's render things to the terminal # now that everything's plotted, let's render things to the terminal
# figure out margin # figure out margin
xmargin = ( xmargin = (
len(ylabel) + (1 if ylabel else 0), len(ylabel_) + (1 if ylabel_ else 0),
sum(grid[0,0].xmargin[:2]), sum(grid[0,0].xmargin[:2]),
) )
ymargin = ( ymargin = (
@@ -1270,7 +1443,7 @@ def main(csv_paths, *,
) )
# draw title? # draw title?
for line in title: for line in title_:
f.writeln('%*s%s' % ( f.writeln('%*s%s' % (
sum(xmargin[:2]), '', sum(xmargin[:2]), '',
line.center(width_-xmargin[1]))) line.center(width_-xmargin[1])))
@@ -1298,8 +1471,8 @@ def main(csv_paths, *,
ymargin[-1], '', ymargin[-1], '',
line.center(height_-sum(ymargin)), line.center(height_-sum(ymargin)),
ymargin[0], ''))[row] ymargin[0], ''))[row]
for line in ylabel) for line in ylabel_)
if ylabel else '') if ylabel_ else '')
for x_ in range(grid.width): for x_ in range(grid.width):
# figure out the grid x/y position # figure out the grid x/y position
@@ -1315,82 +1488,83 @@ def main(csv_paths, *,
# header # header
if subrow < s.ymargin[-1]: if subrow < s.ymargin[-1]:
# draw subtitle? # draw subtitle?
if subrow < len(s.title): if subrow < len(s.title_):
f.write('%*s%s' % ( f.write('%*s%s' % (
sum(s.xmargin[:2]), '', sum(s.xmargin[:2]), '',
s.title[subrow].center(s.width))) s.title_[subrow].center(s.width_)))
else: else:
f.write('%*s%*s' % ( f.write('%*s%*s' % (
sum(s.xmargin[:2]), '', sum(s.xmargin[:2]), '',
s.width, '')) s.width_, ''))
# draw plot? # draw plot?
elif subrow-s.ymargin[-1] < s.height: elif subrow-s.ymargin[-1] < s.height_:
subrow = subrow-s.ymargin[-1] subrow = subrow-s.ymargin[-1]
# draw ysublabel? # draw ysublabel?
f.write('%-*s' % ( f.write('%-*s' % (
s.xmargin[0], s.xmargin[0],
'%s ' % ''.join( '%s ' % ''.join(
line.center(s.height)[subrow] line.center(s.height_)[subrow]
for line in s.ylabel) for line in s.ylabel_)
if s.ylabel else '')) if s.ylabel_ else ''))
# draw yunits? # draw yunits?
if subrow == 0 and s.yticklabels != []: if subrow == 0 and s.yticklabels_ != []:
f.write('%*s' % ( f.write('%*s' % (
s.xmargin[1], s.xmargin[1],
((si2 if s.y2 else si)(s.ylim[1]) + s.yunits ((si2 if s.y2 else si)(s.ylim_[1]) + s.yunits
if s.yticklabels is None if s.yticklabels_ is None
else s.yticklabels[1]) else s.yticklabels_[1])
+ ' ')) + ' '))
elif subrow == s.height-1 and s.yticklabels != []: elif subrow == s.height_-1 and s.yticklabels_ != []:
f.write('%*s' % ( f.write('%*s' % (
s.xmargin[1], s.xmargin[1],
((si2 if s.y2 else si)(s.ylim[0]) + s.yunits ((si2 if s.y2 else si)(s.ylim_[0]) + s.yunits
if s.yticklabels is None if s.yticklabels_ is None
else s.yticklabels[0]) else s.yticklabels_[0])
+ ' ')) + ' '))
else: else:
f.write('%*s' % ( f.write('%*s' % (
s.xmargin[1], '')) s.xmargin[1], ''))
# draw plot! # draw plot!
f.write(s.plot.draw(subrow, color=color)) f.write(s.plot_.draw(subrow, color=color))
# footer # footer
else: else:
subrow = subrow-s.ymargin[-1]-s.height subrow = subrow-s.ymargin[-1]-s.height_
# draw xunits? # draw xunits?
if subrow < (1 if s.xticklabels != [] else 0): if subrow < (1 if s.xticklabels_ != [] else 0):
f.write('%*s%-*s%*s%*s' % ( f.write('%*s%-*s%*s%*s' % (
sum(s.xmargin[:2]), '', sum(s.xmargin[:2]), '',
(5 if s.x2 else 4) + len(s.xunits) (5 if s.x2 else 4) + len(s.xunits)
if s.xticklabels is None if s.xticklabels_ is None
else len(s.xticklabels[0]), else len(s.xticklabels_[0]),
(si2 if s.x2 else si)(s.xlim[0]) + s.xunits (si2 if s.x2 else si)(s.xlim_[0]) + s.xunits
if s.xticklabels is None if s.xticklabels_ is None
else s.xticklabels[0], else s.xticklabels_[0],
s.width - (2*((5 if s.x2 else 4)+len(s.xunits)) s.width_ - (2*((5 if s.x2 else 4)+len(s.xunits))
if s.xticklabels is None if s.xticklabels_ is None
else sum(len(t) else sum(len(t)
for t in s.xticklabels)), '', for t in s.xticklabels_)), '',
(5 if s.x2 else 4) + len(s.xunits) (5 if s.x2 else 4) + len(s.xunits)
if s.xticklabels is None if s.xticklabels_ is None
else len(s.xticklabels[1]), else len(s.xticklabels_[1]),
(si2 if s.x2 else si)(s.xlim[1]) + s.xunits (si2 if s.x2 else si)(s.xlim_[1]) + s.xunits
if s.xticklabels is None if s.xticklabels_ is None
else s.xticklabels[1])) else s.xticklabels_[1]))
# draw xsublabel? # draw xsublabel?
elif (subrow < s.ymargin[1] elif (subrow < s.ymargin[1]
or subrow-s.ymargin[1] >= len(s.xlabel)): or subrow-s.ymargin[1] >= len(s.xlabel_)):
f.write('%*s%*s' % ( f.write('%*s%*s' % (
sum(s.xmargin[:2]), '', sum(s.xmargin[:2]), '',
s.width, '')) s.width_, ''))
else: else:
f.write('%*s%s' % ( f.write('%*s%s' % (
sum(s.xmargin[:2]), '', sum(s.xmargin[:2]), '',
s.xlabel[subrow-s.ymargin[1]].center(s.width))) s.xlabel_[subrow-s.ymargin[1]]
.center(s.width_)))
# draw legend_right? # draw legend_right?
if (legend_right and legend_ if (legend_right and legend_
@@ -1405,7 +1579,7 @@ def main(csv_paths, *,
f.writeln() f.writeln()
# draw xlabel? # draw xlabel?
for line in xlabel: for line in xlabel_:
f.writeln('%*s%s' % ( f.writeln('%*s%s' % (
sum(xmargin[:2]), '', sum(xmargin[:2]), '',
line.center(width_-xmargin[1]))) line.center(width_-xmargin[1])))
@@ -1494,17 +1668,56 @@ if __name__ == "__main__":
help="Only include results where this field is this value. May " help="Only include results where this field is this value. May "
"include comma-separated options.") "include comma-separated options.")
parser.add_argument( parser.add_argument(
'-L', '--label', '-L', '--add-label',
dest='labels',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs: ( lambda ks, v: (
k.strip(), tuple(k.strip() for k in ks.split(',')),
tuple(v.strip() for v in vs.split(','))) v.strip())
)(*re.split(r'(?<!%)=', x, 1)), )(*x.split('=', 1))
help="Use this label for a given group, where a group is roughly " if '=' in x else x.strip(),
"the comma-separated values in the -b/--by, -x, and -y " help="Add a label to use. Can be assigned to a specific group "
"fields. Also provides an ordering. Accepts %= and other " "where a group is the comma-separated 'by' fields. Accepts %% "
"%-escaped codes.") "modifiers. Also provides an ordering.")
parser.add_argument(
'-*', '--add-char', '--chars',
dest='chars',
action='append',
type=lambda x: (
lambda ks, v: (
tuple(k.strip() for k in ks.split(',')),
v.strip())
)(*x.split('=', 1))
if '=' in x else x.strip(),
help="Add characters to use for points. Can be assigned to a "
"specific group where a group is the comma-separated "
"'by' fields.")
parser.add_argument(
'-_', '--add-line-char', '--line-chars',
dest='line_chars',
action='append',
type=lambda x: (
lambda ks, v: (
tuple(k.strip() for k in ks.split(',')),
v.strip())
)(*x.split('=', 1))
if '=' in x else x.strip(),
help="Add characters to use for lines. Can be assigned to a "
"specific group where a group is the comma-separated "
"'by' fields.")
parser.add_argument(
'-C', '--add-color',
dest='colors',
action='append',
type=lambda x: (
lambda ks, v: (
tuple(k.strip() for k in ks.split(',')),
v.strip())
)(*x.split('=', 1))
if '=' in x else x.strip(),
help="Add a color to use. Can be assigned to a specific group "
"where a group is the comma-separated 'by' fields.")
parser.add_argument( parser.add_argument(
'--color', '--color',
choices=['never', 'always', 'auto'], choices=['never', 'always', 'auto'],
@@ -1523,16 +1736,6 @@ if __name__ == "__main__":
'-!', '--points-and-lines', '-!', '--points-and-lines',
action='store_true', action='store_true',
help="Draw data points and lines.") help="Draw data points and lines.")
parser.add_argument(
'--colors',
type=lambda x: [x.strip() for x in x.split(',')],
help="Comma-separated colors to use.")
parser.add_argument(
'--chars',
help="Characters to use for points.")
parser.add_argument(
'--line-chars',
help="Characters to use for lines.")
parser.add_argument( parser.add_argument(
'-W', '--width', '-W', '--width',
nargs='?', nargs='?',
@@ -1582,25 +1785,25 @@ if __name__ == "__main__":
help="Units for the y-axis.") help="Units for the y-axis.")
parser.add_argument( parser.add_argument(
'--xlabel', '--xlabel',
help="Add a label to the x-axis. Accepts %-escaped codes.") help="Add a label to the x-axis. Accepts %% modifiers.")
parser.add_argument( parser.add_argument(
'--ylabel', '--ylabel',
help="Add a label to the y-axis. Accepts %-escaped codes.") help="Add a label to the y-axis. Accepts %% modifiers.")
parser.add_argument( parser.add_argument(
'--xticklabels', '--xticklabels',
type=lambda x: [x.strip() for x in re.split(r'(?<!%),', x)] type=lambda x: [x.strip() for x in re.split(r'(?<!%),', x)]
if x.strip() else [], if x.strip() else [],
help="Comma separated xticklabels. Accepts %, and other " help="Comma separated xticklabels. Accepts %%, and other "
"%-escaped codes.") "%% modifiers.")
parser.add_argument( parser.add_argument(
'--yticklabels', '--yticklabels',
type=lambda x: [x.strip() for x in re.split(r'(?<!%),', x)] type=lambda x: [x.strip() for x in re.split(r'(?<!%),', x)]
if x.strip() else [], if x.strip() else [],
help="Comma separated yticklabels. Accepts %, and other " help="Comma separated yticklabels. Accepts %%, and other "
"%-escaped codes.") "%% modifiers.")
parser.add_argument( parser.add_argument(
'--title', '--title',
help="Add a title. Accepts %-escaped codes.") help="Add a title. Accepts %% modifiers.")
parser.add_argument( parser.add_argument(
'-l', '--legend', '--legend-right', '-l', '--legend', '--legend-right',
dest='legend_right', dest='legend_right',
scripts/plotmpl.py
+256 -110
View File
@@ -15,6 +15,7 @@ if __name__ == "__main__":
import collections as co import collections as co
import csv import csv
import fnmatch
import io import io
import itertools as it import itertools as it
import logging import logging
@@ -131,31 +132,6 @@ def si2(x):
s = s.rstrip('.') s = s.rstrip('.')
return '%s%s%s' % ('-' if x < 0 else '', s, SI2_PREFIXES[p]) return '%s%s%s' % ('-' if x < 0 else '', s, SI2_PREFIXES[p])
# parse %-escaped strings
def unescape(s):
pattern = re.compile(
'%[%=,abfnrtv0]'
'|' '%x..'
'|' '%u....'
'|' '%U........')
def unescape(m):
if m.group()[1] == '%': return '%'
elif m.group()[1] == '=': return '='
elif m.group()[1] == ',': return ','
elif m.group()[1] == 'a': return '\a'
elif m.group()[1] == 'b': return '\b'
elif m.group()[1] == 'f': return '\f'
elif m.group()[1] == 'n': return '\n'
elif m.group()[1] == 'r': return '\r'
elif m.group()[1] == 't': return '\t'
elif m.group()[1] == 'v': return '\v'
elif m.group()[1] == '0': return '\0'
elif m.group()[1] == 'x': return chr(int(m.group()[2:], 16))
elif m.group()[1] == 'u': return chr(int(m.group()[2:], 16))
elif m.group()[1] == 'U': return chr(int(m.group()[2:], 16))
else: assert False
return re.sub(pattern, unescape, s)
# we want to use MaxNLocator, but since MaxNLocator forces multiples of 10 # we want to use MaxNLocator, but since MaxNLocator forces multiples of 10
# to be an option, we can't really... # to be an option, we can't really...
class AutoMultipleLocator(mpl.ticker.MultipleLocator): class AutoMultipleLocator(mpl.ticker.MultipleLocator):
@@ -217,6 +193,12 @@ def dat(x):
# else give up # else give up
raise ValueError("invalid dat %r" % x) raise ValueError("invalid dat %r" % x)
def try_dat(x):
try:
return dat(x)
except ValueError:
return None
def collect(csv_paths, defines=[]): def collect(csv_paths, defines=[]):
# collect results from CSV files # collect results from CSV files
fields = [] fields = []
@@ -239,7 +221,7 @@ def collect(csv_paths, defines=[]):
return fields, results return fields, results
def fold(results, by=None, x=None, y=None, defines=[], labels=None): def fold(results, by=None, x=None, y=None, defines=[]):
# filter by matching defines # filter by matching defines
if defines: if defines:
results_ = [] results_ = []
@@ -257,11 +239,13 @@ def fold(results, by=None, x=None, y=None, defines=[], labels=None):
# collect all datasets # collect all datasets
datasets = co.OrderedDict() datasets = co.OrderedDict()
dataattrs = co.OrderedDict()
for key in (keys if by else [()]): for key in (keys if by else [()]):
for x_ in (x if x else [None]): for x_ in (x if x else [None]):
for y_ in y: for y_ in y:
# organize by 'by', x, and y # organize by 'by', x, and y
dataset = [] dataset = []
dataattr = {}
i = 0 i = 0
for r in results: for r in results:
# filter by 'by' # filter by 'by'
@@ -298,6 +282,10 @@ def fold(results, by=None, x=None, y=None, defines=[], labels=None):
# incorrect and misleading results # incorrect and misleading results
dataset.append((x__, y__)) dataset.append((x__, y__))
# include all fields in dataattrs in case we use
# them for % modifiers
dataattr.update(r)
# hide x/y if there is only one field # hide x/y if there is only one field
key_ = key key_ = key
if len(x or []) > 1: if len(x or []) > 1:
@@ -305,20 +293,144 @@ def fold(results, by=None, x=None, y=None, defines=[], labels=None):
if len(y or []) > 1 or not key_: if len(y or []) > 1 or not key_:
key_ += (y_,) key_ += (y_,)
datasets[key_] = dataset datasets[key_] = dataset
dataattrs[key_] = dataattr
# order by labels return datasets, dataattrs
if labels:
datasets_ = co.OrderedDict()
for _, key in labels:
if key in datasets:
datasets_[key] = datasets[key]
# include unlabeled data to help with debugging
for key, dataset in datasets.items():
if key not in datasets_:
datasets_[key] = datasets[key]
datasets = datasets_
return datasets # a representation of optionally key-mapped attrs
class Attr:
def __init__(self, attrs, *,
defaults=None):
# include defaults?
if (defaults is not None
and not any(
not isinstance(attr, tuple)
or attr[0] in {None, (), ('*',)}
for attr in (attrs or []))):
attrs = defaults + (attrs or [])
# normalize
self.attrs = []
self.keyed = co.OrderedDict()
for attr in (attrs or []):
if not isinstance(attr, tuple):
attr = ((), attr)
elif attr[0] in {None, (), ('*',)}:
attr = ((), attr[1])
self.attrs.append(attr)
if attr[0] not in self.keyed:
self.keyed[attr[0]] = []
self.keyed[attr[0]].append(attr[1])
def __repr__(self):
return 'Attr(%r)' % [
(','.join(attr[0]), attr[1])
for attr in self.attrs]
def __iter__(self):
return it.cycle(self.keyed[()])
def __bool__(self):
return bool(self.attrs)
def __getitem__(self, key):
if isinstance(key, tuple):
if len(key) > 0 and not isinstance(key[0], str):
i, key = key
else:
i, key = 0, key
else:
i, key = key, ()
# try to lookup by key
best = None
for ks, vs in self.keyed.items():
prefix = []
for j, k in enumerate(ks):
if j < len(key) and fnmatch.fnmatchcase(key[j], k):
prefix.append(k)
else:
prefix = None
break
if prefix is not None and (
best is None or len(prefix) >= len(best[0])):
best = (prefix, vs)
if best is not None:
# cycle based on index
return best[1][i % len(best[1])]
return None
def __contains__(self, key):
return self.__getitem__(key) is not None
# a key function for sorting by key order
def key(self, key):
# allow key to be a tuple to make sorting dicts easier
if (isinstance(key, tuple)
and len(key) >= 1
and isinstance(key[0], tuple)):
key = key[0]
best = None
for i, ks in enumerate(self.keyed.keys()):
prefix = []
for j, k in enumerate(ks):
if j < len(key) and (not k or key[j] == k):
prefix.append(k)
else:
prefix = None
break
if prefix is not None and (
best is None or len(prefix) >= len(best[0])):
best = (prefix, i)
if best is not None:
return best[1]
return len(self.keyed)
# parse %-escaped strings
def punescape(s, attrs=None):
if attrs is None:
attrs = {}
if isinstance(attrs, dict):
attrs_ = attrs
attrs = lambda k: attrs_[k]
pattern = re.compile(
'%[%n]'
'|' '%x..'
'|' '%u....'
'|' '%U........'
'|' '%\((?P<field>[^)]*)\)'
'(?P<format>[+\- #0-9\.]*[scdboxXfFeEgG])')
def unescape(m):
if m.group()[1] == '%': return '%'
elif m.group()[1] == 'n': return '\n'
elif m.group()[1] == 'x': return chr(int(m.group()[2:], 16))
elif m.group()[1] == 'u': return chr(int(m.group()[2:], 16))
elif m.group()[1] == 'U': return chr(int(m.group()[2:], 16))
elif m.group()[1] == '(':
try:
v = attrs(m.group('field'))
except KeyError:
return m.group()
if m.group('format')[-1] in 'dboxXfFeEgG':
if isinstance(v, str):
v = try_dat(v) or 0
else:
if not isinstance(v, str):
v = str(v)
# note we need Python's new format syntax for binary
f = '{:%s}' % m.group('format')
return f.format(v)
else: assert False
return re.sub(pattern, unescape, s)
# some classes for organizing subplots into a grid # some classes for organizing subplots into a grid
@@ -593,11 +705,11 @@ def main(csv_paths, output, *,
x=None, x=None,
y=None, y=None,
define=[], define=[],
label=None, labels=[],
colors=[],
formats=[],
points=False, points=False,
points_and_lines=False, points_and_lines=False,
colors=None,
formats=None,
width=WIDTH, width=WIDTH,
height=HEIGHT, height=HEIGHT,
xlim=(None,None), xlim=(None,None),
@@ -637,21 +749,14 @@ def main(csv_paths, output, *,
svg = True svg = True
# what colors/alphas/formats to use? # what colors/alphas/formats to use?
if colors is not None: colors_ = Attr(colors, defaults=COLORS_DARK if dark else COLORS)
colors_ = colors
elif dark:
colors_ = COLORS_DARK
else:
colors_ = COLORS
if formats is not None: formats_ = Attr(formats, defaults=(
formats_ = [unescape(f) for f in formats] FORMATS_POINTS_AND_LINES if points_and_lines
elif points_and_lines: else FORMATS_POINTS if points
formats_ = FORMATS_POINTS_AND_LINES else FORMATS))
elif points:
formats_ = FORMATS_POINTS labels_ = Attr(labels)
else:
formats_ = FORMATS
if font_color is not None: if font_color is not None:
font_color_ = font_color font_color_ = font_color
@@ -750,9 +855,6 @@ def main(csv_paths, output, *,
subplots_get('define', **subplot, subplots=subplots)): subplots_get('define', **subplot, subplots=subplots)):
all_defines[k] |= vs all_defines[k] |= vs
all_defines = sorted(all_defines.items()) all_defines = sorted(all_defines.items())
all_labels = [(unescape(k), vs) for k, vs in (
(label or [])
+ subplots_get('label', **subplot, subplots=subplots))]
if not all_by and not all_y: if not all_by and not all_y:
print("error: needs --by or -y to figure out fields", print("error: needs --by or -y to figure out fields",
@@ -760,7 +862,7 @@ def main(csv_paths, output, *,
sys.exit(-1) sys.exit(-1)
# first collect results from CSV files # first collect results from CSV files
fields_, results = collect(csv_paths, all_defines) fields_, results = collect(csv_paths)
# if y not specified, guess it's anything not in by/defines/x # if y not specified, guess it's anything not in by/defines/x
if not all_y: if not all_y:
@@ -771,16 +873,27 @@ def main(csv_paths, output, *,
# then extract the requested datasets # then extract the requested datasets
# #
# note we don't need to filter by defines again # note we don't need to filter by defines again
datasets_ = fold(results, all_by, all_x, all_y, None, all_labels) datasets_, dataattrs_ = fold(results, all_by, all_x, all_y)
# order by labels
datasets_ = co.OrderedDict(sorted(
datasets_.items(),
key=labels_.key))
# and merge dataattrs
mergedattrs_ = {k: v
for dataattr in dataattrs_.values()
for k, v in dataattr.items()}
# figure out formats/colors here so that subplot defines don't change # figure out formats/colors here so that subplot defines don't change
# them later, that'd be bad # them later, that'd be bad
dataformats_ = { dataformats_ = {name: formats_[i, name]
name: formats_[i % len(formats_)]
for i, name in enumerate(datasets_.keys())} for i, name in enumerate(datasets_.keys())}
datacolors_ = { datacolors_ = {name: colors_[i, name]
name: colors_[i % len(colors_)]
for i, name in enumerate(datasets_.keys())} for i, name in enumerate(datasets_.keys())}
datalabels_ = {name: punescape(labels_[i, name], dataattrs_[name])
for i, name in enumerate(datasets_.keys())
if (i, name) in labels_}
# create a grid of subplots # create a grid of subplots
grid = Grid.fromargs(**subplot, subplots=subplots) grid = Grid.fromargs(**subplot, subplots=subplots)
@@ -846,13 +959,27 @@ def main(csv_paths, output, *,
# data can be constrained by subplot-specific defines, # data can be constrained by subplot-specific defines,
# so re-extract for each plot # so re-extract for each plot
subdatasets = fold(results, all_by, all_x, all_y, define_, all_labels) subdatasets, subdataattrs = fold(
results, all_by, all_x, all_y, define_)
# order by labels
subdatasets = co.OrderedDict(sorted(
subdatasets.items(),
key=labels_.key))
# filter by subplot x/y # filter by subplot x/y
subdatasets = co.OrderedDict([(name, dataset) subdatasets = co.OrderedDict([(name, dataset)
for name, dataset in subdatasets.items() for name, dataset in subdatasets.items()
if len(all_x) <= 1 or name[-(1 if len(all_y) <= 1 else 2)] in x_ if len(all_x) <= 1 or name[-(1 if len(all_y) <= 1 else 2)] in x_
if len(all_y) <= 1 or name[-1] in y_]) if len(all_y) <= 1 or name[-1] in y_])
subdataattrs = co.OrderedDict([(name, dataattr)
for name, dataattr in subdataattrs.items()
if len(all_x) <= 1 or name[-(1 if len(all_y) <= 1 else 2)] in x_
if len(all_y) <= 1 or name[-1] in y_])
# and merge dataattrs
submergedattrs = {k: v
for dataattr in subdataattrs.values()
for k, v in dataattr.items()}
# plot! # plot!
ax = s.ax ax = s.ax
@@ -898,7 +1025,8 @@ def main(csv_paths, output, *,
ax.xaxis.set_major_formatter(lambda x, pos: ax.xaxis.set_major_formatter(lambda x, pos:
si2(x)+(xunits_ if xunits_ else '')) si2(x)+(xunits_ if xunits_ else ''))
if xticklabels_ is not None: if xticklabels_ is not None:
ax.xaxis.set_ticklabels([unescape(l) for l in xticklabels_]) ax.xaxis.set_ticklabels([punescape(l, submergedattrs)
for l in xticklabels_])
if xticks_ is None: if xticks_ is None:
ax.xaxis.set_major_locator(AutoMultipleLocator(2)) ax.xaxis.set_major_locator(AutoMultipleLocator(2))
elif isinstance(xticks_, list): elif isinstance(xticks_, list):
@@ -911,7 +1039,8 @@ def main(csv_paths, output, *,
ax.xaxis.set_major_formatter(lambda x, pos: ax.xaxis.set_major_formatter(lambda x, pos:
si(x)+(xunits_ if xunits_ else '')) si(x)+(xunits_ if xunits_ else ''))
if xticklabels_ is not None: if xticklabels_ is not None:
ax.xaxis.set_ticklabels([unescape(l) for l in xticklabels_]) ax.xaxis.set_ticklabels([punescape(l, submergedattrs)
for l in xticklabels_])
if xticks_ is None: if xticks_ is None:
ax.xaxis.set_major_locator(mpl.ticker.AutoLocator()) ax.xaxis.set_major_locator(mpl.ticker.AutoLocator())
elif isinstance(xticks_, list): elif isinstance(xticks_, list):
@@ -924,7 +1053,8 @@ def main(csv_paths, output, *,
ax.yaxis.set_major_formatter(lambda x, pos: ax.yaxis.set_major_formatter(lambda x, pos:
si2(x)+(yunits_ if yunits_ else '')) si2(x)+(yunits_ if yunits_ else ''))
if yticklabels_ is not None: if yticklabels_ is not None:
ax.yaxis.set_ticklabels([unescape(l) for l in yticklabels_]) ax.xaxis.set_ticklabels([punescape(l, submergedattrs)
for l in yticklabels_])
if yticks_ is None: if yticks_ is None:
ax.yaxis.set_major_locator(AutoMultipleLocator(2)) ax.yaxis.set_major_locator(AutoMultipleLocator(2))
elif isinstance(yticks_, list): elif isinstance(yticks_, list):
@@ -937,7 +1067,8 @@ def main(csv_paths, output, *,
ax.yaxis.set_major_formatter(lambda x, pos: ax.yaxis.set_major_formatter(lambda x, pos:
si(x)+(yunits_ if yunits_ else '')) si(x)+(yunits_ if yunits_ else ''))
if yticklabels_ is not None: if yticklabels_ is not None:
ax.yaxis.set_ticklabels([unescape(l) for l in yticklabels_]) ax.xaxis.set_ticklabels([punescape(l, submergedattrs)
for l in yticklabels_])
if yticks_ is None: if yticks_ is None:
ax.yaxis.set_major_locator(mpl.ticker.AutoLocator()) ax.yaxis.set_major_locator(mpl.ticker.AutoLocator())
elif isinstance(yticks_, list): elif isinstance(yticks_, list):
@@ -951,11 +1082,11 @@ def main(csv_paths, output, *,
# axes subplot labels # axes subplot labels
if xsublabel is not None: if xsublabel is not None:
ax.set_xlabel(unescape(xsublabel)) ax.set_xlabel(punescape(xsublabel, submergedattrs))
if ysublabel is not None: if ysublabel is not None:
ax.set_ylabel(unescape(ysublabel)) ax.set_ylabel(punescape(ysublabel, submergedattrs))
if subtitle is not None: if subtitle is not None:
ax.set_title(unescape(subtitle)) ax.set_title(punescape(subtitle, submergedattrs))
# add a legend? a bit tricky with matplotlib # add a legend? a bit tricky with matplotlib
# #
@@ -968,16 +1099,14 @@ def main(csv_paths, output, *,
for s in grid: for s in grid:
for h, l in zip(*s.ax.get_legend_handles_labels()): for h, l in zip(*s.ax.get_legend_handles_labels()):
legend[l] = h legend[l] = h
if all_labels:
all_labels_ = {key: l for l, key in all_labels}
# sort in dataset order # sort in dataset order
legend_ = [] legend_ = []
for name in datasets_.keys(): for i, name in enumerate(datasets_.keys()):
name_ = ','.join(name) name_ = ','.join(name)
if name_ in legend: if name_ in legend:
if all_labels and name in all_labels_: if name in datalabels_:
if all_labels_[name]: if datalabels_[name]:
legend_.append((all_labels_[name], legend[name_])) legend_.append((datalabels_[name], legend[name_]))
else: else:
legend_.append((name_, legend[name_])) legend_.append((name_, legend[name_]))
legend = legend_ legend = legend_
@@ -1026,7 +1155,7 @@ def main(csv_paths, output, *,
# big hack to get xlabel above the legend! but hey this # big hack to get xlabel above the legend! but hey this
# works really well actually # works really well actually
if xlabel: if xlabel:
ax.set_title(unescape(xlabel), ax.set_title(punescape(xlabel, mergedattrs_),
size=plt.rcParams['axes.labelsize'], size=plt.rcParams['axes.labelsize'],
weight=plt.rcParams['axes.labelweight']) weight=plt.rcParams['axes.labelweight'])
@@ -1056,11 +1185,11 @@ def main(csv_paths, output, *,
# axes labels, NOTE we reposition these below # axes labels, NOTE we reposition these below
if xlabel is not None and not legend_below: if xlabel is not None and not legend_below:
fig.supxlabel(unescape(xlabel)) fig.supxlabel(punescape(xlabel, mergedattrs_))
if ylabel is not None: if ylabel is not None:
fig.supylabel(unescape(ylabel)) fig.supylabel(punescape(ylabel, mergedattrs_))
if title is not None: if title is not None:
fig.suptitle(unescape(title)) fig.suptitle(punescape(title, mergedattrs_))
# precompute constrained layout and find midpoints to adjust things # precompute constrained layout and find midpoints to adjust things
# that should be centered so they are actually centered # that should be centered so they are actually centered
@@ -1069,11 +1198,11 @@ def main(csv_paths, output, *,
ymid = (grid[0,0].ax.get_position().y0 + grid[0,-1].ax.get_position().y1)/2 ymid = (grid[0,0].ax.get_position().y0 + grid[0,-1].ax.get_position().y1)/2
if xlabel is not None and not legend_below: if xlabel is not None and not legend_below:
fig.supxlabel(unescape(xlabel), x=xmid) fig.supxlabel(punescape(xlabel, mergedattrs_), x=xmid)
if ylabel is not None: if ylabel is not None:
fig.supylabel(unescape(ylabel), y=ymid) fig.supylabel(punescape(ylabel, mergedattrs_), y=ymid)
if title is not None: if title is not None:
fig.suptitle(unescape(title), x=xmid) fig.suptitle(punescape(title, mergedattrs_), x=xmid)
# write the figure! # write the figure!
@@ -1137,17 +1266,43 @@ if __name__ == "__main__":
help="Only include results where this field is this value. May " help="Only include results where this field is this value. May "
"include comma-separated options.") "include comma-separated options.")
parser.add_argument( parser.add_argument(
'-L', '--label', '-L', '--add-label',
dest='labels',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs: ( lambda ks, v: (
k.strip(), tuple(k.strip() for k in ks.split(',')),
tuple(v.strip() for v in vs.split(','))) v.strip())
)(*re.split(r'(?<!%)=', x, 1)), )(*x.split('=', 1))
help="Use this label for a given group, where a group is roughly " if '=' in x else x.strip(),
"the comma-separated values in the -b/--by, -x, and -y " help="Add a label to use. Can be assigned to a specific group "
"fields. Also provides an ordering. Accepts %= and other " "where a group is the comma-separated 'by' fields. Accepts %% "
"%-escaped codes.") "modifiers. Also provides an ordering.")
parser.add_argument(
'-C', '--add-color',
dest='colors',
action='append',
type=lambda x: (
lambda ks, v: (
tuple(k.strip() for k in ks.split(',')),
v.strip())
)(*x.split('=', 1))
if '=' in x else x.strip(),
help="Add a color to use. Can be assigned to a specific group "
"where a group is the comma-separated 'by' fields.")
parser.add_argument(
'-F', '--add-format',
dest='formats',
action='append',
type=lambda x: (
lambda ks, v: (
tuple(k.strip() for k in ks.split(',')),
v.strip())
)(*x.split('=', 1))
if '=' in x else x.strip(),
help="Add a matplotlib format to use. Can be assigned to a "
"specific group where a group is the comma-separated 'by' "
"fields.")
parser.add_argument( parser.add_argument(
'-.', '--points', '-.', '--points',
action='store_true', action='store_true',
@@ -1156,15 +1311,6 @@ if __name__ == "__main__":
'-!', '--points-and-lines', '-!', '--points-and-lines',
action='store_true', action='store_true',
help="Draw data points and lines.") help="Draw data points and lines.")
parser.add_argument(
'--colors',
type=lambda x: [x.strip() for x in x.split(',')],
help="Comma-separated hex colors to use.")
parser.add_argument(
'--formats',
type=lambda x: [x.strip() for x in re.split(r'(?<!%),', x)],
help="Comma-separated matplotlib formats to use. Accepts %, and "
"other %-escaped codes.")
parser.add_argument( parser.add_argument(
'-W', '--width', '-W', '--width',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
@@ -1221,25 +1367,25 @@ if __name__ == "__main__":
help="Units for the y-axis.") help="Units for the y-axis.")
parser.add_argument( parser.add_argument(
'--xlabel', '--xlabel',
help="Add a label to the x-axis. Accepts %-escaped codes.") help="Add a label to the x-axis. Accepts %% modifiers.")
parser.add_argument( parser.add_argument(
'--ylabel', '--ylabel',
help="Add a label to the y-axis. Accepts %-escaped codes.") help="Add a label to the y-axis. Accepts %% modifiers.")
parser.add_argument( parser.add_argument(
'--xticklabels', '--xticklabels',
type=lambda x: [x.strip() for x in re.split(r'(?<!%),', x)] type=lambda x: [x.strip() for x in re.split(r'(?<!%),', x)]
if x.strip() else [], if x.strip() else [],
help="Comma separated xticklabels. Accepts %, and other " help="Comma separated xticklabels. Accepts %%, and other "
"%-escaped codes.") "%%-escaped codes.")
parser.add_argument( parser.add_argument(
'--yticklabels', '--yticklabels',
type=lambda x: [x.strip() for x in re.split(r'(?<!%),', x)] type=lambda x: [x.strip() for x in re.split(r'(?<!%),', x)]
if x.strip() else [], if x.strip() else [],
help="Comma separated yticklabels. Accepts %, and other " help="Comma separated yticklabels. Accepts %%, and other "
"%-escaped codes.") "%%-escaped codes.")
parser.add_argument( parser.add_argument(
'--title', '--title',
help="Add a title. Accepts %-escaped codes.") help="Add a title. Accepts %% modifiers.")
parser.add_argument( parser.add_argument(
'-l', '--legend', '--legend-right', '-l', '--legend', '--legend-right',
dest='legend_right', dest='legend_right',
scripts/test.py
+2 -2
View File
@@ -897,11 +897,11 @@ def find_ids(runner, test_ids=[], **args):
if '*' in name: if '*' in name:
test_ids__.extend(suite test_ids__.extend(suite
for suite in expected_suite_perms.keys() for suite in expected_suite_perms.keys()
if fnmatch.fnmatch(suite, name)) if fnmatch.fnmatchcase(suite, name))
if not test_ids__: if not test_ids__:
test_ids__.extend(case_ test_ids__.extend(case_
for case_ in expected_case_perms.keys() for case_ in expected_case_perms.keys()
if fnmatch.fnmatch(case_, name)) if fnmatch.fnmatchcase(case_, name))
# literal suite # literal suite
elif name in expected_suite_perms: elif name in expected_suite_perms:
test_ids__.append(id) test_ids__.append(id)
scripts/treemap.py
+36 -8
View File
@@ -11,6 +11,7 @@ if __name__ == "__main__":
import bisect import bisect
import collections as co import collections as co
import csv import csv
import fnmatch
import itertools as it import itertools as it
import math as mt import math as mt
import re import re
@@ -163,7 +164,7 @@ class Attr:
if (defaults is not None if (defaults is not None
and not any( and not any(
not isinstance(attr, tuple) not isinstance(attr, tuple)
or attr[0] in {None, (), ('',)} or attr[0] in {None, (), ('*',)}
for attr in (attrs or []))): for attr in (attrs or []))):
attrs = defaults + (attrs or []) attrs = defaults + (attrs or [])
@@ -173,7 +174,7 @@ class Attr:
for attr in (attrs or []): for attr in (attrs or []):
if not isinstance(attr, tuple): if not isinstance(attr, tuple):
attr = ((), attr) attr = ((), attr)
elif attr[0] in {None, (), ('',)}: elif attr[0] in {None, (), ('*',)}:
attr = ((), attr[1]) attr = ((), attr[1])
self.attrs.append(attr) self.attrs.append(attr)
@@ -206,7 +207,7 @@ class Attr:
for ks, vs in self.keyed.items(): for ks, vs in self.keyed.items():
prefix = [] prefix = []
for j, k in enumerate(ks): for j, k in enumerate(ks):
if j < len(key) and (not k or key[j] == k): if j < len(key) and fnmatch.fnmatchcase(key[j], k):
prefix.append(k) prefix.append(k)
else: else:
prefix = None prefix = None
@@ -225,6 +226,33 @@ class Attr:
def __contains__(self, key): def __contains__(self, key):
return self.__getitem__(key) is not None return self.__getitem__(key) is not None
# a key function for sorting by key order
def key(self, key):
# allow key to be a tuple to make sorting dicts easier
if (isinstance(key, tuple)
and len(key) >= 1
and isinstance(key[0], tuple)):
key = key[0]
best = None
for i, ks in enumerate(self.keyed.keys()):
prefix = []
for j, k in enumerate(ks):
if j < len(key) and (not k or key[j] == k):
prefix.append(k)
else:
prefix = None
break
if prefix is not None and (
best is None or len(prefix) >= len(best[0])):
best = (prefix, i)
if best is not None:
return best[1]
return len(self.keyed)
# parse %-escaped strings # parse %-escaped strings
def punescape(s, attrs=None): def punescape(s, attrs=None):
if attrs is None: if attrs is None:
@@ -673,9 +701,9 @@ def main(csv_paths, *,
by=None, by=None,
fields=None, fields=None,
defines=[], defines=[],
labels=None, labels=[],
chars=None, chars=[],
colors=None, colors=[],
color=False, color=False,
dots=False, dots=False,
braille=False, braille=False,
@@ -698,7 +726,7 @@ def main(csv_paths, *,
# what chars/colors/labels to use? # what chars/colors/labels to use?
chars_ = [] chars_ = []
for char in (chars or []): for char in chars:
if isinstance(char, tuple): if isinstance(char, tuple):
chars_.extend((char[0], c) for c in char[1]) chars_.extend((char[0], c) for c in char[1])
else: else:
@@ -1004,7 +1032,7 @@ if __name__ == "__main__":
"where a group is the comma-separated 'by' fields. Accepts %% " "where a group is the comma-separated 'by' fields. Accepts %% "
"modifiers.") "modifiers.")
parser.add_argument( parser.add_argument(
'-.', '--add-char', '--chars', '-*', '--add-char', '--chars',
dest='chars', dest='chars',
action='append', action='append',
type=lambda x: ( type=lambda x: (
scripts/treemapd3.py
+33 -5
View File
@@ -11,6 +11,7 @@ if __name__ == "__main__":
import bisect import bisect
import collections as co import collections as co
import csv import csv
import fnmatch
import itertools as it import itertools as it
import math as mt import math as mt
import re import re
@@ -179,7 +180,7 @@ class Attr:
if (defaults is not None if (defaults is not None
and not any( and not any(
not isinstance(attr, tuple) not isinstance(attr, tuple)
or attr[0] in {None, (), ('',)} or attr[0] in {None, (), ('*',)}
for attr in (attrs or []))): for attr in (attrs or []))):
attrs = defaults + (attrs or []) attrs = defaults + (attrs or [])
@@ -189,7 +190,7 @@ class Attr:
for attr in (attrs or []): for attr in (attrs or []):
if not isinstance(attr, tuple): if not isinstance(attr, tuple):
attr = ((), attr) attr = ((), attr)
elif attr[0] in {None, (), ('',)}: elif attr[0] in {None, (), ('*',)}:
attr = ((), attr[1]) attr = ((), attr[1])
self.attrs.append(attr) self.attrs.append(attr)
@@ -222,7 +223,7 @@ class Attr:
for ks, vs in self.keyed.items(): for ks, vs in self.keyed.items():
prefix = [] prefix = []
for j, k in enumerate(ks): for j, k in enumerate(ks):
if j < len(key) and (not k or key[j] == k): if j < len(key) and fnmatch.fnmatchcase(key[j], k):
prefix.append(k) prefix.append(k)
else: else:
prefix = None prefix = None
@@ -241,6 +242,33 @@ class Attr:
def __contains__(self, key): def __contains__(self, key):
return self.__getitem__(key) is not None return self.__getitem__(key) is not None
# a key function for sorting by key order
def key(self, key):
# allow key to be a tuple to make sorting dicts easier
if (isinstance(key, tuple)
and len(key) >= 1
and isinstance(key[0], tuple)):
key = key[0]
best = None
for i, ks in enumerate(self.keyed.keys()):
prefix = []
for j, k in enumerate(ks):
if j < len(key) and (not k or key[j] == k):
prefix.append(k)
else:
prefix = None
break
if prefix is not None and (
best is None or len(prefix) >= len(best[0])):
best = (prefix, i)
if best is not None:
return best[1]
return len(self.keyed)
# parse %-escaped strings # parse %-escaped strings
def punescape(s, attrs=None): def punescape(s, attrs=None):
if attrs is None: if attrs is None:
@@ -526,8 +554,8 @@ def main(csv_paths, output, *,
by=None, by=None,
fields=None, fields=None,
defines=[], defines=[],
labels=None, labels=[],
colors=None, colors=[],
width=None, width=None,
height=None, height=None,
no_header=False, no_header=False,