Github/littlefs
1
Fork 0
Code Actions Activity

scripts: Dropped field renames from plot.py/plotmpl.py/amor.py/avg.py

Browse Source 
This is now inconsistent with csv.py, and I don't really want to add a
full expr parser to every script that might want to rename fields.

Field renaming (or any expr really!) can be accomplished with
intermediate calls to csv.py anyways. No reason to make these scripts
more complicated than they need to be.
This commit is contained in:
Christopher Haster
2024-11-12 14:09:12 -06:00
parent 6714e2869f
commit 02881faf6f
4 changed files with 31 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/amor.py
+6 -40
View File
@@ -47,7 +47,7 @@ def dat(x):
# else give up # else give up
raise ValueError("invalid dat %r" % x) raise ValueError("invalid dat %r" % x)
def collect(csv_paths, renames=[], defines=[]): def collect(csv_paths, defines=[]):
# collect results from CSV files # collect results from CSV files
fields = [] fields = []
results = [] results = []
@@ -59,15 +59,6 @@ def collect(csv_paths, renames=[], defines=[]):
k for k in reader.fieldnames k for k in reader.fieldnames
if k not in fields) if k not in fields)
for r in reader: for r in reader:
# apply any renames
if renames:
# make a copy so renames can overlap
r_ = {}
for new_k, old_k in renames:
if old_k in r:
r_[new_k] = r[old_k]
r.update(r_)
# filter by matching defines # filter by matching defines
if not all(k in r and r[k] in vs for k, vs in defines): if not all(k in r and r[k] in vs for k, vs in defines):
continue continue
@@ -91,41 +82,30 @@ def main(csv_paths, output, *,
if not amor and not per: if not amor and not per:
amor = True amor = True
# separate out renames
renames = list(it.chain.from_iterable(
((k, v) for v in vs)
for k, vs in it.chain(by or [], fields or [])))
if by is not None:
by = [k for k, _ in by]
if fields is not None:
fields = [k for k, _ in fields]
if by is None and fields is None: if by is None and fields is None:
print("error: needs --by or --fields to figure out fields", print("error: needs --by or --fields to figure out fields",
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
# collect results from csv files # collect results from csv files
fields_, results = collect(csv_paths, renames, defines) fields_, results = collect(csv_paths, defines)
# if by not specified, guess it's anything not in # if by not specified, guess it's anything not in
# iter/size/fields/renames/defines # iter/size/fields/defines
if by is None: if by is None:
by = [k for k in fields_ by = [k for k in fields_
if k != iter if k != iter
and k != size and k != size
and k not in (fields or []) and k not in (fields or [])
and not any(k == old_k for _, old_k in renames)
and not any(k == k_ for k_, _ in defines)] and not any(k == k_ for k_, _ in defines)]
# if fields not specified, guess it's anything not in # if fields not specified, guess it's anything not in
# by/iter/size/renames/defines # by/iter/size/defines
if fields is None: if fields is None:
fields = [k for k in fields_ fields = [k for k in fields_
if k not in (by or []) if k not in (by or [])
and k != iter and k != iter
and k != size and k != size
and not any(k == old_k for _, old_k in renames)
and not any(k == k_ for k_, _ in defines)] and not any(k == k_ for k_, _ in defines)]
# add meas to by if it isn't already present # add meas to by if it isn't already present
@@ -209,14 +189,7 @@ if __name__ == "__main__":
parser.add_argument( parser.add_argument(
'-b', '--by', '-b', '--by',
action='append', action='append',
type=lambda x: ( help="Group by this field.")
lambda k, vs=None: (
k.strip(),
tuple(v.strip() for v in vs.split(','))
if vs is not None else ())
)(*x.split('=', 1)),
help="Group by this field. Can rename fields with "
"new_name=old_name.")
parser.add_argument( parser.add_argument(
'-m', '--meas', '-m', '--meas',
help="Optional name of measurement name field. If provided, the " help="Optional name of measurement name field. If provided, the "
@@ -232,14 +205,7 @@ if __name__ == "__main__":
'-f', '--field', '-f', '--field',
dest='fields', dest='fields',
action='append', action='append',
type=lambda x: ( help="Field to amortize.")
lambda k, vs=None: (
k.strip(),
tuple(v.strip() for v in vs.split(','))
if vs is not None else ())
)(*x.split('=', 1)),
help="Field to amortize. Can rename fields with "
"new_name=old_name.")
parser.add_argument( parser.add_argument(
'-D', '--define', '-D', '--define',
dest='defines', dest='defines',
scripts/avg.py
+7 -50
View File
@@ -47,7 +47,7 @@ def dat(x):
# else give up # else give up
raise ValueError("invalid dat %r" % x) raise ValueError("invalid dat %r" % x)
def collect(csv_paths, renames=[], defines=[]): def collect(csv_paths, defines=[]):
# collect results from CSV files # collect results from CSV files
fields = [] fields = []
results = [] results = []
@@ -59,15 +59,6 @@ def collect(csv_paths, renames=[], defines=[]):
k for k in reader.fieldnames k for k in reader.fieldnames
if k not in fields) if k not in fields)
for r in reader: for r in reader:
# apply any renames
if renames:
# make a copy so renames can overlap
r_ = {}
for new_k, old_k in renames:
if old_k in r:
r_[new_k] = r[old_k]
r.update(r_)
# filter by matching defines # filter by matching defines
if not all(k in r and r[k] in vs for k, vs in defines): if not all(k in r and r[k] in vs for k, vs in defines):
continue continue
@@ -109,41 +100,28 @@ def main(csv_paths, output, *,
and not gstddev): and not gstddev):
avg = True avg = True
# separate out renames
renames = list(it.chain.from_iterable(
((k, v) for v in vs)
for k, vs in it.chain(by or [], seeds or [], fields or [])))
if by is not None:
by = [k for k, _ in by]
if seeds is not None:
seeds = [k for k, _ in seeds]
if fields is not None:
fields = [k for k, _ in fields]
if by is None and fields is None: if by is None and fields is None:
print("error: needs --by or --fields to figure out fields", print("error: needs --by or --fields to figure out fields",
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
# collect results from csv files # collect results from csv files
fields_, results = collect(csv_paths, renames, defines) fields_, results = collect(csv_paths, defines)
# if by not specified, guess it's anything not in # if by not specified, guess it's anything not in
# seeds/fields/renames/defines # seeds/fields/defines
if by is None: if by is None:
by = [k for k in fields_ by = [k for k in fields_
if k not in (seeds or []) if k not in (seeds or [])
and k not in (fields or []) and k not in (fields or [])
and not any(k == old_k for _, old_k in renames)
and not any(k == k_ for k_, _ in defines)] and not any(k == k_ for k_, _ in defines)]
# if fields not specified, guess it's anything not in # if fields not specified, guess it's anything not in
# by/seeds/renames/defines # by/seeds/defines
if fields is None: if fields is None:
fields = [k for k in fields_ fields = [k for k in fields_
if k not in (by or []) if k not in (by or [])
and k not in (seeds or []) and k not in (seeds or [])
and not any(k == old_k for _, old_k in renames)
and not any(k == k_ for k_, _ in defines)] and not any(k == k_ for k_, _ in defines)]
# add meas to by if it isn't already present # add meas to by if it isn't already present
@@ -261,14 +239,7 @@ if __name__ == "__main__":
parser.add_argument( parser.add_argument(
'-b', '--by', '-b', '--by',
action='append', action='append',
type=lambda x: ( help="Group by this field.")
lambda k, vs=None: (
k.strip(),
tuple(v.strip() for v in vs.split(','))
if vs is not None else ())
)(*x.split('=', 1)),
help="Group by this field. Can rename fields with "
"new_name=old_name.")
parser.add_argument( parser.add_argument(
'-m', '--meas', '-m', '--meas',
help="Optional name of measurement name field. If provided, the " help="Optional name of measurement name field. If provided, the "
@@ -277,26 +248,12 @@ if __name__ == "__main__":
'-s', '--seed', '-s', '--seed',
dest='seeds', dest='seeds',
action='append', action='append',
type=lambda x: ( help="Field to ignore when averaging.")
lambda k, vs=None: (
k.strip(),
tuple(v.strip() for v in vs.split(','))
if vs is not None else ())
)(*x.split('=', 1)),
help="Field to ignore when averaging. Can rename fields with "
"new_name=old_name.")
parser.add_argument( parser.add_argument(
'-f', '--field', '-f', '--field',
dest='fields', dest='fields',
action='append', action='append',
type=lambda x: ( help="Field to amortize.")
lambda k, vs=None: (
k.strip(),
tuple(v.strip() for v in vs.split(','))
if vs is not None else ())
)(*x.split('=', 1)),
help="Field to amortize. Can rename fields with "
"new_name=old_name.")
parser.add_argument( parser.add_argument(
'-D', '--define', '-D', '--define',
dest='defines', dest='defines',
scripts/plot.py
+9 -48
View File
@@ -454,7 +454,7 @@ class Plot:
return ''.join(row_) return ''.join(row_)
def collect(csv_paths, renames=[], defines=[]): def collect(csv_paths, defines=[]):
# collect results from CSV files # collect results from CSV files
fields = [] fields = []
results = [] results = []
@@ -466,15 +466,6 @@ def collect(csv_paths, renames=[], defines=[]):
k for k in reader.fieldnames k for k in reader.fieldnames
if k not in fields) if k not in fields)
for r in reader: for r in reader:
# apply any renames
if renames:
# make a copy so renames can overlap
r_ = {}
for new_k, old_k in renames:
if old_k in r:
r_[new_k] = r[old_k]
r.update(r_)
# filter by matching defines # filter by matching defines
if not all(k in r and r[k] in vs for k, vs in defines): if not all(k in r and r[k] in vs for k, vs in defines):
continue continue
@@ -918,14 +909,6 @@ def main(csv_paths, *,
all_labels = ((label or []) all_labels = ((label or [])
+ subplots_get('label', **subplot, subplots=subplots)) + subplots_get('label', **subplot, subplots=subplots))
# separate out renames
all_renames = list(it.chain.from_iterable(
((k, v) for v in vs)
for k, vs in it.chain(all_by, all_x, all_y)))
all_by = [k for k, _ in all_by]
all_x = [k for k, _ in all_x]
all_y = [k for k, _ in all_y]
if not all_by and not all_y: if not all_by and not all_y:
print("error: needs --by or -y to figure out fields", print("error: needs --by or -y to figure out fields",
file=sys.stderr) file=sys.stderr)
@@ -1014,15 +997,14 @@ def main(csv_paths, *,
f.writeln = writeln f.writeln = writeln
# first collect results from CSV files # first collect results from CSV files
fields_, results = collect(csv_paths, all_renames, all_defines) fields_, results = collect(csv_paths, all_defines)
# if y not specified, guess it's anything not in by/defines/x/renames # if y not specified, guess it's anything not in by/defines/x
all_y_ = all_y all_y_ = all_y
if not all_y: if not all_y:
all_y_ = [k for k in fields_ all_y_ = [k for k in fields_
if k not in all_by if k not in all_by
and not any(k == k_ for k_, _ in all_defines) and not any(k == k_ for k_, _ in all_defines)]
and not any(k == old_k for _, old_k in all_renames)]
# then extract the requested datasets # then extract the requested datasets
datasets_ = fold(results, all_by, all_x, all_y_, None, all_labels) datasets_ = fold(results, all_by, all_x, all_y_, None, all_labels)
@@ -1160,8 +1142,8 @@ def main(csv_paths, *,
# create a plot for each subplot # create a plot for each subplot
for s in grid: for s in grid:
# allow subplot params to override global params # allow subplot params to override global params
x_ = {k for k,_ in (x or []) + s.args.get('x', [])} x_ = set((x or []) + s.args.get('x', []))
y_ = {k for k,_ in (y or []) + s.args.get('y', [])} y_ = set((y or []) + s.args.get('y', []))
define_ = define + s.args.get('define', []) define_ = define + s.args.get('define', [])
xlim_ = s.args.get('xlim', xlim) xlim_ = s.args.get('xlim', xlim)
ylim_ = s.args.get('ylim', ylim) ylim_ = s.args.get('ylim', ylim)
@@ -1459,36 +1441,15 @@ if __name__ == "__main__":
parser.add_argument( parser.add_argument(
'-b', '--by', '-b', '--by',
action='append', action='append',
type=lambda x: ( help="Group by this field.")
lambda k, vs=None: (
k.strip(),
tuple(v.strip() for v in vs.split(','))
if vs is not None else ())
)(*x.split('=', 1)),
help="Group by this field. Can rename fields with "
"new_name=old_name.")
parser.add_argument( parser.add_argument(
'-x', '-x',
action='append', action='append',
type=lambda x: ( help="Field to use for the x-axis.")
lambda k, vs=None: (
k.strip(),
tuple(v.strip() for v in vs.split(','))
if vs is not None else ())
)(*x.split('=', 1)),
help="Field to use for the x-axis. Can rename fields with "
"new_name=old_name.")
parser.add_argument( parser.add_argument(
'-y', '-y',
action='append', action='append',
type=lambda x: ( help="Field to use for the y-axis.")
lambda k, vs=None: (
k.strip(),
tuple(v.strip() for v in vs.split(','))
if vs is not None else ())
)(*x.split('=', 1)),
help="Field to use for the y-axis. Can rename fields with "
"new_name=old_name.")
parser.add_argument( parser.add_argument(
'-D', '--define', '-D', '--define',
type=lambda x: ( type=lambda x: (
scripts/plotmpl.py
+9 -48
View File
@@ -194,7 +194,7 @@ def dat(x):
# else give up # else give up
raise ValueError("invalid dat %r" % x) raise ValueError("invalid dat %r" % x)
def collect(csv_paths, renames=[], defines=[]): def collect(csv_paths, defines=[]):
# collect results from CSV files # collect results from CSV files
fields = [] fields = []
results = [] results = []
@@ -206,15 +206,6 @@ def collect(csv_paths, renames=[], defines=[]):
k for k in reader.fieldnames k for k in reader.fieldnames
if k not in fields) if k not in fields)
for r in reader: for r in reader:
# apply any renames
if renames:
# make a copy so renames can overlap
r_ = {}
for new_k, old_k in renames:
if old_k in r:
r_[new_k] = r[old_k]
r.update(r_)
# filter by matching defines # filter by matching defines
if not all(k in r and r[k] in vs for k, vs in defines): if not all(k in r and r[k] in vs for k, vs in defines):
continue continue
@@ -733,28 +724,19 @@ def main(csv_paths, output, *,
all_labels = ((label or []) all_labels = ((label or [])
+ subplots_get('label', **subplot, subplots=subplots)) + subplots_get('label', **subplot, subplots=subplots))
# separate out renames
all_renames = list(it.chain.from_iterable(
((k, v) for v in vs)
for k, vs in it.chain(all_by, all_x, all_y)))
all_by = [k for k, _ in all_by]
all_x = [k for k, _ in all_x]
all_y = [k for k, _ in all_y]
if not all_by and not all_y: if not all_by and not all_y:
print("error: needs --by or -y to figure out fields", print("error: needs --by or -y to figure out fields",
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
# first collect results from CSV files # first collect results from CSV files
fields_, results = collect(csv_paths, all_renames, all_defines) fields_, results = collect(csv_paths, all_defines)
# if y not specified, guess it's anything not in by/defines/x/renames # if y not specified, guess it's anything not in by/defines/x
if not all_y: if not all_y:
all_y = [k for k in fields_ all_y = [k for k in fields_
if k not in all_by if k not in all_by
and not any(k == k_ for k_, _ in all_defines) and not any(k == k_ for k_, _ in all_defines)]
and not any(k == old_k for _, old_k in all_renames)]
# then extract the requested datasets # then extract the requested datasets
# #
@@ -805,8 +787,8 @@ def main(csv_paths, output, *,
# now plot each subplot # now plot each subplot
for s in grid: for s in grid:
# allow subplot params to override global params # allow subplot params to override global params
x_ = {k for k,_ in (x or []) + s.args.get('x', [])} x_ = set((x or []) + s.args.get('x', []))
y_ = {k for k,_ in (y or []) + s.args.get('y', [])} y_ = set((y or []) + s.args.get('y', []))
define_ = define + s.args.get('define', []) define_ = define + s.args.get('define', [])
xlim_ = s.args.get('xlim', xlim) xlim_ = s.args.get('xlim', xlim)
ylim_ = s.args.get('ylim', ylim) ylim_ = s.args.get('ylim', ylim)
@@ -1105,36 +1087,15 @@ if __name__ == "__main__":
parser.add_argument( parser.add_argument(
'-b', '--by', '-b', '--by',
action='append', action='append',
type=lambda x: ( help="Group by this field.")
lambda k, vs=None: (
k.strip(),
tuple(v.strip() for v in vs.split(','))
if vs is not None else ())
)(*x.split('=', 1)),
help="Group by this field. Can rename fields with "
"new_name=old_name.")
parser.add_argument( parser.add_argument(
'-x', '-x',
action='append', action='append',
type=lambda x: ( help="Field to use for the x-axis.")
lambda k, vs=None: (
k.strip(),
tuple(v.strip() for v in vs.split(','))
if vs is not None else ())
)(*x.split('=', 1)),
help="Field to use for the x-axis. Can rename fields with "
"new_name=old_name.")
parser.add_argument( parser.add_argument(
'-y', '-y',
action='append', action='append',
type=lambda x: ( help="Field to use for the y-axis.")
lambda k, vs=None: (
k.strip(),
tuple(v.strip() for v in vs.split(','))
if vs is not None else ())
)(*x.split('=', 1)),
help="Field to use for the y-axis. Can rename fields with "
"new_name=old_name.")
parser.add_argument( parser.add_argument(
'-D', '--define', '-D', '--define',
type=lambda x: ( type=lambda x: (