Github/littlefs
1
Fork 0
Code Actions Activity

scripts: Tweaked csv read/writing to pass notes through

Browse Source 
This was, uh, half-implemented in csv.py's collect_csv, but completely
ignored in read_csv/write_csv. Adding support to read_csv/write_csv
wasn't too hard, so maybe we should keep this?

There's an argument notes should not be included in csv output, as the
nested commas (for multiple results) can make a mess of csv's simplicty.
(There's a different argument that csv is a terrible format, but I'm not
sure I agree.)

But for now, including notes doesn't seem to harm anything.

---

Note this also includes a fix for filtering empty notes in csv.py's
collect_csv.
This commit is contained in:
Christopher Haster
2026-03-05 10:06:50 -06:00
parent e521e21764
commit ecc72e3dba
9 changed files with 163 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/code.py
+18 -5
View File
@@ -937,10 +937,10 @@ def read_csv(path, Result, *,
with openio(path, 'r') as f: with openio(path, 'r') as f:
# csv or json? assume json starts with [ # csv or json? assume json starts with [
is_json = (f.buffer.peek(1)[:1] == b'[') json = (f.buffer.peek(1)[:1] == b'[')
# read csv? # read csv?
if not is_json: if not json:
results = [] results = []
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
for r in reader: for r in reader:
@@ -955,7 +955,13 @@ def read_csv(path, Result, *,
and r[k].strip()} and r[k].strip()}
| {k: r[prefix+k] for k in fields | {k: r[prefix+k] for k in fields
if prefix+k in r if prefix+k in r
and r[prefix+k].strip()}))) and r[prefix+k].strip()}
| ({Result._notes: set(n.strip()
for n in r[Result._notes].split(',')
if n.strip())}
if hasattr(Result, '_notes')
and Result._notes in r
else {}))))
except TypeError: except TypeError:
pass pass
return results return results
@@ -1021,7 +1027,9 @@ def write_csv(path, Result, results, *,
writer = csv.DictWriter(f, list( writer = csv.DictWriter(f, list(
co.OrderedDict.fromkeys(it.chain( co.OrderedDict.fromkeys(it.chain(
by, by,
(prefix+k for k in fields))).keys())) (prefix+k for k in fields),
[Result._notes] if hasattr(Result, '_notes')
else [])).keys()))
writer.writeheader() writer.writeheader()
for r in results: for r in results:
# note this allows by/fields to overlap # note this allows by/fields to overlap
@@ -1031,7 +1039,12 @@ def write_csv(path, Result, results, *,
if getattr(r, k) is not None} if getattr(r, k) is not None}
| {prefix+k: getattr(r, k).__csv__() | {prefix+k: getattr(r, k).__csv__()
for k in fields for k in fields
if getattr(r, k) is not None}) if getattr(r, k) is not None}
| ({Result._notes: ','.join(
getattr(r, Result._notes))}
if hasattr(Result, '_notes')
and getattr(r, Result._notes)
else {}))
# write json? # write json?
else: else:
scripts/cov.py
+18 -5
View File
@@ -806,10 +806,10 @@ def read_csv(path, Result, *,
with openio(path, 'r') as f: with openio(path, 'r') as f:
# csv or json? assume json starts with [ # csv or json? assume json starts with [
is_json = (f.buffer.peek(1)[:1] == b'[') json = (f.buffer.peek(1)[:1] == b'[')
# read csv? # read csv?
if not is_json: if not json:
results = [] results = []
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
for r in reader: for r in reader:
@@ -824,7 +824,13 @@ def read_csv(path, Result, *,
and r[k].strip()} and r[k].strip()}
| {k: r[prefix+k] for k in fields | {k: r[prefix+k] for k in fields
if prefix+k in r if prefix+k in r
and r[prefix+k].strip()}))) and r[prefix+k].strip()}
| ({Result._notes: set(n.strip()
for n in r[Result._notes].split(',')
if n.strip())}
if hasattr(Result, '_notes')
and Result._notes in r
else {}))))
except TypeError: except TypeError:
pass pass
return results return results
@@ -890,7 +896,9 @@ def write_csv(path, Result, results, *,
writer = csv.DictWriter(f, list( writer = csv.DictWriter(f, list(
co.OrderedDict.fromkeys(it.chain( co.OrderedDict.fromkeys(it.chain(
by, by,
(prefix+k for k in fields))).keys())) (prefix+k for k in fields),
[Result._notes] if hasattr(Result, '_notes')
else [])).keys()))
writer.writeheader() writer.writeheader()
for r in results: for r in results:
# note this allows by/fields to overlap # note this allows by/fields to overlap
@@ -900,7 +908,12 @@ def write_csv(path, Result, results, *,
if getattr(r, k) is not None} if getattr(r, k) is not None}
| {prefix+k: getattr(r, k).__csv__() | {prefix+k: getattr(r, k).__csv__()
for k in fields for k in fields
if getattr(r, k) is not None}) if getattr(r, k) is not None}
| ({Result._notes: ','.join(
getattr(r, Result._notes))}
if hasattr(Result, '_notes')
and getattr(r, Result._notes)
else {}))
# write json? # write json?
else: else:
scripts/csv.py
+19 -4
View File
@@ -1894,7 +1894,9 @@ def collect_csv(csv_paths, *,
and v.strip()} and v.strip()}
# special handling for notes field # special handling for notes field
if notes is not None and notes in r: if notes is not None and notes in r:
r_[notes] = set(r[notes].split(',')) r_[notes] = set(n.strip()
for n in r[notes].split(',')
if n.strip())
results.append(r_) results.append(r_)
# read json? # read json?
@@ -2632,7 +2634,13 @@ def read_csv(path, Result, *,
and r[k].strip()} and r[k].strip()}
| {k: r[prefix+k] for k in fields | {k: r[prefix+k] for k in fields
if prefix+k in r if prefix+k in r
and r[prefix+k].strip()}))) and r[prefix+k].strip()}
| ({Result._notes: set(n.strip()
for n in r[Result._notes].split(',')
if n.strip())}
if hasattr(Result, '_notes')
and Result._notes in r
else {}))))
except TypeError: except TypeError:
pass pass
return results return results
@@ -2698,7 +2706,9 @@ def write_csv(path, Result, results, *,
writer = csv.DictWriter(f, list( writer = csv.DictWriter(f, list(
co.OrderedDict.fromkeys(it.chain( co.OrderedDict.fromkeys(it.chain(
by, by,
(prefix+k for k in fields))).keys())) (prefix+k for k in fields),
[Result._notes] if hasattr(Result, '_notes')
else [])).keys()))
writer.writeheader() writer.writeheader()
for r in results: for r in results:
# note this allows by/fields to overlap # note this allows by/fields to overlap
@@ -2708,7 +2718,12 @@ def write_csv(path, Result, results, *,
if getattr(r, k) is not None} if getattr(r, k) is not None}
| {prefix+k: getattr(r, k).__csv__() | {prefix+k: getattr(r, k).__csv__()
for k in fields for k in fields
if getattr(r, k) is not None}) if getattr(r, k) is not None}
| ({Result._notes: ','.join(
getattr(r, Result._notes))}
if hasattr(Result, '_notes')
and getattr(r, Result._notes)
else {}))
# write json? # write json?
else: else:
scripts/ctx.py
+18 -5
View File
@@ -1196,10 +1196,10 @@ def read_csv(path, Result, *,
with openio(path, 'r') as f: with openio(path, 'r') as f:
# csv or json? assume json starts with [ # csv or json? assume json starts with [
is_json = (f.buffer.peek(1)[:1] == b'[') json = (f.buffer.peek(1)[:1] == b'[')
# read csv? # read csv?
if not is_json: if not json:
results = [] results = []
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
for r in reader: for r in reader:
@@ -1214,7 +1214,13 @@ def read_csv(path, Result, *,
and r[k].strip()} and r[k].strip()}
| {k: r[prefix+k] for k in fields | {k: r[prefix+k] for k in fields
if prefix+k in r if prefix+k in r
and r[prefix+k].strip()}))) and r[prefix+k].strip()}
| ({Result._notes: set(n.strip()
for n in r[Result._notes].split(',')
if n.strip())}
if hasattr(Result, '_notes')
and Result._notes in r
else {}))))
except TypeError: except TypeError:
pass pass
return results return results
@@ -1280,7 +1286,9 @@ def write_csv(path, Result, results, *,
writer = csv.DictWriter(f, list( writer = csv.DictWriter(f, list(
co.OrderedDict.fromkeys(it.chain( co.OrderedDict.fromkeys(it.chain(
by, by,
(prefix+k for k in fields))).keys())) (prefix+k for k in fields),
[Result._notes] if hasattr(Result, '_notes')
else [])).keys()))
writer.writeheader() writer.writeheader()
for r in results: for r in results:
# note this allows by/fields to overlap # note this allows by/fields to overlap
@@ -1290,7 +1298,12 @@ def write_csv(path, Result, results, *,
if getattr(r, k) is not None} if getattr(r, k) is not None}
| {prefix+k: getattr(r, k).__csv__() | {prefix+k: getattr(r, k).__csv__()
for k in fields for k in fields
if getattr(r, k) is not None}) if getattr(r, k) is not None}
| ({Result._notes: ','.join(
getattr(r, Result._notes))}
if hasattr(Result, '_notes')
and getattr(r, Result._notes)
else {}))
# write json? # write json?
else: else:
scripts/data.py
+18 -5
View File
@@ -937,10 +937,10 @@ def read_csv(path, Result, *,
with openio(path, 'r') as f: with openio(path, 'r') as f:
# csv or json? assume json starts with [ # csv or json? assume json starts with [
is_json = (f.buffer.peek(1)[:1] == b'[') json = (f.buffer.peek(1)[:1] == b'[')
# read csv? # read csv?
if not is_json: if not json:
results = [] results = []
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
for r in reader: for r in reader:
@@ -955,7 +955,13 @@ def read_csv(path, Result, *,
and r[k].strip()} and r[k].strip()}
| {k: r[prefix+k] for k in fields | {k: r[prefix+k] for k in fields
if prefix+k in r if prefix+k in r
and r[prefix+k].strip()}))) and r[prefix+k].strip()}
| ({Result._notes: set(n.strip()
for n in r[Result._notes].split(',')
if n.strip())}
if hasattr(Result, '_notes')
and Result._notes in r
else {}))))
except TypeError: except TypeError:
pass pass
return results return results
@@ -1021,7 +1027,9 @@ def write_csv(path, Result, results, *,
writer = csv.DictWriter(f, list( writer = csv.DictWriter(f, list(
co.OrderedDict.fromkeys(it.chain( co.OrderedDict.fromkeys(it.chain(
by, by,
(prefix+k for k in fields))).keys())) (prefix+k for k in fields),
[Result._notes] if hasattr(Result, '_notes')
else [])).keys()))
writer.writeheader() writer.writeheader()
for r in results: for r in results:
# note this allows by/fields to overlap # note this allows by/fields to overlap
@@ -1031,7 +1039,12 @@ def write_csv(path, Result, results, *,
if getattr(r, k) is not None} if getattr(r, k) is not None}
| {prefix+k: getattr(r, k).__csv__() | {prefix+k: getattr(r, k).__csv__()
for k in fields for k in fields
if getattr(r, k) is not None}) if getattr(r, k) is not None}
| ({Result._notes: ','.join(
getattr(r, Result._notes))}
if hasattr(Result, '_notes')
and getattr(r, Result._notes)
else {}))
# write json? # write json?
else: else:
scripts/perf.py
+18 -5
View File
@@ -1295,10 +1295,10 @@ def read_csv(path, Result, *,
with openio(path, 'r') as f: with openio(path, 'r') as f:
# csv or json? assume json starts with [ # csv or json? assume json starts with [
is_json = (f.buffer.peek(1)[:1] == b'[') json = (f.buffer.peek(1)[:1] == b'[')
# read csv? # read csv?
if not is_json: if not json:
results = [] results = []
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
for r in reader: for r in reader:
@@ -1313,7 +1313,13 @@ def read_csv(path, Result, *,
and r[k].strip()} and r[k].strip()}
| {k: r[prefix+k] for k in fields | {k: r[prefix+k] for k in fields
if prefix+k in r if prefix+k in r
and r[prefix+k].strip()}))) and r[prefix+k].strip()}
| ({Result._notes: set(n.strip()
for n in r[Result._notes].split(',')
if n.strip())}
if hasattr(Result, '_notes')
and Result._notes in r
else {}))))
except TypeError: except TypeError:
pass pass
return results return results
@@ -1379,7 +1385,9 @@ def write_csv(path, Result, results, *,
writer = csv.DictWriter(f, list( writer = csv.DictWriter(f, list(
co.OrderedDict.fromkeys(it.chain( co.OrderedDict.fromkeys(it.chain(
by, by,
(prefix+k for k in fields))).keys())) (prefix+k for k in fields),
[Result._notes] if hasattr(Result, '_notes')
else [])).keys()))
writer.writeheader() writer.writeheader()
for r in results: for r in results:
# note this allows by/fields to overlap # note this allows by/fields to overlap
@@ -1389,7 +1397,12 @@ def write_csv(path, Result, results, *,
if getattr(r, k) is not None} if getattr(r, k) is not None}
| {prefix+k: getattr(r, k).__csv__() | {prefix+k: getattr(r, k).__csv__()
for k in fields for k in fields
if getattr(r, k) is not None}) if getattr(r, k) is not None}
| ({Result._notes: ','.join(
getattr(r, Result._notes))}
if hasattr(Result, '_notes')
and getattr(r, Result._notes)
else {}))
# write json? # write json?
else: else:
scripts/perfbd.py
+18 -5
View File
@@ -1269,10 +1269,10 @@ def read_csv(path, Result, *,
with openio(path, 'r') as f: with openio(path, 'r') as f:
# csv or json? assume json starts with [ # csv or json? assume json starts with [
is_json = (f.buffer.peek(1)[:1] == b'[') json = (f.buffer.peek(1)[:1] == b'[')
# read csv? # read csv?
if not is_json: if not json:
results = [] results = []
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
for r in reader: for r in reader:
@@ -1287,7 +1287,13 @@ def read_csv(path, Result, *,
and r[k].strip()} and r[k].strip()}
| {k: r[prefix+k] for k in fields | {k: r[prefix+k] for k in fields
if prefix+k in r if prefix+k in r
and r[prefix+k].strip()}))) and r[prefix+k].strip()}
| ({Result._notes: set(n.strip()
for n in r[Result._notes].split(',')
if n.strip())}
if hasattr(Result, '_notes')
and Result._notes in r
else {}))))
except TypeError: except TypeError:
pass pass
return results return results
@@ -1353,7 +1359,9 @@ def write_csv(path, Result, results, *,
writer = csv.DictWriter(f, list( writer = csv.DictWriter(f, list(
co.OrderedDict.fromkeys(it.chain( co.OrderedDict.fromkeys(it.chain(
by, by,
(prefix+k for k in fields))).keys())) (prefix+k for k in fields),
[Result._notes] if hasattr(Result, '_notes')
else [])).keys()))
writer.writeheader() writer.writeheader()
for r in results: for r in results:
# note this allows by/fields to overlap # note this allows by/fields to overlap
@@ -1363,7 +1371,12 @@ def write_csv(path, Result, results, *,
if getattr(r, k) is not None} if getattr(r, k) is not None}
| {prefix+k: getattr(r, k).__csv__() | {prefix+k: getattr(r, k).__csv__()
for k in fields for k in fields
if getattr(r, k) is not None}) if getattr(r, k) is not None}
| ({Result._notes: ','.join(
getattr(r, Result._notes))}
if hasattr(Result, '_notes')
and getattr(r, Result._notes)
else {}))
# write json? # write json?
else: else:
scripts/stack.py
+18 -5
View File
@@ -937,10 +937,10 @@ def read_csv(path, Result, *,
with openio(path, 'r') as f: with openio(path, 'r') as f:
# csv or json? assume json starts with [ # csv or json? assume json starts with [
is_json = (f.buffer.peek(1)[:1] == b'[') json = (f.buffer.peek(1)[:1] == b'[')
# read csv? # read csv?
if not is_json: if not json:
results = [] results = []
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
for r in reader: for r in reader:
@@ -955,7 +955,13 @@ def read_csv(path, Result, *,
and r[k].strip()} and r[k].strip()}
| {k: r[prefix+k] for k in fields | {k: r[prefix+k] for k in fields
if prefix+k in r if prefix+k in r
and r[prefix+k].strip()}))) and r[prefix+k].strip()}
| ({Result._notes: set(n.strip()
for n in r[Result._notes].split(',')
if n.strip())}
if hasattr(Result, '_notes')
and Result._notes in r
else {}))))
except TypeError: except TypeError:
pass pass
return results return results
@@ -1021,7 +1027,9 @@ def write_csv(path, Result, results, *,
writer = csv.DictWriter(f, list( writer = csv.DictWriter(f, list(
co.OrderedDict.fromkeys(it.chain( co.OrderedDict.fromkeys(it.chain(
by, by,
(prefix+k for k in fields))).keys())) (prefix+k for k in fields),
[Result._notes] if hasattr(Result, '_notes')
else [])).keys()))
writer.writeheader() writer.writeheader()
for r in results: for r in results:
# note this allows by/fields to overlap # note this allows by/fields to overlap
@@ -1031,7 +1039,12 @@ def write_csv(path, Result, results, *,
if getattr(r, k) is not None} if getattr(r, k) is not None}
| {prefix+k: getattr(r, k).__csv__() | {prefix+k: getattr(r, k).__csv__()
for k in fields for k in fields
if getattr(r, k) is not None}) if getattr(r, k) is not None}
| ({Result._notes: ','.join(
getattr(r, Result._notes))}
if hasattr(Result, '_notes')
and getattr(r, Result._notes)
else {}))
# write json? # write json?
else: else:
scripts/structs.py
+18 -5
View File
@@ -1085,10 +1085,10 @@ def read_csv(path, Result, *,
with openio(path, 'r') as f: with openio(path, 'r') as f:
# csv or json? assume json starts with [ # csv or json? assume json starts with [
is_json = (f.buffer.peek(1)[:1] == b'[') json = (f.buffer.peek(1)[:1] == b'[')
# read csv? # read csv?
if not is_json: if not json:
results = [] results = []
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
for r in reader: for r in reader:
@@ -1103,7 +1103,13 @@ def read_csv(path, Result, *,
and r[k].strip()} and r[k].strip()}
| {k: r[prefix+k] for k in fields | {k: r[prefix+k] for k in fields
if prefix+k in r if prefix+k in r
and r[prefix+k].strip()}))) and r[prefix+k].strip()}
| ({Result._notes: set(n.strip()
for n in r[Result._notes].split(',')
if n.strip())}
if hasattr(Result, '_notes')
and Result._notes in r
else {}))))
except TypeError: except TypeError:
pass pass
return results return results
@@ -1169,7 +1175,9 @@ def write_csv(path, Result, results, *,
writer = csv.DictWriter(f, list( writer = csv.DictWriter(f, list(
co.OrderedDict.fromkeys(it.chain( co.OrderedDict.fromkeys(it.chain(
by, by,
(prefix+k for k in fields))).keys())) (prefix+k for k in fields),
[Result._notes] if hasattr(Result, '_notes')
else [])).keys()))
writer.writeheader() writer.writeheader()
for r in results: for r in results:
# note this allows by/fields to overlap # note this allows by/fields to overlap
@@ -1179,7 +1187,12 @@ def write_csv(path, Result, results, *,
if getattr(r, k) is not None} if getattr(r, k) is not None}
| {prefix+k: getattr(r, k).__csv__() | {prefix+k: getattr(r, k).__csv__()
for k in fields for k in fields
if getattr(r, k) is not None}) if getattr(r, k) is not None}
| ({Result._notes: ','.join(
getattr(r, Result._notes))}
if hasattr(Result, '_notes')
and getattr(r, Result._notes)
else {}))
# write json? # write json?
else: else: