scripts: csv.py: Simplified --prefix, moved to collect_csv
The current... attempt at an approach was broken and becoming horribly unmaintainable. Two issues found without even looking: 1. Field inference didn't understand prefixes, leading to duplicate by/field fields when attempting to infer by fields with --prefix. 2. Sort wasn't working for some reason, probably because they behavior of sort, defines, etc are really weird since they apply to both by fields and field fields. I considered just dropping support for --prefix completely, this really isn't worth the time, but instead found a simple solution of moving prefix handling to one of the first steps in collect_csv. This has the downside of creating conflicts when a prefixed/non-prefixed field has the same name, but I don't care. --prefix is a niche flag that shouldn't mess with the rest of the code like this, and none of the other scripts really handle field conflicts correctly anyways.
This commit is contained in:
+26
-13
@@ -1469,7 +1469,19 @@ def collect_csv(csv_paths, *,
|
|||||||
depth=1,
|
depth=1,
|
||||||
children=None,
|
children=None,
|
||||||
notes=None,
|
notes=None,
|
||||||
|
prefix=None,
|
||||||
**_):
|
**_):
|
||||||
|
# useful function for stripping the optional prefix
|
||||||
|
#
|
||||||
|
# what, it's not like any of the other scripts avoided prefix
|
||||||
|
# conflicts, trying to avoid conflicts until after expr eval
|
||||||
|
# quickly became unmaintainable
|
||||||
|
def stripprefix(k):
|
||||||
|
if prefix is not None and k.startswith(prefix):
|
||||||
|
return k[len(prefix):]
|
||||||
|
else:
|
||||||
|
return k
|
||||||
|
|
||||||
# collect both results and fields from CSV files
|
# collect both results and fields from CSV files
|
||||||
fields = co.OrderedDict()
|
fields = co.OrderedDict()
|
||||||
results = []
|
results = []
|
||||||
@@ -1483,8 +1495,12 @@ def collect_csv(csv_paths, *,
|
|||||||
if not is_json:
|
if not is_json:
|
||||||
reader = csv.DictReader(f, restval='')
|
reader = csv.DictReader(f, restval='')
|
||||||
# collect fields
|
# collect fields
|
||||||
fields.update((k, True) for k in reader.fieldnames or [])
|
fields.update((stripprefix(k), True)
|
||||||
|
for k in reader.fieldnames or [])
|
||||||
for r in reader:
|
for r in reader:
|
||||||
|
# strip prefix early
|
||||||
|
if prefix is not None:
|
||||||
|
r = {stripprefix(k): v for k, v in r.items()}
|
||||||
# strip and drop empty fields
|
# strip and drop empty fields
|
||||||
r_ = {k: v.strip()
|
r_ = {k: v.strip()
|
||||||
for k, v in r.items()
|
for k, v in r.items()
|
||||||
@@ -1501,6 +1517,9 @@ def collect_csv(csv_paths, *,
|
|||||||
def unjsonify(results, depth_):
|
def unjsonify(results, depth_):
|
||||||
results_ = []
|
results_ = []
|
||||||
for r in results:
|
for r in results:
|
||||||
|
# strip prefix early
|
||||||
|
if prefix is not None:
|
||||||
|
r = {stripprefix(k): v for k, v in r.items()}
|
||||||
# collect fields
|
# collect fields
|
||||||
fields.update((k, True) for k in r.keys())
|
fields.update((k, True) for k in r.keys())
|
||||||
# convert to strings, we'll reparse these later
|
# convert to strings, we'll reparse these later
|
||||||
@@ -1544,12 +1563,7 @@ def compile(fields_, results,
|
|||||||
children=None,
|
children=None,
|
||||||
hot=None,
|
hot=None,
|
||||||
notes=None,
|
notes=None,
|
||||||
prefix=None,
|
|
||||||
**_):
|
**_):
|
||||||
# default to no prefix
|
|
||||||
if prefix is None:
|
|
||||||
prefix = ''
|
|
||||||
|
|
||||||
by = by.copy()
|
by = by.copy()
|
||||||
fields = fields.copy()
|
fields = fields.copy()
|
||||||
|
|
||||||
@@ -1580,16 +1594,16 @@ def compile(fields_, results,
|
|||||||
#
|
#
|
||||||
# it's tempting to also allow enumerate fields here, but this
|
# it's tempting to also allow enumerate fields here, but this
|
||||||
# currently doesn't work when hotifying
|
# currently doesn't work when hotifying
|
||||||
if prefix+k not in fields_:
|
if k not in fields_:
|
||||||
print("error: no field %r?" % k,
|
print("error: no field %r?" % k,
|
||||||
file=sys.stderr)
|
file=sys.stderr)
|
||||||
sys.exit(2)
|
sys.exit(2)
|
||||||
|
|
||||||
for t in [CsvInt, CsvFloat, CsvFrac]:
|
for t in [CsvInt, CsvFloat, CsvFrac]:
|
||||||
for r in results:
|
for r in results:
|
||||||
if prefix+k in r and r[prefix+k].strip():
|
if k in r and r[k].strip():
|
||||||
try:
|
try:
|
||||||
t(r[prefix+k])
|
t(r[k])
|
||||||
except ValueError:
|
except ValueError:
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
@@ -1615,9 +1629,9 @@ def compile(fields_, results,
|
|||||||
# create result class
|
# create result class
|
||||||
def __new__(cls, _state=None, **r):
|
def __new__(cls, _state=None, **r):
|
||||||
r_ = r.copy()
|
r_ = r.copy()
|
||||||
# evaluate types, strip prefix
|
# evaluate types
|
||||||
for k, t in types__.items():
|
for k, t in types__.items():
|
||||||
r_[k] = t(r[prefix+k]) if prefix+k in r else t()
|
r_[k] = t(r[k]) if k in r else t()
|
||||||
|
|
||||||
r__ = r_.copy()
|
r__ = r_.copy()
|
||||||
# evaluate exprs
|
# evaluate exprs
|
||||||
@@ -2476,8 +2490,7 @@ def main(csv_paths, *,
|
|||||||
sort=sort,
|
sort=sort,
|
||||||
children=children,
|
children=children,
|
||||||
hot=hot,
|
hot=hot,
|
||||||
notes=notes,
|
notes=notes)
|
||||||
**args)
|
|
||||||
|
|
||||||
# homogenize
|
# homogenize
|
||||||
results = homogenize(Result, results,
|
results = homogenize(Result, results,
|
||||||
|
|||||||
Reference in New Issue
Block a user