Github/littlefs
1
Fork 0
Code Actions Activity

scripts: csv.py: Added enumerate/accumulate exprs

Browse Source 
This adds two new exprs to csv.py, useful for sequential data:

  enumerate()    A number incremented each result
  accumulate(a)  A running sum across results

To make these work required adding support for cross-row state, thus the
new state field in CsvExpr.Expr.eval.

Once you have that cross-row state, implementing enumerate/accumulate is
pretty straightforward. The only complication being that we need to hash
state by the unique Python id (`id(self)`), otherwise multiple exprs
would share state, which would be pretty weird.

Note that csv.py's pipeline is now quite complex, and stage order is
important!

  input --> define    --> expr --> folding --> sorting --> output
            filtering     eval

As a result, it's unfortunately not possible to organize enumerate/
accumulate by by fields. I poked around with the idea but decided it was
too complex (aren't I supposed be building a filesystem?). The guiding
principle behind csv.py is most problems can be solved with more process
substitution.

---

This is a bit clunky since we can't use the existing fold system, but
csv.py is already a pile of hacks, so what's one more?

The reason for the clunkiness is that the original idea behind csv.py
was to treat each folded row independently and order-agnostic. Not the
greatest idea in hindsight, cross-row operations are useful!
This commit is contained in:
Christopher Haster
2026-01-22 14:20:11 -06:00
parent 3978a32156
commit 84b2e73a30
1 changed files with 181 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/csv.py
+181 -187
View File
@@ -540,8 +540,8 @@ class CsvExpr:
def fold(self, types={}): def fold(self, types={}):
return self.a.fold(types) return self.a.fold(types)
def eval(self, fields={}): def eval(self, fields={}, state=None):
return self.a.eval(fields) return self.a.eval(fields, state)
# expr nodes # expr nodes
@@ -556,7 +556,7 @@ class CsvExpr:
def fold(self, types={}): def fold(self, types={}):
return CsvSum, CsvInt return CsvSum, CsvInt
def eval(self, fields={}): def eval(self, fields={}, state=None):
return self.a return self.a
class FloatLit(Expr): class FloatLit(Expr):
@@ -569,7 +569,7 @@ class CsvExpr:
def fold(self, types={}): def fold(self, types={}):
return CsvSum, CsvFloat return CsvSum, CsvFloat
def eval(self, fields={}): def eval(self, fields={}, state=None):
return self.a return self.a
# field expr # field expr
@@ -587,7 +587,7 @@ class CsvExpr:
raise CsvExpr.Error("unfoldable field? %s" % self.a) raise CsvExpr.Error("unfoldable field? %s" % self.a)
return CsvSum, types[self.a] return CsvSum, types[self.a]
def eval(self, fields={}): def eval(self, fields={}, state=None):
if self.a not in fields: if self.a not in fields:
raise CsvExpr.Error("unknown field? %s" % self.a) raise CsvExpr.Error("unknown field? %s" % self.a)
return fields[self.a] return fields[self.a]
@@ -613,8 +613,8 @@ class CsvExpr:
def type(self, types={}): def type(self, types={}):
return CsvInt return CsvInt
def eval(self, fields={}): def eval(self, fields={}, state=None):
return CsvInt(self.a.eval(fields)) return CsvInt(self.a.eval(fields, state))
@func('float', 'a') @func('float', 'a')
class Float(Expr): class Float(Expr):
@@ -622,8 +622,8 @@ class CsvExpr:
def type(self, types={}): def type(self, types={}):
return CsvFloat return CsvFloat
def eval(self, fields={}): def eval(self, fields={}, state=None):
return CsvFloat(self.a.eval(fields)) return CsvFloat(self.a.eval(fields, state))
@func('frac', 'a[, b]') @func('frac', 'a[, b]')
class Frac(Expr): class Frac(Expr):
@@ -631,11 +631,14 @@ class CsvExpr:
def type(self, types={}): def type(self, types={}):
return CsvFrac return CsvFrac
def eval(self, fields={}): def eval(self, fields={}, state=None):
if len(self) == 1: if len(self) == 1:
return CsvFrac(self.a.eval(fields)) return CsvFrac(
self.a.eval(fields, state))
else: else:
return CsvFrac(self.a.eval(fields), self.b.eval(fields)) return CsvFrac(
self.a.eval(fields, state),
self.b.eval(fields, state))
# fold exprs # fold exprs
@func('sum', 'a[, ...]') @func('sum', 'a[, ...]')
@@ -647,11 +650,11 @@ class CsvExpr:
else: else:
return self.a.fold(types) return self.a.fold(types)
def eval(self, fields={}): def eval(self, fields={}, state=None):
if len(self) == 1: if len(self) == 1:
return self.a.eval(fields) return self.a.eval(fields, state)
else: else:
return CsvSum()([v.eval(fields) for v in self]) return CsvSum()([v.eval(fields, state) for v in self])
@func('prod', 'a[, ...]') @func('prod', 'a[, ...]')
class Prod(Expr): class Prod(Expr):
@@ -662,11 +665,11 @@ class CsvExpr:
else: else:
return self.a.fold(types) return self.a.fold(types)
def eval(self, fields={}): def eval(self, fields={}, state=None):
if len(self) == 1: if len(self) == 1:
return self.a.eval(fields) return self.a.eval(fields, state)
else: else:
return Prod()([v.eval(fields) for v in self]) return Prod()([v.eval(fields, state) for v in self])
@func('min', 'a[, ...]') @func('min', 'a[, ...]')
class Min(Expr): class Min(Expr):
@@ -677,11 +680,11 @@ class CsvExpr:
else: else:
return self.a.fold(types) return self.a.fold(types)
def eval(self, fields={}): def eval(self, fields={}, state=None):
if len(self) == 1: if len(self) == 1:
return self.a.eval(fields) return self.a.eval(fields, state)
else: else:
return CsvMin()([v.eval(fields) for v in self]) return CsvMin()([v.eval(fields, state) for v in self])
@func('max', 'a[, ...]') @func('max', 'a[, ...]')
class Max(Expr): class Max(Expr):
@@ -692,11 +695,11 @@ class CsvExpr:
else: else:
return self.a.fold(types) return self.a.fold(types)
def eval(self, fields={}): def eval(self, fields={}, state=None):
if len(self) == 1: if len(self) == 1:
return self.a.eval(fields) return self.a.eval(fields, state)
else: else:
return CsvMax()([v.eval(fields) for v in self]) return CsvMax()([v.eval(fields, state) for v in self])
@func('avg', 'a[, ...]') @func('avg', 'a[, ...]')
class Avg(Expr): class Avg(Expr):
@@ -713,11 +716,11 @@ class CsvExpr:
else: else:
return self.a.fold(types) return self.a.fold(types)
def eval(self, fields={}): def eval(self, fields={}, state=None):
if len(self) == 1: if len(self) == 1:
return self.a.eval(fields) return self.a.eval(fields, state)
else: else:
return CsvAvg()([v.eval(fields) for v in self]) return CsvAvg()([v.eval(fields, state) for v in self])
@func('stddev', 'a[, ...]') @func('stddev', 'a[, ...]')
class Stddev(Expr): class Stddev(Expr):
@@ -734,11 +737,11 @@ class CsvExpr:
else: else:
return self.a.fold(types) return self.a.fold(types)
def eval(self, fields={}): def eval(self, fields={}, state=None):
if len(self) == 1: if len(self) == 1:
return self.a.eval(fields) return self.a.eval(fields, state)
else: else:
return CsvStddev()([v.eval(fields) for v in self]) return CsvStddev()([v.eval(fields, state) for v in self])
@func('gmean', 'a[, ...]') @func('gmean', 'a[, ...]')
class GMean(Expr): class GMean(Expr):
@@ -755,11 +758,11 @@ class CsvExpr:
else: else:
return self.a.fold(types) return self.a.fold(types)
def eval(self, fields={}): def eval(self, fields={}, state=None):
if len(self) == 1: if len(self) == 1:
return self.a.eval(fields) return self.a.eval(fields, state)
else: else:
return CsvGMean()([v.eval(fields) for v in self]) return CsvGMean()([v.eval(fields, state) for v in self])
@func('gstddev', 'a[, ...]') @func('gstddev', 'a[, ...]')
class GStddev(Expr): class GStddev(Expr):
@@ -776,11 +779,54 @@ class CsvExpr:
else: else:
return self.a.fold(types) return self.a.fold(types)
def eval(self, fields={}): def eval(self, fields={}, state=None):
if len(self) == 1: if len(self) == 1:
return self.a.eval(fields) return self.a.eval(fields, state)
else: else:
return CsvGStddev()([v.eval(fields) for v in self]) return CsvGStddev()([v.eval(fields, state) for v in self])
# enumerate exprs
@func('enumerate', '')
class Enumerate(Expr):
"""A number incremented each result"""
def fields(self):
return set()
def type(self, types={}):
return CsvInt
def fold(self, types={}):
return CsvSum, CsvInt
def eval(self, fields={}, state=None):
if state is None:
return CsvInt(0)
# enumerate
v = state.get(('enumerate', id(self)))
if v is None:
v = 0
else:
v += 1
# keep track of unique enumerate state
state[('enumerate', id(self))] = v
return CsvInt(v)
@func('accumulate', 'a')
class Accumulate(Expr):
"""A running sum across results"""
def eval(self, fields={}, state=None):
v = self.a.eval(fields, state)
if state is None:
return v
# accumulate
v_ = state.get(('accumulate', id(self)))
if v_ is None:
v_ = v
else:
v_ += v
# keep track of unique accumulate state
state[('accumulate', id(self))] = v_
return v_
# functions # functions
@func('ratio', 'a') @func('ratio', 'a')
@@ -789,8 +835,8 @@ class CsvExpr:
def type(self, types={}): def type(self, types={}):
return CsvFloat return CsvFloat
def eval(self, fields={}): def eval(self, fields={}, state=None):
v = CsvFrac(self.a.eval(fields)) v = CsvFrac(self.a.eval(fields, state))
if not float(v.b) and not float(v.a): if not float(v.b) and not float(v.a):
return CsvFloat(1) return CsvFloat(1)
elif not float(v.b): elif not float(v.b):
@@ -804,14 +850,14 @@ class CsvExpr:
def type(self, types={}): def type(self, types={}):
return CsvInt return CsvInt
def eval(self, fields={}): def eval(self, fields={}, state=None):
return CsvFrac(self.a.eval(fields)).b return CsvFrac(self.a.eval(fields, state)).b
@func('abs', 'a') @func('abs', 'a')
class Abs(Expr): class Abs(Expr):
"""Absolute value""" """Absolute value"""
def eval(self, fields={}): def eval(self, fields={}, state=None):
return abs(self.a.eval(fields)) return abs(self.a.eval(fields, state))
@func('ceil', 'a') @func('ceil', 'a')
class Ceil(Expr): class Ceil(Expr):
@@ -819,8 +865,8 @@ class CsvExpr:
def type(self, types={}): def type(self, types={}):
return CsvFloat return CsvFloat
def eval(self, fields={}): def eval(self, fields={}, state=None):
return CsvFloat(mt.ceil(float(self.a.eval(fields)))) return CsvFloat(mt.ceil(float(self.a.eval(fields, state))))
@func('floor', 'a') @func('floor', 'a')
class Floor(Expr): class Floor(Expr):
@@ -828,8 +874,8 @@ class CsvExpr:
def type(self, types={}): def type(self, types={}):
return CsvFloat return CsvFloat
def eval(self, fields={}): def eval(self, fields={}, state=None):
return CsvFloat(mt.floor(float(self.a.eval(fields)))) return CsvFloat(mt.floor(float(self.a.eval(fields, state))))
@func('log', 'a[, b]') @func('log', 'a[, b]')
class Log(Expr): class Log(Expr):
@@ -837,14 +883,14 @@ class CsvExpr:
def type(self, types={}): def type(self, types={}):
return CsvFloat return CsvFloat
def eval(self, fields={}): def eval(self, fields={}, state=None):
if len(self) == 1: if len(self) == 1:
return CsvFloat(mt.log( return CsvFloat(mt.log(
float(self.a.eval(fields)))) float(self.a.eval(fields, state))))
else: else:
return CsvFloat(mt.log( return CsvFloat(mt.log(
float(self.a.eval(fields)), float(self.a.eval(fields, state)),
float(self.b.eval(fields)))) float(self.b.eval(fields, state))))
@func('pow', 'a[, b]') @func('pow', 'a[, b]')
class Pow(Expr): class Pow(Expr):
@@ -852,14 +898,14 @@ class CsvExpr:
def type(self, types={}): def type(self, types={}):
return CsvFloat return CsvFloat
def eval(self, fields={}): def eval(self, fields={}, state=None):
if len(self) == 1: if len(self) == 1:
return CsvFloat(mt.exp( return CsvFloat(mt.exp(
float(self.a.eval(fields)))) float(self.a.eval(fields, state))))
else: else:
return CsvFloat(mt.pow( return CsvFloat(mt.pow(
float(self.a.eval(fields)), float(self.a.eval(fields, state)),
float(self.b.eval(fields)))) float(self.b.eval(fields, state))))
@func('sqrt', 'a') @func('sqrt', 'a')
class Sqrt(Expr): class Sqrt(Expr):
@@ -867,8 +913,8 @@ class CsvExpr:
def type(self, types={}): def type(self, types={}):
return CsvFloat return CsvFloat
def eval(self, fields={}): def eval(self, fields={}, state=None):
return CsvFloat(mt.sqrt(float(self.a.eval(fields)))) return CsvFloat(mt.sqrt(float(self.a.eval(fields, state))))
@func('isint', 'a') @func('isint', 'a')
class IsInt(Expr): class IsInt(Expr):
@@ -876,8 +922,8 @@ class CsvExpr:
def type(self, types={}): def type(self, types={}):
return CsvInt return CsvInt
def eval(self, fields={}): def eval(self, fields={}, state=None):
if isinstance(self.a.eval(fields), CsvInt): if isinstance(self.a.eval(fields, state), CsvInt):
return CsvInt(1) return CsvInt(1)
else: else:
return CsvInt(0) return CsvInt(0)
@@ -888,8 +934,8 @@ class CsvExpr:
def type(self, types={}): def type(self, types={}):
return CsvInt return CsvInt
def eval(self, fields={}): def eval(self, fields={}, state=None):
if isinstance(self.a.eval(fields), CsvFloat): if isinstance(self.a.eval(fields, state), CsvFloat):
return CsvInt(1) return CsvInt(1)
else: else:
return CsvInt(0) return CsvInt(0)
@@ -900,8 +946,8 @@ class CsvExpr:
def type(self, types={}): def type(self, types={}):
return CsvInt return CsvInt
def eval(self, fields={}): def eval(self, fields={}, state=None):
if isinstance(self.a.eval(fields), CsvFrac): if isinstance(self.a.eval(fields, state), CsvFrac):
return CsvInt(1) return CsvInt(1)
else: else:
return CsvInt(0) return CsvInt(0)
@@ -912,8 +958,8 @@ class CsvExpr:
def type(self, types={}): def type(self, types={}):
return CsvInt return CsvInt
def eval(self, fields={}): def eval(self, fields={}, state=None):
if mt.isinf(self.a.eval(fields)): if mt.isinf(self.a.eval(fields, state)):
return CsvInt(1) return CsvInt(1)
else: else:
return CsvInt(0) return CsvInt(0)
@@ -924,8 +970,8 @@ class CsvExpr:
def type(self, types={}): def type(self, types={}):
return CsvInt return CsvInt
def eval(self, fields={}): def eval(self, fields={}, state=None):
if mt.isnan(self.a.eval(fields)): if mt.isnan(self.a.eval(fields, state)):
return CsvInt(1) return CsvInt(1)
else: else:
return CsvInt(0) return CsvInt(0)
@@ -947,14 +993,14 @@ class CsvExpr:
@uop('+') @uop('+')
class Pos(Expr): class Pos(Expr):
"""Non-negation""" """Non-negation"""
def eval(self, fields={}): def eval(self, fields={}, state=None):
return +self.a.eval(fields) return +self.a.eval(fields, state)
@uop('-') @uop('-')
class Neg(Expr): class Neg(Expr):
"""Negation""" """Negation"""
def eval(self, fields={}): def eval(self, fields={}, state=None):
return -self.a.eval(fields) return -self.a.eval(fields, state)
@uop('!') @uop('!')
class NotNot(Expr): class NotNot(Expr):
@@ -962,8 +1008,8 @@ class CsvExpr:
def type(self, types={}): def type(self, types={}):
return CsvInt return CsvInt
def eval(self, fields={}): def eval(self, fields={}, state=None):
if self.a.eval(fields): if self.a.eval(fields, state):
return CsvInt(0) return CsvInt(0)
else: else:
return CsvInt(1) return CsvInt(1)
@@ -988,40 +1034,38 @@ class CsvExpr:
@bop('*', 10) @bop('*', 10)
class Mul(Expr): class Mul(Expr):
"""Multiplication""" """Multiplication"""
def eval(self, fields={}): def eval(self, fields={}, state=None):
return self.a.eval(fields) * self.b.eval(fields) return self.a.eval(fields, state) * self.b.eval(fields, state)
@bop('/', 10) @bop('/', 10)
class Div(Expr): class Div(Expr):
"""Division""" """Division"""
def eval(self, fields={}): def eval(self, fields={}, state=None):
return self.a.eval(fields) / self.b.eval(fields) return self.a.eval(fields, state) / self.b.eval(fields, state)
@bop('%', 10) @bop('%', 10)
class Mod(Expr): class Mod(Expr):
"""Modulo""" """Modulo"""
def eval(self, fields={}): def eval(self, fields={}, state=None):
return self.a.eval(fields) % self.b.eval(fields) return self.a.eval(fields, state) % self.b.eval(fields, state)
@bop('+', 9) @bop('+', 9)
class Add(Expr): class Add(Expr):
"""Addition""" """Addition"""
def eval(self, fields={}): def eval(self, fields={}, state=None):
a = self.a.eval(fields) return self.a.eval(fields, state) + self.b.eval(fields, state)
b = self.b.eval(fields)
return a + b
@bop('-', 9) @bop('-', 9)
class Sub(Expr): class Sub(Expr):
"""Subtraction""" """Subtraction"""
def eval(self, fields={}): def eval(self, fields={}, state=None):
return self.a.eval(fields) - self.b.eval(fields) return self.a.eval(fields, state) - self.b.eval(fields, state)
@bop('==', 4) @bop('==', 4)
class Eq(Expr): class Eq(Expr):
"""1 if a equals b, otherwise 0""" """1 if a equals b, otherwise 0"""
def eval(self, fields={}): def eval(self, fields={}, state=None):
if self.a.eval(fields) == self.b.eval(fields): if self.a.eval(fields, state) == self.b.eval(fields, state):
return CsvInt(1) return CsvInt(1)
else: else:
return CsvInt(0) return CsvInt(0)
@@ -1029,8 +1073,8 @@ class CsvExpr:
@bop('!=', 4) @bop('!=', 4)
class Ne(Expr): class Ne(Expr):
"""1 if a does not equal b, otherwise 0""" """1 if a does not equal b, otherwise 0"""
def eval(self, fields={}): def eval(self, fields={}, state=None):
if self.a.eval(fields) != self.b.eval(fields): if self.a.eval(fields, state) != self.b.eval(fields, state):
return CsvInt(1) return CsvInt(1)
else: else:
return CsvInt(0) return CsvInt(0)
@@ -1038,8 +1082,8 @@ class CsvExpr:
@bop('<', 4) @bop('<', 4)
class Lt(Expr): class Lt(Expr):
"""1 if a is less than b""" """1 if a is less than b"""
def eval(self, fields={}): def eval(self, fields={}, state=None):
if self.a.eval(fields) < self.b.eval(fields): if self.a.eval(fields, state) < self.b.eval(fields, state):
return CsvInt(1) return CsvInt(1)
else: else:
return CsvInt(0) return CsvInt(0)
@@ -1047,8 +1091,8 @@ class CsvExpr:
@bop('<=', 4) @bop('<=', 4)
class Le(Expr): class Le(Expr):
"""1 if a is less than or equal to b""" """1 if a is less than or equal to b"""
def eval(self, fields={}): def eval(self, fields={}, state=None):
if self.a.eval(fields) <= self.b.eval(fields): if self.a.eval(fields, state) <= self.b.eval(fields, state):
return CsvInt(1) return CsvInt(1)
else: else:
return CsvInt(0) return CsvInt(0)
@@ -1056,8 +1100,8 @@ class CsvExpr:
@bop('>', 4) @bop('>', 4)
class Gt(Expr): class Gt(Expr):
"""1 if a is greater than b""" """1 if a is greater than b"""
def eval(self, fields={}): def eval(self, fields={}, state=None):
if self.a.eval(fields) > self.b.eval(fields): if self.a.eval(fields, state) > self.b.eval(fields, state):
return CsvInt(1) return CsvInt(1)
else: else:
return CsvInt(0) return CsvInt(0)
@@ -1065,8 +1109,8 @@ class CsvExpr:
@bop('>=', 4) @bop('>=', 4)
class Ge(Expr): class Ge(Expr):
"""1 if a is greater than or equal to b""" """1 if a is greater than or equal to b"""
def eval(self, fields={}): def eval(self, fields={}, state=None):
if self.a.eval(fields) >= self.b.eval(fields): if self.a.eval(fields, state) >= self.b.eval(fields, state):
return CsvInt(1) return CsvInt(1)
else: else:
return CsvInt(0) return CsvInt(0)
@@ -1074,22 +1118,22 @@ class CsvExpr:
@bop('&&', 3) @bop('&&', 3)
class AndAnd(Expr): class AndAnd(Expr):
"""b if a is non-zero, otherwise a""" """b if a is non-zero, otherwise a"""
def eval(self, fields={}): def eval(self, fields={}, state=None):
a = self.a.eval(fields) a = self.a.eval(fields, state)
if a: if a:
return self.b.eval(fields) return self.b.eval(fields, state)
else: else:
return a return a
@bop('||', 2) @bop('||', 2)
class OrOr(Expr): class OrOr(Expr):
"""a if a is non-zero, otherwise b""" """a if a is non-zero, otherwise b"""
def eval(self, fields={}): def eval(self, fields={}, state=None):
a = self.a.eval(fields) a = self.a.eval(fields, state)
if a: if a:
return a return a
else: else:
return self.b.eval(fields) return self.b.eval(fields, state)
# ternary expr help # ternary expr help
def top(tops, tprecs): def top(tops, tprecs):
@@ -1121,12 +1165,12 @@ class CsvExpr:
def fold(self, types={}): def fold(self, types={}):
return self.b.fold(types) return self.b.fold(types)
def eval(self, fields={}): def eval(self, fields={}, state=None):
a = self.a.eval(fields) a = self.a.eval(fields, state)
if a: if a:
return self.b.eval(fields) return self.b.eval(fields, state)
else: else:
return self.c.eval(fields) return self.c.eval(fields, state)
# show expr help text # show expr help text
@classmethod @classmethod
@@ -1178,17 +1222,17 @@ class CsvExpr:
raise CsvExpr.Error("unknown function? %s" % a) raise CsvExpr.Error("unknown function? %s" % a)
args = [] args = []
while True: while True:
b = p_expr(p) if not p.match('\)'):
args.append(b) b = p_expr(p)
if p.match(','): args.append(b)
p.chomp() if p.match(','):
continue p.chomp()
else: continue
if not p.match('\)'): if not p.match('\)'):
raise CsvExpr.Error("mismatched parens? %s" % p) raise CsvExpr.Error("mismatched parens? %s" % p)
p.chomp() p.chomp()
a = CsvExpr.funcs[a](*args) a = CsvExpr.funcs[a](*args)
break break
else: else:
a = CsvExpr.Field(a) a = CsvExpr.Field(a)
@@ -1296,9 +1340,9 @@ class CsvExpr:
sys.exit(3) sys.exit(3)
# recursive evaluate the expr # recursive evaluate the expr
def eval(self, fields={}): def eval(self, fields={}, state=None):
try: try:
return self.tree.eval(fields) return self.tree.eval(fields, state)
except CsvExpr.Error as e: except CsvExpr.Error as e:
print('error: in expr: %s' % self.expr, print('error: in expr: %s' % self.expr,
file=sys.stderr) file=sys.stderr)
@@ -1410,15 +1454,6 @@ def punescape_help():
'%(field)[fFeEgG]', 'An existing field formatted as a float')) '%(field)[fFeEgG]', 'An existing field formatted as a float'))
# a couple marker classes
class CsvEnumerate:
pass
class CsvAccumulate:
def __init__(self, expr):
self.expr = expr
# open with '-' for stdin/stdout # open with '-' for stdin/stdout
def openio(path, mode='r', buffering=-1): def openio(path, mode='r', buffering=-1):
import os import os
@@ -1577,7 +1612,7 @@ def compile(fields_, results,
folds___ = {k: (f(), t) for k, (f, t) in folds___.items()} folds___ = {k: (f(), t) for k, (f, t) in folds___.items()}
# create result class # create result class
def __new__(cls, **r): def __new__(cls, _state=None, **r):
r_ = r.copy() r_ = r.copy()
# evaluate types, strip prefix # evaluate types, strip prefix
for k, t in types__.items(): for k, t in types__.items():
@@ -1586,7 +1621,7 @@ def compile(fields_, results,
r__ = r_.copy() r__ = r_.copy()
# evaluate exprs # evaluate exprs
for k, expr in exprs.items(): for k, expr in exprs.items():
r__[k] = expr.eval(r_) r__[k] = expr.eval(r_, _state)
# evaluate mods # evaluate mods
for k, m in mods.items(): for k, m in mods.items():
r__[k] = punescape(m, r_) r__[k] = punescape(m, r_)
@@ -1656,18 +1691,12 @@ def compile(fields_, results,
def homogenize(Result, results, *, def homogenize(Result, results, *,
enumerates=None, enumerates=None,
accumulates=None,
defines=[], defines=[],
depth=1, depth=1,
**_): **_):
# running result state
state = {}
# convert all (possibly recursive) results to our result type # convert all (possibly recursive) results to our result type
# prepare accumulators
accumulators = {
k: {'i': CsvInt(0), 'a': CsvInt(0), 'b': CsvInt(0), 'i': CsvInt(0)}
for k, v in accumulates}
# homogenize results
results_ = [] results_ = []
for r in results: for r in results:
# filter by matching defines # filter by matching defines
@@ -1682,26 +1711,24 @@ def homogenize(Result, results, *,
continue continue
# append a result # append a result
results_.append(Result(**( results_.append(Result(
r **(r
# enumerate? # enumerate?
| ({e: len(results_) for e in enumerates} | ({e: len(results_) for e in enumerates}
if enumerates is not None if enumerates is not None
else {}) else {})
# accumulate?
# TODO
# recurse? # recurse?
| ({Result._children: homogenize( | ({Result._children: homogenize(
Result, r[Result._children], Result, r[Result._children],
# only filter defines at the top level! # only filter defines at the top level!
enumerates=enumerates, enumerates=enumerates,
accumulates=accumulates,
depth=depth-1)} depth=depth-1)}
if hasattr(Result, '_children') if hasattr(Result, '_children')
and Result._children in r and Result._children in r
and r[Result._children] is not None and r[Result._children] is not None
and depth > 1 and depth > 1
else {})))) else {})),
_state=state))
return results_ return results_
@@ -2366,25 +2393,20 @@ def main(csv_paths, *,
# separate out enumerates/mods/exprs # separate out enumerates/mods/exprs
# #
# enumerate enumerates: -ia # enumerate enumerates: -ia
# accumulate supports exprs: -ga=0.99*g+0.01*b
# by supports mods: -ba=%(b)s # by supports mods: -ba=%(b)s
# fields/sort/etc supports exprs: -fa=b+c # fields/sort/etc supports exprs: -fa=b+c
# #
enumerates = [k enumerates = [k
for (k, v), hidden in (by or []) for (k, v), hidden in (by or [])
if isinstance(v, CsvEnumerate)] if v == enumerate]
accumulates = [(k, v)
for (k, v), hidden in (fields or [])
if isinstance(v, CsvAccumulate)]
mods = [(k, v) mods = [(k, v)
for k, v in it.chain( for k, v in it.chain(
((k, v) for (k, v), hidden in (by or []) ((k, v) for (k, v), hidden in (by or [])
if not isinstance(v, CsvEnumerate))) if v != enumerate))
if v is not None] if v is not None]
exprs = [(k, v) exprs = [(k, v)
for k, v in it.chain( for k, v in it.chain(
((k, v) for (k, v), hidden in (fields or []) ((k, v) for (k, v), hidden in (fields or [])),
if not isinstance(v, CsvAccumulate)),
((k, v) for (k, v), reverse in (sort or [])), ((k, v) for (k, v), reverse in (sort or [])),
((k, v) for (k, v), reverse in (hot or []))) ((k, v) for (k, v), reverse in (hot or [])))
if v is not None] if v is not None]
@@ -2457,7 +2479,6 @@ def main(csv_paths, *,
# homogenize # homogenize
results = homogenize(Result, results, results = homogenize(Result, results,
enumerates=enumerates, enumerates=enumerates,
accumulates=accumulates,
defines=defines, defines=defines,
depth=depth) depth=depth)
@@ -2495,7 +2516,6 @@ def main(csv_paths, *,
# homogenize # homogenize
diff_results = homogenize(Result, diff_results, diff_results = homogenize(Result, diff_results,
enumerates=enumerates, enumerates=enumerates,
accumulates=accumulates,
defines=defines, defines=defines,
depth=depth) depth=depth)
@@ -2594,16 +2614,16 @@ if __name__ == "__main__":
'-i', '--enumerate', '-i', '--enumerate',
action=AppendBy, action=AppendBy,
nargs='?', nargs='?',
type=lambda x: (x, CsvEnumerate()), type=lambda x: (x, enumerate),
const=('i', CsvEnumerate()), const=('i', enumerate),
help="Enumerate results with this field. This will prevent " help="Enumerate results with this field, equivalent to "
"result folding.") " -bi -Fi=enumerate(). This will prevent result folding.")
parser.add_argument( parser.add_argument(
'-I', '--hidden-enumerate', '-I', '--hidden-enumerate',
action=AppendBy, action=AppendBy,
nargs='?', nargs='?',
type=lambda x: (x, CsvEnumerate()), type=lambda x: (x, enumerate),
const=('i', CsvEnumerate()), const=('i', enumerate),
help="Like -i/--enumerate, but hidden from the table renderer, " help="Like -i/--enumerate, but hidden from the table renderer, "
"and doesn't affect -b/--by defaults.") "and doesn't affect -b/--by defaults.")
parser.add_argument( parser.add_argument(
@@ -2631,8 +2651,7 @@ if __name__ == "__main__":
if namespace.fields is None: if namespace.fields is None:
namespace.fields = [] namespace.fields = []
namespace.fields.append((value, option in { namespace.fields.append((value, option in {
'-F', '--hidden-field', '-F', '--hidden-field'}))
'-G', '--hidden-accumulate'}))
parser.add_argument( parser.add_argument(
'-f', '--field', '-f', '--field',
dest='fields', dest='fields',
@@ -2655,31 +2674,6 @@ if __name__ == "__main__":
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Like -f/--field, but hidden from the table renderer, " help="Like -f/--field, but hidden from the table renderer, "
"and doesn't affect -f/--field defaults.") "and doesn't affect -f/--field defaults.")
parser.add_argument(
'-g', '--accumulate',
dest='fields',
action=AppendField,
type=lambda x: (
lambda k, v=None: (
k.strip(),
CsvAccumulate(CsvExpr(v) if v is not None else None))
)(*x.split('=', 1)),
help="Accumulate this field. Note accumulation is dependent "
"on input row order, and may need a second pass after "
"sorting. Can include an expression, but the expression "
"is evaluated early with i, a, b, and g as arguments. "
"Default behavior matches the expression (i==0)?b:g+b.")
parser.add_argument(
'-G', '--hidden-accumulate',
dest='fields',
action=AppendField,
type=lambda x: (
lambda k, v=None: (
k.strip(),
CsvAccumulate(CsvExpr(v) if v is not None else None))
)(*x.split('=', 1)),
help="Like -g/--accumulate, but hidden from the table renderer, "
"and doesn't affect -f/--field defaults.")
parser.add_argument( parser.add_argument(
'-D', '--define', '-D', '--define',
dest='defines', dest='defines',