Github/littlefs
1
Fork 0
Code Actions Activity

scripts: Adopted double-indent on multiline expressions

Browse Source 
This matches the style used in C, which is good for consistency:

  a_really_long_function_name(
          double_indent_after_first_newline(
              single_indent_nested_newlines))

We were already doing this for multiline control-flow statements, simply
because I'm not sure how else you could indent this without making
things really confusing:

  if a_really_long_function_name(
          double_indent_after_first_newline(
              single_indent_nested_newlines)):
      do_the_thing()

This was the only real difference style-wise between the Python code and
C code, so now both should be following roughly the same style (80 cols,
double-indent multiline exprs, prefix multiline binary ops, etc).
This commit is contained in:
Christopher Haster
2024-11-06 15:31:17 -06:00
parent 48c2e7784b
commit 007ac97bec
34 changed files with 6290 additions and 6109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/amor.py
+81 -81
View File
@@ -53,8 +53,8 @@ def collect(csv_paths, renames=[], defines=[]):
with openio(path) as f: with openio(path) as f:
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
fields.extend( fields.extend(
k for k in reader.fieldnames k for k in reader.fieldnames
if k not in fields) if k not in fields)
for r in reader: for r in reader:
# apply any renames # apply any renames
if renames: if renames:
@@ -90,8 +90,8 @@ def main(csv_paths, output, *,
# separate out renames # separate out renames
renames = list(it.chain.from_iterable( renames = list(it.chain.from_iterable(
((k, v) for v in vs) ((k, v) for v in vs)
for k, vs in it.chain(by or [], fields or []))) for k, vs in it.chain(by or [], fields or [])))
if by is not None: if by is not None:
by = [k for k, _ in by] by = [k for k, _ in by]
if fields is not None: if fields is not None:
@@ -99,7 +99,7 @@ def main(csv_paths, output, *,
if by is None and fields is None: if by is None and fields is None:
print("error: needs --by or --fields to figure out fields", print("error: needs --by or --fields to figure out fields",
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
# collect results from csv files # collect results from csv files
@@ -108,24 +108,22 @@ def main(csv_paths, output, *,
# if by not specified, guess it's anything not in # if by not specified, guess it's anything not in
# iter/size/fields/renames/defines # iter/size/fields/renames/defines
if by is None: if by is None:
by = [ by = [k for k in fields_
k for k in fields_ if k != iter
if k != iter and k != size
and k != size and k not in (fields or [])
and k not in (fields or []) and not any(k == old_k for _, old_k in renames)
and not any(k == old_k for _, old_k in renames) and not any(k == k_ for k_, _ in defines)]
and not any(k == k_ for k_, _ in defines)]
# if fields not specified, guess it's anything not in # if fields not specified, guess it's anything not in
# by/iter/size/renames/defines # by/iter/size/renames/defines
if fields is None: if fields is None:
fields = [ fields = [k for k in fields_
k for k in fields_ if k not in (by or [])
if k not in (by or []) and k != iter
and k != iter and k != size
and k != size and not any(k == old_k for _, old_k in renames)
and not any(k == old_k for _, old_k in renames) and not any(k == k_ for k_, _ in defines)]
and not any(k == k_ for k_, _ in defines)]
# add meas to by if it isn't already present # add meas to by if it isn't already present
if meas is not None and meas not in by: if meas is not None and meas not in by:
@@ -161,23 +159,23 @@ def main(csv_paths, output, *,
# find amortized results # find amortized results
if amor: if amor:
amors.append(r amors.append(r
| {f: sums[f] / size_ for f in fields} | {f: sums[f] / size_ for f in fields}
| ({} if meas is None | ({} if meas is None
else {meas: r[meas]+'+amor'} if meas in r else {meas: r[meas]+'+amor'} if meas in r
else {meas: 'amor'})) else {meas: 'amor'}))
# also find per-byte results # also find per-byte results
if per: if per:
amors.append(r amors.append(r
| {f: r.get(f, 0) / size_ for f in fields} | {f: r.get(f, 0) / size_ for f in fields}
| ({} if meas is None | ({} if meas is None
else {meas: r[meas]+'+per'} if meas in r else {meas: r[meas]+'+per'} if meas in r
else {meas: 'per'})) else {meas: 'per'}))
# write results to CSV # write results to CSV
with openio(output, 'w') as f: with openio(output, 'w') as f:
writer = csv.DictWriter(f, writer = csv.DictWriter(f,
by + [iter] + ([size] if size is not None else []) + fields) by + [iter] + ([size] if size is not None else []) + fields)
writer.writeheader() writer.writeheader()
for r in amors: for r in amors:
writer.writerow(r) writer.writerow(r)
@@ -187,68 +185,70 @@ if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Amortize benchmark measurements.", description="Amortize benchmark measurements.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'csv_paths', 'csv_paths',
nargs='*', nargs='*',
help="Input *.csv files.") help="Input *.csv files.")
parser.add_argument( parser.add_argument(
'-o', '--output', '-o', '--output',
required=True, required=True,
help="*.csv file to write amortized measurements to.") help="*.csv file to write amortized measurements to.")
parser.add_argument( parser.add_argument(
'--amor', '--amor',
action='store_true', action='store_true',
help="Compute amortized results.") help="Compute amortized results.")
parser.add_argument( parser.add_argument(
'--per', '--per',
action='store_true', action='store_true',
help="Compute per-byte results.") help="Compute per-byte results.")
parser.add_argument( parser.add_argument(
'-b', '--by', '-b', '--by',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs=None: ( lambda k, vs=None: (
k.strip(), k.strip(),
tuple(v.strip() for v in vs.split(',')) tuple(v.strip() for v in vs.split(','))
if vs is not None else ()) if vs is not None else ())
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Group by this field. Can rename fields with new_name=old_name.") help="Group by this field. Can rename fields with "
"new_name=old_name.")
parser.add_argument( parser.add_argument(
'-m', '--meas', '-m', '--meas',
help="Optional name of measurement name field. If provided, the name " help="Optional name of measurement name field. If provided, the "
"will be modified with +amor or +per.") "name will be modified with +amor or +per.")
parser.add_argument( parser.add_argument(
'-i', '--iter', '-i', '--iter',
required=True, required=True,
help="Name of iteration field.") help="Name of iteration field.")
parser.add_argument( parser.add_argument(
'-n', '--size', '-n', '--size',
help="Optional name of size field.") help="Optional name of size field.")
parser.add_argument( parser.add_argument(
'-f', '--field', '-f', '--field',
dest='fields', dest='fields',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs=None: ( lambda k, vs=None: (
k.strip(), k.strip(),
tuple(v.strip() for v in vs.split(',')) tuple(v.strip() for v in vs.split(','))
if vs is not None else ()) if vs is not None else ())
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Field to amortize. Can rename fields with new_name=old_name.") help="Field to amortize. Can rename fields with "
"new_name=old_name.")
parser.add_argument( parser.add_argument(
'-D', '--define', '-D', '--define',
dest='defines', dest='defines',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs: ( lambda k, vs: (
k.strip(), k.strip(),
{v.strip() for v in vs.split(',')}) {v.strip() for v in vs.split(',')})
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Only include results where this field is this value. May include " help="Only include results where this field is this value. May "
"comma-separated options.") "include comma-separated options.")
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/avg.py
+110 -111
View File
@@ -53,8 +53,8 @@ def collect(csv_paths, renames=[], defines=[]):
with openio(path) as f: with openio(path) as f:
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
fields.extend( fields.extend(
k for k in reader.fieldnames k for k in reader.fieldnames
if k not in fields) if k not in fields)
for r in reader: for r in reader:
# apply any renames # apply any renames
if renames: if renames:
@@ -108,8 +108,8 @@ def main(csv_paths, output, *,
# separate out renames # separate out renames
renames = list(it.chain.from_iterable( renames = list(it.chain.from_iterable(
((k, v) for v in vs) ((k, v) for v in vs)
for k, vs in it.chain(by or [], seeds or [], fields or []))) for k, vs in it.chain(by or [], seeds or [], fields or [])))
if by is not None: if by is not None:
by = [k for k, _ in by] by = [k for k, _ in by]
if seeds is not None: if seeds is not None:
@@ -119,7 +119,7 @@ def main(csv_paths, output, *,
if by is None and fields is None: if by is None and fields is None:
print("error: needs --by or --fields to figure out fields", print("error: needs --by or --fields to figure out fields",
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
# collect results from csv files # collect results from csv files
@@ -128,22 +128,20 @@ def main(csv_paths, output, *,
# if by not specified, guess it's anything not in # if by not specified, guess it's anything not in
# seeds/fields/renames/defines # seeds/fields/renames/defines
if by is None: if by is None:
by = [ by = [k for k in fields_
k for k in fields_ if k not in (seeds or [])
if k not in (seeds or []) and k not in (fields or [])
and k not in (fields or []) and not any(k == old_k for _, old_k in renames)
and not any(k == old_k for _, old_k in renames) and not any(k == k_ for k_, _ in defines)]
and not any(k == k_ for k_, _ in defines)]
# if fields not specified, guess it's anything not in # if fields not specified, guess it's anything not in
# by/seeds/renames/defines # by/seeds/renames/defines
if fields is None: if fields is None:
fields = [ fields = [k for k in fields_
k for k in fields_ if k not in (by or [])
if k not in (by or []) and k not in (seeds or [])
and k not in (seeds or []) and not any(k == old_k for _, old_k in renames)
and not any(k == old_k for _, old_k in renames) and not any(k == k_ for k_, _ in defines)]
and not any(k == k_ for k_, _ in defines)]
# add meas to by if it isn't already present # add meas to by if it isn't already present
if meas is not None and meas not in by: if meas is not None and meas not in by:
@@ -174,12 +172,11 @@ def main(csv_paths, output, *,
meas__ = r[meas] meas__ = r[meas]
def append(meas_, f_): def append(meas_, f_):
avgs.append( avgs.append({k: v for k, v in zip(by, key)}
{k: v for k, v in zip(by, key)} | {f: f_(vs_) for f, vs_ in vs.items()}
| {f: f_(vs_) for f, vs_ in vs.items()} | ({} if meas is None
| ({} if meas is None else {meas: meas_} if meas__ is None
else {meas: meas_} if meas__ is None else {meas: meas__+'+'+meas_}))
else {meas: meas__+'+'+meas_}))
if sum_: append('sum', lambda vs: sum(vs)) if sum_: append('sum', lambda vs: sum(vs))
if prod: append('prod', lambda vs: mt.prod(vs)) if prod: append('prod', lambda vs: mt.prod(vs))
@@ -189,16 +186,16 @@ def main(csv_paths, output, *,
if bnd: append('bnd', lambda vs: max(vs, default=0)) if bnd: append('bnd', lambda vs: max(vs, default=0))
if avg: append('avg', lambda vs: sum(vs) / max(len(vs), 1)) if avg: append('avg', lambda vs: sum(vs) / max(len(vs), 1))
if stddev: append('stddev', lambda vs: ( if stddev: append('stddev', lambda vs: (
lambda avg: mt.sqrt( lambda avg: mt.sqrt(
sum((v - avg)**2 for v in vs) / max(len(vs), 1)) sum((v - avg)**2 for v in vs) / max(len(vs), 1))
)(sum(vs) / max(len(vs), 1))) )(sum(vs) / max(len(vs), 1)))
if gmean: append('gmean', lambda vs: if gmean: append('gmean', lambda vs:
mt.prod(float(v) for v in vs)**(1 / max(len(vs), 1))) mt.prod(float(v) for v in vs)**(1 / max(len(vs), 1)))
if gstddev: append('gstddev', lambda vs: ( if gstddev: append('gstddev', lambda vs: (
lambda gmean: mt.exp(mt.sqrt( lambda gmean: mt.exp(mt.sqrt(
sum(mt.log(v/gmean)**2 for v in vs) / max(len(vs), 1))) sum(mt.log(v/gmean)**2 for v in vs) / max(len(vs), 1)))
if gmean else mt.inf if gmean else mt.inf
)(mt.prod(float(v) for v in vs)**(1 / max(len(vs), 1)))) )(mt.prod(float(v) for v in vs)**(1 / max(len(vs), 1))))
# write results to CSVS # write results to CSVS
with openio(output, 'w') as f: with openio(output, 'w') as f:
@@ -212,101 +209,103 @@ if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Compute averages/etc of benchmark measurements.", description="Compute averages/etc of benchmark measurements.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'csv_paths', 'csv_paths',
nargs='*', nargs='*',
help="Input *.csv files.") help="Input *.csv files.")
parser.add_argument( parser.add_argument(
'-o', '--output', '-o', '--output',
required=True, required=True,
help="*.csv file to write amortized measurements to.") help="*.csv file to write amortized measurements to.")
parser.add_argument( parser.add_argument(
'--sum', '--sum',
action='store_true', action='store_true',
help="Compute the sum.") help="Compute the sum.")
parser.add_argument( parser.add_argument(
'--prod', '--prod',
action='store_true', action='store_true',
help="Compute the product.") help="Compute the product.")
parser.add_argument( parser.add_argument(
'--min', '--min',
action='store_true', action='store_true',
help="Compute the min.") help="Compute the min.")
parser.add_argument( parser.add_argument(
'--max', '--max',
action='store_true', action='store_true',
help="Compute the max.") help="Compute the max.")
parser.add_argument( parser.add_argument(
'--bnd', '--bnd',
action='store_true', action='store_true',
help="Compute the bounds (min+max concatenated).") help="Compute the bounds (min+max concatenated).")
parser.add_argument( parser.add_argument(
'--avg', '--mean', '--avg', '--mean',
action='store_true', action='store_true',
help="Compute the average (the default).") help="Compute the average (the default).")
parser.add_argument( parser.add_argument(
'--stddev', '--stddev',
action='store_true', action='store_true',
help="Compute the standard deviation.") help="Compute the standard deviation.")
parser.add_argument( parser.add_argument(
'--gmean', '--gmean',
action='store_true', action='store_true',
help="Compute the geometric mean.") help="Compute the geometric mean.")
parser.add_argument( parser.add_argument(
'--gstddev', '--gstddev',
action='store_true', action='store_true',
help="Compute the geometric standard deviation.") help="Compute the geometric standard deviation.")
parser.add_argument( parser.add_argument(
'-b', '--by', '-b', '--by',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs=None: ( lambda k, vs=None: (
k.strip(), k.strip(),
tuple(v.strip() for v in vs.split(',')) tuple(v.strip() for v in vs.split(','))
if vs is not None else ()) if vs is not None else ())
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Group by this field. Can rename fields with new_name=old_name.") help="Group by this field. Can rename fields with "
"new_name=old_name.")
parser.add_argument( parser.add_argument(
'-m', '--meas', '-m', '--meas',
help="Optional name of measurement name field. If provided, the name " help="Optional name of measurement name field. If provided, the "
"will be modified with +amor or +per.") "name will be modified with +amor or +per.")
parser.add_argument( parser.add_argument(
'-s', '--seed', '-s', '--seed',
dest='seeds', dest='seeds',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs=None: ( lambda k, vs=None: (
k.strip(), k.strip(),
tuple(v.strip() for v in vs.split(',')) tuple(v.strip() for v in vs.split(','))
if vs is not None else ()) if vs is not None else ())
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Field to ignore when averaging. Can rename fields with " help="Field to ignore when averaging. Can rename fields with "
"new_name=old_name.") "new_name=old_name.")
parser.add_argument( parser.add_argument(
'-f', '--field', '-f', '--field',
dest='fields', dest='fields',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs=None: ( lambda k, vs=None: (
k.strip(), k.strip(),
tuple(v.strip() for v in vs.split(',')) tuple(v.strip() for v in vs.split(','))
if vs is not None else ()) if vs is not None else ())
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Field to amortize. Can rename fields with new_name=old_name.") help="Field to amortize. Can rename fields with "
"new_name=old_name.")
parser.add_argument( parser.add_argument(
'-D', '--define', '-D', '--define',
dest='defines', dest='defines',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs: ( lambda k, vs: (
k.strip(), k.strip(),
{v.strip() for v in vs.split(',')}) {v.strip() for v in vs.split(',')})
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Only include results where this field is this value. May include " help="Only include results where this field is this value. May "
"comma-separated options.") "include comma-separated options.")
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/bench.py
+508 -501
View File
File diff suppressed because it is too large Load Diff
scripts/changeprefix.py
+49 -47
View File
@@ -21,6 +21,7 @@ import shutil
import subprocess import subprocess
import tempfile import tempfile
GIT_PATH = ['git'] GIT_PATH = ['git']
@@ -36,17 +37,17 @@ def openio(path, mode='r', buffering=-1):
def changeprefix(from_prefix, to_prefix, line): def changeprefix(from_prefix, to_prefix, line):
line, count1 = re.subn( line, count1 = re.subn(
'\\b'+from_prefix, '\\b'+from_prefix,
to_prefix, to_prefix,
line) line)
line, count2 = re.subn( line, count2 = re.subn(
'\\b'+from_prefix.upper(), '\\b'+from_prefix.upper(),
to_prefix.upper(), to_prefix.upper(),
line) line)
line, count3 = re.subn( line, count3 = re.subn(
'\\B-D'+from_prefix.upper(), '\\B-D'+from_prefix.upper(),
'-D'+to_prefix.upper(), '-D'+to_prefix.upper(),
line) line)
return line, count1+count2+count3 return line, count1+count2+count3
def changefile(from_prefix, to_prefix, from_path, to_path, *, def changefile(from_prefix, to_prefix, from_path, to_path, *,
@@ -79,8 +80,9 @@ def changefile(from_prefix, to_prefix, from_path, to_path, *,
# Summary # Summary
print('%s: %d replacements' % ( print('%s: %d replacements' % (
'%s -> %s' % (from_path, to_path) if not to_path_temp else from_path, '%s -> %s' % (from_path, to_path) if not to_path_temp
count)) else from_path,
count))
def main(from_prefix, to_prefix, paths=[], *, def main(from_prefix, to_prefix, paths=[], *,
verbose=False, verbose=False,
@@ -111,7 +113,7 @@ def main(from_prefix, to_prefix, paths=[], *,
# rename contents # rename contents
changefile(from_prefix, to_prefix, from_path, to_path, changefile(from_prefix, to_prefix, from_path, to_path,
no_replacements=no_replacements) no_replacements=no_replacements)
# stage? # stage?
if git and not no_stage: if git and not no_stage:
@@ -130,49 +132,49 @@ if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Change prefixes in files/filenames. Useful for creating " description="Change prefixes in files/filenames. Useful for "
"different versions of a codebase that don't conflict at compile " "creating different versions of a codebase that don't "
"time.", "conflict at compile time.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'from_prefix', 'from_prefix',
help="Prefix to replace.") help="Prefix to replace.")
parser.add_argument( parser.add_argument(
'to_prefix', 'to_prefix',
help="Prefix to replace with.") help="Prefix to replace with.")
parser.add_argument( parser.add_argument(
'paths', 'paths',
nargs='*', nargs='*',
help="Files to operate on.") help="Files to operate on.")
parser.add_argument( parser.add_argument(
'-v', '--verbose', '-v', '--verbose',
action='store_true', action='store_true',
help="Output commands that run behind the scenes.") help="Output commands that run behind the scenes.")
parser.add_argument( parser.add_argument(
'-o', '--output', '-o', '--output',
help="Output file.") help="Output file.")
parser.add_argument( parser.add_argument(
'-N', '--no-replacements', '-N', '--no-replacements',
action='store_true', action='store_true',
help="Don't change prefixes in files") help="Don't change prefixes in files")
parser.add_argument( parser.add_argument(
'-R', '--no-renames', '-R', '--no-renames',
action='store_true', action='store_true',
help="Don't rename files") help="Don't rename files")
parser.add_argument( parser.add_argument(
'--git', '--git',
action='store_true', action='store_true',
help="Use git to find/update files.") help="Use git to find/update files.")
parser.add_argument( parser.add_argument(
'--no-stage', '--no-stage',
action='store_true', action='store_true',
help="Don't stage changes with git.") help="Don't stage changes with git.")
parser.add_argument( parser.add_argument(
'--git-path', '--git-path',
type=lambda x: x.split(), type=lambda x: x.split(),
default=GIT_PATH, default=GIT_PATH,
help="Path to git executable, may include flags. " help="Path to git executable, may include flags. "
"Defaults to %r." % GIT_PATH) "Defaults to %r." % GIT_PATH)
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/code.py
+207 -202
View File
@@ -115,11 +115,11 @@ class CodeResult(co.namedtuple('CodeResult', [
__slots__ = () __slots__ = ()
def __new__(cls, file='', function='', size=0): def __new__(cls, file='', function='', size=0):
return super().__new__(cls, file, function, return super().__new__(cls, file, function,
RInt(size)) RInt(size))
def __add__(self, other): def __add__(self, other):
return CodeResult(self.file, self.function, return CodeResult(self.file, self.function,
self.size + other.size) self.size + other.size)
def openio(path, mode='r', buffering=-1): def openio(path, mode='r', buffering=-1):
@@ -140,18 +140,18 @@ def collect(obj_paths, *,
everything=False, everything=False,
**args): **args):
size_pattern = re.compile( size_pattern = re.compile(
'^(?P<size>[0-9a-fA-F]+)' + '^(?P<size>[0-9a-fA-F]+)'
' (?P<type>[%s])' % re.escape(nm_types) + + ' (?P<type>[%s])' % re.escape(nm_types)
' (?P<func>.+?)$') + ' (?P<func>.+?)$')
line_pattern = re.compile( line_pattern = re.compile(
'^\s+(?P<no>[0-9]+)' '^\s+(?P<no>[0-9]+)'
'(?:\s+(?P<dir>[0-9]+))?' '(?:\s+(?P<dir>[0-9]+))?'
'\s+.*' '\s+.*'
'\s+(?P<path>[^\s]+)$') '\s+(?P<path>[^\s]+)$')
info_pattern = re.compile( info_pattern = re.compile(
'^(?:.*(?P<tag>DW_TAG_[a-z_]+).*' '^(?:.*(?P<tag>DW_TAG_[a-z_]+).*'
'|.*DW_AT_name.*:\s*(?P<name>[^:\s]+)\s*' '|.*DW_AT_name.*:\s*(?P<name>[^:\s]+)\s*'
'|.*DW_AT_decl_file.*:\s*(?P<file>[0-9]+)\s*)$') '|.*DW_AT_decl_file.*:\s*(?P<file>[0-9]+)\s*)$')
results = [] results = []
for path in obj_paths: for path in obj_paths:
@@ -165,11 +165,11 @@ def collect(obj_paths, *,
if args.get('verbose'): if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd)) print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd, proc = sp.Popen(cmd,
stdout=sp.PIPE, stdout=sp.PIPE,
stderr=None if args.get('verbose') else sp.DEVNULL, stderr=None if args.get('verbose') else sp.DEVNULL,
universal_newlines=True, universal_newlines=True,
errors='replace', errors='replace',
close_fds=False) close_fds=False)
for line in proc.stdout: for line in proc.stdout:
m = size_pattern.match(line) m = size_pattern.match(line)
if m: if m:
@@ -178,8 +178,8 @@ def collect(obj_paths, *,
if not everything and func.startswith('__'): if not everything and func.startswith('__'):
continue continue
results_.append(CodeResult( results_.append(CodeResult(
file, func, file, func,
int(m.group('size'), 16))) int(m.group('size'), 16)))
proc.wait() proc.wait()
if proc.returncode != 0: if proc.returncode != 0:
if not args.get('verbose'): if not args.get('verbose'):
@@ -196,11 +196,11 @@ def collect(obj_paths, *,
if args.get('verbose'): if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd)) print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd, proc = sp.Popen(cmd,
stdout=sp.PIPE, stdout=sp.PIPE,
stderr=None if args.get('verbose') else sp.DEVNULL, stderr=None if args.get('verbose') else sp.DEVNULL,
universal_newlines=True, universal_newlines=True,
errors='replace', errors='replace',
close_fds=False) close_fds=False)
for line in proc.stdout: for line in proc.stdout:
# note that files contain references to dirs, which we # note that files contain references to dirs, which we
# dereference as soon as we see them as each file table follows a # dereference as soon as we see them as each file table follows a
@@ -215,8 +215,8 @@ def collect(obj_paths, *,
dir = int(m.group('dir')) dir = int(m.group('dir'))
if dir in dirs: if dir in dirs:
files[int(m.group('no'))] = os.path.join( files[int(m.group('no'))] = os.path.join(
dirs[dir], dirs[dir],
m.group('path')) m.group('path'))
else: else:
files[int(m.group('no'))] = m.group('path') files[int(m.group('no'))] = m.group('path')
proc.wait() proc.wait()
@@ -241,11 +241,11 @@ def collect(obj_paths, *,
if args.get('verbose'): if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd)) print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd, proc = sp.Popen(cmd,
stdout=sp.PIPE, stdout=sp.PIPE,
stderr=None if args.get('verbose') else sp.DEVNULL, stderr=None if args.get('verbose') else sp.DEVNULL,
universal_newlines=True, universal_newlines=True,
errors='replace', errors='replace',
close_fds=False) close_fds=False)
for line in proc.stdout: for line in proc.stdout:
# state machine here to find definitions # state machine here to find definitions
m = info_pattern.match(line) m = info_pattern.match(line)
@@ -279,17 +279,16 @@ def collect(obj_paths, *,
file = defs[r.function] file = defs[r.function]
else: else:
_, file = max( _, file = max(
defs.items(), defs.items(),
key=lambda d: difflib.SequenceMatcher(None, key=lambda d: difflib.SequenceMatcher(None,
d[0], d[0],
r.function, False).ratio()) r.function, False).ratio())
else: else:
file = r.file file = r.file
# ignore filtered sources # ignore filtered sources
if sources is not None: if sources is not None:
if not any( if not any(os.path.abspath(file) == os.path.abspath(s)
os.path.abspath(file) == os.path.abspath(s)
for s in sources): for s in sources):
continue continue
else: else:
@@ -319,7 +318,7 @@ def fold(Result, results, by=None, defines=[]):
for k in it.chain(by or [], (k for k, _ in defines)): for k in it.chain(by or [], (k for k, _ in defines)):
if k not in Result._by and k not in Result._fields: if k not in Result._by and k not in Result._fields:
print("error: could not find field %r?" % k, print("error: could not find field %r?" % k,
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
# filter by matching defines # filter by matching defines
@@ -368,52 +367,55 @@ def table(Result, results, diff_results=None, *,
# organize by name # organize by name
table = { table = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in results} for r in results}
diff_table = { diff_table = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in diff_results or []} for r in diff_results or []}
names = [name names = [name
for name in table.keys() | diff_table.keys() for name in table.keys() | diff_table.keys()
if diff_results is None if diff_results is None
or all_ or all_
or any( or any(
types[k].ratio( types[k].ratio(
getattr(table.get(name), k, None), getattr(table.get(name), k, None),
getattr(diff_table.get(name), k, None)) getattr(diff_table.get(name), k, None))
for k in fields)] for k in fields)]
# sort again, now with diff info, note that python's sort is stable # sort again, now with diff info, note that python's sort is stable
names.sort() names.sort()
if diff_results is not None: if diff_results is not None:
names.sort(key=lambda n: tuple( names.sort(
types[k].ratio( key=lambda n: tuple(
getattr(table.get(n), k, None), types[k].ratio(
getattr(diff_table.get(n), k, None)) getattr(table.get(n), k, None),
for k in fields), getattr(diff_table.get(n), k, None))
reverse=True) for k in fields),
reverse=True)
if sort: if sort:
for k, reverse in reversed(sort): for k, reverse in reversed(sort):
names.sort( names.sort(
key=lambda n: tuple( key=lambda n: tuple(
(getattr(table[n], k),) (getattr(table[n], k),)
if getattr(table.get(n), k, None) is not None else () if getattr(table.get(n), k, None) is not None
for k in ([k] if k else [ else ()
k for k in Result._sort if k in fields])), for k in (
reverse=reverse ^ (not k or k in Result._fields)) [k] if k else [
k for k in Result._sort
if k in fields])),
reverse=reverse ^ (not k or k in Result._fields))
# build up our lines # build up our lines
lines = [] lines = []
# header # header
header = [ header = ['%s%s' % (
'%s%s' % ( ','.join(by),
','.join(by), ' (%d added, %d removed)' % (
' (%d added, %d removed)' % ( sum(1 for n in table if n not in diff_table),
sum(1 for n in table if n not in diff_table), sum(1 for n in diff_table if n not in table))
sum(1 for n in diff_table if n not in table)) if diff_results is not None and not percent else '')
if diff_results is not None and not percent else '')
if not summary else ''] if not summary else '']
if diff_results is None: if diff_results is None:
for k in fields: for k in fields:
@@ -436,43 +438,43 @@ def table(Result, results, diff_results=None, *,
if diff_results is None: if diff_results is None:
for k in fields: for k in fields:
entry.append( entry.append(
(getattr(r, k).table(), (getattr(r, k).table(),
getattr(getattr(r, k), 'notes', lambda: [])()) getattr(getattr(r, k), 'notes', lambda: [])())
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none) else types[k].none)
elif percent: elif percent:
for k in fields: for k in fields:
entry.append( entry.append(
(getattr(r, k).table() (getattr(r, k).table()
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none, else types[k].none,
(lambda t: ['+∞%'] if t == +mt.inf (lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)])( else ['%+.1f%%' % (100*t)])(
types[k].ratio( types[k].ratio(
getattr(r, k, None), getattr(r, k, None),
getattr(diff_r, k, None))))) getattr(diff_r, k, None)))))
else: else:
for k in fields: for k in fields:
entry.append(getattr(diff_r, k).table() entry.append(getattr(diff_r, k).table()
if getattr(diff_r, k, None) is not None if getattr(diff_r, k, None) is not None
else types[k].none) else types[k].none)
for k in fields: for k in fields:
entry.append(getattr(r, k).table() entry.append(getattr(r, k).table()
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none) else types[k].none)
for k in fields: for k in fields:
entry.append( entry.append(
(types[k].diff( (types[k].diff(
getattr(r, k, None),
getattr(diff_r, k, None)),
(lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)] if t
else [])(
types[k].ratio(
getattr(r, k, None), getattr(r, k, None),
getattr(diff_r, k, None))))) getattr(diff_r, k, None)),
(lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)] if t
else [])(
types[k].ratio(
getattr(r, k, None),
getattr(diff_r, k, None)))))
return entry return entry
# entries # entries
@@ -495,8 +497,8 @@ def table(Result, results, diff_results=None, *,
# homogenize # homogenize
lines = [ lines = [
[x if isinstance(x, tuple) else (x, []) for x in line] [x if isinstance(x, tuple) else (x, []) for x in line]
for line in lines] for line in lines]
# find the best widths, note that column 0 contains the names and is # find the best widths, note that column 0 contains the names and is
# handled a bit differently # handled a bit differently
@@ -510,11 +512,11 @@ def table(Result, results, diff_results=None, *,
# print our table # print our table
for line in lines: for line in lines:
print('%-*s %s' % ( print('%-*s %s' % (
widths[0], line[0][0], widths[0], line[0][0],
' '.join('%*s%-*s' % ( ' '.join('%*s%-*s' % (
widths[i], x[0], widths[i], x[0],
notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '')
for i, x in enumerate(line[1:], 1)))) for i, x in enumerate(line[1:], 1))))
def main(obj_paths, *, def main(obj_paths, *,
@@ -540,10 +542,10 @@ def main(obj_paths, *,
continue continue
try: try:
results.append(CodeResult( results.append(CodeResult(
**{k: r[k] for k in CodeResult._by **{k: r[k] for k in CodeResult._by
if k in r and r[k].strip()}, if k in r and r[k].strip()},
**{k: r[k] for k in CodeResult._fields **{k: r[k] for k in CodeResult._fields
if k in r and r[k].strip()})) if k in r and r[k].strip()}))
except TypeError: except TypeError:
pass pass
@@ -555,25 +557,27 @@ def main(obj_paths, *,
if sort: if sort:
for k, reverse in reversed(sort): for k, reverse in reversed(sort):
results.sort( results.sort(
key=lambda r: tuple( key=lambda r: tuple(
(getattr(r, k),) if getattr(r, k) is not None else () (getattr(r, k),) if getattr(r, k) is not None else ()
for k in ([k] if k else CodeResult._sort)), for k in ([k] if k else CodeResult._sort)),
reverse=reverse ^ (not k or k in CodeResult._fields)) reverse=reverse ^ (not k or k in CodeResult._fields))
# write results to CSV # write results to CSV
if args.get('output'): if args.get('output'):
with openio(args['output'], 'w') as f: with openio(args['output'], 'w') as f:
writer = csv.DictWriter(f, writer = csv.DictWriter(f,
(by if by is not None else CodeResult._by) (by if by is not None else CodeResult._by)
+ [k for k in ( + [k for k in (
fields if fields is not None else CodeResult._fields)]) fields if fields is not None
else CodeResult._fields)])
writer.writeheader() writer.writeheader()
for r in results: for r in results:
writer.writerow( writer.writerow(
{k: getattr(r, k) for k in ( {k: getattr(r, k) for k in (
by if by is not None else CodeResult._by)} by if by is not None else CodeResult._by)}
| {k: getattr(r, k) for k in ( | {k: getattr(r, k) for k in (
fields if fields is not None else CodeResult._fields)}) fields if fields is not None
else CodeResult._fields)})
# find previous results? # find previous results?
if args.get('diff'): if args.get('diff'):
@@ -591,10 +595,10 @@ def main(obj_paths, *,
continue continue
try: try:
diff_results.append(CodeResult( diff_results.append(CodeResult(
**{k: r[k] for k in CodeResult._by **{k: r[k] for k in CodeResult._by
if k in r and r[k].strip()}, if k in r and r[k].strip()},
**{k: r[k] for k in CodeResult._fields **{k: r[k] for k in CodeResult._fields
if k in r and r[k].strip()})) if k in r and r[k].strip()}))
except TypeError: except TypeError:
pass pass
except FileNotFoundError: except FileNotFoundError:
@@ -606,115 +610,116 @@ def main(obj_paths, *,
# print table # print table
if not args.get('quiet'): if not args.get('quiet'):
table(CodeResult, results, table(CodeResult, results,
diff_results if args.get('diff') else None, diff_results if args.get('diff') else None,
by=by if by is not None else ['function'], by=by if by is not None else ['function'],
fields=fields, fields=fields,
sort=sort, sort=sort,
**args) **args)
if __name__ == "__main__": if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Find code size at the function level.", description="Find code size at the function level.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'obj_paths', 'obj_paths',
nargs='*', nargs='*',
help="Input *.o files.") help="Input *.o files.")
parser.add_argument( parser.add_argument(
'-v', '--verbose', '-v', '--verbose',
action='store_true', action='store_true',
help="Output commands that run behind the scenes.") help="Output commands that run behind the scenes.")
parser.add_argument( parser.add_argument(
'-q', '--quiet', '-q', '--quiet',
action='store_true', action='store_true',
help="Don't show anything, useful with -o.") help="Don't show anything, useful with -o.")
parser.add_argument( parser.add_argument(
'-o', '--output', '-o', '--output',
help="Specify CSV file to store results.") help="Specify CSV file to store results.")
parser.add_argument( parser.add_argument(
'-u', '--use', '-u', '--use',
help="Don't parse anything, use this CSV file.") help="Don't parse anything, use this CSV file.")
parser.add_argument( parser.add_argument(
'-d', '--diff', '-d', '--diff',
help="Specify CSV file to diff against.") help="Specify CSV file to diff against.")
parser.add_argument( parser.add_argument(
'-a', '--all', '-a', '--all',
action='store_true', action='store_true',
help="Show all, not just the ones that changed.") help="Show all, not just the ones that changed.")
parser.add_argument( parser.add_argument(
'-p', '--percent', '-p', '--percent',
action='store_true', action='store_true',
help="Only show percentage change, not a full diff.") help="Only show percentage change, not a full diff.")
parser.add_argument( parser.add_argument(
'-b', '--by', '-b', '--by',
action='append', action='append',
choices=CodeResult._by, choices=CodeResult._by,
help="Group by this field.") help="Group by this field.")
parser.add_argument( parser.add_argument(
'-f', '--field', '-f', '--field',
dest='fields', dest='fields',
action='append', action='append',
choices=CodeResult._fields, choices=CodeResult._fields,
help="Show this field.") help="Show this field.")
parser.add_argument( parser.add_argument(
'-D', '--define', '-D', '--define',
dest='defines', dest='defines',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs: ( lambda k, vs: (
k.strip(), k.strip(),
{v.strip() for v in vs.split(',')}) {v.strip() for v in vs.split(',')})
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Only include results where this field is this value.") help="Only include results where this field is this value.")
class AppendSort(argparse.Action): class AppendSort(argparse.Action):
def __call__(self, parser, namespace, value, option): def __call__(self, parser, namespace, value, option):
if namespace.sort is None: if namespace.sort is None:
namespace.sort = [] namespace.sort = []
namespace.sort.append((value, True if option == '-S' else False)) namespace.sort.append((value, True if option == '-S' else False))
parser.add_argument( parser.add_argument(
'-s', '--sort', '-s', '--sort',
nargs='?', nargs='?',
action=AppendSort, action=AppendSort,
help="Sort by this field.") help="Sort by this field.")
parser.add_argument( parser.add_argument(
'-S', '--reverse-sort', '-S', '--reverse-sort',
nargs='?', nargs='?',
action=AppendSort, action=AppendSort,
help="Sort by this field, but backwards.") help="Sort by this field, but backwards.")
parser.add_argument( parser.add_argument(
'-Y', '--summary', '-Y', '--summary',
action='store_true', action='store_true',
help="Only show the total.") help="Only show the total.")
parser.add_argument( parser.add_argument(
'-F', '--source', '-F', '--source',
dest='sources', dest='sources',
action='append', action='append',
help="Only consider definitions in this file. Defaults to anything " help="Only consider definitions in this file. Defaults to "
"in the current directory.") "anything in the current directory.")
parser.add_argument( parser.add_argument(
'--everything', '--everything',
action='store_true', action='store_true',
help="Include builtin and libc specific symbols.") help="Include builtin and libc specific symbols.")
parser.add_argument( parser.add_argument(
'--nm-types', '--nm-types',
default=NM_TYPES, default=NM_TYPES,
help="Type of symbols to report, this uses the same single-character " help="Type of symbols to report, this uses the same "
"type-names emitted by nm. Defaults to %r." % NM_TYPES) "single-character type-names emitted by nm. Defaults to "
"%r." % NM_TYPES)
parser.add_argument( parser.add_argument(
'--nm-path', '--nm-path',
type=lambda x: x.split(), type=lambda x: x.split(),
default=NM_PATH, default=NM_PATH,
help="Path to the nm executable, may include flags. " help="Path to the nm executable, may include flags. "
"Defaults to %r." % NM_PATH) "Defaults to %r." % NM_PATH)
parser.add_argument( parser.add_argument(
'--objdump-path', '--objdump-path',
type=lambda x: x.split(), type=lambda x: x.split(),
default=OBJDUMP_PATH, default=OBJDUMP_PATH,
help="Path to the objdump executable, may include flags. " help="Path to the objdump executable, may include flags. "
"Defaults to %r." % OBJDUMP_PATH) "Defaults to %r." % OBJDUMP_PATH)
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/cov.py
+245 -241
View File
@@ -4,8 +4,8 @@
# #
# Example: # Example:
# ./scripts/cov.py \ # ./scripts/cov.py \
# lfs.t.a.gcda lfs_util.t.a.gcda \ # lfs.t.a.gcda lfs_util.t.a.gcda \
# -Flfs.c -Flfs_util.c -slines # -Flfs.c -Flfs_util.c -slines
# #
# Copyright (c) 2022, The littlefs authors. # Copyright (c) 2022, The littlefs authors.
# Copyright (c) 2020, Arm Limited. All rights reserved. # Copyright (c) 2020, Arm Limited. All rights reserved.
@@ -22,6 +22,7 @@ import re
import shlex import shlex
import subprocess as sp import subprocess as sp
# TODO use explode_asserts to avoid counting assert branches? # TODO use explode_asserts to avoid counting assert branches?
# TODO use dwarf=info to find functions for inline functions? # TODO use dwarf=info to find functions for inline functions?
@@ -128,15 +129,15 @@ class RFrac(co.namedtuple('RFrac', 'a,b')):
def notes(self): def notes(self):
t = self.a.x/self.b.x if self.b.x else 1.0 t = self.a.x/self.b.x if self.b.x else 1.0
return ['∞%' if t == +mt.inf return ['∞%' if t == +mt.inf
else '-∞%' if t == -mt.inf else '-∞%' if t == -mt.inf
else '%.1f%%' % (100*t)] else '%.1f%%' % (100*t)]
def diff(self, other): def diff(self, other):
new_a, new_b = self if self else (RInt(0), RInt(0)) new_a, new_b = self if self else (RInt(0), RInt(0))
old_a, old_b = other if other else (RInt(0), RInt(0)) old_a, old_b = other if other else (RInt(0), RInt(0))
return '%11s' % ('%s/%s' % ( return '%11s' % ('%s/%s' % (
new_a.diff(old_a).strip(), new_a.diff(old_a).strip(),
new_b.diff(old_b).strip())) new_b.diff(old_b).strip()))
def ratio(self, other): def ratio(self, other):
new_a, new_b = self if self else (RInt(0), RInt(0)) new_a, new_b = self if self else (RInt(0), RInt(0))
@@ -184,23 +185,23 @@ class CovResult(co.namedtuple('CovResult', [
_fields = ['calls', 'hits', 'funcs', 'lines', 'branches'] _fields = ['calls', 'hits', 'funcs', 'lines', 'branches']
_sort = ['funcs', 'lines', 'branches', 'hits', 'calls'] _sort = ['funcs', 'lines', 'branches', 'hits', 'calls']
_types = { _types = {
'calls': RInt, 'hits': RInt, 'calls': RInt, 'hits': RInt,
'funcs': RFrac, 'lines': RFrac, 'branches': RFrac} 'funcs': RFrac, 'lines': RFrac, 'branches': RFrac}
__slots__ = () __slots__ = ()
def __new__(cls, file='', function='', line=0, def __new__(cls, file='', function='', line=0,
calls=0, hits=0, funcs=0, lines=0, branches=0): calls=0, hits=0, funcs=0, lines=0, branches=0):
return super().__new__(cls, file, function, int(RInt(line)), return super().__new__(cls, file, function, int(RInt(line)),
RInt(calls), RInt(hits), RInt(calls), RInt(hits),
RFrac(funcs), RFrac(lines), RFrac(branches)) RFrac(funcs), RFrac(lines), RFrac(branches))
def __add__(self, other): def __add__(self, other):
return CovResult(self.file, self.function, self.line, return CovResult(self.file, self.function, self.line,
max(self.calls, other.calls), max(self.calls, other.calls),
max(self.hits, other.hits), max(self.hits, other.hits),
self.funcs + other.funcs, self.funcs + other.funcs,
self.lines + other.lines, self.lines + other.lines,
self.branches + other.branches) self.branches + other.branches)
def openio(path, mode='r', buffering=-1): def openio(path, mode='r', buffering=-1):
@@ -226,11 +227,11 @@ def collect(gcda_paths, *,
if args.get('verbose'): if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd)) print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd, proc = sp.Popen(cmd,
stdout=sp.PIPE, stdout=sp.PIPE,
stderr=None if args.get('verbose') else sp.DEVNULL, stderr=None if args.get('verbose') else sp.DEVNULL,
universal_newlines=True, universal_newlines=True,
errors='replace', errors='replace',
close_fds=False) close_fds=False)
data = json.load(proc.stdout) data = json.load(proc.stdout)
proc.wait() proc.wait()
if proc.returncode != 0: if proc.returncode != 0:
@@ -243,8 +244,7 @@ def collect(gcda_paths, *,
for file in data['files']: for file in data['files']:
# ignore filtered sources # ignore filtered sources
if sources is not None: if sources is not None:
if not any( if not any(os.path.abspath(file['file']) == os.path.abspath(s)
os.path.abspath(file['file']) == os.path.abspath(s)
for s in sources): for s in sources):
continue continue
else: else:
@@ -272,11 +272,11 @@ def collect(gcda_paths, *,
# go ahead and add functions, later folding will merge this if # go ahead and add functions, later folding will merge this if
# there are other hits on this line # there are other hits on this line
results.append(CovResult( results.append(CovResult(
file_name, func_name, func['start_line'], file_name, func_name, func['start_line'],
func['execution_count'], 0, func['execution_count'], 0,
RFrac(1 if func['execution_count'] > 0 else 0, 1), RFrac(1 if func['execution_count'] > 0 else 0, 1),
0, 0,
0)) 0))
for line in file['lines']: for line in file['lines']:
func_name = line.get('function_name', '(inlined)') func_name = line.get('function_name', '(inlined)')
@@ -288,14 +288,14 @@ def collect(gcda_paths, *,
# go ahead and add lines, later folding will merge this if # go ahead and add lines, later folding will merge this if
# there are other hits on this line # there are other hits on this line
results.append(CovResult( results.append(CovResult(
file_name, func_name, line['line_number'], file_name, func_name, line['line_number'],
0, line['count'], 0, line['count'],
0, 0,
RFrac(1 if line['count'] > 0 else 0, 1), RFrac(1 if line['count'] > 0 else 0, 1),
RFrac( RFrac(
sum(1 if branch['count'] > 0 else 0 sum(1 if branch['count'] > 0 else 0
for branch in line['branches']), for branch in line['branches']),
len(line['branches'])))) len(line['branches']))))
return results return results
@@ -307,7 +307,7 @@ def fold(Result, results, by=None, defines=[]):
for k in it.chain(by or [], (k for k, _ in defines)): for k in it.chain(by or [], (k for k, _ in defines)):
if k not in Result._by and k not in Result._fields: if k not in Result._by and k not in Result._fields:
print("error: could not find field %r?" % k, print("error: could not find field %r?" % k,
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
# filter by matching defines # filter by matching defines
@@ -356,52 +356,55 @@ def table(Result, results, diff_results=None, *,
# organize by name # organize by name
table = { table = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in results} for r in results}
diff_table = { diff_table = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in diff_results or []} for r in diff_results or []}
names = [name names = [name
for name in table.keys() | diff_table.keys() for name in table.keys() | diff_table.keys()
if diff_results is None if diff_results is None
or all_ or all_
or any( or any(
types[k].ratio( types[k].ratio(
getattr(table.get(name), k, None), getattr(table.get(name), k, None),
getattr(diff_table.get(name), k, None)) getattr(diff_table.get(name), k, None))
for k in fields)] for k in fields)]
# sort again, now with diff info, note that python's sort is stable # sort again, now with diff info, note that python's sort is stable
names.sort() names.sort()
if diff_results is not None: if diff_results is not None:
names.sort(key=lambda n: tuple( names.sort(
types[k].ratio( key=lambda n: tuple(
getattr(table.get(n), k, None), types[k].ratio(
getattr(diff_table.get(n), k, None)) getattr(table.get(n), k, None),
for k in fields), getattr(diff_table.get(n), k, None))
reverse=True) for k in fields),
reverse=True)
if sort: if sort:
for k, reverse in reversed(sort): for k, reverse in reversed(sort):
names.sort( names.sort(
key=lambda n: tuple( key=lambda n: tuple(
(getattr(table[n], k),) (getattr(table[n], k),)
if getattr(table.get(n), k, None) is not None else () if getattr(table.get(n), k, None) is not None
for k in ([k] if k else [ else ()
k for k in Result._sort if k in fields])), for k in (
reverse=reverse ^ (not k or k in Result._fields)) [k] if k else [
k for k in Result._sort
if k in fields])),
reverse=reverse ^ (not k or k in Result._fields))
# build up our lines # build up our lines
lines = [] lines = []
# header # header
header = [ header = ['%s%s' % (
'%s%s' % ( ','.join(by),
','.join(by), ' (%d added, %d removed)' % (
' (%d added, %d removed)' % ( sum(1 for n in table if n not in diff_table),
sum(1 for n in table if n not in diff_table), sum(1 for n in diff_table if n not in table))
sum(1 for n in diff_table if n not in table)) if diff_results is not None and not percent else '')
if diff_results is not None and not percent else '')
if not summary else ''] if not summary else '']
if diff_results is None: if diff_results is None:
for k in fields: for k in fields:
@@ -424,43 +427,43 @@ def table(Result, results, diff_results=None, *,
if diff_results is None: if diff_results is None:
for k in fields: for k in fields:
entry.append( entry.append(
(getattr(r, k).table(), (getattr(r, k).table(),
getattr(getattr(r, k), 'notes', lambda: [])()) getattr(getattr(r, k), 'notes', lambda: [])())
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none) else types[k].none)
elif percent: elif percent:
for k in fields: for k in fields:
entry.append( entry.append(
(getattr(r, k).table() (getattr(r, k).table()
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none, else types[k].none,
(lambda t: ['+∞%'] if t == +mt.inf (lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)])( else ['%+.1f%%' % (100*t)])(
types[k].ratio( types[k].ratio(
getattr(r, k, None), getattr(r, k, None),
getattr(diff_r, k, None))))) getattr(diff_r, k, None)))))
else: else:
for k in fields: for k in fields:
entry.append(getattr(diff_r, k).table() entry.append(getattr(diff_r, k).table()
if getattr(diff_r, k, None) is not None if getattr(diff_r, k, None) is not None
else types[k].none) else types[k].none)
for k in fields: for k in fields:
entry.append(getattr(r, k).table() entry.append(getattr(r, k).table()
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none) else types[k].none)
for k in fields: for k in fields:
entry.append( entry.append(
(types[k].diff( (types[k].diff(
getattr(r, k, None),
getattr(diff_r, k, None)),
(lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)] if t
else [])(
types[k].ratio(
getattr(r, k, None), getattr(r, k, None),
getattr(diff_r, k, None))))) getattr(diff_r, k, None)),
(lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)] if t
else [])(
types[k].ratio(
getattr(r, k, None),
getattr(diff_r, k, None)))))
return entry return entry
# entries # entries
@@ -483,8 +486,8 @@ def table(Result, results, diff_results=None, *,
# homogenize # homogenize
lines = [ lines = [
[x if isinstance(x, tuple) else (x, []) for x in line] [x if isinstance(x, tuple) else (x, []) for x in line]
for line in lines] for line in lines]
# find the best widths, note that column 0 contains the names and is # find the best widths, note that column 0 contains the names and is
# handled a bit differently # handled a bit differently
@@ -498,11 +501,11 @@ def table(Result, results, diff_results=None, *,
# print our table # print our table
for line in lines: for line in lines:
print('%-*s %s' % ( print('%-*s %s' % (
widths[0], line[0][0], widths[0], line[0][0],
' '.join('%*s%-*s' % ( ' '.join('%*s%-*s' % (
widths[i], x[0], widths[i], x[0],
notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '')
for i, x in enumerate(line[1:], 1)))) for i, x in enumerate(line[1:], 1))))
def annotate(Result, results, *, def annotate(Result, results, *,
@@ -529,14 +532,14 @@ def annotate(Result, results, *,
or (branches and r.branches.a < r.branches.b)): or (branches and r.branches.a < r.branches.b)):
if last is not None and line - last.stop <= args['context']: if last is not None and line - last.stop <= args['context']:
last = range( last = range(
last.start, last.start,
line+1+args['context']) line+1+args['context'])
else: else:
if last is not None: if last is not None:
spans.append((last, func)) spans.append((last, func))
last = range( last = range(
line-args['context'], line-args['context'],
line+1+args['context']) line+1+args['context'])
func = r.function func = r.function
if last is not None: if last is not None:
spans.append((last, func)) spans.append((last, func))
@@ -552,11 +555,11 @@ def annotate(Result, results, *,
if skipped: if skipped:
skipped = False skipped = False
print('%s@@ %s:%d: %s @@%s' % ( print('%s@@ %s:%d: %s @@%s' % (
'\x1b[36m' if args['color'] else '', '\x1b[36m' if args['color'] else '',
path, path,
i+1, i+1,
next(iter(f for _, f in spans)), next(iter(f for _, f in spans)),
'\x1b[m' if args['color'] else '')) '\x1b[m' if args['color'] else ''))
# build line # build line
if line.endswith('\n'): if line.endswith('\n'):
@@ -565,11 +568,11 @@ def annotate(Result, results, *,
if i+1 in table: if i+1 in table:
r = table[i+1] r = table[i+1]
line = '%-*s // %s hits%s' % ( line = '%-*s // %s hits%s' % (
args['width'], args['width'],
line, line,
r.hits, r.hits,
', %s branches' % (r.branches,) ', %s branches' % (r.branches,)
if int(r.branches.b) else '') if int(r.branches.b) else '')
if args['color']: if args['color']:
if lines and int(r.hits) == 0: if lines and int(r.hits) == 0:
@@ -612,11 +615,11 @@ def main(gcda_paths, *,
continue continue
try: try:
results.append(CovResult( results.append(CovResult(
**{k: r[k] for k in CovResult._by **{k: r[k] for k in CovResult._by
if k in r and r[k].strip()}, if k in r and r[k].strip()},
**{k: r[k] **{k: r[k]
for k in CovResult._fields for k in CovResult._fields
if k in r and r[k].strip()})) if k in r and r[k].strip()}))
except TypeError: except TypeError:
pass pass
@@ -628,25 +631,27 @@ def main(gcda_paths, *,
if sort: if sort:
for k, reverse in reversed(sort): for k, reverse in reversed(sort):
results.sort( results.sort(
key=lambda r: tuple( key=lambda r: tuple(
(getattr(r, k),) if getattr(r, k) is not None else () (getattr(r, k),) if getattr(r, k) is not None else ()
for k in ([k] if k else CovResult._sort)), for k in ([k] if k else CovResult._sort)),
reverse=reverse ^ (not k or k in CovResult._fields)) reverse=reverse ^ (not k or k in CovResult._fields))
# write results to CSV # write results to CSV
if args.get('output'): if args.get('output'):
with openio(args['output'], 'w') as f: with openio(args['output'], 'w') as f:
writer = csv.DictWriter(f, writer = csv.DictWriter(f,
(by if by is not None else CovResult._by) (by if by is not None else CovResult._by)
+ [k for k in ( + [k for k in (
fields if fields is not None else CovResult._fields)]) fields if fields is not None
else CovResult._fields)])
writer.writeheader() writer.writeheader()
for r in results: for r in results:
writer.writerow( writer.writerow(
{k: getattr(r, k) for k in ( {k: getattr(r, k) for k in (
by if by is not None else CovResult._by)} by if by is not None else CovResult._by)}
| {k: getattr(r, k) for k in ( | {k: getattr(r, k) for k in (
fields if fields is not None else CovResult._fields)}) fields if fields is not None
else CovResult._fields)})
# find previous results? # find previous results?
if args.get('diff'): if args.get('diff'):
@@ -664,19 +669,17 @@ def main(gcda_paths, *,
continue continue
try: try:
diff_results.append(CovResult( diff_results.append(CovResult(
**{k: r[k] for k in CovResult._by **{k: r[k] for k in CovResult._by
if k in r and r[k].strip()}, if k in r and r[k].strip()},
**{k: r[k] **{k: r[k] for k in CovResult._fields
for k in CovResult._fields if k in r and r[k].strip()}))
if k in r and r[k].strip()}))
except TypeError: except TypeError:
pass pass
except FileNotFoundError: except FileNotFoundError:
pass pass
# fold # fold
diff_results = fold(CovResult, diff_results, diff_results = fold(CovResult, diff_results, by=by, defines=defines)
by=by, defines=defines)
# print table # print table
if not args.get('quiet'): if not args.get('quiet'):
@@ -688,13 +691,13 @@ def main(gcda_paths, *,
else: else:
# print table # print table
table(CovResult, results, table(CovResult, results,
diff_results if args.get('diff') else None, diff_results if args.get('diff') else None,
by=by if by is not None else ['function'], by=by if by is not None else ['function'],
fields=fields if fields is not None fields=fields if fields is not None
else ['lines', 'branches'] if not hits else ['lines', 'branches'] if not hits
else ['calls', 'hits'], else ['calls', 'hits'],
sort=sort, sort=sort,
**args) **args)
# catch lack of coverage # catch lack of coverage
if args.get('error_on_lines') and any( if args.get('error_on_lines') and any(
@@ -709,132 +712,133 @@ if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Find coverage info after running tests.", description="Find coverage info after running tests.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'gcda_paths', 'gcda_paths',
nargs='*', nargs='*',
help="Input *.gcda files.") help="Input *.gcda files.")
parser.add_argument( parser.add_argument(
'-v', '--verbose', '-v', '--verbose',
action='store_true', action='store_true',
help="Output commands that run behind the scenes.") help="Output commands that run behind the scenes.")
parser.add_argument( parser.add_argument(
'-q', '--quiet', '-q', '--quiet',
action='store_true', action='store_true',
help="Don't show anything, useful with -o.") help="Don't show anything, useful with -o.")
parser.add_argument( parser.add_argument(
'-o', '--output', '-o', '--output',
help="Specify CSV file to store results.") help="Specify CSV file to store results.")
parser.add_argument( parser.add_argument(
'-u', '--use', '-u', '--use',
help="Don't parse anything, use this CSV file.") help="Don't parse anything, use this CSV file.")
parser.add_argument( parser.add_argument(
'-d', '--diff', '-d', '--diff',
help="Specify CSV file to diff against.") help="Specify CSV file to diff against.")
parser.add_argument( parser.add_argument(
'-a', '--all', '-a', '--all',
action='store_true', action='store_true',
help="Show all, not just the ones that changed.") help="Show all, not just the ones that changed.")
parser.add_argument( parser.add_argument(
'-p', '--percent', '-p', '--percent',
action='store_true', action='store_true',
help="Only show percentage change, not a full diff.") help="Only show percentage change, not a full diff.")
parser.add_argument( parser.add_argument(
'-b', '--by', '-b', '--by',
action='append', action='append',
choices=CovResult._by, choices=CovResult._by,
help="Group by this field.") help="Group by this field.")
parser.add_argument( parser.add_argument(
'-f', '--field', '-f', '--field',
dest='fields', dest='fields',
action='append', action='append',
choices=CovResult._fields, choices=CovResult._fields,
help="Show this field.") help="Show this field.")
parser.add_argument( parser.add_argument(
'-D', '--define', '-D', '--define',
dest='defines', dest='defines',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs: ( lambda k, vs: (
k.strip(), k.strip(),
{v.strip() for v in vs.split(',')}) {v.strip() for v in vs.split(',')})
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Only include results where this field is this value.") help="Only include results where this field is this value.")
class AppendSort(argparse.Action): class AppendSort(argparse.Action):
def __call__(self, parser, namespace, value, option): def __call__(self, parser, namespace, value, option):
if namespace.sort is None: if namespace.sort is None:
namespace.sort = [] namespace.sort = []
namespace.sort.append((value, True if option == '-S' else False)) namespace.sort.append((value, True if option == '-S' else False))
parser.add_argument( parser.add_argument(
'-s', '--sort', '-s', '--sort',
nargs='?', nargs='?',
action=AppendSort, action=AppendSort,
help="Sort by this field.") help="Sort by this field.")
parser.add_argument( parser.add_argument(
'-S', '--reverse-sort', '-S', '--reverse-sort',
nargs='?', nargs='?',
action=AppendSort, action=AppendSort,
help="Sort by this field, but backwards.") help="Sort by this field, but backwards.")
parser.add_argument( parser.add_argument(
'-Y', '--summary', '-Y', '--summary',
action='store_true', action='store_true',
help="Only show the total.") help="Only show the total.")
parser.add_argument( parser.add_argument(
'-F', '--source', '-F', '--source',
dest='sources', dest='sources',
action='append', action='append',
help="Only consider definitions in this file. Defaults to anything " help="Only consider definitions in this file. Defaults to "
"in the current directory.") "anything in the current directory.")
parser.add_argument( parser.add_argument(
'--everything', '--everything',
action='store_true', action='store_true',
help="Include builtin and libc specific symbols.") help="Include builtin and libc specific symbols.")
parser.add_argument( parser.add_argument(
'--hits', '--hits',
action='store_true', action='store_true',
help="Show total hits instead of coverage.") help="Show total hits instead of coverage.")
parser.add_argument( parser.add_argument(
'-A', '--annotate', '-A', '--annotate',
action='store_true', action='store_true',
help="Show source files annotated with coverage info.") help="Show source files annotated with coverage info.")
parser.add_argument( parser.add_argument(
'-L', '--lines', '-L', '--lines',
action='store_true', action='store_true',
help="Show uncovered lines.") help="Show uncovered lines.")
parser.add_argument( parser.add_argument(
'-B', '--branches', '-B', '--branches',
action='store_true', action='store_true',
help="Show uncovered branches.") help="Show uncovered branches.")
parser.add_argument( parser.add_argument(
'-C', '--context', '-C', '--context',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
default=3, default=3,
help="Show n additional lines of context. Defaults to 3.") help="Show n additional lines of context. Defaults to 3.")
parser.add_argument( parser.add_argument(
'-W', '--width', '-W', '--width',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
default=80, default=80,
help="Assume source is styled with this many columns. Defaults to 80.") help="Assume source is styled with this many columns. Defaults "
"to 80.")
parser.add_argument( parser.add_argument(
'--color', '--color',
choices=['never', 'always', 'auto'], choices=['never', 'always', 'auto'],
default='auto', default='auto',
help="When to use terminal colors. Defaults to 'auto'.") help="When to use terminal colors. Defaults to 'auto'.")
parser.add_argument( parser.add_argument(
'-e', '--error-on-lines', '-e', '--error-on-lines',
action='store_true', action='store_true',
help="Error if any lines are not covered.") help="Error if any lines are not covered.")
parser.add_argument( parser.add_argument(
'-E', '--error-on-branches', '-E', '--error-on-branches',
action='store_true', action='store_true',
help="Error if any branches are not covered.") help="Error if any branches are not covered.")
parser.add_argument( parser.add_argument(
'--gcov-path', '--gcov-path',
default=GCOV_PATH, default=GCOV_PATH,
type=lambda x: x.split(), type=lambda x: x.split(),
help="Path to the gcov executable, may include paths. " help="Path to the gcov executable, may include paths. "
"Defaults to %r." % GCOV_PATH) "Defaults to %r." % GCOV_PATH)
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/crc32c.py
+14 -13
View File
@@ -57,24 +57,25 @@ def main(paths, **args):
else: else:
print('%08x' % crc) print('%08x' % crc)
if __name__ == "__main__": if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Calculates crc32cs.", description="Calculates crc32cs.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'paths', 'paths',
nargs='*', nargs='*',
help="Paths to read. Reads stdin by default.") help="Paths to read. Reads stdin by default.")
parser.add_argument( parser.add_argument(
'-x', '--hex', '-x', '--hex',
action='store_true', action='store_true',
help="Interpret as a sequence of hex bytes.") help="Interpret as a sequence of hex bytes.")
parser.add_argument( parser.add_argument(
'-s', '--string', '-s', '--string',
action='store_true', action='store_true',
help="Interpret as strings.") help="Interpret as strings.")
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/data.py
+207 -202
View File
@@ -115,11 +115,11 @@ class DataResult(co.namedtuple('DataResult', [
__slots__ = () __slots__ = ()
def __new__(cls, file='', function='', size=0): def __new__(cls, file='', function='', size=0):
return super().__new__(cls, file, function, return super().__new__(cls, file, function,
RInt(size)) RInt(size))
def __add__(self, other): def __add__(self, other):
return DataResult(self.file, self.function, return DataResult(self.file, self.function,
self.size + other.size) self.size + other.size)
def openio(path, mode='r', buffering=-1): def openio(path, mode='r', buffering=-1):
@@ -140,18 +140,18 @@ def collect(obj_paths, *,
everything=False, everything=False,
**args): **args):
size_pattern = re.compile( size_pattern = re.compile(
'^(?P<size>[0-9a-fA-F]+)' + '^(?P<size>[0-9a-fA-F]+)'
' (?P<type>[%s])' % re.escape(nm_types) + + ' (?P<type>[%s])' % re.escape(nm_types)
' (?P<func>.+?)$') + ' (?P<func>.+?)$')
line_pattern = re.compile( line_pattern = re.compile(
'^\s+(?P<no>[0-9]+)' '^\s+(?P<no>[0-9]+)'
'(?:\s+(?P<dir>[0-9]+))?' '(?:\s+(?P<dir>[0-9]+))?'
'\s+.*' '\s+.*'
'\s+(?P<path>[^\s]+)$') '\s+(?P<path>[^\s]+)$')
info_pattern = re.compile( info_pattern = re.compile(
'^(?:.*(?P<tag>DW_TAG_[a-z_]+).*' '^(?:.*(?P<tag>DW_TAG_[a-z_]+).*'
'|.*DW_AT_name.*:\s*(?P<name>[^:\s]+)\s*' '|.*DW_AT_name.*:\s*(?P<name>[^:\s]+)\s*'
'|.*DW_AT_decl_file.*:\s*(?P<file>[0-9]+)\s*)$') '|.*DW_AT_decl_file.*:\s*(?P<file>[0-9]+)\s*)$')
results = [] results = []
for path in obj_paths: for path in obj_paths:
@@ -165,11 +165,11 @@ def collect(obj_paths, *,
if args.get('verbose'): if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd)) print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd, proc = sp.Popen(cmd,
stdout=sp.PIPE, stdout=sp.PIPE,
stderr=None if args.get('verbose') else sp.DEVNULL, stderr=None if args.get('verbose') else sp.DEVNULL,
universal_newlines=True, universal_newlines=True,
errors='replace', errors='replace',
close_fds=False) close_fds=False)
for line in proc.stdout: for line in proc.stdout:
m = size_pattern.match(line) m = size_pattern.match(line)
if m: if m:
@@ -178,8 +178,8 @@ def collect(obj_paths, *,
if not everything and func.startswith('__'): if not everything and func.startswith('__'):
continue continue
results_.append(DataResult( results_.append(DataResult(
file, func, file, func,
int(m.group('size'), 16))) int(m.group('size'), 16)))
proc.wait() proc.wait()
if proc.returncode != 0: if proc.returncode != 0:
if not args.get('verbose'): if not args.get('verbose'):
@@ -196,11 +196,11 @@ def collect(obj_paths, *,
if args.get('verbose'): if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd)) print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd, proc = sp.Popen(cmd,
stdout=sp.PIPE, stdout=sp.PIPE,
stderr=None if args.get('verbose') else sp.DEVNULL, stderr=None if args.get('verbose') else sp.DEVNULL,
universal_newlines=True, universal_newlines=True,
errors='replace', errors='replace',
close_fds=False) close_fds=False)
for line in proc.stdout: for line in proc.stdout:
# note that files contain references to dirs, which we # note that files contain references to dirs, which we
# dereference as soon as we see them as each file table follows a # dereference as soon as we see them as each file table follows a
@@ -215,8 +215,8 @@ def collect(obj_paths, *,
dir = int(m.group('dir')) dir = int(m.group('dir'))
if dir in dirs: if dir in dirs:
files[int(m.group('no'))] = os.path.join( files[int(m.group('no'))] = os.path.join(
dirs[dir], dirs[dir],
m.group('path')) m.group('path'))
else: else:
files[int(m.group('no'))] = m.group('path') files[int(m.group('no'))] = m.group('path')
proc.wait() proc.wait()
@@ -241,11 +241,11 @@ def collect(obj_paths, *,
if args.get('verbose'): if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd)) print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd, proc = sp.Popen(cmd,
stdout=sp.PIPE, stdout=sp.PIPE,
stderr=None if args.get('verbose') else sp.DEVNULL, stderr=None if args.get('verbose') else sp.DEVNULL,
universal_newlines=True, universal_newlines=True,
errors='replace', errors='replace',
close_fds=False) close_fds=False)
for line in proc.stdout: for line in proc.stdout:
# state machine here to find definitions # state machine here to find definitions
m = info_pattern.match(line) m = info_pattern.match(line)
@@ -279,17 +279,16 @@ def collect(obj_paths, *,
file = defs[r.function] file = defs[r.function]
else: else:
_, file = max( _, file = max(
defs.items(), defs.items(),
key=lambda d: difflib.SequenceMatcher(None, key=lambda d: difflib.SequenceMatcher(None,
d[0], d[0],
r.function, False).ratio()) r.function, False).ratio())
else: else:
file = r.file file = r.file
# ignore filtered sources # ignore filtered sources
if sources is not None: if sources is not None:
if not any( if not any(os.path.abspath(file) == os.path.abspath(s)
os.path.abspath(file) == os.path.abspath(s)
for s in sources): for s in sources):
continue continue
else: else:
@@ -319,7 +318,7 @@ def fold(Result, results, by=None, defines=[]):
for k in it.chain(by or [], (k for k, _ in defines)): for k in it.chain(by or [], (k for k, _ in defines)):
if k not in Result._by and k not in Result._fields: if k not in Result._by and k not in Result._fields:
print("error: could not find field %r?" % k, print("error: could not find field %r?" % k,
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
# filter by matching defines # filter by matching defines
@@ -368,52 +367,55 @@ def table(Result, results, diff_results=None, *,
# organize by name # organize by name
table = { table = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in results} for r in results}
diff_table = { diff_table = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in diff_results or []} for r in diff_results or []}
names = [name names = [name
for name in table.keys() | diff_table.keys() for name in table.keys() | diff_table.keys()
if diff_results is None if diff_results is None
or all_ or all_
or any( or any(
types[k].ratio( types[k].ratio(
getattr(table.get(name), k, None), getattr(table.get(name), k, None),
getattr(diff_table.get(name), k, None)) getattr(diff_table.get(name), k, None))
for k in fields)] for k in fields)]
# sort again, now with diff info, note that python's sort is stable # sort again, now with diff info, note that python's sort is stable
names.sort() names.sort()
if diff_results is not None: if diff_results is not None:
names.sort(key=lambda n: tuple( names.sort(
types[k].ratio( key=lambda n: tuple(
getattr(table.get(n), k, None), types[k].ratio(
getattr(diff_table.get(n), k, None)) getattr(table.get(n), k, None),
for k in fields), getattr(diff_table.get(n), k, None))
reverse=True) for k in fields),
reverse=True)
if sort: if sort:
for k, reverse in reversed(sort): for k, reverse in reversed(sort):
names.sort( names.sort(
key=lambda n: tuple( key=lambda n: tuple(
(getattr(table[n], k),) (getattr(table[n], k),)
if getattr(table.get(n), k, None) is not None else () if getattr(table.get(n), k, None) is not None
for k in ([k] if k else [ else ()
k for k in Result._sort if k in fields])), for k in (
reverse=reverse ^ (not k or k in Result._fields)) [k] if k else [
k for k in Result._sort
if k in fields])),
reverse=reverse ^ (not k or k in Result._fields))
# build up our lines # build up our lines
lines = [] lines = []
# header # header
header = [ header = ['%s%s' % (
'%s%s' % ( ','.join(by),
','.join(by), ' (%d added, %d removed)' % (
' (%d added, %d removed)' % ( sum(1 for n in table if n not in diff_table),
sum(1 for n in table if n not in diff_table), sum(1 for n in diff_table if n not in table))
sum(1 for n in diff_table if n not in table)) if diff_results is not None and not percent else '')
if diff_results is not None and not percent else '')
if not summary else ''] if not summary else '']
if diff_results is None: if diff_results is None:
for k in fields: for k in fields:
@@ -436,43 +438,43 @@ def table(Result, results, diff_results=None, *,
if diff_results is None: if diff_results is None:
for k in fields: for k in fields:
entry.append( entry.append(
(getattr(r, k).table(), (getattr(r, k).table(),
getattr(getattr(r, k), 'notes', lambda: [])()) getattr(getattr(r, k), 'notes', lambda: [])())
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none) else types[k].none)
elif percent: elif percent:
for k in fields: for k in fields:
entry.append( entry.append(
(getattr(r, k).table() (getattr(r, k).table()
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none, else types[k].none,
(lambda t: ['+∞%'] if t == +mt.inf (lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)])( else ['%+.1f%%' % (100*t)])(
types[k].ratio( types[k].ratio(
getattr(r, k, None), getattr(r, k, None),
getattr(diff_r, k, None))))) getattr(diff_r, k, None)))))
else: else:
for k in fields: for k in fields:
entry.append(getattr(diff_r, k).table() entry.append(getattr(diff_r, k).table()
if getattr(diff_r, k, None) is not None if getattr(diff_r, k, None) is not None
else types[k].none) else types[k].none)
for k in fields: for k in fields:
entry.append(getattr(r, k).table() entry.append(getattr(r, k).table()
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none) else types[k].none)
for k in fields: for k in fields:
entry.append( entry.append(
(types[k].diff( (types[k].diff(
getattr(r, k, None),
getattr(diff_r, k, None)),
(lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)] if t
else [])(
types[k].ratio(
getattr(r, k, None), getattr(r, k, None),
getattr(diff_r, k, None))))) getattr(diff_r, k, None)),
(lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)] if t
else [])(
types[k].ratio(
getattr(r, k, None),
getattr(diff_r, k, None)))))
return entry return entry
# entries # entries
@@ -495,8 +497,8 @@ def table(Result, results, diff_results=None, *,
# homogenize # homogenize
lines = [ lines = [
[x if isinstance(x, tuple) else (x, []) for x in line] [x if isinstance(x, tuple) else (x, []) for x in line]
for line in lines] for line in lines]
# find the best widths, note that column 0 contains the names and is # find the best widths, note that column 0 contains the names and is
# handled a bit differently # handled a bit differently
@@ -510,11 +512,11 @@ def table(Result, results, diff_results=None, *,
# print our table # print our table
for line in lines: for line in lines:
print('%-*s %s' % ( print('%-*s %s' % (
widths[0], line[0][0], widths[0], line[0][0],
' '.join('%*s%-*s' % ( ' '.join('%*s%-*s' % (
widths[i], x[0], widths[i], x[0],
notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '')
for i, x in enumerate(line[1:], 1)))) for i, x in enumerate(line[1:], 1))))
def main(obj_paths, *, def main(obj_paths, *,
@@ -537,10 +539,10 @@ def main(obj_paths, *,
try: try:
results.append(DataResult( results.append(DataResult(
**{k: r[k] for k in DataResult._by **{k: r[k] for k in DataResult._by
if k in r and r[k].strip()}, if k in r and r[k].strip()},
**{k: r[k] for k in DataResult._fields **{k: r[k] for k in DataResult._fields
if k in r and r[k].strip()})) if k in r and r[k].strip()}))
except TypeError: except TypeError:
pass pass
@@ -552,25 +554,27 @@ def main(obj_paths, *,
if sort: if sort:
for k, reverse in reversed(sort): for k, reverse in reversed(sort):
results.sort( results.sort(
key=lambda r: tuple( key=lambda r: tuple(
(getattr(r, k),) if getattr(r, k) is not None else () (getattr(r, k),) if getattr(r, k) is not None else ()
for k in ([k] if k else DataResult._sort)), for k in ([k] if k else DataResult._sort)),
reverse=reverse ^ (not k or k in DataResult._fields)) reverse=reverse ^ (not k or k in DataResult._fields))
# write results to CSV # write results to CSV
if args.get('output'): if args.get('output'):
with openio(args['output'], 'w') as f: with openio(args['output'], 'w') as f:
writer = csv.DictWriter(f, writer = csv.DictWriter(f,
(by if by is not None else DataResult._by) (by if by is not None else DataResult._by)
+ [k for k in ( + [k for k in (
fields if fields is not None else DataResult._fields)]) fields if fields is not None
else DataResult._fields)])
writer.writeheader() writer.writeheader()
for r in results: for r in results:
writer.writerow( writer.writerow(
{k: getattr(r, k) for k in ( {k: getattr(r, k) for k in (
by if by is not None else DataResult._by)} by if by is not None else DataResult._by)}
| {k: getattr(r, k) for k in ( | {k: getattr(r, k) for k in (
fields if fields is not None else DataResult._fields)}) fields if fields is not None
else DataResult._fields)})
# find previous results? # find previous results?
if args.get('diff'): if args.get('diff'):
@@ -588,10 +592,10 @@ def main(obj_paths, *,
continue continue
try: try:
diff_results.append(DataResult( diff_results.append(DataResult(
**{k: r[k] for k in DataResult._by **{k: r[k] for k in DataResult._by
if k in r and r[k].strip()}, if k in r and r[k].strip()},
**{k: r[k] for k in DataResult._fields **{k: r[k] for k in DataResult._fields
if k in r and r[k].strip()})) if k in r and r[k].strip()}))
except TypeError: except TypeError:
pass pass
except FileNotFoundError: except FileNotFoundError:
@@ -603,115 +607,116 @@ def main(obj_paths, *,
# print table # print table
if not args.get('quiet'): if not args.get('quiet'):
table(DataResult, results, table(DataResult, results,
diff_results if args.get('diff') else None, diff_results if args.get('diff') else None,
by=by if by is not None else ['function'], by=by if by is not None else ['function'],
fields=fields, fields=fields,
sort=sort, sort=sort,
**args) **args)
if __name__ == "__main__": if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Find data size at the function level.", description="Find data size at the function level.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'obj_paths', 'obj_paths',
nargs='*', nargs='*',
help="Input *.o files.") help="Input *.o files.")
parser.add_argument( parser.add_argument(
'-v', '--verbose', '-v', '--verbose',
action='store_true', action='store_true',
help="Output commands that run behind the scenes.") help="Output commands that run behind the scenes.")
parser.add_argument( parser.add_argument(
'-q', '--quiet', '-q', '--quiet',
action='store_true', action='store_true',
help="Don't show anything, useful with -o.") help="Don't show anything, useful with -o.")
parser.add_argument( parser.add_argument(
'-o', '--output', '-o', '--output',
help="Specify CSV file to store results.") help="Specify CSV file to store results.")
parser.add_argument( parser.add_argument(
'-u', '--use', '-u', '--use',
help="Don't parse anything, use this CSV file.") help="Don't parse anything, use this CSV file.")
parser.add_argument( parser.add_argument(
'-d', '--diff', '-d', '--diff',
help="Specify CSV file to diff against.") help="Specify CSV file to diff against.")
parser.add_argument( parser.add_argument(
'-a', '--all', '-a', '--all',
action='store_true', action='store_true',
help="Show all, not just the ones that changed.") help="Show all, not just the ones that changed.")
parser.add_argument( parser.add_argument(
'-p', '--percent', '-p', '--percent',
action='store_true', action='store_true',
help="Only show percentage change, not a full diff.") help="Only show percentage change, not a full diff.")
parser.add_argument( parser.add_argument(
'-b', '--by', '-b', '--by',
action='append', action='append',
choices=DataResult._by, choices=DataResult._by,
help="Group by this field.") help="Group by this field.")
parser.add_argument( parser.add_argument(
'-f', '--field', '-f', '--field',
dest='fields', dest='fields',
action='append', action='append',
choices=DataResult._fields, choices=DataResult._fields,
help="Show this field.") help="Show this field.")
parser.add_argument( parser.add_argument(
'-D', '--define', '-D', '--define',
dest='defines', dest='defines',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs: ( lambda k, vs: (
k.strip(), k.strip(),
{v.strip() for v in vs.split(',')}) {v.strip() for v in vs.split(',')})
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Only include results where this field is this value.") help="Only include results where this field is this value.")
class AppendSort(argparse.Action): class AppendSort(argparse.Action):
def __call__(self, parser, namespace, value, option): def __call__(self, parser, namespace, value, option):
if namespace.sort is None: if namespace.sort is None:
namespace.sort = [] namespace.sort = []
namespace.sort.append((value, True if option == '-S' else False)) namespace.sort.append((value, True if option == '-S' else False))
parser.add_argument( parser.add_argument(
'-s', '--sort', '-s', '--sort',
nargs='?', nargs='?',
action=AppendSort, action=AppendSort,
help="Sort by this field.") help="Sort by this field.")
parser.add_argument( parser.add_argument(
'-S', '--reverse-sort', '-S', '--reverse-sort',
nargs='?', nargs='?',
action=AppendSort, action=AppendSort,
help="Sort by this field, but backwards.") help="Sort by this field, but backwards.")
parser.add_argument( parser.add_argument(
'-Y', '--summary', '-Y', '--summary',
action='store_true', action='store_true',
help="Only show the total.") help="Only show the total.")
parser.add_argument( parser.add_argument(
'-F', '--source', '-F', '--source',
dest='sources', dest='sources',
action='append', action='append',
help="Only consider definitions in this file. Defaults to anything " help="Only consider definitions in this file. Defaults to "
"in the current directory.") "anything in the current directory.")
parser.add_argument( parser.add_argument(
'--everything', '--everything',
action='store_true', action='store_true',
help="Include builtin and libc specific symbols.") help="Include builtin and libc specific symbols.")
parser.add_argument( parser.add_argument(
'--nm-types', '--nm-types',
default=NM_TYPES, default=NM_TYPES,
help="Type of symbols to report, this uses the same single-character " help="Type of symbols to report, this uses the same "
"type-names emitted by nm. Defaults to %r." % NM_TYPES) "single-character type-names emitted by nm. Defaults to "
"%r." % NM_TYPES)
parser.add_argument( parser.add_argument(
'--nm-path', '--nm-path',
type=lambda x: x.split(), type=lambda x: x.split(),
default=NM_PATH, default=NM_PATH,
help="Path to the nm executable, may include flags. " help="Path to the nm executable, may include flags. "
"Defaults to %r." % NM_PATH) "Defaults to %r." % NM_PATH)
parser.add_argument( parser.add_argument(
'--objdump-path', '--objdump-path',
type=lambda x: x.split(), type=lambda x: x.split(),
default=OBJDUMP_PATH, default=OBJDUMP_PATH,
help="Path to the objdump executable, may include flags. " help="Path to the objdump executable, may include flags. "
"Defaults to %r." % OBJDUMP_PATH) "Defaults to %r." % OBJDUMP_PATH)
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/dbgblock.py
+51 -49
View File
@@ -66,12 +66,12 @@ def rbydaddr(s):
def xxd(data, width=16): def xxd(data, width=16):
for i in range(0, len(data), width): for i in range(0, len(data), width):
yield '%-*s %-*s' % ( yield '%-*s %-*s' % (
3*width, 3*width,
' '.join('%02x' % b for b in data[i:i+width]), ' '.join('%02x' % b for b in data[i:i+width]),
width, width,
''.join( ''.join(
b if b >= ' ' and b <= '~' else '.' b if b >= ' ' and b <= '~' else '.'
for b in map(chr, data[i:i+width]))) for b in map(chr, data[i:i+width])))
def crc32c(data, crc=0): def crc32c(data, crc=0):
crc ^= 0xffffffff crc ^= 0xffffffff
@@ -99,7 +99,7 @@ def main(disk, block=None, *,
if len(block) > 1: if len(block) > 1:
print("error: more than one block address?", print("error: more than one block address?",
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
block = block[0] block = block[0]
@@ -112,17 +112,18 @@ def main(disk, block=None, *,
# block may also encode an offset # block may also encode an offset
block, off, size = ( block, off, size = (
block[0] if isinstance(block, tuple) else block, block[0] if isinstance(block, tuple) else block,
off[0] if isinstance(off, tuple) off[0] if isinstance(off, tuple)
else off if off is not None else off if off is not None
else size[0] if isinstance(size, tuple) and len(size) > 1 else size[0] if isinstance(size, tuple) and len(size) > 1
else block[1] if isinstance(block, tuple) else block[1] if isinstance(block, tuple)
else None, else None,
size[1] - size[0] if isinstance(size, tuple) and len(size) > 1 size[1] - size[0] if isinstance(size, tuple) and len(size) > 1
else size[0] if isinstance(size, tuple) else size[0] if isinstance(size, tuple)
else size if size is not None else size if size is not None
else off[1] - off[0] if isinstance(off, tuple) and len(off) > 1 else off[1] - off[0]
else block_size) if isinstance(off, tuple) and len(off) > 1
else block_size)
# read the block # read the block
f.seek((block * block_size) + (off or 0)) f.seek((block * block_size) + (off or 0))
@@ -133,50 +134,51 @@ def main(disk, block=None, *,
# print the header # print the header
print('block %s, size %d, cksum %08x' % ( print('block %s, size %d, cksum %08x' % (
'0x%x.%x' % (block, off) '0x%x.%x' % (block, off)
if off is not None if off is not None
else '0x%x' % block, else '0x%x' % block,
size, size,
cksum)) cksum))
# render the hex view # render the hex view
for o, line in enumerate(xxd(data)): for o, line in enumerate(xxd(data)):
print('%08x: %s' % ((off or 0) + 16*o, line)) print('%08x: %s' % ((off or 0) + 16*o, line))
if __name__ == "__main__": if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Debug block devices.", description="Debug block devices.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'disk', 'disk',
help="File containing the block device.") help="File containing the block device.")
parser.add_argument( parser.add_argument(
'block', 'block',
nargs='?', nargs='?',
type=rbydaddr, type=rbydaddr,
help="Block address.") help="Block address.")
parser.add_argument( parser.add_argument(
'-b', '--block-size', '-b', '--block-size',
type=bdgeom, type=bdgeom,
help="Block size/geometry in bytes.") help="Block size/geometry in bytes.")
parser.add_argument( parser.add_argument(
'--block-count', '--block-count',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Block count in blocks.") help="Block count in blocks.")
parser.add_argument( parser.add_argument(
'--off', '--off',
type=lambda x: tuple( type=lambda x: tuple(
int(x, 0) if x.strip() else None int(x, 0) if x.strip() else None
for x in x.split(',')), for x in x.split(',')),
help="Show a specific offset, may be a range.") help="Show a specific offset, may be a range.")
parser.add_argument( parser.add_argument(
'-n', '--size', '-n', '--size',
type=lambda x: tuple( type=lambda x: tuple(
int(x, 0) if x.strip() else None int(x, 0) if x.strip() else None
for x in x.split(',')), for x in x.split(',')),
help="Show this many bytes, may be a range.") help="Show this many bytes, may be a range.")
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/dbgbmap.py
+229 -220
View File
@@ -58,14 +58,14 @@ COLORS = ['33', '34', '32', '90']
CHARS_DOTS = " .':" CHARS_DOTS = " .':"
CHARS_BRAILLE = ( CHARS_BRAILLE = (
'⠀⢀⡀⣀⠠⢠⡠⣠⠄⢄⡄⣄⠤⢤⡤⣤' '⠐⢐⡐⣐⠰⢰⡰⣰⠔⢔⡔⣔⠴⢴⡴⣴' '⠀⢀⡀⣀⠠⢠⡠⣠⠄⢄⡄⣄⠤⢤⡤⣤' '⠐⢐⡐⣐⠰⢰⡰⣰⠔⢔⡔⣔⠴⢴⡴⣴'
'⠂⢂⡂⣂⠢⢢⡢⣢⠆⢆⡆⣆⠦⢦⡦⣦' '⠒⢒⡒⣒⠲⢲⡲⣲⠖⢖⡖⣖⠶⢶⡶⣶' '⠂⢂⡂⣂⠢⢢⡢⣢⠆⢆⡆⣆⠦⢦⡦⣦' '⠒⢒⡒⣒⠲⢲⡲⣲⠖⢖⡖⣖⠶⢶⡶⣶'
'⠈⢈⡈⣈⠨⢨⡨⣨⠌⢌⡌⣌⠬⢬⡬⣬' '⠘⢘⡘⣘⠸⢸⡸⣸⠜⢜⡜⣜⠼⢼⡼⣼' '⠈⢈⡈⣈⠨⢨⡨⣨⠌⢌⡌⣌⠬⢬⡬⣬' '⠘⢘⡘⣘⠸⢸⡸⣸⠜⢜⡜⣜⠼⢼⡼⣼'
'⠊⢊⡊⣊⠪⢪⡪⣪⠎⢎⡎⣎⠮⢮⡮⣮' '⠚⢚⡚⣚⠺⢺⡺⣺⠞⢞⡞⣞⠾⢾⡾⣾' '⠊⢊⡊⣊⠪⢪⡪⣪⠎⢎⡎⣎⠮⢮⡮⣮' '⠚⢚⡚⣚⠺⢺⡺⣺⠞⢞⡞⣞⠾⢾⡾⣾'
'⠁⢁⡁⣁⠡⢡⡡⣡⠅⢅⡅⣅⠥⢥⡥⣥' '⠑⢑⡑⣑⠱⢱⡱⣱⠕⢕⡕⣕⠵⢵⡵⣵' '⠁⢁⡁⣁⠡⢡⡡⣡⠅⢅⡅⣅⠥⢥⡥⣥' '⠑⢑⡑⣑⠱⢱⡱⣱⠕⢕⡕⣕⠵⢵⡵⣵'
'⠃⢃⡃⣃⠣⢣⡣⣣⠇⢇⡇⣇⠧⢧⡧⣧' '⠓⢓⡓⣓⠳⢳⡳⣳⠗⢗⡗⣗⠷⢷⡷⣷' '⠃⢃⡃⣃⠣⢣⡣⣣⠇⢇⡇⣇⠧⢧⡧⣧' '⠓⢓⡓⣓⠳⢳⡳⣳⠗⢗⡗⣗⠷⢷⡷⣷'
'⠉⢉⡉⣉⠩⢩⡩⣩⠍⢍⡍⣍⠭⢭⡭⣭' '⠙⢙⡙⣙⠹⢹⡹⣹⠝⢝⡝⣝⠽⢽⡽⣽' '⠉⢉⡉⣉⠩⢩⡩⣩⠍⢍⡍⣍⠭⢭⡭⣭' '⠙⢙⡙⣙⠹⢹⡹⣹⠝⢝⡝⣝⠽⢽⡽⣽'
'⠋⢋⡋⣋⠫⢫⡫⣫⠏⢏⡏⣏⠯⢯⡯⣯' '⠛⢛⡛⣛⠻⢻⡻⣻⠟⢟⡟⣟⠿⢿⡿⣿') '⠋⢋⡋⣋⠫⢫⡫⣫⠏⢏⡏⣏⠯⢯⡯⣯' '⠛⢛⡛⣛⠻⢻⡻⣻⠟⢟⡟⣟⠿⢿⡿⣿')
# some ways of block geometry representations # some ways of block geometry representations
@@ -250,8 +250,8 @@ def hilbert_curve(width, height):
yield from hilbert_(x, y, b_x_, b_y_, a_x_, a_y_) yield from hilbert_(x, y, b_x_, b_y_, a_x_, a_y_)
yield from hilbert_(x+b_x_, y+b_y_, a_x, a_y, b_x-b_x_, b_y-b_y_) yield from hilbert_(x+b_x_, y+b_y_, a_x, a_y, b_x-b_x_, b_y-b_y_)
yield from hilbert_( yield from hilbert_(
x+(a_x-a_dx)+(b_x_-b_dx), y+(a_y-a_dy)+(b_y_-b_dy), x+(a_x-a_dx)+(b_x_-b_dx), y+(a_y-a_dy)+(b_y_-b_dy),
-b_x_, -b_y_, -(a_x-a_x_), -(a_y-a_y_)) -b_x_, -b_y_, -(a_x-a_x_), -(a_y-a_y_))
if width >= height: if width >= height:
curve = hilbert_(0, 0, +width, 0, 0, +height) curve = hilbert_(0, 0, +width, 0, 0, +height)
@@ -293,10 +293,10 @@ class Pixel(int):
btree=False, btree=False,
data=False): data=False):
return super().__new__(cls, return super().__new__(cls,
state state
| (1 if mdir else 0) | (1 if mdir else 0)
| (2 if btree else 0) | (2 if btree else 0)
| (4 if data else 0)) | (4 if data else 0))
@property @property
def is_mdir(self): def is_mdir(self):
@@ -367,8 +367,8 @@ class Pixel(int):
# apply colors # apply colors
if f and color: if f and color:
c = '%s%s\x1b[m' % ( c = '%s%s\x1b[m' % (
''.join('\x1b[%sm' % f_ for f_ in f), ''.join('\x1b[%sm' % f_ for f_ in f),
c) c)
return c return c
@@ -434,25 +434,25 @@ class Bmap:
block -= self._block_window.start block -= self._block_window.start
size = (max(self._off_window.start, size = (max(self._off_window.start,
min(self._off_window.stop, off+size)) min(self._off_window.stop, off+size))
- max(self._off_window.start, - max(self._off_window.start,
min(self._off_window.stop, off))) min(self._off_window.stop, off)))
off = (max(self._off_window.start, off = (max(self._off_window.start,
min(self._off_window.stop, off)) min(self._off_window.stop, off))
- self._off_window.start) - self._off_window.start)
if size == 0: if size == 0:
return return
# map to our block space # map to our block space
range_ = range( range_ = range(
block*len(self._off_window) + off, block*len(self._off_window) + off,
block*len(self._off_window) + off+size) block*len(self._off_window) + off+size)
range_ = range( range_ = range(
(range_.start*len(self.pixels)) // self._window, (range_.start*len(self.pixels)) // self._window,
(range_.stop*len(self.pixels)) // self._window) (range_.stop*len(self.pixels)) // self._window)
range_ = range( range_ = range(
range_.start, range_.start,
max(range_.stop, range_.start+1)) max(range_.stop, range_.start+1))
# apply the op # apply the op
for i in range_: for i in range_:
@@ -476,9 +476,9 @@ class Bmap:
width=None, width=None,
height=None): height=None):
block_size = (block_size if block_size is not None block_size = (block_size if block_size is not None
else self.block_size) else self.block_size)
block_count = (block_count if block_count is not None block_count = (block_count if block_count is not None
else self.block_count) else self.block_count)
width = width if width is not None else self.width width = width if width is not None else self.width
height = height if height is not None else self.height height = height if height is not None else self.height
@@ -496,17 +496,17 @@ class Bmap:
for x in range(width*height): for x in range(width*height):
# map into our old bd space # map into our old bd space
range_ = range( range_ = range(
(x*self._window) // (width*height), (x*self._window) // (width*height),
((x+1)*self._window) // (width*height)) ((x+1)*self._window) // (width*height))
range_ = range( range_ = range(
range_.start, range_.start,
max(range_.stop, range_.start+1)) max(range_.stop, range_.start+1))
# aggregate state # aggregate state
pixels.append(ft.reduce( pixels.append(ft.reduce(
Pixel.__or__, Pixel.__or__,
self.pixels[range_.start:range_.stop], self.pixels[range_.start:range_.stop],
Pixel())) Pixel()))
self.width = width self.width = width
self.height = height self.height = height
@@ -552,12 +552,12 @@ class Bmap:
byte_p |= 1 << i byte_p |= 1 << i
line.append(best_p.draw( line.append(best_p.draw(
CHARS_BRAILLE[byte_p], CHARS_BRAILLE[byte_p],
braille=True, braille=True,
mdirs=mdirs, mdirs=mdirs,
btrees=btrees, btrees=btrees,
datas=datas, datas=datas,
**args)) **args))
elif dots: elif dots:
# encode into a byte # encode into a byte
for x in range(self.width): for x in range(self.width):
@@ -572,19 +572,19 @@ class Bmap:
byte_p |= 1 << i byte_p |= 1 << i
line.append(best_p.draw( line.append(best_p.draw(
CHARS_DOTS[byte_p], CHARS_DOTS[byte_p],
dots=True, dots=True,
mdirs=mdirs, mdirs=mdirs,
btrees=btrees, btrees=btrees,
datas=datas, datas=datas,
**args)) **args))
else: else:
for x in range(self.width): for x in range(self.width):
line.append(grid[x + row*self.width].draw( line.append(grid[x + row*self.width].draw(
mdirs=mdirs, mdirs=mdirs,
btrees=btrees, btrees=btrees,
datas=datas, datas=datas,
**args)) **args))
return ''.join(line) return ''.join(line)
@@ -610,9 +610,9 @@ class Rbyd:
return '0x%x.%x' % (self.block, self.trunk) return '0x%x.%x' % (self.block, self.trunk)
else: else:
return '0x{%x,%s}.%x' % ( return '0x{%x,%s}.%x' % (
self.block, self.block,
','.join('%x' % block for block in self.redund_blocks), ','.join('%x' % block for block in self.redund_blocks),
self.trunk) self.trunk)
@classmethod @classmethod
def fetch(cls, f, block_size, blocks, trunk=None): def fetch(cls, f, block_size, blocks, trunk=None):
@@ -621,21 +621,23 @@ class Rbyd:
if len(blocks) > 1: if len(blocks) > 1:
# fetch all blocks # fetch all blocks
rbyds = [cls.fetch(f, block_size, block, trunk) for block in blocks] rbyds = [cls.fetch(f, block_size, block, trunk)
for block in blocks]
# determine most recent revision # determine most recent revision
i = 0 i = 0
for i_, rbyd in enumerate(rbyds): for i_, rbyd in enumerate(rbyds):
# compare with sequence arithmetic # compare with sequence arithmetic
if rbyd and ( if rbyd and (
not rbyds[i] not rbyds[i]
or not ((rbyd.rev - rbyds[i].rev) & 0x80000000) or not ((rbyd.rev - rbyds[i].rev) & 0x80000000)
or (rbyd.rev == rbyds[i].rev or (rbyd.rev == rbyds[i].rev
and rbyd.trunk > rbyds[i].trunk)): and rbyd.trunk > rbyds[i].trunk)):
i = i_ i = i_
# keep track of the other blocks # keep track of the other blocks
rbyd = rbyds[i] rbyd = rbyds[i]
rbyd.redund_blocks = [rbyds[(i+1+j) % len(rbyds)].block rbyd.redund_blocks = [
for j in range(len(rbyds)-1)] rbyds[(i+1+j) % len(rbyds)].block
for j in range(len(rbyds)-1)]
return rbyd return rbyd
else: else:
# block may encode a trunk # block may encode a trunk
@@ -789,7 +791,9 @@ class Rbyd:
done = not tag_ or (rid_, tag_) < (rid, tag) done = not tag_ or (rid_, tag_) < (rid, tag)
return done, rid_, tag_, w_, j, d, self.data[j+d:j+d+jump], path return (done, rid_, tag_, w_, j, d,
self.data[j+d:j+d+jump],
path)
def __bool__(self): def __bool__(self):
return bool(self.trunk) return bool(self.trunk)
@@ -834,7 +838,7 @@ class Rbyd:
w = 0 w = 0
for i in it.count(): for i in it.count():
done, rid__, tag, w_, j, d, data, _ = rbyd.lookup( done, rid__, tag, w_, j, d, data, _ = rbyd.lookup(
rid_, tag+0x1) rid_, tag+0x1)
if done or (i != 0 and rid__ != rid_): if done or (i != 0 and rid__ != rid_):
break break
@@ -880,14 +884,15 @@ class Rbyd:
# lookup our mbid # lookup our mbid
done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup( done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup(
f, block_size, mbid) f, block_size, mbid)
if done: if done:
return True, -1, 0, None return True, -1, 0, None
mdir = next(((tag, j, d, data) mdir = next(
for tag, j, d, data in tags ((tag, j, d, data)
if tag == TAG_MDIR), for tag, j, d, data in tags
None) if tag == TAG_MDIR),
None)
if not mdir: if not mdir:
return True, -1, 0, None return True, -1, 0, None
@@ -979,7 +984,7 @@ def main(disk, mroots=None, *,
if any(isinstance(b, list) and len(b) > 1 for b in block): if any(isinstance(b, list) and len(b) > 1 for b in block):
print("error: more than one block address?", print("error: more than one block address?",
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
if isinstance(block[0], list): if isinstance(block[0], list):
block = (block[0][0], *block[1:]) block = (block[0][0], *block[1:])
@@ -1034,16 +1039,17 @@ def main(disk, mroots=None, *,
# create our block device representation # create our block device representation
bmap = Bmap( bmap = Bmap(
block_size=block_size, block_size=block_size,
block_count=block_count, block_count=block_count,
block_window=block_window, block_window=block_window,
off_window=off_window, off_window=off_window,
# scale if we're printing with dots or braille # scale if we're printing with dots or braille
width=2*width_ if braille else width_, width=2*width_ if braille else width_,
height=max(1, height=max(
4*height_ if braille 1,
else 2*height_ if dots 4*height_ if braille
else height_)) else 2*height_ if dots
else height_))
# keep track of how many blocks are in use # keep track of how many blocks are in use
mdirs_ = 0 mdirs_ = 0
@@ -1063,16 +1069,16 @@ def main(disk, mroots=None, *,
block_size = f.tell() block_size = f.tell()
block_count = 1 block_count = 1
bmap.resize( bmap.resize(
block_size=block_size, block_size=block_size,
block_count=block_count) block_count=block_count)
# if block_count is omitted, derive the block_count from our file size # if block_count is omitted, derive the block_count from our file size
if block_count is None: if block_count is None:
f.seek(0, os.SEEK_END) f.seek(0, os.SEEK_END)
block_count = f.tell() // block_size block_count = f.tell() // block_size
bmap.resize( bmap.resize(
block_size=block_size, block_size=block_size,
block_count=block_count) block_count=block_count)
#### traverse the filesystem #### traverse the filesystem
@@ -1090,7 +1096,7 @@ def main(disk, mroots=None, *,
# mark mroots in our bmap # mark mroots in our bmap
for block in mroot.blocks: for block in mroot.blocks:
bmap.mdir(block, bmap.mdir(block,
mroot.eoff if args.get('in_use') else block_size) mroot.eoff if args.get('in_use') else block_size)
mdirs_ += 1; mdirs_ += 1;
# find any file btrees in our mroot # find any file btrees in our mroot
@@ -1129,7 +1135,8 @@ def main(disk, mroots=None, *,
# mark mdir in our bmap # mark mdir in our bmap
for block in mdir.blocks: for block in mdir.blocks:
bmap.mdir(block, bmap.mdir(block,
mdir.eoff if args.get('in_use') else block_size) mdir.eoff if args.get('in_use')
else block_size)
mdirs_ += 1 mdirs_ += 1
# find any file btrees in our mdir # find any file btrees in our mdir
@@ -1153,8 +1160,8 @@ def main(disk, mroots=None, *,
ppath = [] ppath = []
while True: while True:
done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup( done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup(
f, block_size, mbid+1, f, block_size, mbid+1,
depth=args.get('depth', mdepth)-mdepth) depth=args.get('depth', mdepth)-mdepth)
if done: if done:
break break
@@ -1176,8 +1183,8 @@ def main(disk, mroots=None, *,
d, (mid_, w_, rbyd_, rid_, tags_) = x d, (mid_, w_, rbyd_, rid_, tags_) = x
for block in rbyd_.blocks: for block in rbyd_.blocks:
bmap.btree(block, bmap.btree(block,
rbyd_.eoff if args.get('in_use') rbyd_.eoff if args.get('in_use')
else block_size) else block_size)
btrees_ += 1 btrees_ += 1
ppath = path ppath = path
@@ -1190,10 +1197,11 @@ def main(disk, mroots=None, *,
mdir__ = None mdir__ = None
if (not args.get('depth') if (not args.get('depth')
or mdepth+len(path) < args.get('depth')): or mdepth+len(path) < args.get('depth')):
mdir__ = next(((tag, j, d, data) mdir__ = next(
for tag, j, d, data in tags ((tag, j, d, data)
if tag == TAG_MDIR), for tag, j, d, data in tags
None) if tag == TAG_MDIR),
None)
if mdir__: if mdir__:
# fetch the mdir # fetch the mdir
@@ -1208,8 +1216,8 @@ def main(disk, mroots=None, *,
# mark mdir in our bmap # mark mdir in our bmap
for block in mdir_.blocks: for block in mdir_.blocks:
bmap.mdir(block, 0, bmap.mdir(block, 0,
mdir_.eoff if args.get('in_use') mdir_.eoff if args.get('in_use')
else block_size) else block_size)
mdirs_ += 1 mdirs_ += 1
# find any file btrees in our mdir # find any file btrees in our mdir
@@ -1233,8 +1241,8 @@ def main(disk, mroots=None, *,
size, block, off = frombptr(data) size, block, off = frombptr(data)
# mark block in our bmap # mark block in our bmap
bmap.data(block, bmap.data(block,
off if args.get('in_use') else 0, off if args.get('in_use') else 0,
size if args.get('in_use') else block_size) size if args.get('in_use') else block_size)
datas_ += 1 datas_ += 1
continue continue
@@ -1258,9 +1266,9 @@ def main(disk, mroots=None, *,
ppath = [] ppath = []
while True: while True:
(done, bid, w, rbyd, rid, tags, path (done, bid, w, rbyd, rid, tags, path
) = btree.btree_lookup( ) = btree.btree_lookup(
f, block_size, bid+1, f, block_size, bid+1,
depth=args.get('depth', mdepth)-mdepth) depth=args.get('depth', mdepth)-mdepth)
if done: if done:
break break
@@ -1285,8 +1293,8 @@ def main(disk, mroots=None, *,
continue continue
for block in rbyd_.blocks: for block in rbyd_.blocks:
bmap.btree(block, bmap.btree(block,
rbyd_.eoff if args.get('in_use') rbyd_.eoff if args.get('in_use')
else block_size) else block_size)
btrees_ += 1 btrees_ += 1
ppath = path ppath = path
@@ -1299,10 +1307,11 @@ def main(disk, mroots=None, *,
bptr__ = None bptr__ = None
if (not args.get('depth') if (not args.get('depth')
or mdepth+len(path) < args.get('depth')): or mdepth+len(path) < args.get('depth')):
bptr__ = next(((tag, j, d, data) bptr__ = next(
for tag, j, d, data in tags ((tag, j, d, data)
if tag & 0xfff == TAG_BLOCK), for tag, j, d, data in tags
None) if tag & 0xfff == TAG_BLOCK),
None)
if bptr__: if bptr__:
# fetch the block # fetch the block
@@ -1311,8 +1320,8 @@ def main(disk, mroots=None, *,
# mark blocks in our bmap # mark blocks in our bmap
bmap.data(block, bmap.data(block,
off if args.get('in_use') else 0, off if args.get('in_use') else 0,
size if args.get('in_use') else block_size) size if args.get('in_use') else block_size)
datas_ += 1 datas_ += 1
#### actual rendering begins here #### actual rendering begins here
@@ -1320,29 +1329,29 @@ def main(disk, mroots=None, *,
# print some information about the bmap # print some information about the bmap
if not no_header: if not no_header:
print('bd %dx%d%s%s%s' % ( print('bd %dx%d%s%s%s' % (
block_size, block_count, block_size, block_count,
', %6s mdir' % ('%.1f%%' % (100*mdirs_ / block_count)) ', %6s mdir' % ('%.1f%%' % (100*mdirs_ / block_count))
if mdirs else '', if mdirs else '',
', %6s btree' % ('%.1f%%' % (100*btrees_ / block_count)) ', %6s btree' % ('%.1f%%' % (100*btrees_ / block_count))
if btrees else '', if btrees else '',
', %6s data' % ('%.1f%%' % (100*datas_ / block_count)) ', %6s data' % ('%.1f%%' % (100*datas_ / block_count))
if datas else '')) if datas else ''))
# and then print the bmap # and then print the bmap
for row in range( for row in range(
mt.ceil(bmap.height/4) if braille mt.ceil(bmap.height/4) if braille
else mt.ceil(bmap.height/2) if dots else mt.ceil(bmap.height/2) if dots
else bmap.height): else bmap.height):
line = bmap.draw(row, line = bmap.draw(row,
mdirs=mdirs, mdirs=mdirs,
btrees=btrees, btrees=btrees,
datas=datas, datas=datas,
color=color, color=color,
dots=dots, dots=dots,
braille=braille, braille=braille,
hilbert=hilbert, hilbert=hilbert,
lebesgue=lebesgue, lebesgue=lebesgue,
**args) **args)
print(line) print(line)
if args.get('error_on_corrupt') and corrupted: if args.get('error_on_corrupt') and corrupted:
@@ -1353,122 +1362,122 @@ if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Render currently used blocks in a littlefs image.", description="Render currently used blocks in a littlefs image.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'disk', 'disk',
help="File containing the block device.") help="File containing the block device.")
parser.add_argument( parser.add_argument(
'mroots', 'mroots',
nargs='*', nargs='*',
type=rbydaddr, type=rbydaddr,
help="Block address of the mroots. Defaults to 0x{0,1}.") help="Block address of the mroots. Defaults to 0x{0,1}.")
parser.add_argument( parser.add_argument(
'-b', '--block-size', '-b', '--block-size',
type=bdgeom, type=bdgeom,
help="Block size/geometry in bytes.") help="Block size/geometry in bytes.")
parser.add_argument( parser.add_argument(
'--block-count', '--block-count',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Block count in blocks.") help="Block count in blocks.")
parser.add_argument( parser.add_argument(
'-@', '--block', '-@', '--block',
nargs='?', nargs='?',
type=lambda x: tuple( type=lambda x: tuple(
rbydaddr(x) if x.strip() else None rbydaddr(x) if x.strip() else None
for x in x.split(',')), for x in x.split(',')),
help="Optional block to show, may be a range.") help="Optional block to show, may be a range.")
parser.add_argument( parser.add_argument(
'--off', '--off',
type=lambda x: tuple( type=lambda x: tuple(
int(x, 0) if x.strip() else None int(x, 0) if x.strip() else None
for x in x.split(',')), for x in x.split(',')),
help="Show a specific offset, may be a range.") help="Show a specific offset, may be a range.")
parser.add_argument( parser.add_argument(
'--size', '--size',
type=lambda x: tuple( type=lambda x: tuple(
int(x, 0) if x.strip() else None int(x, 0) if x.strip() else None
for x in x.split(',')), for x in x.split(',')),
help="Show this many bytes, may be a range.") help="Show this many bytes, may be a range.")
parser.add_argument( parser.add_argument(
'-M', '--mdirs', '-M', '--mdirs',
action='store_true', action='store_true',
help="Render mdir blocks.") help="Render mdir blocks.")
parser.add_argument( parser.add_argument(
'-B', '--btrees', '-B', '--btrees',
action='store_true', action='store_true',
help="Render btree blocks.") help="Render btree blocks.")
parser.add_argument( parser.add_argument(
'-D', '--datas', '-D', '--datas',
action='store_true', action='store_true',
help="Render data blocks.") help="Render data blocks.")
parser.add_argument( parser.add_argument(
'-N', '--no-header', '-N', '--no-header',
action='store_true', action='store_true',
help="Don't show the header.") help="Don't show the header.")
parser.add_argument( parser.add_argument(
'--color', '--color',
choices=['never', 'always', 'auto'], choices=['never', 'always', 'auto'],
default='auto', default='auto',
help="When to use terminal colors. Defaults to 'auto'.") help="When to use terminal colors. Defaults to 'auto'.")
parser.add_argument( parser.add_argument(
'-:', '--dots', '-:', '--dots',
action='store_true', action='store_true',
help="Use 1x2 ascii dot characters.") help="Use 1x2 ascii dot characters.")
parser.add_argument( parser.add_argument(
'-⣿', '--braille', '-⣿', '--braille',
action='store_true', action='store_true',
help="Use 2x4 unicode braille characters. Note that braille characters " help="Use 2x4 unicode braille characters. Note that braille "
"sometimes suffer from inconsistent widths.") "characters sometimes suffer from inconsistent widths.")
parser.add_argument( parser.add_argument(
'--chars', '--chars',
help="Characters to use for mdir, btree, data, unused blocks.") help="Characters to use for mdir, btree, data, unused blocks.")
parser.add_argument( parser.add_argument(
'--colors', '--colors',
type=lambda x: [x.strip() for x in x.split(',')], type=lambda x: [x.strip() for x in x.split(',')],
help="Colors to use for mdir, btree, data, unused blocks.") help="Colors to use for mdir, btree, data, unused blocks.")
parser.add_argument( parser.add_argument(
'-W', '--width', '-W', '--width',
nargs='?', nargs='?',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
const=0, const=0,
help="Width in columns. 0 uses the terminal width. Defaults to " help="Width in columns. 0 uses the terminal width. Defaults to "
"min(terminal, 80).") "min(terminal, 80).")
parser.add_argument( parser.add_argument(
'-H', '--height', '-H', '--height',
nargs='?', nargs='?',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
const=0, const=0,
help="Height in rows. 0 uses the terminal height. Defaults to 1.") help="Height in rows. 0 uses the terminal height. Defaults to 1.")
parser.add_argument( parser.add_argument(
'-n', '--lines', '-n', '--lines',
nargs='?', nargs='?',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
const=0, const=0,
help="Show this many lines of history. 0 uses the terminal height. " help="Show this many lines of history. 0 uses the terminal "
"Defaults to 5.") "height. Defaults to 5.")
parser.add_argument( parser.add_argument(
'-U', '--hilbert', '-U', '--hilbert',
action='store_true', action='store_true',
help="Render as a space-filling Hilbert curve.") help="Render as a space-filling Hilbert curve.")
parser.add_argument( parser.add_argument(
'-Z', '--lebesgue', '-Z', '--lebesgue',
action='store_true', action='store_true',
help="Render as a space-filling Z-curve.") help="Render as a space-filling Z-curve.")
parser.add_argument( parser.add_argument(
'-i', '--in-use', '-i', '--in-use',
action='store_true', action='store_true',
help="Show how much of each block is in use.") help="Show how much of each block is in use.")
parser.add_argument( parser.add_argument(
'-z', '--depth', '-z', '--depth',
nargs='?', nargs='?',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
const=0, const=0,
help="Depth of the filesystem tree to parse.") help="Depth of the filesystem tree to parse.")
parser.add_argument( parser.add_argument(
'-e', '--error-on-corrupt', '-e', '--error-on-corrupt',
action='store_true', action='store_true',
help="Error if the filesystem is corrupt.") help="Error if the filesystem is corrupt.")
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/dbgbtree.py
+202 -195
View File
@@ -154,108 +154,108 @@ def frombranch(data):
def xxd(data, width=16): def xxd(data, width=16):
for i in range(0, len(data), width): for i in range(0, len(data), width):
yield '%-*s %-*s' % ( yield '%-*s %-*s' % (
3*width, 3*width,
' '.join('%02x' % b for b in data[i:i+width]), ' '.join('%02x' % b for b in data[i:i+width]),
width, width,
''.join( ''.join(
b if b >= ' ' and b <= '~' else '.' b if b >= ' ' and b <= '~' else '.'
for b in map(chr, data[i:i+width]))) for b in map(chr, data[i:i+width])))
def tagrepr(tag, w=None, size=None, off=None): def tagrepr(tag, w=None, size=None, off=None):
if (tag & 0x6fff) == TAG_NULL: if (tag & 0x6fff) == TAG_NULL:
return '%snull%s%s' % ( return '%snull%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if size else '') ' %d' % size if size else '')
elif (tag & 0x6f00) == TAG_CONFIG: elif (tag & 0x6f00) == TAG_CONFIG:
return '%s%s%s%s' % ( return '%s%s%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
'magic' if (tag & 0xfff) == TAG_MAGIC 'magic' if (tag & 0xfff) == TAG_MAGIC
else 'version' if (tag & 0xfff) == TAG_VERSION else 'version' if (tag & 0xfff) == TAG_VERSION
else 'rcompat' if (tag & 0xfff) == TAG_RCOMPAT else 'rcompat' if (tag & 0xfff) == TAG_RCOMPAT
else 'wcompat' if (tag & 0xfff) == TAG_WCOMPAT else 'wcompat' if (tag & 0xfff) == TAG_WCOMPAT
else 'ocompat' if (tag & 0xfff) == TAG_OCOMPAT else 'ocompat' if (tag & 0xfff) == TAG_OCOMPAT
else 'geometry' if (tag & 0xfff) == TAG_GEOMETRY else 'geometry' if (tag & 0xfff) == TAG_GEOMETRY
else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT
else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT
else 'config 0x%02x' % (tag & 0xff), else 'config 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x6f00) == TAG_GDELTA: elif (tag & 0x6f00) == TAG_GDELTA:
return '%s%s%s%s' % ( return '%s%s%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA
else 'gdelta 0x%02x' % (tag & 0xff), else 'gdelta 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x6f00) == TAG_NAME: elif (tag & 0x6f00) == TAG_NAME:
return '%s%s%s%s' % ( return '%s%s%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
'name' if (tag & 0xfff) == TAG_NAME 'name' if (tag & 0xfff) == TAG_NAME
else 'reg' if (tag & 0xfff) == TAG_REG else 'reg' if (tag & 0xfff) == TAG_REG
else 'dir' if (tag & 0xfff) == TAG_DIR else 'dir' if (tag & 0xfff) == TAG_DIR
else 'orphan' if (tag & 0xfff) == TAG_ORPHAN else 'orphan' if (tag & 0xfff) == TAG_ORPHAN
else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK
else 'name 0x%02x' % (tag & 0xff), else 'name 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x6f00) == TAG_STRUCT: elif (tag & 0x6f00) == TAG_STRUCT:
return '%s%s%s%s' % ( return '%s%s%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
'data' if (tag & 0xfff) == TAG_DATA 'data' if (tag & 0xfff) == TAG_DATA
else 'block' if (tag & 0xfff) == TAG_BLOCK else 'block' if (tag & 0xfff) == TAG_BLOCK
else 'bshrub' if (tag & 0xfff) == TAG_BSHRUB else 'bshrub' if (tag & 0xfff) == TAG_BSHRUB
else 'btree' if (tag & 0xfff) == TAG_BTREE else 'btree' if (tag & 0xfff) == TAG_BTREE
else 'mroot' if (tag & 0xfff) == TAG_MROOT else 'mroot' if (tag & 0xfff) == TAG_MROOT
else 'mdir' if (tag & 0xfff) == TAG_MDIR else 'mdir' if (tag & 0xfff) == TAG_MDIR
else 'mtree' if (tag & 0xfff) == TAG_MTREE else 'mtree' if (tag & 0xfff) == TAG_MTREE
else 'did' if (tag & 0xfff) == TAG_DID else 'did' if (tag & 0xfff) == TAG_DID
else 'branch' if (tag & 0xfff) == TAG_BRANCH else 'branch' if (tag & 0xfff) == TAG_BRANCH
else 'struct 0x%02x' % (tag & 0xff), else 'struct 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x6e00) == TAG_ATTR: elif (tag & 0x6e00) == TAG_ATTR:
return '%s%sattr 0x%02x%s%s' % ( return '%s%sattr 0x%02x%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
's' if tag & 0x100 else 'u', 's' if tag & 0x100 else 'u',
((tag & 0x100) >> 1) ^ (tag & 0xff), ((tag & 0x100) >> 1) ^ (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif tag & TAG_ALT: elif tag & TAG_ALT:
return 'alt%s%s%s%s%s' % ( return 'alt%s%s%s%s%s' % (
'r' if tag & TAG_R else 'b', 'r' if tag & TAG_R else 'b',
'a' if tag & 0x0fff == 0 and tag & TAG_GT 'a' if tag & 0x0fff == 0 and tag & TAG_GT
else 'n' if tag & 0x0fff == 0 else 'n' if tag & 0x0fff == 0
else 'gt' if tag & TAG_GT else 'gt' if tag & TAG_GT
else 'le', else 'le',
' 0x%x' % (tag & 0x0fff) if tag & 0x0fff != 0 else '', ' 0x%x' % (tag & 0x0fff) if tag & 0x0fff != 0 else '',
' w%d' % w if w is not None else '', ' w%d' % w if w is not None else '',
' 0x%x' % (0xffffffff & (off-size)) ' 0x%x' % (0xffffffff & (off-size))
if size and off is not None if size and off is not None
else ' -%d' % size if size else ' -%d' % size if size
else '') else '')
elif (tag & 0x7f00) == TAG_CKSUM: elif (tag & 0x7f00) == TAG_CKSUM:
return 'cksum%s%s%s%s%s' % ( return 'cksum%s%s%s%s%s' % (
'q' if not tag & 0xfc and tag & TAG_Q else '', 'q' if not tag & 0xfc and tag & TAG_Q else '',
'p' if not tag & 0xfc and tag & TAG_P else '', 'p' if not tag & 0xfc and tag & TAG_P else '',
' 0x%02x' % (tag & 0xff) if tag & 0xfc else '', ' 0x%02x' % (tag & 0xff) if tag & 0xfc else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x7f00) == TAG_NOTE: elif (tag & 0x7f00) == TAG_NOTE:
return 'note%s%s%s' % ( return 'note%s%s%s' % (
' 0x%02x' % (tag & 0xff) if tag & 0xff else '', ' 0x%02x' % (tag & 0xff) if tag & 0xff else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x7f00) == TAG_ECKSUM: elif (tag & 0x7f00) == TAG_ECKSUM:
return 'ecksum%s%s%s' % ( return 'ecksum%s%s%s' % (
' 0x%02x' % (tag & 0xff) if tag & 0xff else '', ' 0x%02x' % (tag & 0xff) if tag & 0xff else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
else: else:
return '0x%04x%s%s' % ( return '0x%04x%s%s' % (
tag, tag,
' w%d' % w if w is not None else '', ' w%d' % w if w is not None else '',
' %d' % size if size is not None else '') ' %d' % size if size is not None else '')
# this type is used for tree representations # this type is used for tree representations
@@ -278,9 +278,9 @@ class Rbyd:
return '0x%x.%x' % (self.block, self.trunk) return '0x%x.%x' % (self.block, self.trunk)
else: else:
return '0x{%x,%s}.%x' % ( return '0x{%x,%s}.%x' % (
self.block, self.block,
','.join('%x' % block for block in self.redund_blocks), ','.join('%x' % block for block in self.redund_blocks),
self.trunk) self.trunk)
@classmethod @classmethod
def fetch(cls, f, block_size, blocks, trunk=None): def fetch(cls, f, block_size, blocks, trunk=None):
@@ -289,21 +289,23 @@ class Rbyd:
if len(blocks) > 1: if len(blocks) > 1:
# fetch all blocks # fetch all blocks
rbyds = [cls.fetch(f, block_size, block, trunk) for block in blocks] rbyds = [cls.fetch(f, block_size, block, trunk)
for block in blocks]
# determine most recent revision # determine most recent revision
i = 0 i = 0
for i_, rbyd in enumerate(rbyds): for i_, rbyd in enumerate(rbyds):
# compare with sequence arithmetic # compare with sequence arithmetic
if rbyd and ( if rbyd and (
not rbyds[i] not rbyds[i]
or not ((rbyd.rev - rbyds[i].rev) & 0x80000000) or not ((rbyd.rev - rbyds[i].rev) & 0x80000000)
or (rbyd.rev == rbyds[i].rev or (rbyd.rev == rbyds[i].rev
and rbyd.trunk > rbyds[i].trunk)): and rbyd.trunk > rbyds[i].trunk)):
i = i_ i = i_
# keep track of the other blocks # keep track of the other blocks
rbyd = rbyds[i] rbyd = rbyds[i]
rbyd.redund_blocks = [rbyds[(i+1+j) % len(rbyds)].block rbyd.redund_blocks = [
for j in range(len(rbyds)-1)] rbyds[(i+1+j) % len(rbyds)].block
for j in range(len(rbyds)-1)]
return rbyd return rbyd
else: else:
# block may encode a trunk # block may encode a trunk
@@ -457,7 +459,9 @@ class Rbyd:
done = not tag_ or (rid_, tag_) < (rid, tag) done = not tag_ or (rid_, tag_) < (rid, tag)
return done, rid_, tag_, w_, j, d, self.data[j+d:j+d+jump], path return (done, rid_, tag_, w_, j, d,
self.data[j+d:j+d+jump],
path)
def __bool__(self): def __bool__(self):
return bool(self.trunk) return bool(self.trunk)
@@ -549,8 +553,8 @@ class Rbyd:
else: else:
if 'h' not in alts[j_]: if 'h' not in alts[j_]:
alts[j_]['h'] = max( alts[j_]['h'] = max(
rec_height(alts[j_]['f']), rec_height(alts[j_]['f']),
rec_height(alts[j_]['nf'])) + 1 rec_height(alts[j_]['nf'])) + 1
return alts[j_]['h'] return alts[j_]['h']
for j_ in alts.keys(): for j_ in alts.keys():
@@ -614,10 +618,10 @@ def main(disk, roots=None, *,
# fetch the root # fetch the root
btree = Rbyd.fetch(f, block_size, roots, trunk) btree = Rbyd.fetch(f, block_size, roots, trunk)
print('btree %s w%d, rev %08x, cksum %08x' % ( print('btree %s w%d, rev %08x, cksum %08x' % (
btree.addr(), btree.addr(),
btree.weight, btree.weight,
btree.rev, btree.rev,
btree.cksum)) btree.cksum))
# look up a bid, while keeping track of the search path # look up a bid, while keeping track of the search path
def btree_lookup(bid, *, def btree_lookup(bid, *,
@@ -642,7 +646,7 @@ def main(disk, roots=None, *,
w = 0 w = 0
for i in it.count(): for i in it.count():
done, rid__, tag, w_, j, d, data, _ = rbyd.lookup( done, rid__, tag, w_, j, d, data, _ = rbyd.lookup(
rid_, tag+0x1) rid_, tag+0x1)
if done or (i != 0 and rid__ != rid_): if done or (i != 0 and rid__ != rid_):
break break
@@ -683,7 +687,7 @@ def main(disk, roots=None, *,
bid = -1 bid = -1
while True: while True:
done, bid, w, rbyd, rid, tags, path = btree_lookup( done, bid, w, rbyd, rid, tags, path = btree_lookup(
bid+1, depth=args.get('depth')) bid+1, depth=args.get('depth'))
if done: if done:
break break
@@ -698,7 +702,7 @@ def main(disk, roots=None, *,
bid = -1 bid = -1
while True: while True:
done, bid, w, rbyd, rid, tags, path = btree_lookup( done, bid, w, rbyd, rid, tags, path = btree_lookup(
bid+1, depth=args.get('depth')) bid+1, depth=args.get('depth'))
if done: if done:
break break
@@ -731,8 +735,8 @@ def main(disk, roots=None, *,
# connect our branch to the rbyd's root # connect our branch to the rbyd's root
if leaf is not None: if leaf is not None:
root = min(rtree, root = min(rtree,
key=lambda branch: branch.d, key=lambda branch: branch.d,
default=None) default=None)
if root is not None: if root is not None:
r_rid, r_tag = root.a r_rid, r_tag = root.a
@@ -758,9 +762,10 @@ def main(disk, roots=None, *,
d_ += max(bdepths.get(d, 0), 1) d_ += max(bdepths.get(d, 0), 1)
leaf = (bid-(w-1), d, rid-(w-1), leaf = (bid-(w-1), d, rid-(w-1),
next((tag for tag, _, _, _ in tags next(
if tag & 0xfff == TAG_BRANCH), (tag for tag, _, _, _ in tags
TAG_BRANCH)) if tag & 0xfff == TAG_BRANCH),
TAG_BRANCH))
# remap branches to leaves if we aren't showing inner branches # remap branches to leaves if we aren't showing inner branches
if not args.get('inner'): if not args.get('inner'):
@@ -813,7 +818,7 @@ def main(disk, roots=None, *,
bid = -1 bid = -1
while True: while True:
done, bid, w, rbyd, rid, tags, path = btree_lookup( done, bid, w, rbyd, rid, tags, path = btree_lookup(
bid+1, depth=args.get('depth')) bid+1, depth=args.get('depth'))
if done: if done:
break break
@@ -836,7 +841,7 @@ def main(disk, roots=None, *,
continue continue
b = (bid-(w-1), d, rid-(w-1), b = (bid-(w-1), d, rid-(w-1),
(name if name else tags[0])[0]) (name if name else tags[0])[0])
# remap branches to leaves if we aren't showing # remap branches to leaves if we aren't showing
# inner branches # inner branches
@@ -846,8 +851,8 @@ def main(disk, roots=None, *,
if not tags: if not tags:
continue continue
branches[b] = ( branches[b] = (
bid-(w-1), len(path)-1, rid-(w-1), bid-(w-1), len(path)-1, rid-(w-1),
(name if name else tags[0])[0]) (name if name else tags[0])[0])
b = branches[b] b = branches[b]
# found entry point? # found entry point?
@@ -905,16 +910,16 @@ def main(disk, roots=None, *,
was = None was = None
for d in range(t_depth): for d in range(t_depth):
t, c, was = branchrepr( t, c, was = branchrepr(
(bid-(w-1), bd, rid-(w-1), tag), d, was) (bid-(w-1), bd, rid-(w-1), tag), d, was)
trunk.append('%s%s%s%s' % ( trunk.append('%s%s%s%s' % (
'\x1b[33m' if color and c == 'y' '\x1b[33m' if color and c == 'y'
else '\x1b[31m' if color and c == 'r' else '\x1b[31m' if color and c == 'r'
else '\x1b[90m' if color and c == 'b' else '\x1b[90m' if color and c == 'b'
else '', else '',
t, t,
('>' if was else ' ') if d == t_depth-1 else '', ('>' if was else ' ') if d == t_depth-1 else '',
'\x1b[m' if color and c else '')) '\x1b[m' if color and c else ''))
return '%s ' % ''.join(trunk) return '%s ' % ''.join(trunk)
@@ -931,39 +936,41 @@ def main(disk, roots=None, *,
# show human-readable representation # show human-readable representation
for i, (tag, j, d, data) in enumerate(tags): for i, (tag, j, d, data) in enumerate(tags):
print('%10s %s%*s %-*s %s' % ( print('%10s %s%*s %-*s %s' % (
'%04x.%04x:' % (rbyd.block, rbyd.trunk) '%04x.%04x:' % (rbyd.block, rbyd.trunk)
if prbyd is None or rbyd != prbyd if prbyd is None or rbyd != prbyd
else '', else '',
treerepr(bid, w, bd, rid, tag) treerepr(bid, w, bd, rid, tag)
if args.get('tree') if args.get('tree')
or args.get('rbyd') or args.get('rbyd')
or args.get('btree') else '', or args.get('btree')
2*w_width+1, '' if i != 0 else '',
else '%d-%d' % (bid-(w-1), bid) if w > 1 2*w_width+1, '' if i != 0
else bid if w > 0 else '%d-%d' % (bid-(w-1), bid) if w > 1
else '', else bid if w > 0
21+w_width, tagrepr( else '',
tag, w if i == 0 else 0, len(data), None), 21+w_width, tagrepr(
next(xxd(data, 8), '') tag, w if i == 0 else 0, len(data), None),
if not args.get('raw') and not args.get('no_truncate') next(xxd(data, 8), '')
else '')) if not args.get('raw')
and not args.get('no_truncate')
else ''))
prbyd = rbyd prbyd = rbyd
# show on-disk encoding of tags/data # show on-disk encoding of tags/data
if args.get('raw'): if args.get('raw'):
for o, line in enumerate(xxd(rbyd.data[j:j+d])): for o, line in enumerate(xxd(rbyd.data[j:j+d])):
print('%9s: %*s%*s %s' % ( print('%9s: %*s%*s %s' % (
'%04x' % (j + o*16), '%04x' % (j + o*16),
t_width, '', t_width, '',
2*w_width+1, '', 2*w_width+1, '',
line)) line))
if args.get('raw') or args.get('no_truncate'): if args.get('raw') or args.get('no_truncate'):
for o, line in enumerate(xxd(data)): for o, line in enumerate(xxd(data)):
print('%9s: %*s%*s %s' % ( print('%9s: %*s%*s %s' % (
'%04x' % (j+d + o*16), '%04x' % (j+d + o*16),
t_width, '', t_width, '',
2*w_width+1, '', 2*w_width+1, '',
line)) line))
# traverse and print entries # traverse and print entries
@@ -973,7 +980,7 @@ def main(disk, roots=None, *,
corrupted = False corrupted = False
while True: while True:
done, bid, w, rbyd, rid, tags, path = btree_lookup( done, bid, w, rbyd, rid, tags, path = btree_lookup(
bid+1, depth=args.get('depth')) bid+1, depth=args.get('depth'))
if done: if done:
break break
@@ -997,11 +1004,11 @@ def main(disk, roots=None, *,
# corrupted? try to keep printing the tree # corrupted? try to keep printing the tree
if not rbyd: if not rbyd:
print('%04x.%04x: %*s%s%s%s' % ( print('%04x.%04x: %*s%s%s%s' % (
rbyd.block, rbyd.trunk, rbyd.block, rbyd.trunk,
t_width, '', t_width, '',
'\x1b[31m' if color else '', '\x1b[31m' if color else '',
'(corrupted rbyd %s)' % rbyd.addr(), '(corrupted rbyd %s)' % rbyd.addr(),
'\x1b[m' if color else '')) '\x1b[m' if color else ''))
prbyd = rbyd prbyd = rbyd
corrupted = True corrupted = True
continue continue
@@ -1020,8 +1027,8 @@ def main(disk, roots=None, *,
if name is not None: if name is not None:
tags = [name] + [(tag, j, d, data) tags = [name] + [(tag, j, d, data)
for tag, j, d, data in tags for tag, j, d, data in tags
if tag & 0x7f00 != TAG_NAME] if tag & 0x7f00 != TAG_NAME]
# show the branch # show the branch
dbg_branch(bid, w, rbyd, rid, tags, len(path)-1) dbg_branch(bid, w, rbyd, rid, tags, len(path)-1)
@@ -1034,67 +1041,67 @@ if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Debug rbyd B-trees.", description="Debug rbyd B-trees.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'disk', 'disk',
help="File containing the block device.") help="File containing the block device.")
parser.add_argument( parser.add_argument(
'roots', 'roots',
nargs='*', nargs='*',
type=rbydaddr, type=rbydaddr,
help="Block address of the roots of the tree.") help="Block address of the roots of the tree.")
parser.add_argument( parser.add_argument(
'-b', '--block-size', '-b', '--block-size',
type=bdgeom, type=bdgeom,
help="Block size/geometry in bytes.") help="Block size/geometry in bytes.")
parser.add_argument( parser.add_argument(
'--block-count', '--block-count',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Block count in blocks.") help="Block count in blocks.")
parser.add_argument( parser.add_argument(
'--trunk', '--trunk',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Use this offset as the trunk of the tree.") help="Use this offset as the trunk of the tree.")
parser.add_argument( parser.add_argument(
'--color', '--color',
choices=['never', 'always', 'auto'], choices=['never', 'always', 'auto'],
default='auto', default='auto',
help="When to use terminal colors. Defaults to 'auto'.") help="When to use terminal colors. Defaults to 'auto'.")
parser.add_argument( parser.add_argument(
'-r', '--raw', '-r', '--raw',
action='store_true', action='store_true',
help="Show the raw data including tag encodings.") help="Show the raw data including tag encodings.")
parser.add_argument( parser.add_argument(
'-T', '--no-truncate', '-T', '--no-truncate',
action='store_true', action='store_true',
help="Don't truncate, show the full contents.") help="Don't truncate, show the full contents.")
parser.add_argument( parser.add_argument(
'-t', '--tree', '-t', '--tree',
action='store_true', action='store_true',
help="Show the underlying rbyd trees.") help="Show the underlying rbyd trees.")
parser.add_argument( parser.add_argument(
'-B', '--btree', '-B', '--btree',
action='store_true', action='store_true',
help="Show the B-tree.") help="Show the B-tree.")
parser.add_argument( parser.add_argument(
'-R', '--rbyd', '-R', '--rbyd',
action='store_true', action='store_true',
help="Show the full underlying rbyd trees.") help="Show the full underlying rbyd trees.")
parser.add_argument( parser.add_argument(
'-i', '--inner', '-i', '--inner',
action='store_true', action='store_true',
help="Show inner branches.") help="Show inner branches.")
parser.add_argument( parser.add_argument(
'-z', '--depth', '-z', '--depth',
nargs='?', nargs='?',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
const=0, const=0,
help="Depth of tree to show.") help="Depth of tree to show.")
parser.add_argument( parser.add_argument(
'-e', '--error-on-corrupt', '-e', '--error-on-corrupt',
action='store_true', action='store_true',
help="Error if B-tree is corrupt.") help="Error if B-tree is corrupt.")
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/dbgcat.py
+48 -45
View File
@@ -66,12 +66,12 @@ def rbydaddr(s):
def xxd(data, width=16): def xxd(data, width=16):
for i in range(0, len(data), width): for i in range(0, len(data), width):
yield '%-*s %-*s' % ( yield '%-*s %-*s' % (
3*width, 3*width,
' '.join('%02x' % b for b in data[i:i+width]), ' '.join('%02x' % b for b in data[i:i+width]),
width, width,
''.join( ''.join(
b if b >= ' ' and b <= '~' else '.' b if b >= ' ' and b <= '~' else '.'
for b in map(chr, data[i:i+width]))) for b in map(chr, data[i:i+width])))
def crc32c(data, crc=0): def crc32c(data, crc=0):
crc ^= 0xffffffff crc ^= 0xffffffff
@@ -105,19 +105,21 @@ def main(disk, blocks=None, *,
# blocks may also encode offsets # blocks may also encode offsets
blocks, offs, size = ( blocks, offs, size = (
[block[0] if isinstance(block, tuple) else block [block[0] if isinstance(block, tuple) else block
for block in blocks], for block in blocks],
[off[0] if isinstance(off, tuple) [off[0] if isinstance(off, tuple)
else off if off is not None else off if off is not None
else size[0] if isinstance(size, tuple) and len(size) > 1 else size[0]
else block[1] if isinstance(block, tuple) if isinstance(size, tuple) and len(size) > 1
else None else block[1] if isinstance(block, tuple)
for block in blocks], else None
size[1] - size[0] if isinstance(size, tuple) and len(size) > 1 for block in blocks],
else size[0] if isinstance(size, tuple) size[1] - size[0] if isinstance(size, tuple) and len(size) > 1
else size if size is not None else size[0] if isinstance(size, tuple)
else off[1] - off[0] if isinstance(off, tuple) and len(off) > 1 else size if size is not None
else block_size) else off[1] - off[0]
if isinstance(off, tuple) and len(off) > 1
else block_size)
# cat the blocks # cat the blocks
for block, off in zip(blocks, offs): for block, off in zip(blocks, offs):
@@ -126,40 +128,41 @@ def main(disk, blocks=None, *,
sys.stdout.buffer.write(data) sys.stdout.buffer.write(data)
sys.stdout.flush() sys.stdout.flush()
if __name__ == "__main__": if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Cat data from a block device.", description="Cat data from a block device.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'disk', 'disk',
help="File containing the block device.") help="File containing the block device.")
parser.add_argument( parser.add_argument(
'blocks', 'blocks',
nargs='*', nargs='*',
type=rbydaddr, type=rbydaddr,
help="Block address.") help="Block address.")
parser.add_argument( parser.add_argument(
'-b', '--block-size', '-b', '--block-size',
type=bdgeom, type=bdgeom,
help="Block size/geometry in bytes.") help="Block size/geometry in bytes.")
parser.add_argument( parser.add_argument(
'--block-count', '--block-count',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Block count in blocks.") help="Block count in blocks.")
parser.add_argument( parser.add_argument(
'--off', '--off',
type=lambda x: tuple( type=lambda x: tuple(
int(x, 0) if x.strip() else None int(x, 0) if x.strip() else None
for x in x.split(',')), for x in x.split(',')),
help="Show a specific offset, may be a range.") help="Show a specific offset, may be a range.")
parser.add_argument( parser.add_argument(
'-n', '--size', '-n', '--size',
type=lambda x: tuple( type=lambda x: tuple(
int(x, 0) if x.strip() else None int(x, 0) if x.strip() else None
for x in x.split(',')), for x in x.split(',')),
help="Show this many bytes, may be a range.") help="Show this many bytes, may be a range.")
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/dbgerr.py
+14 -13
View File
@@ -37,9 +37,9 @@ def main(errs, *,
# print # print
for n, e, h in ERRS: for n, e, h in ERRS:
print('%-*s %-*s %s' % ( print('%-*s %-*s %s' % (
w[0], 'LFS_ERR_'+n, w[0], 'LFS_ERR_'+n,
w[1], e, w[1], e,
h)) h))
# find these errors # find these errors
else: else:
@@ -77,20 +77,21 @@ def main(errs, *,
except KeyError: except KeyError:
print('%s ?' % err) print('%s ?' % err)
if __name__ == "__main__": if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Decode littlefs error codes.", description="Decode littlefs error codes.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'errs', 'errs',
nargs='*', nargs='*',
help="Error codes or error names to decode.") help="Error codes or error names to decode.")
parser.add_argument( parser.add_argument(
'-l', '--list', '-l', '--list',
action='store_true', action='store_true',
help="List all known error codes.") help="List all known error codes.")
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/dbglfs.py
+483 -459
View File
File diff suppressed because it is too large Load Diff
scripts/dbgmtree.py
+313 -293
View File
@@ -169,108 +169,108 @@ def frombtree(data):
def xxd(data, width=16): def xxd(data, width=16):
for i in range(0, len(data), width): for i in range(0, len(data), width):
yield '%-*s %-*s' % ( yield '%-*s %-*s' % (
3*width, 3*width,
' '.join('%02x' % b for b in data[i:i+width]), ' '.join('%02x' % b for b in data[i:i+width]),
width, width,
''.join( ''.join(
b if b >= ' ' and b <= '~' else '.' b if b >= ' ' and b <= '~' else '.'
for b in map(chr, data[i:i+width]))) for b in map(chr, data[i:i+width])))
def tagrepr(tag, w=None, size=None, off=None): def tagrepr(tag, w=None, size=None, off=None):
if (tag & 0x6fff) == TAG_NULL: if (tag & 0x6fff) == TAG_NULL:
return '%snull%s%s' % ( return '%snull%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if size else '') ' %d' % size if size else '')
elif (tag & 0x6f00) == TAG_CONFIG: elif (tag & 0x6f00) == TAG_CONFIG:
return '%s%s%s%s' % ( return '%s%s%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
'magic' if (tag & 0xfff) == TAG_MAGIC 'magic' if (tag & 0xfff) == TAG_MAGIC
else 'version' if (tag & 0xfff) == TAG_VERSION else 'version' if (tag & 0xfff) == TAG_VERSION
else 'rcompat' if (tag & 0xfff) == TAG_RCOMPAT else 'rcompat' if (tag & 0xfff) == TAG_RCOMPAT
else 'wcompat' if (tag & 0xfff) == TAG_WCOMPAT else 'wcompat' if (tag & 0xfff) == TAG_WCOMPAT
else 'ocompat' if (tag & 0xfff) == TAG_OCOMPAT else 'ocompat' if (tag & 0xfff) == TAG_OCOMPAT
else 'geometry' if (tag & 0xfff) == TAG_GEOMETRY else 'geometry' if (tag & 0xfff) == TAG_GEOMETRY
else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT
else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT
else 'config 0x%02x' % (tag & 0xff), else 'config 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x6f00) == TAG_GDELTA: elif (tag & 0x6f00) == TAG_GDELTA:
return '%s%s%s%s' % ( return '%s%s%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA
else 'gdelta 0x%02x' % (tag & 0xff), else 'gdelta 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x6f00) == TAG_NAME: elif (tag & 0x6f00) == TAG_NAME:
return '%s%s%s%s' % ( return '%s%s%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
'name' if (tag & 0xfff) == TAG_NAME 'name' if (tag & 0xfff) == TAG_NAME
else 'reg' if (tag & 0xfff) == TAG_REG else 'reg' if (tag & 0xfff) == TAG_REG
else 'dir' if (tag & 0xfff) == TAG_DIR else 'dir' if (tag & 0xfff) == TAG_DIR
else 'orphan' if (tag & 0xfff) == TAG_ORPHAN else 'orphan' if (tag & 0xfff) == TAG_ORPHAN
else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK
else 'name 0x%02x' % (tag & 0xff), else 'name 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x6f00) == TAG_STRUCT: elif (tag & 0x6f00) == TAG_STRUCT:
return '%s%s%s%s' % ( return '%s%s%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
'data' if (tag & 0xfff) == TAG_DATA 'data' if (tag & 0xfff) == TAG_DATA
else 'block' if (tag & 0xfff) == TAG_BLOCK else 'block' if (tag & 0xfff) == TAG_BLOCK
else 'bshrub' if (tag & 0xfff) == TAG_BSHRUB else 'bshrub' if (tag & 0xfff) == TAG_BSHRUB
else 'btree' if (tag & 0xfff) == TAG_BTREE else 'btree' if (tag & 0xfff) == TAG_BTREE
else 'mroot' if (tag & 0xfff) == TAG_MROOT else 'mroot' if (tag & 0xfff) == TAG_MROOT
else 'mdir' if (tag & 0xfff) == TAG_MDIR else 'mdir' if (tag & 0xfff) == TAG_MDIR
else 'mtree' if (tag & 0xfff) == TAG_MTREE else 'mtree' if (tag & 0xfff) == TAG_MTREE
else 'did' if (tag & 0xfff) == TAG_DID else 'did' if (tag & 0xfff) == TAG_DID
else 'branch' if (tag & 0xfff) == TAG_BRANCH else 'branch' if (tag & 0xfff) == TAG_BRANCH
else 'struct 0x%02x' % (tag & 0xff), else 'struct 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x6e00) == TAG_ATTR: elif (tag & 0x6e00) == TAG_ATTR:
return '%s%sattr 0x%02x%s%s' % ( return '%s%sattr 0x%02x%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
's' if tag & 0x100 else 'u', 's' if tag & 0x100 else 'u',
((tag & 0x100) >> 1) ^ (tag & 0xff), ((tag & 0x100) >> 1) ^ (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif tag & TAG_ALT: elif tag & TAG_ALT:
return 'alt%s%s%s%s%s' % ( return 'alt%s%s%s%s%s' % (
'r' if tag & TAG_R else 'b', 'r' if tag & TAG_R else 'b',
'a' if tag & 0x0fff == 0 and tag & TAG_GT 'a' if tag & 0x0fff == 0 and tag & TAG_GT
else 'n' if tag & 0x0fff == 0 else 'n' if tag & 0x0fff == 0
else 'gt' if tag & TAG_GT else 'gt' if tag & TAG_GT
else 'le', else 'le',
' 0x%x' % (tag & 0x0fff) if tag & 0x0fff != 0 else '', ' 0x%x' % (tag & 0x0fff) if tag & 0x0fff != 0 else '',
' w%d' % w if w is not None else '', ' w%d' % w if w is not None else '',
' 0x%x' % (0xffffffff & (off-size)) ' 0x%x' % (0xffffffff & (off-size))
if size and off is not None if size and off is not None
else ' -%d' % size if size else ' -%d' % size if size
else '') else '')
elif (tag & 0x7f00) == TAG_CKSUM: elif (tag & 0x7f00) == TAG_CKSUM:
return 'cksum%s%s%s%s%s' % ( return 'cksum%s%s%s%s%s' % (
'q' if not tag & 0xfc and tag & TAG_Q else '', 'q' if not tag & 0xfc and tag & TAG_Q else '',
'p' if not tag & 0xfc and tag & TAG_P else '', 'p' if not tag & 0xfc and tag & TAG_P else '',
' 0x%02x' % (tag & 0xff) if tag & 0xfc else '', ' 0x%02x' % (tag & 0xff) if tag & 0xfc else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x7f00) == TAG_NOTE: elif (tag & 0x7f00) == TAG_NOTE:
return 'note%s%s%s' % ( return 'note%s%s%s' % (
' 0x%02x' % (tag & 0xff) if tag & 0xff else '', ' 0x%02x' % (tag & 0xff) if tag & 0xff else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x7f00) == TAG_ECKSUM: elif (tag & 0x7f00) == TAG_ECKSUM:
return 'ecksum%s%s%s' % ( return 'ecksum%s%s%s' % (
' 0x%02x' % (tag & 0xff) if tag & 0xff else '', ' 0x%02x' % (tag & 0xff) if tag & 0xff else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
else: else:
return '0x%04x%s%s' % ( return '0x%04x%s%s' % (
tag, tag,
' w%d' % w if w is not None else '', ' w%d' % w if w is not None else '',
' %d' % size if size is not None else '') ' %d' % size if size is not None else '')
# this type is used for tree representations # this type is used for tree representations
@@ -293,9 +293,9 @@ class Rbyd:
return '0x%x.%x' % (self.block, self.trunk) return '0x%x.%x' % (self.block, self.trunk)
else: else:
return '0x{%x,%s}.%x' % ( return '0x{%x,%s}.%x' % (
self.block, self.block,
','.join('%x' % block for block in self.redund_blocks), ','.join('%x' % block for block in self.redund_blocks),
self.trunk) self.trunk)
@classmethod @classmethod
def fetch(cls, f, block_size, blocks, trunk=None): def fetch(cls, f, block_size, blocks, trunk=None):
@@ -304,21 +304,23 @@ class Rbyd:
if len(blocks) > 1: if len(blocks) > 1:
# fetch all blocks # fetch all blocks
rbyds = [cls.fetch(f, block_size, block, trunk) for block in blocks] rbyds = [cls.fetch(f, block_size, block, trunk)
for block in blocks]
# determine most recent revision # determine most recent revision
i = 0 i = 0
for i_, rbyd in enumerate(rbyds): for i_, rbyd in enumerate(rbyds):
# compare with sequence arithmetic # compare with sequence arithmetic
if rbyd and ( if rbyd and (
not rbyds[i] not rbyds[i]
or not ((rbyd.rev - rbyds[i].rev) & 0x80000000) or not ((rbyd.rev - rbyds[i].rev) & 0x80000000)
or (rbyd.rev == rbyds[i].rev or (rbyd.rev == rbyds[i].rev
and rbyd.trunk > rbyds[i].trunk)): and rbyd.trunk > rbyds[i].trunk)):
i = i_ i = i_
# keep track of the other blocks # keep track of the other blocks
rbyd = rbyds[i] rbyd = rbyds[i]
rbyd.redund_blocks = [rbyds[(i+1+j) % len(rbyds)].block rbyd.redund_blocks = [
for j in range(len(rbyds)-1)] rbyds[(i+1+j) % len(rbyds)].block
for j in range(len(rbyds)-1)]
return rbyd return rbyd
else: else:
# block may encode a trunk # block may encode a trunk
@@ -472,7 +474,9 @@ class Rbyd:
done = not tag_ or (rid_, tag_) < (rid, tag) done = not tag_ or (rid_, tag_) < (rid, tag)
return done, rid_, tag_, w_, j, d, self.data[j+d:j+d+jump], path return (done, rid_, tag_, w_, j, d,
self.data[j+d:j+d+jump],
path)
def __bool__(self): def __bool__(self):
return bool(self.trunk) return bool(self.trunk)
@@ -564,8 +568,8 @@ class Rbyd:
else: else:
if 'h' not in alts[j_]: if 'h' not in alts[j_]:
alts[j_]['h'] = max( alts[j_]['h'] = max(
rec_height(alts[j_]['f']), rec_height(alts[j_]['f']),
rec_height(alts[j_]['nf'])) + 1 rec_height(alts[j_]['nf'])) + 1
return alts[j_]['h'] return alts[j_]['h']
for j_ in alts.keys(): for j_ in alts.keys():
@@ -616,7 +620,7 @@ class Rbyd:
w = 0 w = 0
for i in it.count(): for i in it.count():
done, rid__, tag, w_, j, d, data, _ = rbyd.lookup( done, rid__, tag, w_, j, d, data, _ = rbyd.lookup(
rid_, tag+0x1) rid_, tag+0x1)
if done or (i != 0 and rid__ != rid_): if done or (i != 0 and rid__ != rid_):
break break
@@ -659,7 +663,7 @@ class Rbyd:
bid = -1 bid = -1
while True: while True:
done, bid, w, rbyd_, rid, tags, path = self.btree_lookup( done, bid, w, rbyd_, rid, tags, path = self.btree_lookup(
f, block_size, bid+1, depth=depth) f, block_size, bid+1, depth=depth)
if done: if done:
break break
@@ -674,7 +678,7 @@ class Rbyd:
bid = -1 bid = -1
while True: while True:
done, bid, w, rbyd_, rid, tags, path = self.btree_lookup( done, bid, w, rbyd_, rid, tags, path = self.btree_lookup(
f, block_size, bid+1, depth=depth) f, block_size, bid+1, depth=depth)
if done: if done:
break break
@@ -707,8 +711,8 @@ class Rbyd:
# connect our branch to the rbyd's root # connect our branch to the rbyd's root
if leaf is not None: if leaf is not None:
root = min(rtree, root = min(rtree,
key=lambda branch: branch.d, key=lambda branch: branch.d,
default=None) default=None)
if root is not None: if root is not None:
r_rid, r_tag = root.a r_rid, r_tag = root.a
@@ -734,9 +738,10 @@ class Rbyd:
d_ += max(bdepths.get(d, 0), 1) d_ += max(bdepths.get(d, 0), 1)
leaf = (bid-(w-1), d, rid-(w-1), leaf = (bid-(w-1), d, rid-(w-1),
next((tag for tag, _, _, _ in tags next(
if tag & 0xfff == TAG_BRANCH), (tag for tag, _, _, _ in tags
TAG_BRANCH)) if tag & 0xfff == TAG_BRANCH),
TAG_BRANCH))
# remap branches to leaves if we aren't showing inner branches # remap branches to leaves if we aren't showing inner branches
if not inner: if not inner:
@@ -793,7 +798,7 @@ class Rbyd:
bid = -1 bid = -1
while True: while True:
done, bid, w, rbyd, rid, tags, path = self.btree_lookup( done, bid, w, rbyd, rid, tags, path = self.btree_lookup(
f, block_size, bid+1, depth=depth) f, block_size, bid+1, depth=depth)
if done: if done:
break break
@@ -816,7 +821,7 @@ class Rbyd:
continue continue
b = (bid-(w-1), d, rid-(w-1), b = (bid-(w-1), d, rid-(w-1),
(name if name else tags[0])[0]) (name if name else tags[0])[0])
# remap branches to leaves if we aren't showing # remap branches to leaves if we aren't showing
# inner branches # inner branches
@@ -826,8 +831,8 @@ class Rbyd:
if not tags: if not tags:
continue continue
branches[b] = ( branches[b] = (
bid-(w-1), len(path)-1, rid-(w-1), bid-(w-1), len(path)-1, rid-(w-1),
(name if name else tags[0])[0]) (name if name else tags[0])[0])
b = branches[b] b = branches[b]
# found entry point? # found entry point?
@@ -935,8 +940,8 @@ def main(disk, mroots=None, *,
mbid = -1 mbid = -1
while True: while True:
done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup( done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup(
f, block_size, mbid+1, f, block_size, mbid+1,
depth=args.get('depth', mdepth)-mdepth) depth=args.get('depth', mdepth)-mdepth)
if done: if done:
break break
@@ -947,10 +952,11 @@ def main(disk, mroots=None, *,
mdir__ = None mdir__ = None
if (not args.get('depth') if (not args.get('depth')
or mdepth+len(path) < args.get('depth')): or mdepth+len(path) < args.get('depth')):
mdir__ = next(((tag, j, d, data) mdir__ = next(
for tag, j, d, data in tags ((tag, j, d, data)
if tag == TAG_MDIR), for tag, j, d, data in tags
None) if tag == TAG_MDIR),
None)
if mdir__: if mdir__:
# fetch the mdir # fetch the mdir
@@ -981,8 +987,8 @@ def main(disk, mroots=None, *,
# connect branch to our root # connect branch to our root
if d > 0: if d > 0:
root = min(rtree, root = min(rtree,
key=lambda branch: branch.d, key=lambda branch: branch.d,
default=None) default=None)
if root: if root:
r_rid, r_tag = root.a r_rid, r_tag = root.a
@@ -1026,8 +1032,8 @@ def main(disk, mroots=None, *,
# connect branch to our root # connect branch to our root
root = min(rtree, root = min(rtree,
key=lambda branch: branch.d, key=lambda branch: branch.d,
default=None) default=None)
if root: if root:
r_rid, r_tag = root.a r_rid, r_tag = root.a
@@ -1054,10 +1060,10 @@ def main(disk, mroots=None, *,
# compute the mtree's rbyd-tree if there is one # compute the mtree's rbyd-tree if there is one
if mtree: if mtree:
tree_, tdepth = mtree.btree_tree( tree_, tdepth = mtree.btree_tree(
f, block_size, f, block_size,
depth=args.get('depth', mdepth)-mdepth, depth=args.get('depth', mdepth)-mdepth,
inner=args.get('inner'), inner=args.get('inner'),
rbyd=args.get('rbyd')) rbyd=args.get('rbyd'))
# connect a branch to the root of the tree # connect a branch to the root of the tree
root = min(tree_, key=lambda branch: branch.d, default=None) root = min(tree_, key=lambda branch: branch.d, default=None)
@@ -1086,8 +1092,8 @@ def main(disk, mroots=None, *,
mbid = -1 mbid = -1
while True: while True:
done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup( done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup(
f, block_size, mbid+1, f, block_size, mbid+1,
depth=args.get('depth', mdepth)-mdepth) depth=args.get('depth', mdepth)-mdepth)
if done: if done:
break break
@@ -1098,10 +1104,11 @@ def main(disk, mroots=None, *,
mdir__ = None mdir__ = None
if (not args.get('depth') if (not args.get('depth')
or mdepth+len(path) < args.get('depth')): or mdepth+len(path) < args.get('depth')):
mdir__ = next(((tag, j, d, data) mdir__ = next(
for tag, j, d, data in tags ((tag, j, d, data)
if tag == TAG_MDIR), for tag, j, d, data in tags
None) if tag == TAG_MDIR),
None)
if mdir__: if mdir__:
# fetch the mdir # fetch the mdir
@@ -1116,8 +1123,8 @@ def main(disk, mroots=None, *,
mbid = -1 mbid = -1
while True: while True:
done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup( done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup(
f, block_size, mbid+1, f, block_size, mbid+1,
depth=args.get('depth', mdepth)-mdepth) depth=args.get('depth', mdepth)-mdepth)
if done: if done:
break break
@@ -1128,10 +1135,11 @@ def main(disk, mroots=None, *,
mdir__ = None mdir__ = None
if (not args.get('depth') if (not args.get('depth')
or mdepth+len(path) < args.get('depth')): or mdepth+len(path) < args.get('depth')):
mdir__ = next(((tag, j, d, data) mdir__ = next(
for tag, j, d, data in tags ((tag, j, d, data)
if tag == TAG_MDIR), for tag, j, d, data in tags
None) if tag == TAG_MDIR),
None)
if mdir__: if mdir__:
# fetch the mdir # fetch the mdir
@@ -1143,19 +1151,19 @@ def main(disk, mroots=None, *,
# connect the root to the mtree # connect the root to the mtree
branch = max( branch = max(
(branch for branch in tree (branch for branch in tree
if branch.b[0] == mbid-(mw-1)), if branch.b[0] == mbid-(mw-1)),
key=lambda branch: branch.d,
default=None)
if branch:
root = min(rtree,
key=lambda branch: branch.d, key=lambda branch: branch.d,
default=None) default=None)
if branch:
root = min(rtree,
key=lambda branch: branch.d,
default=None)
if root: if root:
r_rid, r_tag = root.a r_rid, r_tag = root.a
else: else:
_, r_rid, r_tag, _, _, _, _, _ = ( _, r_rid, r_tag, _, _, _, _, _ = (
mdir_.lookup(-1, 0x1)) mdir_.lookup(-1, 0x1))
tree.add(TBranch( tree.add(TBranch(
a=branch.b, a=branch.b,
b=(mbid-(mw-1), len(path), 0, r_rid, r_tag), b=(mbid-(mw-1), len(path), 0, r_rid, r_tag),
@@ -1183,7 +1191,7 @@ def main(disk, mroots=None, *,
# keep track of the original bids, unfortunately because we # keep track of the original bids, unfortunately because we
# store the bids in the branches we overwrite these # store the bids in the branches we overwrite these
tree = {(branch.b[0] - branch.b[2], branch) tree = {(branch.b[0] - branch.b[2], branch)
for branch in tree} for branch in tree}
for bd in reversed(range(b_depth-1)): for bd in reversed(range(b_depth-1)):
# find leaf-roots at this level # find leaf-roots at this level
@@ -1274,9 +1282,9 @@ def main(disk, mroots=None, *,
# compute the mtree's B-tree if there is one # compute the mtree's B-tree if there is one
if mtree: if mtree:
tree_, tdepth = mtree.btree_btree( tree_, tdepth = mtree.btree_btree(
f, block_size, f, block_size,
depth=args.get('depth', mdepth)-mdepth, depth=args.get('depth', mdepth)-mdepth,
inner=args.get('inner')) inner=args.get('inner'))
# connect a branch to the root of the tree # connect a branch to the root of the tree
root = min(tree_, key=lambda branch: branch.d, default=None) root = min(tree_, key=lambda branch: branch.d, default=None)
@@ -1306,8 +1314,8 @@ def main(disk, mroots=None, *,
while True: while True:
done, mbid, mw, rbyd, rid, tags, path = ( done, mbid, mw, rbyd, rid, tags, path = (
mtree.btree_lookup( mtree.btree_lookup(
f, block_size, mbid+1, f, block_size, mbid+1,
depth=args.get('depth', mdepth)-mdepth)) depth=args.get('depth', mdepth)-mdepth))
if done: if done:
break break
@@ -1318,10 +1326,11 @@ def main(disk, mroots=None, *,
mdir__ = None mdir__ = None
if (not args.get('depth') if (not args.get('depth')
or mdepth+len(path) < args.get('depth')): or mdepth+len(path) < args.get('depth')):
mdir__ = next(((tag, j, d, data) mdir__ = next(
for tag, j, d, data in tags ((tag, j, d, data)
if tag == TAG_MDIR), for tag, j, d, data in tags
None) if tag == TAG_MDIR),
None)
if mdir__: if mdir__:
# fetch the mdir # fetch the mdir
@@ -1332,7 +1341,7 @@ def main(disk, mroots=None, *,
# find the first entry in the mdir, map branches # find the first entry in the mdir, map branches
# to this entry # to this entry
done, rid, tag, _, j, d, data, _ = ( done, rid, tag, _, j, d, data, _ = (
mdir_.lookup(-1, 0x1)) mdir_.lookup(-1, 0x1))
tree_ = set() tree_ = set()
for branch in tree: for branch in tree:
@@ -1373,8 +1382,8 @@ def main(disk, mroots=None, *,
for branch in tree): for branch in tree):
return '+-', branch.c, branch.c return '+-', branch.c, branch.c
elif any(branch.d == d elif any(branch.d == d
and x > min(branch.a, branch.b) and x > min(branch.a, branch.b)
and x < max(branch.a, branch.b) and x < max(branch.a, branch.b)
for branch in tree): for branch in tree):
return '|-', branch.c, branch.c return '|-', branch.c, branch.c
elif branch.a < branch.b: elif branch.a < branch.b:
@@ -1397,17 +1406,18 @@ def main(disk, mroots=None, *,
was = None was = None
for d in range(t_depth): for d in range(t_depth):
t, c, was = branchrepr( t, c, was = branchrepr(
(mbid-max(mw-1, 0), md, mrid-max(mw-1, 0), rid, tag), (mbid-max(mw-1, 0), md,
d, was) mrid-max(mw-1, 0), rid, tag),
d, was)
trunk.append('%s%s%s%s' % ( trunk.append('%s%s%s%s' % (
'\x1b[33m' if color and c == 'y' '\x1b[33m' if color and c == 'y'
else '\x1b[31m' if color and c == 'r' else '\x1b[31m' if color and c == 'r'
else '\x1b[90m' if color and c == 'b' else '\x1b[90m' if color and c == 'b'
else '', else '',
t, t,
('>' if was else ' ') if d == t_depth-1 else '', ('>' if was else ' ') if d == t_depth-1 else '',
'\x1b[m' if color and c else '')) '\x1b[m' if color and c else ''))
return '%s ' % ''.join(trunk) return '%s ' % ''.join(trunk)
@@ -1416,41 +1426,45 @@ def main(disk, mroots=None, *,
for i, (rid, tag, w, j, d, data) in enumerate(mdir): for i, (rid, tag, w, j, d, data) in enumerate(mdir):
# show human-readable tag representation # show human-readable tag representation
print('%12s %s%s' % ( print('%12s %s%s' % (
'{%s}:' % ','.join('%04x' % block '{%s}:' % ','.join('%04x' % block
for block in it.chain([mdir.block], for block in it.chain(
mdir.redund_blocks)) [mdir.block],
if i == 0 else '', mdir.redund_blocks))
treerepr(mbid-max(mw-1, 0), 0, md, 0, rid, tag) if i == 0 else '',
if args.get('tree') treerepr(mbid-max(mw-1, 0), 0, md, 0, rid, tag)
or args.get('rbyd') if args.get('tree')
or args.get('btree') else '', or args.get('rbyd')
'%*s %-*s%s' % ( or args.get('btree')
2*w_width+1, '%d.%d-%d' % ( else '',
mbid//mleaf_weight, rid-(w-1), rid) '%*s %-*s%s' % (
if w > 1 else '%d.%d' % (mbid//mleaf_weight, rid) 2*w_width+1, '%d.%d-%d' % (
if w > 0 or i == 0 else '', mbid//mleaf_weight, rid-(w-1), rid)
21+w_width, tagrepr(tag, w, len(data), j), if w > 1
' %s' % next(xxd(data, 8), '') else '%d.%d' % (mbid//mleaf_weight, rid)
if not args.get('raw') if w > 0 or i == 0
and not args.get('no_truncate') else '',
else ''))) 21+w_width, tagrepr(tag, w, len(data), j),
' %s' % next(xxd(data, 8), '')
if not args.get('raw')
and not args.get('no_truncate')
else '')))
# show on-disk encoding of tags # show on-disk encoding of tags
if args.get('raw'): if args.get('raw'):
for o, line in enumerate(xxd(mdir.data[j:j+d])): for o, line in enumerate(xxd(mdir.data[j:j+d])):
print('%11s: %*s%*s %s' % ( print('%11s: %*s%*s %s' % (
'%04x' % (j + o*16), '%04x' % (j + o*16),
t_width, '', t_width, '',
2*w_width+1, '', 2*w_width+1, '',
line)) line))
if args.get('raw') or args.get('no_truncate'): if args.get('raw') or args.get('no_truncate'):
if not tag & TAG_ALT: if not tag & TAG_ALT:
for o, line in enumerate(xxd(data)): for o, line in enumerate(xxd(data)):
print('%11s: %*s%*s %s' % ( print('%11s: %*s%*s %s' % (
'%04x' % (j+d + o*16), '%04x' % (j+d + o*16),
t_width, '', t_width, '',
2*w_width+1, '', 2*w_width+1, '',
line)) line))
# prbyd here means the last rendered rbyd, we update # prbyd here means the last rendered rbyd, we update
# in dbg_branch to always print interleaved addresses # in dbg_branch to always print interleaved addresses
@@ -1461,59 +1475,61 @@ def main(disk, mroots=None, *,
# show human-readable representation # show human-readable representation
for i, (tag, j, d, data) in enumerate(tags): for i, (tag, j, d, data) in enumerate(tags):
print('%12s %s%*s %-*s %s' % ( print('%12s %s%*s %-*s %s' % (
'%04x.%04x:' % (rbyd.block, rbyd.trunk) '%04x.%04x:' % (rbyd.block, rbyd.trunk)
if prbyd is None or rbyd != prbyd if prbyd is None or rbyd != prbyd
else '', else '',
treerepr(bid, w, bd, rid, 0, tag) treerepr(bid, w, bd, rid, 0, tag)
if args.get('tree') if args.get('tree')
or args.get('rbyd') or args.get('rbyd')
or args.get('btree') else '', or args.get('btree')
2*w_width+1, '' if i != 0 else '',
else '%d-%d' % ( 2*w_width+1, '' if i != 0
else '%d-%d' % (
(bid-(w-1))//mleaf_weight, (bid-(w-1))//mleaf_weight,
bid//mleaf_weight) bid//mleaf_weight)
if (w//mleaf_weight) > 1 if (w//mleaf_weight) > 1
else bid//mleaf_weight if w > 0 else bid//mleaf_weight if w > 0
else '', else '',
21+w_width, tagrepr( 21+w_width, tagrepr(
tag, w if i == 0 else 0, len(data), None), tag, w if i == 0 else 0, len(data), None),
next(xxd(data, 8), '') next(xxd(data, 8), '')
if not args.get('raw') and not args.get('no_truncate') if not args.get('raw')
else '')) and not args.get('no_truncate')
else ''))
prbyd = rbyd prbyd = rbyd
# show on-disk encoding of tags/data # show on-disk encoding of tags/data
if args.get('raw'): if args.get('raw'):
for o, line in enumerate(xxd(rbyd.data[j:j+d])): for o, line in enumerate(xxd(rbyd.data[j:j+d])):
print('%11s: %*s%*s %s' % ( print('%11s: %*s%*s %s' % (
'%04x' % (j + o*16), '%04x' % (j + o*16),
t_width, '', t_width, '',
2*w_width+1, '', 2*w_width+1, '',
line)) line))
if args.get('raw') or args.get('no_truncate'): if args.get('raw') or args.get('no_truncate'):
for o, line in enumerate(xxd(data)): for o, line in enumerate(xxd(data)):
print('%11s: %*s%*s %s' % ( print('%11s: %*s%*s %s' % (
'%04x' % (j+d + o*16), '%04x' % (j+d + o*16),
t_width, '', t_width, '',
2*w_width+1, '', 2*w_width+1, '',
line)) line))
#### actual debugging begins here #### actual debugging begins here
# print some information about the mtree # print some information about the mtree
print('mtree %s w%d.%d, rev %08x, cksum %08x' % ( print('mtree %s w%d.%d, rev %08x, cksum %08x' % (
mroot.addr(), mroot.addr(),
bweight//mleaf_weight, 1*mleaf_weight, bweight//mleaf_weight, 1*mleaf_weight,
mroot.rev, mroot.rev,
mroot.cksum)) mroot.cksum))
# dynamically size the id field # dynamically size the id field
w_width = max( w_width = max(
mt.ceil(mt.log10(max(1, bweight//mleaf_weight)+1)), mt.ceil(mt.log10(max(1, bweight//mleaf_weight)+1)),
mt.ceil(mt.log10(max(1, rweight)+1)), mt.ceil(mt.log10(max(1, rweight)+1)),
# in case of -1.-1 # in case of -1.-1
2) 2)
# show each mroot # show each mroot
prbyd = None prbyd = None
@@ -1525,12 +1541,13 @@ def main(disk, mroots=None, *,
# corrupted? # corrupted?
if not mroot: if not mroot:
print('{%s}: %s%s%s' % ( print('{%s}: %s%s%s' % (
','.join('%04x' % block ','.join('%04x' % block
for block in it.chain([mroot.block], for block in it.chain(
mroot.redund_blocks)), [mroot.block],
'\x1b[31m' if color else '', mroot.redund_blocks)),
'(corrupted mroot %s)' % mroot.addr(), '\x1b[31m' if color else '',
'\x1b[m' if color else '')) '(corrupted mroot %s)' % mroot.addr(),
'\x1b[m' if color else ''))
corrupted = True corrupted = True
break break
else: else:
@@ -1560,12 +1577,13 @@ def main(disk, mroots=None, *,
# corrupted? # corrupted?
if not mdir: if not mdir:
print('{%s}: %s%s%s' % ( print('{%s}: %s%s%s' % (
','.join('%04x' % block ','.join('%04x' % block
for block in it.chain([mdir.block], for block in it.chain(
mdir.redund_blocks)), [mdir.block],
'\x1b[31m' if color else '', mdir.redund_blocks)),
'(corrupted mdir %s)' % mdir.addr(), '\x1b[31m' if color else '',
'\x1b[m' if color else '')) '(corrupted mdir %s)' % mdir.addr(),
'\x1b[m' if color else ''))
corrupted = True corrupted = True
else: else:
# show the mdir # show the mdir
@@ -1582,8 +1600,8 @@ def main(disk, mroots=None, *,
mbid = -1 mbid = -1
while True: while True:
done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup( done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup(
f, block_size, mbid+1, f, block_size, mbid+1,
depth=args.get('depth', mdepth)-mdepth) depth=args.get('depth', mdepth)-mdepth)
if done: if done:
break break
@@ -1607,11 +1625,11 @@ def main(disk, mroots=None, *,
# corrupted? try to keep printing the tree # corrupted? try to keep printing the tree
if not rbyd: if not rbyd:
print('%11s: %*s%s%s%s' % ( print('%11s: %*s%s%s%s' % (
'%04x.%04x' % (rbyd.block, rbyd.trunk), '%04x.%04x' % (rbyd.block, rbyd.trunk),
t_width, '', t_width, '',
'\x1b[31m' if color else '', '\x1b[31m' if color else '',
'(corrupted rbyd %s)' % rbyd.addr(), '(corrupted rbyd %s)' % rbyd.addr(),
'\x1b[m' if color else '')) '\x1b[m' if color else ''))
prbyd = rbyd prbyd = rbyd
corrupted = True corrupted = True
continue continue
@@ -1630,17 +1648,18 @@ def main(disk, mroots=None, *,
if name is not None: if name is not None:
tags = [name] + [(tag, j, d, data) tags = [name] + [(tag, j, d, data)
for tag, j, d, data in tags for tag, j, d, data in tags
if tag & 0x7f00 != TAG_NAME] if tag & 0x7f00 != TAG_NAME]
# found an mdir in the tags? # found an mdir in the tags?
mdir__ = None mdir__ = None
if (not args.get('depth') if (not args.get('depth')
or mdepth+len(path) < args.get('depth')): or mdepth+len(path) < args.get('depth')):
mdir__ = next(((tag, j, d, data) mdir__ = next(
for tag, j, d, data in tags ((tag, j, d, data)
if tag == TAG_MDIR), for tag, j, d, data in tags
None) if tag == TAG_MDIR),
None)
# show other btree entries in certain cases # show other btree entries in certain cases
if args.get('inner') or not mdir__: if args.get('inner') or not mdir__:
@@ -1657,13 +1676,14 @@ def main(disk, mroots=None, *,
# corrupted? # corrupted?
if not mdir_: if not mdir_:
print('{%s}: %*s%s%s%s' % ( print('{%s}: %*s%s%s%s' % (
','.join('%04x' % block ','.join('%04x' % block
for block in it.chain([mdir_.block], for block in it.chain(
mdir_.redund_blocks)), [mdir_.block],
t_width, '', mdir_.redund_blocks)),
'\x1b[31m' if color else '', t_width, '',
'(corrupted mdir %s)' % mdir_.addr(), '\x1b[31m' if color else '',
'\x1b[m' if color else '')) '(corrupted mdir %s)' % mdir_.addr(),
'\x1b[m' if color else ''))
corrupted = True corrupted = True
else: else:
# show the mdir # show the mdir
@@ -1680,63 +1700,63 @@ if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Debug littlefs's metadata tree.", description="Debug littlefs's metadata tree.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'disk', 'disk',
help="File containing the block device.") help="File containing the block device.")
parser.add_argument( parser.add_argument(
'mroots', 'mroots',
nargs='*', nargs='*',
type=rbydaddr, type=rbydaddr,
help="Block address of the mroots. Defaults to 0x{0,1}.") help="Block address of the mroots. Defaults to 0x{0,1}.")
parser.add_argument( parser.add_argument(
'-b', '--block-size', '-b', '--block-size',
type=bdgeom, type=bdgeom,
help="Block size/geometry in bytes.") help="Block size/geometry in bytes.")
parser.add_argument( parser.add_argument(
'--block-count', '--block-count',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Block count in blocks.") help="Block count in blocks.")
parser.add_argument( parser.add_argument(
'--color', '--color',
choices=['never', 'always', 'auto'], choices=['never', 'always', 'auto'],
default='auto', default='auto',
help="When to use terminal colors. Defaults to 'auto'.") help="When to use terminal colors. Defaults to 'auto'.")
parser.add_argument( parser.add_argument(
'-r', '--raw', '-r', '--raw',
action='store_true', action='store_true',
help="Show the raw data including tag encodings.") help="Show the raw data including tag encodings.")
parser.add_argument( parser.add_argument(
'-T', '--no-truncate', '-T', '--no-truncate',
action='store_true', action='store_true',
help="Don't truncate, show the full contents.") help="Don't truncate, show the full contents.")
parser.add_argument( parser.add_argument(
'-t', '--tree', '-t', '--tree',
action='store_true', action='store_true',
help="Show the underlying rbyd trees.") help="Show the underlying rbyd trees.")
parser.add_argument( parser.add_argument(
'-B', '--btree', '-B', '--btree',
action='store_true', action='store_true',
help="Show the underlying B-trees.") help="Show the underlying B-trees.")
parser.add_argument( parser.add_argument(
'-R', '--rbyd', '-R', '--rbyd',
action='store_true', action='store_true',
help="Show the full underlying rbyd trees.") help="Show the full underlying rbyd trees.")
parser.add_argument( parser.add_argument(
'-i', '--inner', '-i', '--inner',
action='store_true', action='store_true',
help="Show inner branches.") help="Show inner branches.")
parser.add_argument( parser.add_argument(
'-z', '--depth', '-z', '--depth',
nargs='?', nargs='?',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
const=0, const=0,
help="Depth of tree to show.") help="Depth of tree to show.")
parser.add_argument( parser.add_argument(
'-e', '--error-on-corrupt', '-e', '--error-on-corrupt',
action='store_true', action='store_true',
help="Error if the filesystem is corrupt.") help="Error if the filesystem is corrupt.")
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/dbgrbyd.py
+248 -239
View File
@@ -7,6 +7,7 @@ import math as mt
import os import os
import struct import struct
COLORS = [ COLORS = [
'34', # blue '34', # blue
'31', # red '31', # red
@@ -156,108 +157,108 @@ def fromtag(data):
def xxd(data, width=16): def xxd(data, width=16):
for i in range(0, len(data), width): for i in range(0, len(data), width):
yield '%-*s %-*s' % ( yield '%-*s %-*s' % (
3*width, 3*width,
' '.join('%02x' % b for b in data[i:i+width]), ' '.join('%02x' % b for b in data[i:i+width]),
width, width,
''.join( ''.join(
b if b >= ' ' and b <= '~' else '.' b if b >= ' ' and b <= '~' else '.'
for b in map(chr, data[i:i+width]))) for b in map(chr, data[i:i+width])))
def tagrepr(tag, w=None, size=None, off=None): def tagrepr(tag, w=None, size=None, off=None):
if (tag & 0x6fff) == TAG_NULL: if (tag & 0x6fff) == TAG_NULL:
return '%snull%s%s' % ( return '%snull%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if size else '') ' %d' % size if size else '')
elif (tag & 0x6f00) == TAG_CONFIG: elif (tag & 0x6f00) == TAG_CONFIG:
return '%s%s%s%s' % ( return '%s%s%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
'magic' if (tag & 0xfff) == TAG_MAGIC 'magic' if (tag & 0xfff) == TAG_MAGIC
else 'version' if (tag & 0xfff) == TAG_VERSION else 'version' if (tag & 0xfff) == TAG_VERSION
else 'rcompat' if (tag & 0xfff) == TAG_RCOMPAT else 'rcompat' if (tag & 0xfff) == TAG_RCOMPAT
else 'wcompat' if (tag & 0xfff) == TAG_WCOMPAT else 'wcompat' if (tag & 0xfff) == TAG_WCOMPAT
else 'ocompat' if (tag & 0xfff) == TAG_OCOMPAT else 'ocompat' if (tag & 0xfff) == TAG_OCOMPAT
else 'geometry' if (tag & 0xfff) == TAG_GEOMETRY else 'geometry' if (tag & 0xfff) == TAG_GEOMETRY
else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT
else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT
else 'config 0x%02x' % (tag & 0xff), else 'config 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x6f00) == TAG_GDELTA: elif (tag & 0x6f00) == TAG_GDELTA:
return '%s%s%s%s' % ( return '%s%s%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA
else 'gdelta 0x%02x' % (tag & 0xff), else 'gdelta 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x6f00) == TAG_NAME: elif (tag & 0x6f00) == TAG_NAME:
return '%s%s%s%s' % ( return '%s%s%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
'name' if (tag & 0xfff) == TAG_NAME 'name' if (tag & 0xfff) == TAG_NAME
else 'reg' if (tag & 0xfff) == TAG_REG else 'reg' if (tag & 0xfff) == TAG_REG
else 'dir' if (tag & 0xfff) == TAG_DIR else 'dir' if (tag & 0xfff) == TAG_DIR
else 'orphan' if (tag & 0xfff) == TAG_ORPHAN else 'orphan' if (tag & 0xfff) == TAG_ORPHAN
else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK
else 'name 0x%02x' % (tag & 0xff), else 'name 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x6f00) == TAG_STRUCT: elif (tag & 0x6f00) == TAG_STRUCT:
return '%s%s%s%s' % ( return '%s%s%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
'data' if (tag & 0xfff) == TAG_DATA 'data' if (tag & 0xfff) == TAG_DATA
else 'block' if (tag & 0xfff) == TAG_BLOCK else 'block' if (tag & 0xfff) == TAG_BLOCK
else 'bshrub' if (tag & 0xfff) == TAG_BSHRUB else 'bshrub' if (tag & 0xfff) == TAG_BSHRUB
else 'btree' if (tag & 0xfff) == TAG_BTREE else 'btree' if (tag & 0xfff) == TAG_BTREE
else 'mroot' if (tag & 0xfff) == TAG_MROOT else 'mroot' if (tag & 0xfff) == TAG_MROOT
else 'mdir' if (tag & 0xfff) == TAG_MDIR else 'mdir' if (tag & 0xfff) == TAG_MDIR
else 'mtree' if (tag & 0xfff) == TAG_MTREE else 'mtree' if (tag & 0xfff) == TAG_MTREE
else 'did' if (tag & 0xfff) == TAG_DID else 'did' if (tag & 0xfff) == TAG_DID
else 'branch' if (tag & 0xfff) == TAG_BRANCH else 'branch' if (tag & 0xfff) == TAG_BRANCH
else 'struct 0x%02x' % (tag & 0xff), else 'struct 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x6e00) == TAG_ATTR: elif (tag & 0x6e00) == TAG_ATTR:
return '%s%sattr 0x%02x%s%s' % ( return '%s%sattr 0x%02x%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
's' if tag & 0x100 else 'u', 's' if tag & 0x100 else 'u',
((tag & 0x100) >> 1) ^ (tag & 0xff), ((tag & 0x100) >> 1) ^ (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif tag & TAG_ALT: elif tag & TAG_ALT:
return 'alt%s%s%s%s%s' % ( return 'alt%s%s%s%s%s' % (
'r' if tag & TAG_R else 'b', 'r' if tag & TAG_R else 'b',
'a' if tag & 0x0fff == 0 and tag & TAG_GT 'a' if tag & 0x0fff == 0 and tag & TAG_GT
else 'n' if tag & 0x0fff == 0 else 'n' if tag & 0x0fff == 0
else 'gt' if tag & TAG_GT else 'gt' if tag & TAG_GT
else 'le', else 'le',
' 0x%x' % (tag & 0x0fff) if tag & 0x0fff != 0 else '', ' 0x%x' % (tag & 0x0fff) if tag & 0x0fff != 0 else '',
' w%d' % w if w is not None else '', ' w%d' % w if w is not None else '',
' 0x%x' % (0xffffffff & (off-size)) ' 0x%x' % (0xffffffff & (off-size))
if size and off is not None if size and off is not None
else ' -%d' % size if size else ' -%d' % size if size
else '') else '')
elif (tag & 0x7f00) == TAG_CKSUM: elif (tag & 0x7f00) == TAG_CKSUM:
return 'cksum%s%s%s%s%s' % ( return 'cksum%s%s%s%s%s' % (
'q' if not tag & 0xfc and tag & TAG_Q else '', 'q' if not tag & 0xfc and tag & TAG_Q else '',
'p' if not tag & 0xfc and tag & TAG_P else '', 'p' if not tag & 0xfc and tag & TAG_P else '',
' 0x%02x' % (tag & 0xff) if tag & 0xfc else '', ' 0x%02x' % (tag & 0xff) if tag & 0xfc else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x7f00) == TAG_NOTE: elif (tag & 0x7f00) == TAG_NOTE:
return 'note%s%s%s' % ( return 'note%s%s%s' % (
' 0x%02x' % (tag & 0xff) if tag & 0xff else '', ' 0x%02x' % (tag & 0xff) if tag & 0xff else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x7f00) == TAG_ECKSUM: elif (tag & 0x7f00) == TAG_ECKSUM:
return 'ecksum%s%s%s' % ( return 'ecksum%s%s%s' % (
' 0x%02x' % (tag & 0xff) if tag & 0xff else '', ' 0x%02x' % (tag & 0xff) if tag & 0xff else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
else: else:
return '0x%04x%s%s' % ( return '0x%04x%s%s' % (
tag, tag,
' w%d' % w if w is not None else '', ' w%d' % w if w is not None else '',
' %d' % size if size is not None else '') ' %d' % size if size is not None else '')
def dbg_log(data, block_size, rev, eoff, weight, *, def dbg_log(data, block_size, rev, eoff, weight, *,
@@ -291,9 +292,9 @@ def dbg_log(data, block_size, rev, eoff, weight, *,
x = 0 x = 0
while any( while any(
max(a, b) >= min(a_, b_) max(a, b) >= min(a_, b_)
and max(a_, b_) >= min(a, b) and max(a_, b_) >= min(a, b)
and x == x_ and x == x_
for a_, b_, x_, _ in jumps[:j]): for a_, b_, x_, _ in jumps[:j]):
x += 1 x += 1
jumps[j] = a, b, x, c jumps[j] = a, b, x, c
@@ -303,9 +304,9 @@ def dbg_log(data, block_size, rev, eoff, weight, *,
for a, b, x, c in jumps: for a, b, x, c in jumps:
c_start = ( c_start = (
'\x1b[33m' if color and c == 'y' '\x1b[33m' if color and c == 'y'
else '\x1b[31m' if color and c == 'r' else '\x1b[31m' if color and c == 'r'
else '\x1b[90m' if color else '\x1b[90m' if color
else '') else '')
c_stop = '\x1b[m' if color else '' c_stop = '\x1b[m' if color else ''
if j == a: if j == a:
@@ -321,7 +322,7 @@ def dbg_log(data, block_size, rev, eoff, weight, *,
chars[2*x+1] = '%s|%s' % (c_start, c_stop) chars[2*x+1] = '%s|%s' % (c_start, c_stop)
return ''.join(chars.get(x, ' ') return ''.join(chars.get(x, ' ')
for x in range(max(chars.keys(), default=0)+1)) for x in range(max(chars.keys(), default=0)+1))
# preprocess lifetimes # preprocess lifetimes
lifetime_width = 0 lifetime_width = 0
@@ -333,7 +334,7 @@ def dbg_log(data, block_size, rev, eoff, weight, *,
self.tags = set() self.tags = set()
self.color = COLORS[self.__class__.color_i] self.color = COLORS[self.__class__.color_i]
self.__class__.color_i = ( self.__class__.color_i = (
self.__class__.color_i + 1) % len(COLORS) self.__class__.color_i + 1) % len(COLORS)
def add(self, j): def add(self, j):
self.tags.add(j) self.tags.add(j)
@@ -357,8 +358,8 @@ def dbg_log(data, block_size, rev, eoff, weight, *,
def checkpoint(j, weights, lifetimes, grows, shrinks, tags): def checkpoint(j, weights, lifetimes, grows, shrinks, tags):
checkpoint_js.append(j) checkpoint_js.append(j)
checkpoints.append(( checkpoints.append((
weights.copy(), lifetimes.copy(), weights.copy(), lifetimes.copy(),
grows, shrinks, tags)) grows, shrinks, tags))
lower_, upper_ = 0, 0 lower_, upper_ = 0, 0
weight_ = 0 weight_ = 0
@@ -397,7 +398,7 @@ def dbg_log(data, block_size, rev, eoff, weight, *,
if rid_ > 0: if rid_ > 0:
weights[i:i+1] = [rid_, delta, weights[i]-rid_] weights[i:i+1] = [rid_, delta, weights[i]-rid_]
lifetimes[i:i+1] = [ lifetimes[i:i+1] = [
lifetimes[i], Lifetime(j), lifetimes[i]] lifetimes[i], Lifetime(j), lifetimes[i]]
else: else:
weights[i:i] = [delta] weights[i:i] = [delta]
lifetimes[i:i] = [Lifetime(j)] lifetimes[i:i] = [Lifetime(j)]
@@ -438,9 +439,9 @@ def dbg_log(data, block_size, rev, eoff, weight, *,
checkpoint(j, weights, lifetimes, set(), set(), {i}) checkpoint(j, weights, lifetimes, set(), set(), {i})
lifetime_width = 2*max(( lifetime_width = 2*max((
sum(1 for lifetime in lifetimes if lifetime) sum(1 for lifetime in lifetimes if lifetime)
for _, lifetimes, _, _, _ in checkpoints), for _, lifetimes, _, _, _ in checkpoints),
default=0) default=0)
def lifetimerepr(j): def lifetimerepr(j):
x = bisect.bisect(checkpoint_js, j)-1 x = bisect.bisect(checkpoint_js, j)-1
@@ -476,12 +477,12 @@ def dbg_log(data, block_size, rev, eoff, weight, *,
colors.append(lifetime.color) colors.append(lifetime.color)
return '%s%*s' % ( return '%s%*s' % (
''.join('%s%s%s' % ( ''.join('%s%s%s' % (
'\x1b[%sm' % c if color else '', '\x1b[%sm' % c if color else '',
r, r,
'\x1b[m' if color else '') '\x1b[m' if color else '')
for r, c in zip(reprs, colors)), for r, c in zip(reprs, colors)),
lifetime_width - sum(len(r) for r in reprs), '') lifetime_width - sum(len(r) for r in reprs), '')
# dynamically size the id field # dynamically size the id field
@@ -518,10 +519,10 @@ def dbg_log(data, block_size, rev, eoff, weight, *,
# print revision count # print revision count
if args.get('raw'): if args.get('raw'):
print('%8s: %*s%*s %s' % ( print('%8s: %*s%*s %s' % (
'%04x' % 0, '%04x' % 0,
lifetime_width, '', lifetime_width, '',
2*w_width+1, '', 2*w_width+1, '',
next(xxd(data[0:4])))) next(xxd(data[0:4]))))
# print tags # print tags
cksum = crc32c(data[0:4]) cksum = crc32c(data[0:4])
@@ -582,44 +583,49 @@ def dbg_log(data, block_size, rev, eoff, weight, *,
# show human-readable tag representation # show human-readable tag representation
print('%s%08x:%s %*s%s%*s %-*s%s%s%s' % ( print('%s%08x:%s %*s%s%*s %-*s%s%s%s' % (
'\x1b[90m' if color and j >= eoff else '', '\x1b[90m' if color and j >= eoff else '',
j, j,
'\x1b[m' if color and j >= eoff else '', '\x1b[m' if color and j >= eoff else '',
lifetime_width, lifetimerepr(j) if args.get('lifetimes') else '', lifetime_width, lifetimerepr(j)
'\x1b[90m' if color and j >= eoff else '', if args.get('lifetimes')
2*w_width+1, '' if (tag & 0xe000) != 0x0000 else '',
else '%d-%d' % (rid-(w-1), rid) if w > 1 '\x1b[90m' if color and j >= eoff else '',
else rid, 2*w_width+1, '' if (tag & 0xe000) != 0x0000
56+w_width, '%-*s %s' % ( else '%d-%d' % (rid-(w-1), rid) if w > 1
21+w_width, tagrepr(tag, w, size, j), else rid,
next(xxd(data[j+d:j+d+min(size, 8)], 8), '') 56+w_width, '%-*s %s' % (
if not args.get('raw') and not args.get('no_truncate') 21+w_width, tagrepr(tag, w, size, j),
and not tag & TAG_ALT else ''), next(xxd(data[j+d:j+d+min(size, 8)], 8), '')
' (%s)' % ', '.join(notes) if notes else '', if not args.get('raw')
'\x1b[m' if color and j >= eoff else '', and not args.get('no_truncate')
' %s' % jumprepr(j) and not tag & TAG_ALT
if args.get('jumps') and not notes else '')) else ''),
' (%s)' % ', '.join(notes) if notes else '',
'\x1b[m' if color and j >= eoff else '',
' %s' % jumprepr(j)
if args.get('jumps') and not notes
else ''))
# show on-disk encoding of tags # show on-disk encoding of tags
if args.get('raw'): if args.get('raw'):
for o, line in enumerate(xxd(data[j:j+d])): for o, line in enumerate(xxd(data[j:j+d])):
print('%s%8s: %*s%*s %s%s' % ( print('%s%8s: %*s%*s %s%s' % (
'\x1b[90m' if color and j >= eoff else '',
'%04x' % (j + o*16),
lifetime_width, '',
2*w_width+1, '',
line,
'\x1b[m' if color and j >= eoff else ''))
if args.get('raw') or args.get('no_truncate'):
if not tag & TAG_ALT:
for o, line in enumerate(xxd(data[j+d:j+d+size])):
print('%s%8s: %*s%*s %s%s' % (
'\x1b[90m' if color and j >= eoff else '', '\x1b[90m' if color and j >= eoff else '',
'%04x' % (j+d + o*16), '%04x' % (j + o*16),
lifetime_width, '', lifetime_width, '',
2*w_width+1, '', 2*w_width+1, '',
line, line,
'\x1b[m' if color and j >= eoff else '')) '\x1b[m' if color and j >= eoff else ''))
if args.get('raw') or args.get('no_truncate'):
if not tag & TAG_ALT:
for o, line in enumerate(xxd(data[j+d:j+d+size])):
print('%s%8s: %*s%*s %s%s' % (
'\x1b[90m' if color and j >= eoff else '',
'%04x' % (j+d + o*16),
lifetime_width, '',
2*w_width+1, '',
line,
'\x1b[m' if color and j >= eoff else ''))
def dbg_tree(data, block_size, rev, trunk, weight, *, def dbg_tree(data, block_size, rev, trunk, weight, *,
@@ -757,8 +763,8 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
else: else:
if 'h' not in alts[j_]: if 'h' not in alts[j_]:
alts[j_]['h'] = max( alts[j_]['h'] = max(
rec_height(alts[j_]['f']), rec_height(alts[j_]['f']),
rec_height(alts[j_]['nf'])) + 1 rec_height(alts[j_]['nf'])) + 1
return alts[j_]['h'] return alts[j_]['h']
for j_ in alts.keys(): for j_ in alts.keys():
@@ -801,8 +807,8 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
for branch in tree): for branch in tree):
return '+-', branch.c, branch.c return '+-', branch.c, branch.c
elif any(branch.d == d elif any(branch.d == d
and x > min(branch.a, branch.b) and x > min(branch.a, branch.b)
and x < max(branch.a, branch.b) and x < max(branch.a, branch.b)
for branch in tree): for branch in tree):
return '|-', branch.c, branch.c return '|-', branch.c, branch.c
elif branch.a < branch.b: elif branch.a < branch.b:
@@ -827,13 +833,13 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
t, c, was = branchrepr((rid, tag), d, was) t, c, was = branchrepr((rid, tag), d, was)
trunk.append('%s%s%s%s' % ( trunk.append('%s%s%s%s' % (
'\x1b[33m' if color and c == 'y' '\x1b[33m' if color and c == 'y'
else '\x1b[31m' if color and c == 'r' else '\x1b[31m' if color and c == 'r'
else '\x1b[90m' if color and c == 'b' else '\x1b[90m' if color and c == 'b'
else '', else '',
t, t,
('>' if was else ' ') if d == t_depth-1 else '', ('>' if was else ' ') if d == t_depth-1 else '',
'\x1b[m' if color and c else '')) '\x1b[m' if color and c else ''))
return '%s ' % ''.join(trunk) return '%s ' % ''.join(trunk)
@@ -850,33 +856,36 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
# show human-readable tag representation # show human-readable tag representation
print('%08x: %s%*s %-*s %s' % ( print('%08x: %s%*s %-*s %s' % (
j, j,
treerepr(rid, tag) treerepr(rid, tag)
if args.get('tree') or args.get('rbyd') else '', if args.get('tree') or args.get('rbyd')
2*w_width+1, '%d-%d' % (rid-(w-1), rid) else '',
if w > 1 else rid 2*w_width+1, '%d-%d' % (rid-(w-1), rid) if w > 1
if w > 0 or i == 0 else '', else rid if w > 0 or i == 0
21+w_width, tagrepr(tag, w, size, j), else '',
next(xxd(data[j+d:j+d+min(size, 8)], 8), '') 21+w_width, tagrepr(tag, w, size, j),
if not args.get('raw') and not args.get('no_truncate') next(xxd(data[j+d:j+d+min(size, 8)], 8), '')
and not tag & TAG_ALT else '')) if not args.get('raw')
and not args.get('no_truncate')
and not tag & TAG_ALT
else ''))
# show on-disk encoding of tags # show on-disk encoding of tags
if args.get('raw'): if args.get('raw'):
for o, line in enumerate(xxd(data[j:j+d])): for o, line in enumerate(xxd(data[j:j+d])):
print('%8s: %*s%*s %s' % ( print('%8s: %*s%*s %s' % (
'%04x' % (j + o*16), '%04x' % (j + o*16),
t_width, '', t_width, '',
2*w_width+1, '', 2*w_width+1, '',
line)) line))
if args.get('raw') or args.get('no_truncate'): if args.get('raw') or args.get('no_truncate'):
if not tag & TAG_ALT: if not tag & TAG_ALT:
for o, line in enumerate(xxd(data[j+d:j+d+size])): for o, line in enumerate(xxd(data[j+d:j+d+size])):
print('%8s: %*s%*s %s' % ( print('%8s: %*s%*s %s' % (
'%04x' % (j+d + o*16), '%04x' % (j+d + o*16),
t_width, '', t_width, '',
2*w_width+1, '', 2*w_width+1, '',
line)) line))
def main(disk, blocks=None, *, def main(disk, blocks=None, *,
@@ -912,12 +921,12 @@ def main(disk, blocks=None, *,
# blocks may also encode trunks # blocks may also encode trunks
blocks, trunks = ( blocks, trunks = (
[block[0] if isinstance(block, tuple) else block [block[0] if isinstance(block, tuple) else block
for block in blocks], for block in blocks],
[trunk if trunk is not None [trunk if trunk is not None
else block[1] if isinstance(block, tuple) else block[1] if isinstance(block, tuple)
else None else None
for block in blocks]) for block in blocks])
# read each block # read each block
datas = [] datas = []
@@ -1020,41 +1029,41 @@ def main(disk, blocks=None, *,
# compare with sequence arithmetic # compare with sequence arithmetic
if trunk_ and ( if trunk_ and (
not trunks_[i] not trunks_[i]
or not ((rev - revs[i]) & 0x80000000) or not ((rev - revs[i]) & 0x80000000)
or (rev == revs[i] and trunk_ > trunks_[i])): or (rev == revs[i] and trunk_ > trunks_[i])):
i = i_ i = i_
# print contents of the winning metadata block # print contents of the winning metadata block
block, data, rev, eoff, trunk_, weight, cksum = ( block, data, rev, eoff, trunk_, weight, cksum = (
blocks[i], blocks[i],
datas[i], datas[i],
revs[i], revs[i],
eoffs[i], eoffs[i],
trunks_[i], trunks_[i],
weights[i], weights[i],
cksums[i]) cksums[i])
print('rbyd %s w%d, rev %08x, size %d, cksum %08x' % ( print('rbyd %s w%d, rev %08x, size %d, cksum %08x' % (
'0x%x.%x' % (block, trunk_) '0x%x.%x' % (block, trunk_)
if len(blocks) == 1 if len(blocks) == 1
else '0x{%x,%s}.%x' % ( else '0x{%x,%s}.%x' % (
block, block,
','.join('%x' % blocks[(i+1+j) % len(blocks)] ','.join('%x' % blocks[(i+1+j) % len(blocks)]
for j in range(len(blocks)-1)), for j in range(len(blocks)-1)),
trunk_), trunk_),
weight, weight,
rev, rev,
eoff, eoff,
cksum)) cksum))
if args.get('log'): if args.get('log'):
dbg_log(data, block_size, rev, eoff, weight, dbg_log(data, block_size, rev, eoff, weight,
color=color, color=color,
**args) **args)
else: else:
dbg_tree(data, block_size, rev, trunk_, weight, dbg_tree(data, block_size, rev, trunk_, weight,
color=color, color=color,
**args) **args)
if args.get('error_on_corrupt') and eoff == 0: if args.get('error_on_corrupt') and eoff == 0:
sys.exit(2) sys.exit(2)
@@ -1064,69 +1073,69 @@ if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Debug rbyd metadata.", description="Debug rbyd metadata.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'disk', 'disk',
help="File containing the block device.") help="File containing the block device.")
parser.add_argument( parser.add_argument(
'blocks', 'blocks',
nargs='*', nargs='*',
type=rbydaddr, type=rbydaddr,
help="Block address of metadata blocks.") help="Block address of metadata blocks.")
parser.add_argument( parser.add_argument(
'-b', '--block-size', '-b', '--block-size',
type=bdgeom, type=bdgeom,
help="Block size/geometry in bytes.") help="Block size/geometry in bytes.")
parser.add_argument( parser.add_argument(
'--block-count', '--block-count',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Block count in blocks.") help="Block count in blocks.")
parser.add_argument( parser.add_argument(
'--trunk', '--trunk',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Use this offset as the trunk of the tree.") help="Use this offset as the trunk of the tree.")
parser.add_argument( parser.add_argument(
'--color', '--color',
choices=['never', 'always', 'auto'], choices=['never', 'always', 'auto'],
default='auto', default='auto',
help="When to use terminal colors. Defaults to 'auto'.") help="When to use terminal colors. Defaults to 'auto'.")
parser.add_argument( parser.add_argument(
'-a', '--all', '-a', '--all',
action='store_true', action='store_true',
help="Don't stop parsing on bad commits.") help="Don't stop parsing on bad commits.")
parser.add_argument( parser.add_argument(
'-l', '--log', '-l', '--log',
action='store_true', action='store_true',
help="Show the raw tags as they appear in the log.") help="Show the raw tags as they appear in the log.")
parser.add_argument( parser.add_argument(
'-r', '--raw', '-r', '--raw',
action='store_true', action='store_true',
help="Show the raw data including tag encodings.") help="Show the raw data including tag encodings.")
parser.add_argument( parser.add_argument(
'-T', '--no-truncate', '-T', '--no-truncate',
action='store_true', action='store_true',
help="Don't truncate, show the full contents.") help="Don't truncate, show the full contents.")
parser.add_argument( parser.add_argument(
'-t', '--tree', '-t', '--tree',
action='store_true', action='store_true',
help="Show the rbyd tree.") help="Show the rbyd tree.")
parser.add_argument( parser.add_argument(
'-R', '--rbyd', '-R', '--rbyd',
action='store_true', action='store_true',
help="Show the full rbyd tree.") help="Show the full rbyd tree.")
parser.add_argument( parser.add_argument(
'-j', '--jumps', '-j', '--jumps',
action='store_true', action='store_true',
help="Show alt pointer jumps in the margin.") help="Show alt pointer jumps in the margin.")
parser.add_argument( parser.add_argument(
'-g', '--lifetimes', '-g', '--lifetimes',
action='store_true', action='store_true',
help="Show inserts/deletes of ids in the margin.") help="Show inserts/deletes of ids in the margin.")
parser.add_argument( parser.add_argument(
'-e', '--error-on-corrupt', '-e', '--error-on-corrupt',
action='store_true', action='store_true',
help="Error if no valid commit is found.") help="Error if no valid commit is found.")
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/dbgtag.py
+107 -106
View File
@@ -121,98 +121,98 @@ def fromleb128(data):
def tagrepr(tag, w=None, size=None, off=None): def tagrepr(tag, w=None, size=None, off=None):
if (tag & 0x6fff) == TAG_NULL: if (tag & 0x6fff) == TAG_NULL:
return '%snull%s%s' % ( return '%snull%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if size else '') ' %d' % size if size else '')
elif (tag & 0x6f00) == TAG_CONFIG: elif (tag & 0x6f00) == TAG_CONFIG:
return '%s%s%s%s' % ( return '%s%s%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
'magic' if (tag & 0xfff) == TAG_MAGIC 'magic' if (tag & 0xfff) == TAG_MAGIC
else 'version' if (tag & 0xfff) == TAG_VERSION else 'version' if (tag & 0xfff) == TAG_VERSION
else 'rcompat' if (tag & 0xfff) == TAG_RCOMPAT else 'rcompat' if (tag & 0xfff) == TAG_RCOMPAT
else 'wcompat' if (tag & 0xfff) == TAG_WCOMPAT else 'wcompat' if (tag & 0xfff) == TAG_WCOMPAT
else 'ocompat' if (tag & 0xfff) == TAG_OCOMPAT else 'ocompat' if (tag & 0xfff) == TAG_OCOMPAT
else 'geometry' if (tag & 0xfff) == TAG_GEOMETRY else 'geometry' if (tag & 0xfff) == TAG_GEOMETRY
else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT
else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT
else 'config 0x%02x' % (tag & 0xff), else 'config 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x6f00) == TAG_GDELTA: elif (tag & 0x6f00) == TAG_GDELTA:
return '%s%s%s%s' % ( return '%s%s%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA
else 'gdelta 0x%02x' % (tag & 0xff), else 'gdelta 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x6f00) == TAG_NAME: elif (tag & 0x6f00) == TAG_NAME:
return '%s%s%s%s' % ( return '%s%s%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
'name' if (tag & 0xfff) == TAG_NAME 'name' if (tag & 0xfff) == TAG_NAME
else 'reg' if (tag & 0xfff) == TAG_REG else 'reg' if (tag & 0xfff) == TAG_REG
else 'dir' if (tag & 0xfff) == TAG_DIR else 'dir' if (tag & 0xfff) == TAG_DIR
else 'orphan' if (tag & 0xfff) == TAG_ORPHAN else 'orphan' if (tag & 0xfff) == TAG_ORPHAN
else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK
else 'name 0x%02x' % (tag & 0xff), else 'name 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x6f00) == TAG_STRUCT: elif (tag & 0x6f00) == TAG_STRUCT:
return '%s%s%s%s' % ( return '%s%s%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
'data' if (tag & 0xfff) == TAG_DATA 'data' if (tag & 0xfff) == TAG_DATA
else 'block' if (tag & 0xfff) == TAG_BLOCK else 'block' if (tag & 0xfff) == TAG_BLOCK
else 'bshrub' if (tag & 0xfff) == TAG_BSHRUB else 'bshrub' if (tag & 0xfff) == TAG_BSHRUB
else 'btree' if (tag & 0xfff) == TAG_BTREE else 'btree' if (tag & 0xfff) == TAG_BTREE
else 'mroot' if (tag & 0xfff) == TAG_MROOT else 'mroot' if (tag & 0xfff) == TAG_MROOT
else 'mdir' if (tag & 0xfff) == TAG_MDIR else 'mdir' if (tag & 0xfff) == TAG_MDIR
else 'mtree' if (tag & 0xfff) == TAG_MTREE else 'mtree' if (tag & 0xfff) == TAG_MTREE
else 'did' if (tag & 0xfff) == TAG_DID else 'did' if (tag & 0xfff) == TAG_DID
else 'branch' if (tag & 0xfff) == TAG_BRANCH else 'branch' if (tag & 0xfff) == TAG_BRANCH
else 'struct 0x%02x' % (tag & 0xff), else 'struct 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x6e00) == TAG_ATTR: elif (tag & 0x6e00) == TAG_ATTR:
return '%s%sattr 0x%02x%s%s' % ( return '%s%sattr 0x%02x%s%s' % (
'shrub' if tag & TAG_SHRUB else '', 'shrub' if tag & TAG_SHRUB else '',
's' if tag & 0x100 else 'u', 's' if tag & 0x100 else 'u',
((tag & 0x100) >> 1) ^ (tag & 0xff), ((tag & 0x100) >> 1) ^ (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif tag & TAG_ALT: elif tag & TAG_ALT:
return 'alt%s%s%s%s%s' % ( return 'alt%s%s%s%s%s' % (
'r' if tag & TAG_R else 'b', 'r' if tag & TAG_R else 'b',
'a' if tag & 0x0fff == 0 and tag & TAG_GT 'a' if tag & 0x0fff == 0 and tag & TAG_GT
else 'n' if tag & 0x0fff == 0 else 'n' if tag & 0x0fff == 0
else 'gt' if tag & TAG_GT else 'gt' if tag & TAG_GT
else 'le', else 'le',
' 0x%x' % (tag & 0x0fff) if tag & 0x0fff != 0 else '', ' 0x%x' % (tag & 0x0fff) if tag & 0x0fff != 0 else '',
' w%d' % w if w is not None else '', ' w%d' % w if w is not None else '',
' 0x%x' % (0xffffffff & (off-size)) ' 0x%x' % (0xffffffff & (off-size))
if size and off is not None if size and off is not None
else ' -%d' % size if size else ' -%d' % size if size
else '') else '')
elif (tag & 0x7f00) == TAG_CKSUM: elif (tag & 0x7f00) == TAG_CKSUM:
return 'cksum%s%s%s%s%s' % ( return 'cksum%s%s%s%s%s' % (
'q' if not tag & 0xfc and tag & TAG_Q else '', 'q' if not tag & 0xfc and tag & TAG_Q else '',
'p' if not tag & 0xfc and tag & TAG_P else '', 'p' if not tag & 0xfc and tag & TAG_P else '',
' 0x%02x' % (tag & 0xff) if tag & 0xfc else '', ' 0x%02x' % (tag & 0xff) if tag & 0xfc else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x7f00) == TAG_NOTE: elif (tag & 0x7f00) == TAG_NOTE:
return 'note%s%s%s' % ( return 'note%s%s%s' % (
' 0x%02x' % (tag & 0xff) if tag & 0xff else '', ' 0x%02x' % (tag & 0xff) if tag & 0xff else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
elif (tag & 0x7f00) == TAG_ECKSUM: elif (tag & 0x7f00) == TAG_ECKSUM:
return 'ecksum%s%s%s' % ( return 'ecksum%s%s%s' % (
' 0x%02x' % (tag & 0xff) if tag & 0xff else '', ' 0x%02x' % (tag & 0xff) if tag & 0xff else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %s' % size if size is not None else '') ' %s' % size if size is not None else '')
else: else:
return '0x%04x%s%s' % ( return '0x%04x%s%s' % (
tag, tag,
' w%d' % w if w is not None else '', ' w%d' % w if w is not None else '',
' %d' % size if size is not None else '') ' %d' % size if size is not None else '')
def list_tags(): def list_tags():
@@ -221,7 +221,7 @@ def list_tags():
import re import re
tags = [] tags = []
tag_pattern = re.compile( tag_pattern = re.compile(
'^(?P<name>TAG_[^ ]*) *= *(?P<tag>[^ #]+) *#+ *(?P<comment>.*)$') '^(?P<name>TAG_[^ ]*) *= *(?P<tag>[^ #]+) *#+ *(?P<comment>.*)$')
for line in inspect.getsourcelines( for line in inspect.getsourcelines(
inspect.getmodule(inspect.currentframe()))[0]: inspect.getmodule(inspect.currentframe()))[0]:
m = tag_pattern.match(line) m = tag_pattern.match(line)
@@ -236,8 +236,8 @@ def list_tags():
# print # print
for n, t, c in tags: for n, t, c in tags:
print('%-*s %s' % ( print('%-*s %s' % (
w[0], 'LFSR_'+n, w[0], 'LFSR_'+n,
c)) c))
def dbg_tag(data): def dbg_tag(data):
if isinstance(data, int): if isinstance(data, int):
@@ -305,12 +305,12 @@ def main(tags, *,
# blocks may also encode offsets # blocks may also encode offsets
blocks, offs = ( blocks, offs = (
[block[0] if isinstance(block, tuple) else block [block[0] if isinstance(block, tuple) else block
for block in blocks], for block in blocks],
[off if off is not None [off if off is not None
else block[1] if isinstance(block, tuple) else block[1] if isinstance(block, tuple)
else None else None
for block in blocks]) for block in blocks])
# read each tag # read each tag
for block, off in zip(blocks, offs): for block, off in zip(blocks, offs):
@@ -319,40 +319,41 @@ def main(tags, *,
data = f.read(2+5+5) data = f.read(2+5+5)
dbg_tag(data) dbg_tag(data)
if __name__ == "__main__": if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Decode littlefs tags.", description="Decode littlefs tags.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'tags', 'tags',
nargs='*', nargs='*',
help="Tags to decode.") help="Tags to decode.")
parser.add_argument( parser.add_argument(
'-l', '--list', '-l', '--list',
action='store_true', action='store_true',
help="List all known tags.") help="List all known tags.")
parser.add_argument( parser.add_argument(
'-x', '--hex', '-x', '--hex',
action='store_true', action='store_true',
help="Interpret as a sequence of hex bytes.") help="Interpret as a sequence of hex bytes.")
parser.add_argument( parser.add_argument(
'-s', '--string', '-s', '--string',
action='store_true', action='store_true',
help="Interpret as strings.") help="Interpret as strings.")
parser.add_argument( parser.add_argument(
'-b', '--block-size', '-b', '--block-size',
type=bdgeom, type=bdgeom,
help="Block size/geometry in bytes.") help="Block size/geometry in bytes.")
parser.add_argument( parser.add_argument(
'--block-count', '--block-count',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Block count in blocks.") help="Block count in blocks.")
parser.add_argument( parser.add_argument(
'--off', '--off',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Use this offset.") help="Use this offset.")
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/parity.py
+20 -19
View File
@@ -29,17 +29,17 @@ def main(paths, **args):
# interpret as sequence of hex bytes # interpret as sequence of hex bytes
if args.get('hex'): if args.get('hex'):
print('%01x' % parity(ft.reduce( print('%01x' % parity(ft.reduce(
op.xor, op.xor,
bytes(int(path, 16) for path in paths), bytes(int(path, 16) for path in paths),
0))) 0)))
# interpret as strings # interpret as strings
elif args.get('string'): elif args.get('string'):
for path in paths: for path in paths:
print('%01x' % parity(ft.reduce( print('%01x' % parity(ft.reduce(
op.xor, op.xor,
path.encode('utf8'), path.encode('utf8'),
0))) 0)))
# default to interpreting as paths # default to interpreting as paths
else: else:
@@ -63,24 +63,25 @@ def main(paths, **args):
else: else:
print('%01x' % parity(xor)) print('%01x' % parity(xor))
if __name__ == "__main__": if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Calculates parity.", description="Calculates parity.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'paths', 'paths',
nargs='*', nargs='*',
help="Paths to read. Reads stdin by default.") help="Paths to read. Reads stdin by default.")
parser.add_argument( parser.add_argument(
'-x', '--hex', '-x', '--hex',
action='store_true', action='store_true',
help="Interpret as a sequence of hex bytes.") help="Interpret as a sequence of hex bytes.")
parser.add_argument( parser.add_argument(
'-s', '--string', '-s', '--string',
action='store_true', action='store_true',
help="Interpret as strings.") help="Interpret as strings.")
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/perf.py
+384 -372
View File
File diff suppressed because it is too large Load Diff
scripts/perfbd.py
+357 -344
View File
@@ -118,15 +118,15 @@ class PerfBdResult(co.namedtuple('PerfBdResult', [
readed=0, proged=0, erased=0, readed=0, proged=0, erased=0,
children=[]): children=[]):
return super().__new__(cls, file, function, int(RInt(line)), return super().__new__(cls, file, function, int(RInt(line)),
RInt(readed), RInt(proged), RInt(erased), RInt(readed), RInt(proged), RInt(erased),
children) children)
def __add__(self, other): def __add__(self, other):
return PerfBdResult(self.file, self.function, self.line, return PerfBdResult(self.file, self.function, self.line,
self.readed + other.readed, self.readed + other.readed,
self.proged + other.proged, self.proged + other.proged,
self.erased + other.erased, self.erased + other.erased,
self.children + other.children) self.children + other.children)
def openio(path, mode='r', buffering=-1): def openio(path, mode='r', buffering=-1):
@@ -143,27 +143,27 @@ def collect_syms_and_lines(obj_path, *,
objdump_path=None, objdump_path=None,
**args): **args):
symbol_pattern = re.compile( symbol_pattern = re.compile(
'^(?P<addr>[0-9a-fA-F]+)' '^(?P<addr>[0-9a-fA-F]+)'
'\s+.*'
'\s+(?P<size>[0-9a-fA-F]+)'
'\s+(?P<name>[^\s]+)\s*$')
line_pattern = re.compile(
'^\s+(?:'
# matches dir/file table
'(?P<no>[0-9]+)'
'(?:\s+(?P<dir>[0-9]+))?'
'\s+.*' '\s+.*'
'\s+(?P<path>[^\s]+)' '\s+(?P<size>[0-9a-fA-F]+)'
# matches line opcodes '\s+(?P<name>[^\s]+)\s*$')
'|' '\[[^\]]*\]\s+' line_pattern = re.compile(
'(?:' '^\s+(?:'
# matches dir/file table
'(?P<no>[0-9]+)'
'(?:\s+(?P<dir>[0-9]+))?'
'\s+.*'
'\s+(?P<path>[^\s]+)'
# matches line opcodes
'|' '\[[^\]]*\]\s+' '(?:'
'(?P<op_special>Special)' '(?P<op_special>Special)'
'|' '(?P<op_copy>Copy)' '|' '(?P<op_copy>Copy)'
'|' '(?P<op_end>End of Sequence)' '|' '(?P<op_end>End of Sequence)'
'|' 'File .*?to (?:entry )?(?P<op_file>\d+)' '|' 'File .*?to (?:entry )?(?P<op_file>\d+)'
'|' 'Line .*?to (?P<op_line>[0-9]+)' '|' 'Line .*?to (?P<op_line>[0-9]+)'
'|' '(?:Address|PC) .*?to (?P<op_addr>[0x0-9a-fA-F]+)' '|' '(?:Address|PC) .*?to (?P<op_addr>[0x0-9a-fA-F]+)'
'|' '.' ')*' '|' '.'
')*'
')$', re.IGNORECASE) ')$', re.IGNORECASE)
# figure out symbol addresses # figure out symbol addresses
@@ -173,11 +173,11 @@ def collect_syms_and_lines(obj_path, *,
if args.get('verbose'): if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd)) print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd, proc = sp.Popen(cmd,
stdout=sp.PIPE, stdout=sp.PIPE,
stderr=None if args.get('verbose') else sp.DEVNULL, stderr=None if args.get('verbose') else sp.DEVNULL,
universal_newlines=True, universal_newlines=True,
errors='replace', errors='replace',
close_fds=False) close_fds=False)
for line in proc.stdout: for line in proc.stdout:
m = symbol_pattern.match(line) m = symbol_pattern.match(line)
if m: if m:
@@ -222,11 +222,11 @@ def collect_syms_and_lines(obj_path, *,
if args.get('verbose'): if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd)) print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd, proc = sp.Popen(cmd,
stdout=sp.PIPE, stdout=sp.PIPE,
stderr=None if args.get('verbose') else sp.DEVNULL, stderr=None if args.get('verbose') else sp.DEVNULL,
universal_newlines=True, universal_newlines=True,
errors='replace', errors='replace',
close_fds=False) close_fds=False)
for line in proc.stdout: for line in proc.stdout:
m = line_pattern.match(line) m = line_pattern.match(line)
if m: if m:
@@ -238,8 +238,8 @@ def collect_syms_and_lines(obj_path, *,
dir = int(m.group('dir')) dir = int(m.group('dir'))
if dir in dirs: if dir in dirs:
files[int(m.group('no'))] = os.path.join( files[int(m.group('no'))] = os.path.join(
dirs[dir], dirs[dir],
m.group('path')) m.group('path'))
else: else:
files[int(m.group('no'))] = m.group('path') files[int(m.group('no'))] = m.group('path')
else: else:
@@ -296,25 +296,27 @@ def collect_job(path, start, stop, syms, sym_at, lines, line_at, *,
depth=1, depth=1,
**args): **args):
trace_pattern = re.compile( trace_pattern = re.compile(
'^(?P<file>[^:]*):(?P<line>[0-9]+):trace:\s*(?P<prefix>[^\s]*?bd_)(?:' '^(?P<file>[^:]*):(?P<line>[0-9]+):trace:\s*'
'(?P<read>read)\(' '(?P<prefix>[^\s]*?bd_)(?:'
'\s*(?P<read_ctx>\w+)' '\s*,' '(?P<read>read)\('
'\s*(?P<read_block>\w+)' '\s*,' '\s*(?P<read_ctx>\w+)' '\s*,'
'\s*(?P<read_off>\w+)' '\s*,' '\s*(?P<read_block>\w+)' '\s*,'
'\s*(?P<read_buffer>\w+)' '\s*,' '\s*(?P<read_off>\w+)' '\s*,'
'\s*(?P<read_size>\w+)' '\s*\)' '\s*(?P<read_buffer>\w+)' '\s*,'
'|' '(?P<prog>prog)\(' '\s*(?P<read_size>\w+)' '\s*\)'
'\s*(?P<prog_ctx>\w+)' '\s*,' '|' '(?P<prog>prog)\('
'\s*(?P<prog_block>\w+)' '\s*,' '\s*(?P<prog_ctx>\w+)' '\s*,'
'\s*(?P<prog_off>\w+)' '\s*,' '\s*(?P<prog_block>\w+)' '\s*,'
'\s*(?P<prog_buffer>\w+)' '\s*,' '\s*(?P<prog_off>\w+)' '\s*,'
'\s*(?P<prog_size>\w+)' '\s*\)' '\s*(?P<prog_buffer>\w+)' '\s*,'
'|' '(?P<erase>erase)\(' '\s*(?P<prog_size>\w+)' '\s*\)'
'\s*(?P<erase_ctx>\w+)' '\s*,' '|' '(?P<erase>erase)\('
'\s*(?P<erase_block>\w+)' '\s*(?P<erase_ctx>\w+)' '\s*,'
'\s*\(\s*(?P<erase_size>\w+)\s*\)' '\s*\)' ')\s*$') '\s*(?P<erase_block>\w+)'
'\s*\(\s*(?P<erase_size>\w+)\s*\)' '\s*\)'
')\s*$')
frame_pattern = re.compile( frame_pattern = re.compile(
'^\s+at (?P<addr>\w+)\s*$') '^\s+at (?P<addr>\w+)\s*$')
# parse all of the trace files for read/prog/erase operations # parse all of the trace files for read/prog/erase operations
last_filtered = False last_filtered = False
@@ -338,9 +340,7 @@ def collect_job(path, start, stop, syms, sym_at, lines, line_at, *,
# ignore filtered sources # ignore filtered sources
if sources is not None: if sources is not None:
if not any( if not any(os.path.abspath(file) == os.path.abspath(s)
os.path.abspath(file)
== os.path.abspath(s)
for s in sources): for s in sources):
return return
else: else:
@@ -359,10 +359,10 @@ def collect_job(path, start, stop, syms, sym_at, lines, line_at, *,
file = os.path.abspath(file) file = os.path.abspath(file)
results[(file, sym, line)] = ( results[(file, sym, line)] = (
last_readed, last_readed,
last_proged, last_proged,
last_erased, last_erased,
{}) {})
else: else:
# tail-recursively propagate measurements # tail-recursively propagate measurements
for i in range(len(last_stack)): for i in range(len(last_stack)):
@@ -378,10 +378,10 @@ def collect_job(path, start, stop, syms, sym_at, lines, line_at, *,
else: else:
r, p, e, children = 0, 0, 0, {} r, p, e, children = 0, 0, 0, {}
results_[name] = ( results_[name] = (
r+last_readed, r+last_readed,
p+last_proged, p+last_proged,
e+last_erased, e+last_erased,
children) children)
# recurse # recurse
results_ = results_[name][-1] results_ = results_[name][-1]
@@ -444,7 +444,7 @@ def collect_job(path, start, stop, syms, sym_at, lines, line_at, *,
# of reference # of reference
if last_delta is None: if last_delta is None:
i = bisect.bisect(lines, (last_file, last_line), i = bisect.bisect(lines, (last_file, last_line),
key=lambda x: (x[0], x[1])) key=lambda x: (x[0], x[1]))
if i > 0: if i > 0:
last_delta = lines[i-1][2] - addr_ last_delta = lines[i-1][2] - addr_
else: else:
@@ -474,9 +474,9 @@ def collect_job(path, start, stop, syms, sym_at, lines, line_at, *,
# filter out internal/unknown functions # filter out internal/unknown functions
if not everything and ( if not everything and (
sym.startswith('__') sym.startswith('__')
or sym.startswith('0') or sym.startswith('0')
or sym.startswith('-') or sym.startswith('-')
or sym == '_start'): or sym == '_start'):
at_cache[addr] = None at_cache[addr] = None
continue continue
@@ -492,9 +492,8 @@ def collect_job(path, start, stop, syms, sym_at, lines, line_at, *,
# ignore filtered sources # ignore filtered sources
if sources is not None: if sources is not None:
if not any( if not any(
os.path.abspath(file) os.path.abspath(file) == os.path.abspath(s)
== os.path.abspath(s) for s in sources):
for s in sources):
at_cache[addr] = None at_cache[addr] = None
continue continue
else: else:
@@ -529,8 +528,8 @@ def collect_job(path, start, stop, syms, sym_at, lines, line_at, *,
results_ = [] results_ = []
for name, (r, p, e, children) in results.items(): for name, (r, p, e, children) in results.items():
results_.append(PerfBdResult(*name, results_.append(PerfBdResult(*name,
r, p, e, r, p, e,
children=to_results(children))) children=to_results(children)))
return results_ return results_
return to_results(results) return to_results(results)
@@ -573,9 +572,10 @@ def collect(obj_path, trace_paths, *,
with mp.Pool(jobs) as p: with mp.Pool(jobs) as p:
for results_ in p.imap_unordered( for results_ in p.imap_unordered(
starapply, starapply,
((collect_job, (path, start, stop, ((collect_job,
syms, sym_at, lines, line_at), (path, start, stop,
args) syms, sym_at, lines, line_at),
args)
for path, ranges in zip(trace_paths, trace_ranges) for path, ranges in zip(trace_paths, trace_ranges)
for start, stop in ranges)): for start, stop in ranges)):
results.extend(results_) results.extend(results_)
@@ -583,9 +583,10 @@ def collect(obj_path, trace_paths, *,
else: else:
results = [] results = []
for path in trace_paths: for path in trace_paths:
results.extend(collect_job(path, None, None, results.extend(collect_job(
syms, sym_at, lines, line_at, path, None, None,
**args)) syms, sym_at, lines, line_at,
**args))
return results return results
@@ -597,7 +598,7 @@ def fold(Result, results, by=None, defines=[]):
for k in it.chain(by or [], (k for k, _ in defines)): for k in it.chain(by or [], (k for k, _ in defines)):
if k not in Result._by and k not in Result._fields: if k not in Result._by and k not in Result._fields:
print("error: could not find field %r?" % k, print("error: could not find field %r?" % k,
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
# filter by matching defines # filter by matching defines
@@ -653,74 +654,78 @@ def table(Result, results, diff_results=None, *,
return [] return []
r = max(results_, r = max(results_,
key=lambda r: tuple( key=lambda r: tuple(
tuple( tuple((getattr(r, k),)
(getattr(r, k),) if getattr(r, k, None) is not None
if getattr(r, k, None) is not None else ()
else () for k in (
for k in ([k] if k else [ [k] if k else [
k for k in Result._sort if k in fields]) k for k in Result._sort
if k in fields) if k in fields])
for k in it.chain(hot, [None]))) if k in fields)
for k in it.chain(hot, [None])))
# found a cycle? # found a cycle?
if tuple(getattr(r, k) for k in Result._by) in seen: if tuple(getattr(r, k) for k in Result._by) in seen:
return [] return []
return [r._replace(children=[])] + rec_hot( return [r._replace(children=[])] + rec_hot(
r.children, r.children,
seen | {tuple(getattr(r, k) for k in Result._by)}) seen | {tuple(getattr(r, k) for k in Result._by)})
results = [r._replace(children=rec_hot(r.children)) for r in results] results = [r._replace(children=rec_hot(r.children)) for r in results]
# organize by name # organize by name
table = { table = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in results} for r in results}
diff_table = { diff_table = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in diff_results or []} for r in diff_results or []}
names = [name names = [name
for name in table.keys() | diff_table.keys() for name in table.keys() | diff_table.keys()
if diff_results is None if diff_results is None
or all_ or all_
or any( or any(
types[k].ratio( types[k].ratio(
getattr(table.get(name), k, None), getattr(table.get(name), k, None),
getattr(diff_table.get(name), k, None)) getattr(diff_table.get(name), k, None))
for k in fields)] for k in fields)]
# sort again, now with diff info, note that python's sort is stable # sort again, now with diff info, note that python's sort is stable
names.sort() names.sort()
if diff_results is not None: if diff_results is not None:
names.sort(key=lambda n: tuple( names.sort(
types[k].ratio( key=lambda n: tuple(
getattr(table.get(n), k, None), types[k].ratio(
getattr(diff_table.get(n), k, None)) getattr(table.get(n), k, None),
for k in fields), getattr(diff_table.get(n), k, None))
reverse=True) for k in fields),
reverse=True)
if sort: if sort:
for k, reverse in reversed(sort): for k, reverse in reversed(sort):
names.sort( names.sort(
key=lambda n: tuple( key=lambda n: tuple(
(getattr(table[n], k),) (getattr(table[n], k),)
if getattr(table.get(n), k, None) is not None else () if getattr(table.get(n), k, None) is not None
for k in ([k] if k else [ else ()
k for k in Result._sort if k in fields])), for k in (
reverse=reverse ^ (not k or k in Result._fields)) [k] if k else [
k for k in Result._sort
if k in fields])),
reverse=reverse ^ (not k or k in Result._fields))
# build up our lines # build up our lines
lines = [] lines = []
# header # header
header = [ header = ['%s%s' % (
'%s%s' % ( ','.join(by),
','.join(by), ' (%d added, %d removed)' % (
' (%d added, %d removed)' % ( sum(1 for n in table if n not in diff_table),
sum(1 for n in table if n not in diff_table), sum(1 for n in diff_table if n not in table))
sum(1 for n in diff_table if n not in table)) if diff_results is not None and not percent else '')
if diff_results is not None and not percent else '')
if not summary else ''] if not summary else '']
if diff_results is None: if diff_results is None:
for k in fields: for k in fields:
@@ -743,43 +748,43 @@ def table(Result, results, diff_results=None, *,
if diff_results is None: if diff_results is None:
for k in fields: for k in fields:
entry.append( entry.append(
(getattr(r, k).table(), (getattr(r, k).table(),
getattr(getattr(r, k), 'notes', lambda: [])()) getattr(getattr(r, k), 'notes', lambda: [])())
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none) else types[k].none)
elif percent: elif percent:
for k in fields: for k in fields:
entry.append( entry.append(
(getattr(r, k).table() (getattr(r, k).table()
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none, else types[k].none,
(lambda t: ['+∞%'] if t == +mt.inf (lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)])( else ['%+.1f%%' % (100*t)])(
types[k].ratio( types[k].ratio(
getattr(r, k, None), getattr(r, k, None),
getattr(diff_r, k, None))))) getattr(diff_r, k, None)))))
else: else:
for k in fields: for k in fields:
entry.append(getattr(diff_r, k).table() entry.append(getattr(diff_r, k).table()
if getattr(diff_r, k, None) is not None if getattr(diff_r, k, None) is not None
else types[k].none) else types[k].none)
for k in fields: for k in fields:
entry.append(getattr(r, k).table() entry.append(getattr(r, k).table()
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none) else types[k].none)
for k in fields: for k in fields:
entry.append( entry.append(
(types[k].diff( (types[k].diff(
getattr(r, k, None),
getattr(diff_r, k, None)),
(lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)] if t
else [])(
types[k].ratio(
getattr(r, k, None), getattr(r, k, None),
getattr(diff_r, k, None))))) getattr(diff_r, k, None)),
(lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)] if t
else [])(
types[k].ratio(
getattr(r, k, None),
getattr(diff_r, k, None)))))
return entry return entry
# recursive entry helper # recursive entry helper
@@ -788,8 +793,8 @@ def table(Result, results, diff_results=None, *,
# build the children table at each layer # build the children table at each layer
results_ = fold(Result, results_, by=by) results_ = fold(Result, results_, by=by)
table_ = { table_ = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in results_} for r in results_}
names_ = list(table_.keys()) names_ = list(table_.keys())
# sort the children layer # sort the children layer
@@ -797,13 +802,16 @@ def table(Result, results, diff_results=None, *,
if sort: if sort:
for k, reverse in reversed(sort): for k, reverse in reversed(sort):
names_.sort( names_.sort(
key=lambda n: tuple( key=lambda n: tuple(
(getattr(table_[n], k),) (getattr(table_[n], k),)
if getattr(table_.get(n), k, None) is not None if getattr(table_.get(n), k, None)
else () is not None
for k in ([k] if k else [ else ()
k for k in Result._sort if k in fields])), for k in (
reverse=reverse ^ (not k or k in Result._fields)) [k] if k else [
k for k in Result._sort
if k in fields])),
reverse=reverse ^ (not k or k in Result._fields))
for i, name in enumerate(names_): for i, name in enumerate(names_):
r = table_[name] r = table_[name]
@@ -824,14 +832,13 @@ def table(Result, results, diff_results=None, *,
# recurse? # recurse?
if depth_ > 1: if depth_ > 1:
recurse( recurse(r.children,
r.children, depth_-1,
depth_-1, seen | {name},
seen | {name}, (prefixes[2+is_last] + "|-> ",
(prefixes[2+is_last] + "|-> ", prefixes[2+is_last] + "'-> ",
prefixes[2+is_last] + "'-> ", prefixes[2+is_last] + "| ",
prefixes[2+is_last] + "| ", prefixes[2+is_last] + " "))
prefixes[2+is_last] + " "))
# entries # entries
if not summary: if not summary:
@@ -845,14 +852,13 @@ def table(Result, results, diff_results=None, *,
# recursive entries # recursive entries
if name in table and depth > 1: if name in table and depth > 1:
recurse( recurse(table[name].children,
table[name].children, depth-1,
depth-1, {name},
{name}, ("|-> ",
("|-> ", "'-> ",
"'-> ", "| ",
"| ", " "))
" "))
# total # total
r = next(iter(fold(Result, results, by=[])), None) r = next(iter(fold(Result, results, by=[])), None)
@@ -864,8 +870,8 @@ def table(Result, results, diff_results=None, *,
# homogenize # homogenize
lines = [ lines = [
[x if isinstance(x, tuple) else (x, []) for x in line] [x if isinstance(x, tuple) else (x, []) for x in line]
for line in lines] for line in lines]
# find the best widths, note that column 0 contains the names and is # find the best widths, note that column 0 contains the names and is
# handled a bit differently # handled a bit differently
@@ -879,11 +885,11 @@ def table(Result, results, diff_results=None, *,
# print our table # print our table
for line in lines: for line in lines:
print('%-*s %s' % ( print('%-*s %s' % (
widths[0], line[0][0], widths[0], line[0][0],
' '.join('%*s%-*s' % ( ' '.join('%*s%-*s' % (
widths[i], x[0], widths[i], x[0],
notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '')
for i, x in enumerate(line[1:], 1)))) for i, x in enumerate(line[1:], 1))))
def annotate(Result, results, *, def annotate(Result, results, *,
@@ -944,14 +950,14 @@ def annotate(Result, results, *,
or float(r.erased) / max_erased >= erase_t0): or float(r.erased) / max_erased >= erase_t0):
if last is not None and line - last.stop <= args['context']: if last is not None and line - last.stop <= args['context']:
last = range( last = range(
last.start, last.start,
line+1+args['context']) line+1+args['context'])
else: else:
if last is not None: if last is not None:
spans.append((last, func)) spans.append((last, func))
last = range( last = range(
line-args['context'], line-args['context'],
line+1+args['context']) line+1+args['context'])
func = r.function func = r.function
if last is not None: if last is not None:
spans.append((last, func)) spans.append((last, func))
@@ -967,11 +973,11 @@ def annotate(Result, results, *,
if skipped: if skipped:
skipped = False skipped = False
print('%s@@ %s:%d: %s @@%s' % ( print('%s@@ %s:%d: %s @@%s' % (
'\x1b[36m' if args['color'] else '', '\x1b[36m' if args['color'] else '',
path, path,
i+1, i+1,
next(iter(f for _, f in spans)), next(iter(f for _, f in spans)),
'\x1b[m' if args['color'] else '')) '\x1b[m' if args['color'] else ''))
# build line # build line
if line.endswith('\n'): if line.endswith('\n'):
@@ -980,11 +986,11 @@ def annotate(Result, results, *,
if i+1 in table: if i+1 in table:
r = table[i+1] r = table[i+1]
line = '%-*s // %s readed, %s proged, %s erased' % ( line = '%-*s // %s readed, %s proged, %s erased' % (
args['width'], args['width'],
line, line,
r.readed, r.readed,
r.proged, r.proged,
r.erased) r.erased)
if args['color']: if args['color']:
if (float(r.readed) / max_readed >= read_t1 if (float(r.readed) / max_readed >= read_t1
@@ -1036,10 +1042,10 @@ def report(obj_path='', trace_paths=[], *,
continue continue
try: try:
results.append(PerfBdResult( results.append(PerfBdResult(
**{k: r[k] for k in PerfBdResult._by **{k: r[k] for k in PerfBdResult._by
if k in r and r[k].strip()}, if k in r and r[k].strip()},
**{k: r[k] for k in PerfBdResult._fields **{k: r[k] for k in PerfBdResult._fields
if k in r and r[k].strip()})) if k in r and r[k].strip()}))
except TypeError: except TypeError:
pass pass
@@ -1051,25 +1057,27 @@ def report(obj_path='', trace_paths=[], *,
if sort: if sort:
for k, reverse in reversed(sort): for k, reverse in reversed(sort):
results.sort( results.sort(
key=lambda r: tuple( key=lambda r: tuple(
(getattr(r, k),) if getattr(r, k) is not None else () (getattr(r, k),) if getattr(r, k) is not None else ()
for k in ([k] if k else PerfBdResult._sort)), for k in ([k] if k else PerfBdResult._sort)),
reverse=reverse ^ (not k or k in PerfBdResult._fields)) reverse=reverse ^ (not k or k in PerfBdResult._fields))
# write results to CSV # write results to CSV
if args.get('output'): if args.get('output'):
with openio(args['output'], 'w') as f: with openio(args['output'], 'w') as f:
writer = csv.DictWriter(f, writer = csv.DictWriter(f,
(by if by is not None else PerfBdResult._by) (by if by is not None else PerfBdResult._by)
+ [k for k in ( + [k for k in (
fields if fields is not None else PerfBdResult._fields)]) fields if fields is not None
else PerfBdResult._fields)])
writer.writeheader() writer.writeheader()
for r in results: for r in results:
writer.writerow( writer.writerow(
{k: getattr(r, k) for k in ( {k: getattr(r, k) for k in (
by if by is not None else PerfBdResult._by)} by if by is not None else PerfBdResult._by)}
| {k: getattr(r, k) for k in ( | {k: getattr(r, k) for k in (
fields if fields is not None else PerfBdResult._fields)}) fields if fields is not None
else PerfBdResult._fields)})
# find previous results? # find previous results?
if args.get('diff'): if args.get('diff'):
@@ -1087,10 +1095,10 @@ def report(obj_path='', trace_paths=[], *,
continue continue
try: try:
diff_results.append(PerfBdResult( diff_results.append(PerfBdResult(
**{k: r[k] for k in PerfBdResult._by **{k: r[k] for k in PerfBdResult._by
if k in r and r[k].strip()}, if k in r and r[k].strip()},
**{k: r[k] for k in PerfBdResult._fields **{k: r[k] for k in PerfBdResult._fields
if k in r and r[k].strip()})) if k in r and r[k].strip()}))
except TypeError: except TypeError:
pass pass
except FileNotFoundError: except FileNotFoundError:
@@ -1111,11 +1119,11 @@ def report(obj_path='', trace_paths=[], *,
else: else:
# print table # print table
table(PerfBdResult, results, table(PerfBdResult, results,
diff_results if args.get('diff') else None, diff_results if args.get('diff') else None,
by=by if by is not None else ['function'], by=by if by is not None else ['function'],
fields=fields, fields=fields,
sort=sort, sort=sort,
**args) **args)
def main(**args): def main(**args):
@@ -1129,168 +1137,173 @@ if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Aggregate and report call-stack propagated " description="Aggregate and report call-stack propagated "
"block-device operations from trace output.", "block-device operations from trace output.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'obj_path', 'obj_path',
nargs='?', nargs='?',
help="Input executable for mapping addresses to symbols.") help="Input executable for mapping addresses to symbols.")
parser.add_argument( parser.add_argument(
'trace_paths', 'trace_paths',
nargs='*', nargs='*',
help="Input *.trace files.") help="Input *.trace files.")
parser.add_argument( parser.add_argument(
'-v', '--verbose', '-v', '--verbose',
action='store_true', action='store_true',
help="Output commands that run behind the scenes.") help="Output commands that run behind the scenes.")
parser.add_argument( parser.add_argument(
'-q', '--quiet', '-q', '--quiet',
action='store_true', action='store_true',
help="Don't show anything, useful with -o.") help="Don't show anything, useful with -o.")
parser.add_argument( parser.add_argument(
'-o', '--output', '-o', '--output',
help="Specify CSV file to store results.") help="Specify CSV file to store results.")
parser.add_argument( parser.add_argument(
'-u', '--use', '-u', '--use',
help="Don't parse anything, use this CSV file.") help="Don't parse anything, use this CSV file.")
parser.add_argument( parser.add_argument(
'-d', '--diff', '-d', '--diff',
help="Specify CSV file to diff against.") help="Specify CSV file to diff against.")
parser.add_argument( parser.add_argument(
'-a', '--all', '-a', '--all',
action='store_true', action='store_true',
help="Show all, not just the ones that changed.") help="Show all, not just the ones that changed.")
parser.add_argument( parser.add_argument(
'-p', '--percent', '-p', '--percent',
action='store_true', action='store_true',
help="Only show percentage change, not a full diff.") help="Only show percentage change, not a full diff.")
parser.add_argument( parser.add_argument(
'-b', '--by', '-b', '--by',
action='append', action='append',
choices=PerfBdResult._by, choices=PerfBdResult._by,
help="Group by this field.") help="Group by this field.")
parser.add_argument( parser.add_argument(
'-f', '--field', '-f', '--field',
dest='fields', dest='fields',
action='append', action='append',
choices=PerfBdResult._fields, choices=PerfBdResult._fields,
help="Show this field.") help="Show this field.")
parser.add_argument( parser.add_argument(
'-D', '--define', '-D', '--define',
dest='defines', dest='defines',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs: ( lambda k, vs: (
k.strip(), k.strip(),
{v.strip() for v in vs.split(',')}) {v.strip() for v in vs.split(',')})
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Only include results where this field is this value.") help="Only include results where this field is this value.")
class AppendSort(argparse.Action): class AppendSort(argparse.Action):
def __call__(self, parser, namespace, value, option): def __call__(self, parser, namespace, value, option):
if namespace.sort is None: if namespace.sort is None:
namespace.sort = [] namespace.sort = []
namespace.sort.append((value, True if option == '-S' else False)) namespace.sort.append((value, True if option == '-S' else False))
parser.add_argument( parser.add_argument(
'-s', '--sort', '-s', '--sort',
nargs='?', nargs='?',
action=AppendSort, action=AppendSort,
help="Sort by this field.") help="Sort by this field.")
parser.add_argument( parser.add_argument(
'-S', '--reverse-sort', '-S', '--reverse-sort',
nargs='?', nargs='?',
action=AppendSort, action=AppendSort,
help="Sort by this field, but backwards.") help="Sort by this field, but backwards.")
parser.add_argument( parser.add_argument(
'-Y', '--summary', '-Y', '--summary',
action='store_true', action='store_true',
help="Only show the total.") help="Only show the total.")
parser.add_argument( parser.add_argument(
'-F', '--source', '-F', '--source',
dest='sources', dest='sources',
action='append', action='append',
help="Only consider definitions in this file. Defaults to anything " help="Only consider definitions in this file. Defaults to "
"in the current directory.") "anything in the current directory.")
parser.add_argument( parser.add_argument(
'--everything', '--everything',
action='store_true', action='store_true',
help="Include builtin and libc specific symbols.") help="Include builtin and libc specific symbols.")
parser.add_argument( parser.add_argument(
'-g', '--propagate', '-g', '--propagate',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Depth to propagate samples up the call-stack. 0 propagates up " help="Depth to propagate samples up the call-stack. 0 propagates "
"to the entry point, 1 does no propagation. Defaults to 0.") "up to the entry point, 1 does no propagation. Defaults to 0.")
parser.add_argument( parser.add_argument(
'-z', '--depth', '-z', '--depth',
nargs='?', nargs='?',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
const=0, const=0,
help="Depth of function calls to show. 0 shows all calls unless we " help="Depth of function calls to show. 0 shows all calls unless "
"find a cycle. Defaults to 0.") "we find a cycle. Defaults to 0.")
parser.add_argument( parser.add_argument(
'-t', '--hot', '-t', '--hot',
nargs='?', nargs='?',
action='append', action='append',
help="Show only the hot path for each function call.") help="Show only the hot path for each function call.")
parser.add_argument( parser.add_argument(
'-A', '--annotate', '-A', '--annotate',
action='store_true', action='store_true',
help="Show source files annotated with coverage info.") help="Show source files annotated with coverage info.")
parser.add_argument( parser.add_argument(
'-T', '--threshold', '-T', '--threshold',
nargs='?', nargs='?',
type=lambda x: tuple(float(x) for x in x.split(',')), type=lambda x: tuple(float(x) for x in x.split(',')),
const=THRESHOLD, const=THRESHOLD,
help="Show lines with any ops above this threshold as a percent of " help="Show lines with any ops above this threshold as a percent "
"all lines. Defaults to %s." % ','.join(str(t) for t in THRESHOLD)) "of all lines. Defaults to "
"%s." % ','.join(str(t) for t in THRESHOLD))
parser.add_argument( parser.add_argument(
'--read-threshold', '--read-threshold',
nargs='?', nargs='?',
type=lambda x: tuple(float(x) for x in x.split(',')), type=lambda x: tuple(float(x) for x in x.split(',')),
const=THRESHOLD, const=THRESHOLD,
help="Show lines with reads above this threshold as a percent of " help="Show lines with reads above this threshold as a percent "
"all lines. Defaults to %s." % ','.join(str(t) for t in THRESHOLD)) "of all lines. Defaults to "
"%s." % ','.join(str(t) for t in THRESHOLD))
parser.add_argument( parser.add_argument(
'--prog-threshold', '--prog-threshold',
nargs='?', nargs='?',
type=lambda x: tuple(float(x) for x in x.split(',')), type=lambda x: tuple(float(x) for x in x.split(',')),
const=THRESHOLD, const=THRESHOLD,
help="Show lines with progs above this threshold as a percent of " help="Show lines with progs above this threshold as a percent "
"all lines. Defaults to %s." % ','.join(str(t) for t in THRESHOLD)) "of all lines. Defaults to "
"%s." % ','.join(str(t) for t in THRESHOLD))
parser.add_argument( parser.add_argument(
'--erase-threshold', '--erase-threshold',
nargs='?', nargs='?',
type=lambda x: tuple(float(x) for x in x.split(',')), type=lambda x: tuple(float(x) for x in x.split(',')),
const=THRESHOLD, const=THRESHOLD,
help="Show lines with erases above this threshold as a percent of " help="Show lines with erases above this threshold as a percent "
"all lines. Defaults to %s." % ','.join(str(t) for t in THRESHOLD)) "of all lines. Defaults to "
"%s." % ','.join(str(t) for t in THRESHOLD))
parser.add_argument( parser.add_argument(
'-C', '--context', '-C', '--context',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
default=3, default=3,
help="Show n additional lines of context. Defaults to 3.") help="Show n additional lines of context. Defaults to 3.")
parser.add_argument( parser.add_argument(
'-W', '--width', '-W', '--width',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
default=80, default=80,
help="Assume source is styled with this many columns. Defaults to 80.") help="Assume source is styled with this many columns. Defaults "
"to 80.")
parser.add_argument( parser.add_argument(
'--color', '--color',
choices=['never', 'always', 'auto'], choices=['never', 'always', 'auto'],
default='auto', default='auto',
help="When to use terminal colors. Defaults to 'auto'.") help="When to use terminal colors. Defaults to 'auto'.")
parser.add_argument( parser.add_argument(
'-j', '--jobs', '-j', '--jobs',
nargs='?', nargs='?',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
const=0, const=0,
help="Number of processes to use. 0 spawns one process per core.") help="Number of processes to use. 0 spawns one process per core.")
parser.add_argument( parser.add_argument(
'--objdump-path', '--objdump-path',
type=lambda x: x.split(), type=lambda x: x.split(),
default=OBJDUMP_PATH, default=OBJDUMP_PATH,
help="Path to the objdump executable, may include flags. " help="Path to the objdump executable, may include flags. "
"Defaults to %r." % OBJDUMP_PATH) "Defaults to %r." % OBJDUMP_PATH)
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/plot.py
+395 -383
View File
File diff suppressed because it is too large Load Diff
scripts/plotmpl.py
+324 -317
View File
@@ -25,6 +25,7 @@ import time
import matplotlib as mpl import matplotlib as mpl
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
# some nicer colors borrowed from Seaborn # some nicer colors borrowed from Seaborn
# note these include a non-opaque alpha # note these include a non-opaque alpha
COLORS = [ COLORS = [
@@ -148,8 +149,8 @@ class AutoMultipleLocator(mpl.ticker.MultipleLocator):
nbins = np.clip(self.axis.get_tick_space(), 1, 9) nbins = np.clip(self.axis.get_tick_space(), 1, 9)
# find the best power, use this as our locator's actual base # find the best power, use this as our locator's actual base
scale = self.base scale = (self.base
** (mt.ceil(mt.log((vmax-vmin) / (nbins+1), self.base))) ** (mt.ceil(mt.log((vmax-vmin) / (nbins+1), self.base))))
self.set_params(scale) self.set_params(scale)
return super().__call__() return super().__call__()
@@ -199,8 +200,8 @@ def collect(csv_paths, renames=[], defines=[]):
with openio(path) as f: with openio(path) as f:
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
fields.extend( fields.extend(
k for k in reader.fieldnames k for k in reader.fieldnames
if k not in fields) if k not in fields)
for r in reader: for r in reader:
# apply any renames # apply any renames
if renames: if renames:
@@ -249,7 +250,7 @@ def fold(results, by=None, x=None, y=None, defines=[], labels=None):
# filter by 'by' # filter by 'by'
if by and not all( if by and not all(
k in r and r[k] == v k in r and r[k] == v
for k, v in zip(by, key)): for k, v in zip(by, key)):
continue continue
# find xs # find xs
@@ -353,9 +354,9 @@ class Grid:
self_i = 0 self_i = 0
other_i = 0 other_i = 0
self_xweight = (self_xweights[self_i] self_xweight = (self_xweights[self_i]
if self_i < len(self_xweights) else mt.inf) if self_i < len(self_xweights) else mt.inf)
other_xweight = (other_xweights[other_i] other_xweight = (other_xweights[other_i]
if other_i < len(other_xweights) else mt.inf) if other_i < len(other_xweights) else mt.inf)
while self_i < len(self_xweights) and other_i < len(other_xweights): while self_i < len(self_xweights) and other_i < len(other_xweights):
if other_xweight - self_xweight > 0.0000001: if other_xweight - self_xweight > 0.0000001:
new_xweights.append(self_xweight) new_xweights.append(self_xweight)
@@ -374,7 +375,7 @@ class Grid:
self_i += 1 self_i += 1
self_xweight = (self_xweights[self_i] self_xweight = (self_xweights[self_i]
if self_i < len(self_xweights) else mt.inf) if self_i < len(self_xweights) else mt.inf)
elif self_xweight - other_xweight > 0.0000001: elif self_xweight - other_xweight > 0.0000001:
new_xweights.append(other_xweight) new_xweights.append(other_xweight)
self_xweight -= other_xweight self_xweight -= other_xweight
@@ -392,7 +393,7 @@ class Grid:
other_i += 1 other_i += 1
other_xweight = (other_xweights[other_i] other_xweight = (other_xweights[other_i]
if other_i < len(other_xweights) else mt.inf) if other_i < len(other_xweights) else mt.inf)
else: else:
new_xweights.append(self_xweight) new_xweights.append(self_xweight)
@@ -404,10 +405,10 @@ class Grid:
self_i += 1 self_i += 1
self_xweight = (self_xweights[self_i] self_xweight = (self_xweights[self_i]
if self_i < len(self_xweights) else mt.inf) if self_i < len(self_xweights) else mt.inf)
other_i += 1 other_i += 1
other_xweight = (other_xweights[other_i] other_xweight = (other_xweights[other_i]
if other_i < len(other_xweights) else mt.inf) if other_i < len(other_xweights) else mt.inf)
# squish so ratios are preserved # squish so ratios are preserved
self_h = sum(self.yweights) self_h = sum(self.yweights)
@@ -423,8 +424,9 @@ class Grid:
self.xweights = new_xweights self.xweights = new_xweights
self.yweights = self_yweights + other.yweights self.yweights = self_yweights + other.yweights
self.map = self_map | {(x, y+len(self_yweights)): s self.map = self_map | {
for (x, y), s in other_map.items()} (x, y+len(self_yweights)): s
for (x, y), s in other_map.items()}
else: else:
for s in self.subplots: for s in self.subplots:
s.y += len(other.yweights) s.y += len(other.yweights)
@@ -432,8 +434,9 @@ class Grid:
self.xweights = new_xweights self.xweights = new_xweights
self.yweights = other.yweights + self_yweights self.yweights = other.yweights + self_yweights
self.map = other_map | {(x, y+len(other.yweights)): s self.map = other_map | {
for (x, y), s in self_map.items()} (x, y+len(other.yweights)): s
for (x, y), s in self_map.items()}
if dir in ['right', 'left']: if dir in ['right', 'left']:
# first scale the two grids so they line up # first scale the two grids so they line up
@@ -451,9 +454,9 @@ class Grid:
self_i = 0 self_i = 0
other_i = 0 other_i = 0
self_yweight = (self_yweights[self_i] self_yweight = (self_yweights[self_i]
if self_i < len(self_yweights) else mt.inf) if self_i < len(self_yweights) else mt.inf)
other_yweight = (other_yweights[other_i] other_yweight = (other_yweights[other_i]
if other_i < len(other_yweights) else mt.inf) if other_i < len(other_yweights) else mt.inf)
while self_i < len(self_yweights) and other_i < len(other_yweights): while self_i < len(self_yweights) and other_i < len(other_yweights):
if other_yweight - self_yweight > 0.0000001: if other_yweight - self_yweight > 0.0000001:
new_yweights.append(self_yweight) new_yweights.append(self_yweight)
@@ -472,7 +475,7 @@ class Grid:
self_i += 1 self_i += 1
self_yweight = (self_yweights[self_i] self_yweight = (self_yweights[self_i]
if self_i < len(self_yweights) else mt.inf) if self_i < len(self_yweights) else mt.inf)
elif self_yweight - other_yweight > 0.0000001: elif self_yweight - other_yweight > 0.0000001:
new_yweights.append(other_yweight) new_yweights.append(other_yweight)
self_yweight -= other_yweight self_yweight -= other_yweight
@@ -490,7 +493,7 @@ class Grid:
other_i += 1 other_i += 1
other_yweight = (other_yweights[other_i] other_yweight = (other_yweights[other_i]
if other_i < len(other_yweights) else mt.inf) if other_i < len(other_yweights) else mt.inf)
else: else:
new_yweights.append(self_yweight) new_yweights.append(self_yweight)
@@ -502,10 +505,10 @@ class Grid:
self_i += 1 self_i += 1
self_yweight = (self_yweights[self_i] self_yweight = (self_yweights[self_i]
if self_i < len(self_yweights) else mt.inf) if self_i < len(self_yweights) else mt.inf)
other_i += 1 other_i += 1
other_yweight = (other_yweights[other_i] other_yweight = (other_yweights[other_i]
if other_i < len(other_yweights) else mt.inf) if other_i < len(other_yweights) else mt.inf)
# squish so ratios are preserved # squish so ratios are preserved
self_w = sum(self.xweights) self_w = sum(self.xweights)
@@ -521,8 +524,9 @@ class Grid:
self.xweights = self_xweights + other.xweights self.xweights = self_xweights + other.xweights
self.yweights = new_yweights self.yweights = new_yweights
self.map = self_map | {(x+len(self_xweights), y): s self.map = self_map | {
for (x, y), s in other_map.items()} (x+len(self_xweights), y): s
for (x, y), s in other_map.items()}
else: else:
for s in self.subplots: for s in self.subplots:
s.x += len(other.xweights) s.x += len(other.xweights)
@@ -530,8 +534,9 @@ class Grid:
self.xweights = other.xweights + self_xweights self.xweights = other.xweights + self_xweights
self.yweights = new_yweights self.yweights = new_yweights
self.map = other_map | {(x+len(other.xweights), y): s self.map = other_map | {
for (x, y), s in self_map.items()} (x+len(other.xweights), y): s
for (x, y), s in self_map.items()}
def scale(self, width, height): def scale(self, width, height):
@@ -546,11 +551,11 @@ class Grid:
for dir, subargs in subplots: for dir, subargs in subplots:
subgrid = cls.fromargs( subgrid = cls.fromargs(
width=subargs.pop('width', width=subargs.pop('width',
0.5 if dir in ['right', 'left'] else width), 0.5 if dir in ['right', 'left'] else width),
height=subargs.pop('height', height=subargs.pop('height',
0.5 if dir in ['above', 'below'] else height), 0.5 if dir in ['above', 'below'] else height),
**subargs) **subargs)
grid.merge(subgrid, dir) grid.merge(subgrid, dir)
grid.scale(width, height) grid.scale(width, height)
@@ -668,8 +673,8 @@ def main(csv_paths, output, *,
# fix ggplot when dark # fix ggplot when dark
if ggplot: if ggplot:
plt.rc('axes', plt.rc('axes',
facecolor=foreground_, facecolor=foreground_,
edgecolor=background_) edgecolor=background_)
plt.rc('grid', color=background_) plt.rc('grid', color=background_)
if font is not None: if font is not None:
@@ -677,22 +682,22 @@ def main(csv_paths, output, *,
plt.rc('font', size=font_size) plt.rc('font', size=font_size)
plt.rc('text', color=font_color_) plt.rc('text', color=font_color_)
plt.rc('figure', plt.rc('figure',
titlesize='medium', titlesize='medium',
labelsize='small') labelsize='small')
plt.rc('axes', plt.rc('axes',
titlesize='small', titlesize='small',
labelsize='small', labelsize='small',
labelcolor=font_color_) labelcolor=font_color_)
if not ggplot: if not ggplot:
plt.rc('axes', edgecolor=font_color_) plt.rc('axes', edgecolor=font_color_)
plt.rc('xtick', labelsize='small', color=font_color_) plt.rc('xtick', labelsize='small', color=font_color_)
plt.rc('ytick', labelsize='small', color=font_color_) plt.rc('ytick', labelsize='small', color=font_color_)
plt.rc('legend', plt.rc('legend',
fontsize='small', fontsize='small',
fancybox=False, fancybox=False,
framealpha=None, framealpha=None,
edgecolor=foreground_, edgecolor=foreground_,
borderaxespad=0) borderaxespad=0)
plt.rc('axes.spines', top=False, right=False) plt.rc('axes.spines', top=False, right=False)
plt.rc('figure', facecolor=background_, edgecolor=background_) plt.rc('figure', facecolor=background_, edgecolor=background_)
@@ -723,19 +728,19 @@ def main(csv_paths, output, *,
all_defines[k] |= vs all_defines[k] |= vs
all_defines = sorted(all_defines.items()) all_defines = sorted(all_defines.items())
all_labels = ((label or []) all_labels = ((label or [])
+ subplots_get('label', **subplot, subplots=subplots)) + subplots_get('label', **subplot, subplots=subplots))
# separate out renames # separate out renames
all_renames = list(it.chain.from_iterable( all_renames = list(it.chain.from_iterable(
((k, v) for v in vs) ((k, v) for v in vs)
for k, vs in it.chain(all_by, all_x, all_y))) for k, vs in it.chain(all_by, all_x, all_y)))
all_by = [k for k, _ in all_by] all_by = [k for k, _ in all_by]
all_x = [k for k, _ in all_x] all_x = [k for k, _ in all_x]
all_y = [k for k, _ in all_y] all_y = [k for k, _ in all_y]
if not all_by and not all_y: if not all_by and not all_y:
print("error: needs --by or -y to figure out fields", print("error: needs --by or -y to figure out fields",
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
# first collect results from CSV files # first collect results from CSV files
@@ -743,8 +748,7 @@ def main(csv_paths, output, *,
# if y not specified, guess it's anything not in by/defines/x/renames # if y not specified, guess it's anything not in by/defines/x/renames
if not all_y: if not all_y:
all_y = [ all_y = [k for k in fields_
k for k in fields_
if k not in all_by if k not in all_by
and not any(k == k_ for k_, _ in all_defines) and not any(k == k_ for k_, _ in all_defines)
and not any(k == old_k for _, old_k in all_renames)] and not any(k == old_k for _, old_k in all_renames)]
@@ -757,42 +761,43 @@ def main(csv_paths, output, *,
# figure out formats/colors here so that subplot defines don't change # figure out formats/colors here so that subplot defines don't change
# them later, that'd be bad # them later, that'd be bad
dataformats_ = { dataformats_ = {
name: formats_[i % len(formats_)] name: formats_[i % len(formats_)]
for i, name in enumerate(datasets_.keys())} for i, name in enumerate(datasets_.keys())}
datacolors_ = { datacolors_ = {
name: colors_[i % len(colors_)] name: colors_[i % len(colors_)]
for i, name in enumerate(datasets_.keys())} for i, name in enumerate(datasets_.keys())}
# create a grid of subplots # create a grid of subplots
grid = Grid.fromargs(**subplot, subplots=subplots) grid = Grid.fromargs(**subplot, subplots=subplots)
# create a matplotlib plot # create a matplotlib plot
fig = plt.figure(figsize=( fig = plt.figure(
width/plt.rcParams['figure.dpi'], figsize=(
height/plt.rcParams['figure.dpi']), width/plt.rcParams['figure.dpi'],
layout='constrained', height/plt.rcParams['figure.dpi']),
# we need a linewidth to keep xkcd mode happy layout='constrained',
linewidth=8 if xkcd else 0) # we need a linewidth to keep xkcd mode happy
linewidth=8 if xkcd else 0)
gs = fig.add_gridspec( gs = fig.add_gridspec(
grid.height grid.height
+ (1 if legend_above else 0) + (1 if legend_above else 0)
+ (1 if legend_below else 0), + (1 if legend_below else 0),
grid.width grid.width
+ (1 if legend_right else 0), + (1 if legend_right else 0),
height_ratios=([0.001] if legend_above else []) height_ratios=([0.001] if legend_above else [])
+ [max(s, 0.01) for s in reversed(grid.yweights)] + [max(s, 0.01) for s in reversed(grid.yweights)]
+ ([0.001] if legend_below else []), + ([0.001] if legend_below else []),
width_ratios=[max(s, 0.01) for s in grid.xweights] width_ratios=[max(s, 0.01) for s in grid.xweights]
+ ([0.001] if legend_right else [])) + ([0.001] if legend_right else []))
# first create axes so that plots can interact with each other # first create axes so that plots can interact with each other
for s in grid: for s in grid:
s.ax = fig.add_subplot(gs[ s.ax = fig.add_subplot(gs[
grid.height-(s.y+s.yspan) + (1 if legend_above else 0) grid.height-(s.y+s.yspan) + (1 if legend_above else 0)
: grid.height-s.y + (1 if legend_above else 0), : grid.height-s.y + (1 if legend_above else 0),
s.x s.x
: s.x+s.xspan]) : s.x+s.xspan])
# now plot each subplot # now plot each subplot
for s in grid: for s in grid:
@@ -830,18 +835,18 @@ def main(csv_paths, output, *,
# filter by subplot x/y # filter by subplot x/y
subdatasets = co.OrderedDict([(name, dataset) subdatasets = co.OrderedDict([(name, dataset)
for name, dataset in subdatasets.items() for name, dataset in subdatasets.items()
if len(all_x) <= 1 or name[-(1 if len(all_y) <= 1 else 2)] in x_ if len(all_x) <= 1 or name[-(1 if len(all_y) <= 1 else 2)] in x_
if len(all_y) <= 1 or name[-1] in y_]) if len(all_y) <= 1 or name[-1] in y_])
# plot! # plot!
ax = s.ax ax = s.ax
for name, dataset in subdatasets.items(): for name, dataset in subdatasets.items():
dats = sorted((x,y) for x,y in dataset) dats = sorted((x,y) for x,y in dataset)
ax.plot([x for x,_ in dats], [y for _,y in dats], ax.plot([x for x,_ in dats], [y for _,y in dats],
dataformats_[name], dataformats_[name],
color=datacolors_[name], color=datacolors_[name],
label=','.join(name)) label=','.join(name))
# axes scaling # axes scaling
if xlog_: if xlog_:
@@ -852,31 +857,31 @@ def main(csv_paths, output, *,
ax.yaxis.set_minor_locator(mpl.ticker.NullLocator()) ax.yaxis.set_minor_locator(mpl.ticker.NullLocator())
# axes limits # axes limits
ax.set_xlim( ax.set_xlim(
xlim_[0] if xlim_[0] is not None xlim_[0] if xlim_[0] is not None
else min(it.chain([0], (x else min(it.chain([0], (x
for dataset in subdatasets.values() for dataset in subdatasets.values()
for x, y in dataset for x, y in dataset
if y is not None))), if y is not None))),
xlim_[1] if xlim_[1] is not None xlim_[1] if xlim_[1] is not None
else max(it.chain([0], (x else max(it.chain([0], (x
for r in subdatasets.values() for r in subdatasets.values()
for x, y in dataset for x, y in dataset
if y is not None)))) if y is not None))))
ax.set_ylim( ax.set_ylim(
ylim_[0] if ylim_[0] is not None ylim_[0] if ylim_[0] is not None
else min(it.chain([0], (y else min(it.chain([0], (y
for dataset in subdatasets.values() for dataset in subdatasets.values()
for _, y in dataset for _, y in dataset
if y is not None))), if y is not None))),
ylim_[1] if ylim_[1] is not None ylim_[1] if ylim_[1] is not None
else max(it.chain([0], (y else max(it.chain([0], (y
for dataset in subdatasets.values() for dataset in subdatasets.values()
for _, y in dataset for _, y in dataset
if y is not None)))) if y is not None))))
# axes ticks # axes ticks
if x2_: if x2_:
ax.xaxis.set_major_formatter(lambda x, pos: ax.xaxis.set_major_formatter(lambda x, pos:
si2(x)+(xunits_ if xunits_ else '')) si2(x)+(xunits_ if xunits_ else ''))
if xticklabels_ is not None: if xticklabels_ is not None:
ax.xaxis.set_ticklabels(xticklabels_) ax.xaxis.set_ticklabels(xticklabels_)
if xticks_ is None: if xticks_ is None:
@@ -889,7 +894,7 @@ def main(csv_paths, output, *,
ax.xaxis.set_major_locator(mpl.ticker.NullLocator()) ax.xaxis.set_major_locator(mpl.ticker.NullLocator())
else: else:
ax.xaxis.set_major_formatter(lambda x, pos: ax.xaxis.set_major_formatter(lambda x, pos:
si(x)+(xunits_ if xunits_ else '')) si(x)+(xunits_ if xunits_ else ''))
if xticklabels_ is not None: if xticklabels_ is not None:
ax.xaxis.set_ticklabels(xticklabels_) ax.xaxis.set_ticklabels(xticklabels_)
if xticks_ is None: if xticks_ is None:
@@ -902,7 +907,7 @@ def main(csv_paths, output, *,
ax.xaxis.set_major_locator(mpl.ticker.NullLocator()) ax.xaxis.set_major_locator(mpl.ticker.NullLocator())
if y2_: if y2_:
ax.yaxis.set_major_formatter(lambda x, pos: ax.yaxis.set_major_formatter(lambda x, pos:
si2(x)+(yunits_ if yunits_ else '')) si2(x)+(yunits_ if yunits_ else ''))
if yticklabels_ is not None: if yticklabels_ is not None:
ax.yaxis.set_ticklabels(yticklabels_) ax.yaxis.set_ticklabels(yticklabels_)
if yticks_ is None: if yticks_ is None:
@@ -915,7 +920,7 @@ def main(csv_paths, output, *,
ax.yaxis.set_major_locator(mpl.ticker.NullLocator()) ax.yaxis.set_major_locator(mpl.ticker.NullLocator())
else: else:
ax.yaxis.set_major_formatter(lambda x, pos: ax.yaxis.set_major_formatter(lambda x, pos:
si(x)+(yunits_ if yunits_ else '')) si(x)+(yunits_ if yunits_ else ''))
if yticklabels_ is not None: if yticklabels_ is not None:
ax.yaxis.set_ticklabels(yticklabels_) ax.yaxis.set_ticklabels(yticklabels_)
if yticks_ is None: if yticks_ is None:
@@ -966,11 +971,11 @@ def main(csv_paths, output, *,
ax = fig.add_subplot(gs[(1 if legend_above else 0):,-1]) ax = fig.add_subplot(gs[(1 if legend_above else 0):,-1])
ax.set_axis_off() ax.set_axis_off()
ax.legend( ax.legend(
[h for _,h in legend], [h for _,h in legend],
[l for l,_ in legend], [l for l,_ in legend],
loc='upper left', loc='upper left',
fancybox=False, fancybox=False,
borderaxespad=0) borderaxespad=0)
if legend_above: if legend_above:
ax = fig.add_subplot(gs[0, :grid.width]) ax = fig.add_subplot(gs[0, :grid.width])
@@ -988,12 +993,12 @@ def main(csv_paths, output, *,
legend_ = [l for l in legend_ if l is not None] legend_ = [l for l in legend_ if l is not None]
legend_ = ax.legend( legend_ = ax.legend(
[h for _,h in legend_], [h for _,h in legend_],
[l for l,_ in legend_], [l for l,_ in legend_],
loc='upper center', loc='upper center',
ncol=ncol, ncol=ncol,
fancybox=False, fancybox=False,
borderaxespad=0) borderaxespad=0)
if (legend_.get_window_extent().width if (legend_.get_window_extent().width
<= ax.get_window_extent().width): <= ax.get_window_extent().width):
@@ -1007,8 +1012,8 @@ def main(csv_paths, output, *,
# works really well actually # works really well actually
if xlabel: if xlabel:
ax.set_title(escape(xlabel), ax.set_title(escape(xlabel),
size=plt.rcParams['axes.labelsize'], size=plt.rcParams['axes.labelsize'],
weight=plt.rcParams['axes.labelweight']) weight=plt.rcParams['axes.labelweight'])
# try different column counts until we fit in the axes # try different column counts until we fit in the axes
for ncol in reversed(range(1, len(legend)+1)): for ncol in reversed(range(1, len(legend)+1)):
@@ -1022,12 +1027,12 @@ def main(csv_paths, output, *,
legend_ = [l for l in legend_ if l is not None] legend_ = [l for l in legend_ if l is not None]
legend_ = ax.legend( legend_ = ax.legend(
[h for _,h in legend_], [h for _,h in legend_],
[l for l,_ in legend_], [l for l,_ in legend_],
loc='upper center', loc='upper center',
ncol=ncol, ncol=ncol,
fancybox=False, fancybox=False,
borderaxespad=0) borderaxespad=0)
if (legend_.get_window_extent().width if (legend_.get_window_extent().width
<= ax.get_window_extent().width): <= ax.get_window_extent().width):
@@ -1062,9 +1067,9 @@ def main(csv_paths, output, *,
# some stats # some stats
if not quiet: if not quiet:
print('updated %s, %s datasets, %s points' % ( print('updated %s, %s datasets, %s points' % (
output, output,
len(datasets_), len(datasets_),
sum(len(dataset) for dataset in datasets_.values()))) sum(len(dataset) for dataset in datasets_.values())))
if __name__ == "__main__": if __name__ == "__main__":
@@ -1072,265 +1077,267 @@ if __name__ == "__main__":
import argparse import argparse
import re import re
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Plot CSV files with matplotlib.", description="Plot CSV files with matplotlib.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'csv_paths', 'csv_paths',
nargs='*', nargs='*',
help="Input *.csv files.") help="Input *.csv files.")
output_rule = parser.add_argument( output_rule = parser.add_argument(
'-o', '--output', '-o', '--output',
required=True, required=True,
help="Output *.svg/*.png file.") help="Output *.svg/*.png file.")
parser.add_argument( parser.add_argument(
'--svg', '--svg',
action='store_true', action='store_true',
help="Output an svg file. By default this is infered.") help="Output an svg file. By default this is infered.")
parser.add_argument( parser.add_argument(
'--png', '--png',
action='store_true', action='store_true',
help="Output a png file. By default this is infered.") help="Output a png file. By default this is infered.")
parser.add_argument( parser.add_argument(
'-q', '--quiet', '-q', '--quiet',
action='store_true', action='store_true',
help="Don't print info.") help="Don't print info.")
parser.add_argument( parser.add_argument(
'-b', '--by', '-b', '--by',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs=None: ( lambda k, vs=None: (
k.strip(), k.strip(),
tuple(v.strip() for v in vs.split(',')) tuple(v.strip() for v in vs.split(','))
if vs is not None else ()) if vs is not None else ())
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Group by this field. Can rename fields with new_name=old_name.") help="Group by this field. Can rename fields with "
"new_name=old_name.")
parser.add_argument( parser.add_argument(
'-x', '-x',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs=None: ( lambda k, vs=None: (
k.strip(), k.strip(),
tuple(v.strip() for v in vs.split(',')) tuple(v.strip() for v in vs.split(','))
if vs is not None else ()) if vs is not None else ())
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Field to use for the x-axis. Can rename fields with " help="Field to use for the x-axis. Can rename fields with "
"new_name=old_name.") "new_name=old_name.")
parser.add_argument( parser.add_argument(
'-y', '-y',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs=None: ( lambda k, vs=None: (
k.strip(), k.strip(),
tuple(v.strip() for v in vs.split(',')) tuple(v.strip() for v in vs.split(','))
if vs is not None else ()) if vs is not None else ())
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Field to use for the y-axis. Can rename fields with " help="Field to use for the y-axis. Can rename fields with "
"new_name=old_name.") "new_name=old_name.")
parser.add_argument( parser.add_argument(
'-D', '--define', '-D', '--define',
type=lambda x: ( type=lambda x: (
lambda k, vs: ( lambda k, vs: (
k.strip(), k.strip(),
{v.strip() for v in vs.split(',')}) {v.strip() for v in vs.split(',')})
)(*x.split('=', 1)), )(*x.split('=', 1)),
action='append', action='append',
help="Only include results where this field is this value. May include " help="Only include results where this field is this value. May "
"comma-separated options.") "include comma-separated options.")
parser.add_argument( parser.add_argument(
'-L', '--label', '-L', '--label',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs: ( lambda k, vs: (
re.sub(r'\\([=\\])', r'\1', k.strip()), re.sub(r'\\([=\\])', r'\1', k.strip()),
tuple(v.strip() for v in vs.split(','))) tuple(v.strip() for v in vs.split(',')))
)(*re.split(r'(?<!\\)=', x, 1)), )(*re.split(r'(?<!\\)=', x, 1)),
help="Use this label for a given group, where a group is roughly the " help="Use this label for a given group, where a group is roughly "
"comma-separated values in the -b/--by, -x, and -y fields. Also " "the comma-separated values in the -b/--by, -x, and -y "
"provides an ordering. Accepts escaped equals.") "fields. Also provides an ordering. Accepts escaped equals.")
parser.add_argument( parser.add_argument(
'-.', '--points', '-.', '--points',
action='store_true', action='store_true',
help="Only draw data points.") help="Only draw data points.")
parser.add_argument( parser.add_argument(
'-!', '--points-and-lines', '-!', '--points-and-lines',
action='store_true', action='store_true',
help="Draw data points and lines.") help="Draw data points and lines.")
parser.add_argument( parser.add_argument(
'--colors', '--colors',
type=lambda x: [x.strip() for x in x.split(',')], type=lambda x: [x.strip() for x in x.split(',')],
help="Comma-separated hex colors to use.") help="Comma-separated hex colors to use.")
parser.add_argument( parser.add_argument(
'--formats', '--formats',
type=lambda x: [re.sub(r'\\([,\\])', r'\1', x.strip()) type=lambda x: [re.sub(r'\\([,\\])', r'\1', x.strip())
for x in re.split(r'(?<!\\),', x)], for x in re.split(r'(?<!\\),', x)],
help="Comma-separated matplotlib formats to use. Accepts escaped " help="Comma-separated matplotlib formats to use. Accepts escaped "
"commas.") "commas.")
parser.add_argument( parser.add_argument(
'-W', '--width', '-W', '--width',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Width in pixels. Defaults to %r." % WIDTH) help="Width in pixels. Defaults to %r." % WIDTH)
parser.add_argument( parser.add_argument(
'-H', '--height', '-H', '--height',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Height in pixels. Defaults to %r." % HEIGHT) help="Height in pixels. Defaults to %r." % HEIGHT)
parser.add_argument( parser.add_argument(
'-X', '--xlim', '-X', '--xlim',
type=lambda x: tuple( type=lambda x: tuple(
dat(x) if x.strip() else None dat(x) if x.strip() else None
for x in x.split(',')), for x in x.split(',')),
help="Range for the x-axis.") help="Range for the x-axis.")
parser.add_argument( parser.add_argument(
'-Y', '--ylim', '-Y', '--ylim',
type=lambda x: tuple( type=lambda x: tuple(
dat(x) if x.strip() else None dat(x) if x.strip() else None
for x in x.split(',')), for x in x.split(',')),
help="Range for the y-axis.") help="Range for the y-axis.")
parser.add_argument( parser.add_argument(
'--xlog', '--xlog',
action='store_true', action='store_true',
help="Use a logarithmic x-axis.") help="Use a logarithmic x-axis.")
parser.add_argument( parser.add_argument(
'--ylog', '--ylog',
action='store_true', action='store_true',
help="Use a logarithmic y-axis.") help="Use a logarithmic y-axis.")
parser.add_argument( parser.add_argument(
'--x2', '--x2',
action='store_true', action='store_true',
help="Use base-2 prefixes for the x-axis.") help="Use base-2 prefixes for the x-axis.")
parser.add_argument( parser.add_argument(
'--y2', '--y2',
action='store_true', action='store_true',
help="Use base-2 prefixes for the y-axis.") help="Use base-2 prefixes for the y-axis.")
parser.add_argument( parser.add_argument(
'--xticks', '--xticks',
type=lambda x: int(x, 0) if ',' not in x type=lambda x: int(x, 0) if ',' not in x
else [dat(x) for x in x.split(',')], else [dat(x) for x in x.split(',')],
help="Ticks for the x-axis. This can be explicit comma-separated " help="Ticks for the x-axis. This can be explicit comma-separated "
"ticks, the number of ticks, or 0 to disable.") "ticks, the number of ticks, or 0 to disable.")
parser.add_argument( parser.add_argument(
'--yticks', '--yticks',
type=lambda x: int(x, 0) if ',' not in x type=lambda x: int(x, 0) if ',' not in x
else [dat(x) for x in x.split(',')], else [dat(x) for x in x.split(',')],
help="Ticks for the y-axis. This can be explicit comma-separated " help="Ticks for the y-axis. This can be explicit comma-separated "
"ticks, the number of ticks, or 0 to disable.") "ticks, the number of ticks, or 0 to disable.")
parser.add_argument( parser.add_argument(
'--xunits', '--xunits',
help="Units for the x-axis.") help="Units for the x-axis.")
parser.add_argument( parser.add_argument(
'--yunits', '--yunits',
help="Units for the y-axis.") help="Units for the y-axis.")
parser.add_argument( parser.add_argument(
'--xlabel', '--xlabel',
help="Add a label to the x-axis.") help="Add a label to the x-axis.")
parser.add_argument( parser.add_argument(
'--ylabel', '--ylabel',
help="Add a label to the y-axis.") help="Add a label to the y-axis.")
parser.add_argument( parser.add_argument(
'--xticklabels', '--xticklabels',
type=lambda x: [re.sub(r'\\([,\\])', r'\1', x.strip()) type=lambda x: [re.sub(r'\\([,\\])', r'\1', x.strip())
for x in re.split(r'(?<!\\),', x)] for x in re.split(r'(?<!\\),', x)]
if x.strip() else [], if x.strip() else [],
help="Comma separated xticklabels. Accepts escaped commas.") help="Comma separated xticklabels. Accepts escaped commas.")
parser.add_argument( parser.add_argument(
'--yticklabels', '--yticklabels',
type=lambda x: [re.sub(r'\\([,\\])', r'\1', x.strip()) type=lambda x: [re.sub(r'\\([,\\])', r'\1', x.strip())
for x in re.split(r'(?<!\\),', x)] for x in re.split(r'(?<!\\),', x)]
if x.strip() else [], if x.strip() else [],
help="Comma separated yticklabels. Accepts escaped commas.") help="Comma separated yticklabels. Accepts escaped commas.")
parser.add_argument( parser.add_argument(
'-t', '--title', '-t', '--title',
help="Add a title.") help="Add a title.")
parser.add_argument( parser.add_argument(
'-l', '--legend', '--legend-right', '-l', '--legend', '--legend-right',
dest='legend_right', dest='legend_right',
action='store_true', action='store_true',
help="Place a legend to the right.") help="Place a legend to the right.")
parser.add_argument( parser.add_argument(
'--legend-above', '--legend-above',
action='store_true', action='store_true',
help="Place a legend above.") help="Place a legend above.")
parser.add_argument( parser.add_argument(
'--legend-below', '--legend-below',
action='store_true', action='store_true',
help="Place a legend below.") help="Place a legend below.")
parser.add_argument( parser.add_argument(
'--dark', '--dark',
action='store_true', action='store_true',
help="Use the dark style.") help="Use the dark style.")
parser.add_argument( parser.add_argument(
'--ggplot', '--ggplot',
action='store_true', action='store_true',
help="Use the ggplot style.") help="Use the ggplot style.")
parser.add_argument( parser.add_argument(
'--xkcd', '--xkcd',
action='store_true', action='store_true',
help="Use the xkcd style.") help="Use the xkcd style.")
parser.add_argument( parser.add_argument(
'--font', '--font',
type=lambda x: [x.strip() for x in x.split(',')], type=lambda x: [x.strip() for x in x.split(',')],
help="Font family for matplotlib.") help="Font family for matplotlib.")
parser.add_argument( parser.add_argument(
'--font-size', '--font-size',
help="Font size for matplotlib. Defaults to %r." % FONT_SIZE) help="Font size for matplotlib. Defaults to %r." % FONT_SIZE)
parser.add_argument( parser.add_argument(
'--font-color', '--font-color',
help="Color for the font and other line elements.") help="Color for the font and other line elements.")
parser.add_argument( parser.add_argument(
'--foreground', '--foreground',
help="Foreground color to use.") help="Foreground color to use.")
parser.add_argument( parser.add_argument(
'--background', '--background',
help="Background color to use.") help="Background color to use.")
class AppendSubplot(argparse.Action): class AppendSubplot(argparse.Action):
@staticmethod @staticmethod
def parse(value): def parse(value):
import copy import copy
subparser = copy.deepcopy(parser) subparser = copy.deepcopy(parser)
next(a for a in subparser._actions next(a for a in subparser._actions
if '--output' in a.option_strings).required = False if '--output' in a.option_strings).required = False
next(a for a in subparser._actions next(a for a in subparser._actions
if '--width' in a.option_strings).type = float if '--width' in a.option_strings).type = float
next(a for a in subparser._actions next(a for a in subparser._actions
if '--height' in a.option_strings).type = float if '--height' in a.option_strings).type = float
return subparser.parse_intermixed_args(shlex.split(value or "")) return subparser.parse_intermixed_args(shlex.split(value or ""))
def __call__(self, parser, namespace, value, option): def __call__(self, parser, namespace, value, option):
if not hasattr(namespace, 'subplots'): if not hasattr(namespace, 'subplots'):
namespace.subplots = [] namespace.subplots = []
namespace.subplots.append(( namespace.subplots.append((
option.split('-')[-1], option.split('-')[-1],
self.__class__.parse(value))) self.__class__.parse(value)))
parser.add_argument( parser.add_argument(
'--subplot-above', '--subplot-above',
action=AppendSubplot, action=AppendSubplot,
help="Add subplot above with the same dataset. Takes an arg string to " help="Add subplot above with the same dataset. Takes an arg "
"control the subplot which supports most (but not all) of the " "string to control the subplot which supports most (but "
"parameters listed here. The relative dimensions of the subplot " "not all) of the parameters listed here. The relative "
"can be controlled with -W/-H which now take a percentage.") "dimensions of the subplot can be controlled with -W/-H "
"which now take a percentage.")
parser.add_argument( parser.add_argument(
'--subplot-below', '--subplot-below',
action=AppendSubplot, action=AppendSubplot,
help="Add subplot below with the same dataset.") help="Add subplot below with the same dataset.")
parser.add_argument( parser.add_argument(
'--subplot-left', '--subplot-left',
action=AppendSubplot, action=AppendSubplot,
help="Add subplot left with the same dataset.") help="Add subplot left with the same dataset.")
parser.add_argument( parser.add_argument(
'--subplot-right', '--subplot-right',
action=AppendSubplot, action=AppendSubplot,
help="Add subplot right with the same dataset.") help="Add subplot right with the same dataset.")
parser.add_argument( parser.add_argument(
'--subplot', '--subplot',
type=AppendSubplot.parse, type=AppendSubplot.parse,
help="Add subplot-specific arguments to the main plot.") help="Add subplot-specific arguments to the main plot.")
def dictify(ns): def dictify(ns):
if hasattr(ns, 'subplots'): if hasattr(ns, 'subplots'):
ns.subplots = [(dir, dictify(subplot_ns)) ns.subplots = [(dir, dictify(subplot_ns))
for dir, subplot_ns in ns.subplots] for dir, subplot_ns in ns.subplots]
if ns.subplot is not None: if ns.subplot is not None:
ns.subplot = dictify(ns.subplot) ns.subplot = dictify(ns.subplot)
return {k: v return {k: v
for k, v in vars(ns).items() for k, v in vars(ns).items()
if v is not None} if v is not None}
sys.exit(main(**dictify(parser.parse_intermixed_args()))) sys.exit(main(**dictify(parser.parse_intermixed_args())))
scripts/prettyasserts.py
+58 -61
View File
@@ -13,6 +13,7 @@
import re import re
import sys import sys
LIMIT = 16 LIMIT = 16
CMP = { CMP = {
@@ -133,15 +134,14 @@ def write_header(f, limit=LIMIT):
# write assert macros # write assert macros
for op, cmp in sorted(CMP.items()): for op, cmp in sorted(CMP.items()):
f.writeln("#define __PRETTY_ASSERT_BOOL_%s(lh, rh) do { \\" f.writeln("#define __PRETTY_ASSERT_BOOL_%s(lh, rh) do { \\" % (
% cmp.upper()) cmp.upper()))
f.writeln(" bool _lh = !!(lh); \\") f.writeln(" bool _lh = !!(lh); \\")
f.writeln(" bool _rh = !!(rh); \\") f.writeln(" bool _rh = !!(rh); \\")
f.writeln(" if (!(_lh %s _rh)) { \\" % op) f.writeln(" if (!(_lh %s _rh)) { \\" % op)
f.writeln(" __pretty_assert_print( \\") f.writeln(" __pretty_assert_print( \\")
f.writeln(" __FILE__, __LINE__, \\") f.writeln(" __FILE__, __LINE__, \\")
f.writeln(" __pretty_assert_bool, \"%s\", \\" f.writeln(" __pretty_assert_bool, \"%s\", \\" % cmp)
% cmp)
f.writeln(" &_lh, 0, \\") f.writeln(" &_lh, 0, \\")
f.writeln(" &_rh, 0); \\") f.writeln(" &_rh, 0); \\")
f.writeln(" __builtin_trap(); \\") f.writeln(" __builtin_trap(); \\")
@@ -149,15 +149,14 @@ def write_header(f, limit=LIMIT):
f.writeln("} while (0)") f.writeln("} while (0)")
f.writeln() f.writeln()
for op, cmp in sorted(CMP.items()): for op, cmp in sorted(CMP.items()):
f.writeln("#define __PRETTY_ASSERT_INT_%s(lh, rh) do { \\" f.writeln("#define __PRETTY_ASSERT_INT_%s(lh, rh) do { \\" % (
% cmp.upper()) cmp.upper()))
f.writeln(" __typeof__(rh) _lh = lh; \\") f.writeln(" __typeof__(rh) _lh = lh; \\")
f.writeln(" __typeof__(rh) _rh = rh; \\") f.writeln(" __typeof__(rh) _rh = rh; \\")
f.writeln(" if (!(_lh %s _rh)) { \\" % op) f.writeln(" if (!(_lh %s _rh)) { \\" % op)
f.writeln(" __pretty_assert_print( \\") f.writeln(" __pretty_assert_print( \\")
f.writeln(" __FILE__, __LINE__, \\") f.writeln(" __FILE__, __LINE__, \\")
f.writeln(" __pretty_assert_int, \"%s\", \\" f.writeln(" __pretty_assert_int, \"%s\", \\" % cmp)
% cmp)
f.writeln(" &(intmax_t){(intmax_t)_lh}, 0, \\") f.writeln(" &(intmax_t){(intmax_t)_lh}, 0, \\")
f.writeln(" &(intmax_t){(intmax_t)_rh}, 0); \\") f.writeln(" &(intmax_t){(intmax_t)_rh}, 0); \\")
f.writeln(" __builtin_trap(); \\") f.writeln(" __builtin_trap(); \\")
@@ -165,15 +164,14 @@ def write_header(f, limit=LIMIT):
f.writeln("} while (0)") f.writeln("} while (0)")
f.writeln() f.writeln()
for op, cmp in sorted(CMP.items()): for op, cmp in sorted(CMP.items()):
f.writeln("#define __PRETTY_ASSERT_MEM_%s(lh, rh, size) do { \\" f.writeln("#define __PRETTY_ASSERT_MEM_%s(lh, rh, size) do { \\" % (
% cmp.upper()) cmp.upper()))
f.writeln(" const void *_lh = lh; \\") f.writeln(" const void *_lh = lh; \\")
f.writeln(" const void *_rh = rh; \\") f.writeln(" const void *_rh = rh; \\")
f.writeln(" if (!(memcmp(_lh, _rh, size) %s 0)) { \\" % op) f.writeln(" if (!(memcmp(_lh, _rh, size) %s 0)) { \\" % op)
f.writeln(" __pretty_assert_print( \\") f.writeln(" __pretty_assert_print( \\")
f.writeln(" __FILE__, __LINE__, \\") f.writeln(" __FILE__, __LINE__, \\")
f.writeln(" __pretty_assert_mem, \"%s\", \\" f.writeln(" __pretty_assert_mem, \"%s\", \\" % cmp)
% cmp)
f.writeln(" _lh, size, \\") f.writeln(" _lh, size, \\")
f.writeln(" _rh, size); \\") f.writeln(" _rh, size); \\")
f.writeln(" __builtin_trap(); \\") f.writeln(" __builtin_trap(); \\")
@@ -181,15 +179,14 @@ def write_header(f, limit=LIMIT):
f.writeln("} while (0)") f.writeln("} while (0)")
f.writeln() f.writeln()
for op, cmp in sorted(CMP.items()): for op, cmp in sorted(CMP.items()):
f.writeln("#define __PRETTY_ASSERT_STR_%s(lh, rh) do { \\" f.writeln("#define __PRETTY_ASSERT_STR_%s(lh, rh) do { \\" % (
% cmp.upper()) cmp.upper()))
f.writeln(" const char *_lh = lh; \\") f.writeln(" const char *_lh = lh; \\")
f.writeln(" const char *_rh = rh; \\") f.writeln(" const char *_rh = rh; \\")
f.writeln(" if (!(strcmp(_lh, _rh) %s 0)) { \\" % op) f.writeln(" if (!(strcmp(_lh, _rh) %s 0)) { \\" % op)
f.writeln(" __pretty_assert_print( \\") f.writeln(" __pretty_assert_print( \\")
f.writeln(" __FILE__, __LINE__, \\") f.writeln(" __FILE__, __LINE__, \\")
f.writeln(" __pretty_assert_str, \"%s\", \\" f.writeln(" __pretty_assert_str, \"%s\", \\" % cmp)
% cmp)
f.writeln(" _lh, strlen(_lh), \\") f.writeln(" _lh, strlen(_lh), \\")
f.writeln(" _rh, strlen(_rh)); \\") f.writeln(" _rh, strlen(_rh)); \\")
f.writeln(" __builtin_trap(); \\") f.writeln(" __builtin_trap(); \\")
@@ -206,11 +203,11 @@ def write_header(f, limit=LIMIT):
def mkassert(type, cmp, lh, rh, size=None): def mkassert(type, cmp, lh, rh, size=None):
if size is not None: if size is not None:
return ("__PRETTY_ASSERT_%s_%s(%s, %s, %s)" return ("__PRETTY_ASSERT_%s_%s(%s, %s, %s)" % (
% (type.upper(), cmp.upper(), lh, rh, size)) type.upper(), cmp.upper(), lh, rh, size))
else: else:
return ("__PRETTY_ASSERT_%s_%s(%s, %s)" return ("__PRETTY_ASSERT_%s_%s(%s, %s)" % (
% (type.upper(), cmp.upper(), lh, rh)) type.upper(), cmp.upper(), lh, rh))
def mkunreachable(): def mkunreachable():
return "__PRETTY_ASSERT_UNREACHABLE()" return "__PRETTY_ASSERT_UNREACHABLE()"
@@ -224,12 +221,12 @@ class ParseFailure(Exception):
def __str__(self): def __str__(self):
return "expected %r, found %s..." % ( return "expected %r, found %s..." % (
self.expected, repr(self.found)[:70]) self.expected, repr(self.found)[:70])
class Parser: class Parser:
def __init__(self, in_f, lexemes=LEXEMES): def __init__(self, in_f, lexemes=LEXEMES):
p = '|'.join('(?P<%s>%s)' % (n, '|'.join(l)) p = '|'.join('(?P<%s>%s)' % (n, '|'.join(l))
for n, l in lexemes.items()) for n, l in lexemes.items())
p = re.compile(p, re.DOTALL) p = re.compile(p, re.DOTALL)
data = in_f.read() data = in_f.read()
tokens = [] tokens = []
@@ -496,54 +493,54 @@ if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Preprocessor that makes asserts easier to debug.", description="Preprocessor that makes asserts easier to debug.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'input', 'input',
help="Input C file.") help="Input C file.")
parser.add_argument( parser.add_argument(
'-o', '--output', '-o', '--output',
required=True, required=True,
help="Output C file.") help="Output C file.")
parser.add_argument( parser.add_argument(
'-p', '--prefix', '-p', '--prefix',
action='append', action='append',
help="Additional prefixes for symbols.") help="Additional prefixes for symbols.")
parser.add_argument( parser.add_argument(
'-P', '--prefix-insensitive', '-P', '--prefix-insensitive',
action='append', action='append',
help="Additional prefixes for lower/upper case symbol variants.") help="Additional prefixes for lower/upper case symbol variants.")
parser.add_argument( parser.add_argument(
'--assert', '--assert',
dest='assert_', dest='assert_',
action='append', action='append',
help="Additional symbols for assert statements.") help="Additional symbols for assert statements.")
parser.add_argument( parser.add_argument(
'--unreachable', '--unreachable',
action='append', action='append',
help="Additional symbols for unreachable statements.") help="Additional symbols for unreachable statements.")
parser.add_argument( parser.add_argument(
'--memcmp', '--memcmp',
action='append', action='append',
help="Additional symbols for memcmp expressions.") help="Additional symbols for memcmp expressions.")
parser.add_argument( parser.add_argument(
'--strcmp', '--strcmp',
action='append', action='append',
help="Additional symbols for strcmp expressions.") help="Additional symbols for strcmp expressions.")
parser.add_argument( parser.add_argument(
'-n', '--no-defaults', '-n', '--no-defaults',
action='store_true', action='store_true',
help="Disable default symbols.") help="Disable default symbols.")
parser.add_argument( parser.add_argument(
'--no-arrows', '--no-arrows',
action='store_true', action='store_true',
help="Disable arrow (=>) expressions.") help="Disable arrow (=>) expressions.")
parser.add_argument( parser.add_argument(
'-l', '--limit', '-l', '--limit',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
default=LIMIT, default=LIMIT,
help="Maximum number of characters to display in strcmp and memcmp. " help="Maximum number of characters to display in strcmp and "
"Defaults to %r." % LIMIT) "memcmp. Defaults to %r." % LIMIT)
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/readblock.py
+7 -5
View File
@@ -2,25 +2,27 @@
import subprocess as sp import subprocess as sp
def main(args): def main(args):
with open(args.disk, 'rb') as f: with open(args.disk, 'rb') as f:
f.seek(args.block * args.block_size) f.seek(args.block * args.block_size)
block = (f.read(args.block_size) block = (f.read(args.block_size)
.ljust(args.block_size, b'\xff')) .ljust(args.block_size, b'\xff'))
# what did you expect? # what did you expect?
print("%-8s %-s" % ('off', 'data')) print("%-8s %-s" % ('off', 'data'))
return sp.run(['xxd', '-g1', '-'], input=block).returncode return sp.run(['xxd', '-g1', '-'], input=block).returncode
if __name__ == "__main__": if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Hex dump a specific block in a disk.") description="Hex dump a specific block in a disk.")
parser.add_argument('disk', parser.add_argument('disk',
help="File representing the block device.") help="File representing the block device.")
parser.add_argument('block_size', type=lambda x: int(x, 0), parser.add_argument('block_size', type=lambda x: int(x, 0),
help="Size of a block in bytes.") help="Size of a block in bytes.")
parser.add_argument('block', type=lambda x: int(x, 0), parser.add_argument('block', type=lambda x: int(x, 0),
help="Address of block to dump.") help="Address of block to dump.")
sys.exit(main(parser.parse_args())) sys.exit(main(parser.parse_args()))
scripts/readmdir.py
+44 -41
View File
@@ -5,6 +5,7 @@ import binascii
import sys import sys
import itertools as it import itertools as it
TAG_TYPES = { TAG_TYPES = {
'splice': (0x700, 0x400), 'splice': (0x700, 0x400),
'create': (0x7ff, 0x401), 'create': (0x7ff, 0x401),
@@ -104,8 +105,9 @@ class Tag:
try: try:
if ' ' in type: if ' ' in type:
type1, type3 = type.split() type1, type3 = type.split()
return (self.is_(type1) and return (self.is_(type1)
(self.type & ~TAG_TYPES[type1][0]) == int(type3, 0)) and (self.type & ~TAG_TYPES[type1][0])
== int(type3, 0))
return self.type == int(type, 0) return self.type == int(type, 0)
@@ -114,9 +116,9 @@ class Tag:
def mkmask(self): def mkmask(self):
return Tag( return Tag(
0x700 if self.isunique else 0x7ff, 0x700 if self.isunique else 0x7ff,
0x3ff if self.isattr else 0, 0x3ff if self.isattr else 0,
0) 0)
def chid(self, nid): def chid(self, nid):
ntag = Tag(self.type, nid, self.size) ntag = Tag(self.type, nid, self.size)
@@ -142,7 +144,7 @@ class Tag:
type = reverse_types[mask, self.type & mask] type = reverse_types[mask, self.type & mask]
if prefix > 0: if prefix > 0:
return '%s %#x%s' % ( return '%s %#x%s' % (
type, self.type & ((1 << prefix)-1), crc_status) type, self.type & ((1 << prefix)-1), crc_status)
else: else:
return '%s%s' % (type, crc_status) return '%s%s' % (type, crc_status)
else: else:
@@ -226,7 +228,7 @@ class MetadataPair:
if fcrcdata: if fcrcdata:
fcrcsize, fcrc = fcrcdata fcrcsize, fcrc = fcrcdata
fcrc_ = 0xffffffff ^ binascii.crc32( fcrc_ = 0xffffffff ^ binascii.crc32(
block[off:off+fcrcsize]) block[off:off+fcrcsize])
if fcrc_ == fcrc: if fcrc_ == fcrc:
fcrctag.erased = True fcrctag.erased = True
corrupt = True corrupt = True
@@ -239,8 +241,8 @@ class MetadataPair:
# find active ids # find active ids
self.ids = list(it.takewhile( self.ids = list(it.takewhile(
lambda id: Tag('name', id, 0) in self, lambda id: Tag('name', id, 0) in self,
it.count())) it.count()))
# find most recent tags # find most recent tags
self.tags = [] self.tags = []
@@ -286,16 +288,16 @@ class MetadataPair:
gdiff = 0 gdiff = 0
for tag in reversed(self.log): for tag in reversed(self.log):
if (gmask.id != 0 and tag.is_('splice') and if (gmask.id != 0 and tag.is_('splice')
tag.id <= gtag.id - gdiff): and tag.id <= gtag.id - gdiff):
if tag.is_('create') and tag.id == gtag.id - gdiff: if tag.is_('create') and tag.id == gtag.id - gdiff:
# creation point # creation point
break break
gdiff += tag.schunk gdiff += tag.schunk
if ((int(gmask) & int(tag)) == if ((int(gmask) & int(tag))
(int(gmask) & int(gtag.chid(gtag.id - gdiff)))): == (int(gmask) & int(gtag.chid(gtag.id - gdiff)))):
if tag.size == 0x3ff: if tag.size == 0x3ff:
# deleted # deleted
break break
@@ -306,28 +308,28 @@ class MetadataPair:
def _dump_tags(self, tags, f=sys.stdout, truncate=True): def _dump_tags(self, tags, f=sys.stdout, truncate=True):
f.write("%-8s %-8s %-13s %4s %4s" % ( f.write("%-8s %-8s %-13s %4s %4s" % (
'off', 'tag', 'type', 'id', 'len')) 'off', 'tag', 'type', 'id', 'len'))
if truncate: if truncate:
f.write(' data (truncated)') f.write(' data (truncated)')
f.write('\n') f.write('\n')
for tag in tags: for tag in tags:
f.write("%08x: %08x %-14s %3s %4s" % ( f.write("%08x: %08x %-14s %3s %4s" % (
tag.off, tag, tag.off, tag,
tag.typerepr(), tag.idrepr(), tag.sizerepr())) tag.typerepr(), tag.idrepr(), tag.sizerepr()))
if truncate: if truncate:
f.write(" %-23s %-8s\n" % ( f.write(" %-23s %-8s\n" % (
' '.join('%02x' % c for c in tag.data[:8]), ' '.join('%02x' % c for c in tag.data[:8]),
''.join(c if c >= ' ' and c <= '~' else '.' ''.join(c if c >= ' ' and c <= '~' else '.'
for c in map(chr, tag.data[:8])))) for c in map(chr, tag.data[:8]))))
else: else:
f.write("\n") f.write("\n")
for i in range(0, len(tag.data), 16): for i in range(0, len(tag.data), 16):
f.write(" %08x: %-47s %-16s\n" % ( f.write(" %08x: %-47s %-16s\n" % (
tag.off+i, tag.off+i,
' '.join('%02x' % c for c in tag.data[i:i+16]), ' '.join('%02x' % c for c in tag.data[i:i+16]),
''.join(c if c >= ' ' and c <= '~' else '.' ''.join(c if c >= ' ' and c <= '~' else '.'
for c in map(chr, tag.data[i:i+16])))) for c in map(chr, tag.data[i:i+16]))))
def dump_tags(self, f=sys.stdout, truncate=True): def dump_tags(self, f=sys.stdout, truncate=True):
self._dump_tags(self.tags, f=f, truncate=truncate) self._dump_tags(self.tags, f=f, truncate=truncate)
@@ -346,7 +348,7 @@ def main(args):
continue continue
f.seek(block * args.block_size) f.seek(block * args.block_size)
blocks.append(f.read(args.block_size) blocks.append(f.read(args.block_size)
.ljust(args.block_size, b'\xff')) .ljust(args.block_size, b'\xff'))
# find most recent pair # find most recent pair
mdir = MetadataPair(blocks) mdir = MetadataPair(blocks)
@@ -359,15 +361,15 @@ def main(args):
mdir.tail = None mdir.tail = None
print("mdir {%s} rev %d%s%s%s" % ( print("mdir {%s} rev %d%s%s%s" % (
', '.join('%#x' % b ', '.join('%#x' % b
for b in [args.block1, args.block2] for b in [args.block1, args.block2]
if b is not None), if b is not None),
mdir.rev, mdir.rev,
' (was %s)' % ', '.join('%d' % m.rev for m in mdir.pair[1:]) ' (was %s)' % ', '.join('%d' % m.rev for m in mdir.pair[1:])
if len(mdir.pair) > 1 else '', if len(mdir.pair) > 1 else '',
' (corrupted!)' if not mdir else '', ' (corrupted!)' if not mdir else '',
' -> {%#x, %#x}' % struct.unpack('<II', mdir.tail.data) ' -> {%#x, %#x}' % struct.unpack('<II', mdir.tail.data)
if mdir.tail else '')) if mdir.tail else ''))
if args.all: if args.all:
mdir.dump_all(truncate=not args.no_truncate) mdir.dump_all(truncate=not args.no_truncate)
elif args.log: elif args.log:
@@ -377,23 +379,24 @@ def main(args):
return 0 if mdir else 1 return 0 if mdir else 1
if __name__ == "__main__": if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Dump useful info about metadata pairs in littlefs.") description="Dump useful info about metadata pairs in littlefs.")
parser.add_argument('disk', parser.add_argument('disk',
help="File representing the block device.") help="File representing the block device.")
parser.add_argument('block_size', type=lambda x: int(x, 0), parser.add_argument('block_size', type=lambda x: int(x, 0),
help="Size of a block in bytes.") help="Size of a block in bytes.")
parser.add_argument('block1', type=lambda x: int(x, 0), parser.add_argument('block1', type=lambda x: int(x, 0),
help="First block address for finding the metadata pair.") help="First block address for finding the metadata pair.")
parser.add_argument('block2', nargs='?', type=lambda x: int(x, 0), parser.add_argument('block2', nargs='?', type=lambda x: int(x, 0),
help="Second block address for finding the metadata pair.") help="Second block address for finding the metadata pair.")
parser.add_argument('-l', '--log', action='store_true', parser.add_argument('-l', '--log', action='store_true',
help="Show tags in log.") help="Show tags in log.")
parser.add_argument('-a', '--all', action='store_true', parser.add_argument('-a', '--all', action='store_true',
help="Show all tags in log, included tags in corrupted commits.") help="Show all tags in log, included tags in corrupted commits.")
parser.add_argument('-T', '--no-truncate', action='store_true', parser.add_argument('-T', '--no-truncate', action='store_true',
help="Don't truncate large amounts of data.") help="Don't truncate large amounts of data.")
sys.exit(main(parser.parse_args())) sys.exit(main(parser.parse_args()))
scripts/readtree.py
+35 -29
View File
@@ -7,6 +7,7 @@ import io
import itertools as it import itertools as it
from readmdir import Tag, MetadataPair from readmdir import Tag, MetadataPair
def main(args): def main(args):
superblock = None superblock = None
gstate = b'\0\0\0\0\0\0\0\0\0\0\0\0' gstate = b'\0\0\0\0\0\0\0\0\0\0\0\0'
@@ -31,7 +32,7 @@ def main(args):
for block in tail: for block in tail:
f.seek(block * args.block_size) f.seek(block * args.block_size)
data.append(f.read(args.block_size) data.append(f.read(args.block_size)
.ljust(args.block_size, b'\xff')) .ljust(args.block_size, b'\xff'))
blocks[id(data[-1])] = block blocks[id(data[-1])] = block
mdir = MetadataPair(data) mdir = MetadataPair(data)
@@ -48,7 +49,7 @@ def main(args):
# have superblock? # have superblock?
try: try:
nsuperblock = mdir[ nsuperblock = mdir[
Tag(0x7ff, 0x3ff, 0), Tag('superblock', 0, 0)] Tag(0x7ff, 0x3ff, 0), Tag('superblock', 0, 0)]
superblock = nsuperblock, mdir[Tag('inlinestruct', 0, 0)] superblock = nsuperblock, mdir[Tag('inlinestruct', 0, 0)]
except KeyError: except KeyError:
pass pass
@@ -57,7 +58,7 @@ def main(args):
try: try:
ngstate = mdir[Tag('movestate', 0, 0)] ngstate = mdir[Tag('movestate', 0, 0)]
gstate = bytes((a or 0) ^ (b or 0) gstate = bytes((a or 0) ^ (b or 0)
for a,b in it.zip_longest(gstate, ngstate.data)) for a,b in it.zip_longest(gstate, ngstate.data))
except KeyError: except KeyError:
pass pass
@@ -103,10 +104,13 @@ def main(args):
version = ('?', '?') version = ('?', '?')
if superblock: if superblock:
version = tuple(reversed( version = tuple(reversed(
struct.unpack('<HH', superblock[1].data[0:4].ljust(4, b'\xff')))) struct.unpack('<HH',
print("%-47s%s" % ("littlefs v%s.%s" % version, superblock[1].data[0:4].ljust(4, b'\xff'))))
"data (truncated, if it fits)" print("%-47s%s" % (
if not any([args.no_truncate, args.log, args.all]) else "")) "littlefs v%s.%s" % version,
"data (truncated, if it fits)"
if not any([args.no_truncate, args.log, args.all])
else ""))
# print gstate # print gstate
print("gstate 0x%s" % ''.join('%02x' % c for c in gstate)) print("gstate 0x%s" % ''.join('%02x' % c for c in gstate))
@@ -116,19 +120,19 @@ def main(args):
print(" orphans >=%d" % max(tag.size, 1)) print(" orphans >=%d" % max(tag.size, 1))
if tag.type: if tag.type:
print(" move dir {%#x, %#x} id %d" % ( print(" move dir {%#x, %#x} id %d" % (
blocks[0], blocks[1], tag.id)) blocks[0], blocks[1], tag.id))
# print mdir info # print mdir info
for i, dir in enumerate(dirs): for i, dir in enumerate(dirs):
print("dir %s" % (json.dumps(dir[0].path) print("dir %s" % (json.dumps(dir[0].path)
if hasattr(dir[0], 'path') else '(orphan)')) if hasattr(dir[0], 'path') else '(orphan)'))
for j, mdir in enumerate(dir): for j, mdir in enumerate(dir):
print("mdir {%#x, %#x} rev %d (was %d)%s%s" % ( print("mdir {%#x, %#x} rev %d (was %d)%s%s" % (
mdir.blocks[0], mdir.blocks[1], mdir.rev, mdir.pair[1].rev, mdir.blocks[0], mdir.blocks[1], mdir.rev, mdir.pair[1].rev,
' (corrupted!)' if not mdir else '', ' (corrupted!)' if not mdir else '',
' -> {%#x, %#x}' % struct.unpack('<II', mdir.tail.data) ' -> {%#x, %#x}' % struct.unpack('<II', mdir.tail.data)
if mdir.tail else '')) if mdir.tail else ''))
f = io.StringIO() f = io.StringIO()
if args.log: if args.log:
@@ -141,43 +145,45 @@ def main(args):
lines = list(filter(None, f.getvalue().split('\n'))) lines = list(filter(None, f.getvalue().split('\n')))
for k, line in enumerate(lines): for k, line in enumerate(lines):
print("%s %s" % ( print("%s %s" % (
' ' if j == len(dir)-1 else ' ' if j == len(dir)-1 else
'v' if k == len(lines)-1 else 'v' if k == len(lines)-1 else
'|', '|',
line)) line))
errcode = 0 errcode = 0
for mdir in corrupted: for mdir in corrupted:
errcode = errcode or 1 errcode = errcode or 1
print("*** corrupted mdir {%#x, %#x}! ***" % ( print("*** corrupted mdir {%#x, %#x}! ***" % (
mdir.blocks[0], mdir.blocks[1])) mdir.blocks[0], mdir.blocks[1]))
if cycle: if cycle:
errcode = errcode or 2 errcode = errcode or 2
print("*** cycle detected {%#x, %#x}! ***" % ( print("*** cycle detected {%#x, %#x}! ***" % (
cycle[0], cycle[1])) cycle[0], cycle[1]))
return errcode return errcode
if __name__ == "__main__": if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Dump semantic info about the metadata tree in littlefs") description="Dump semantic info about the metadata tree in "
"littlefs")
parser.add_argument('disk', parser.add_argument('disk',
help="File representing the block device.") help="File representing the block device.")
parser.add_argument('block_size', type=lambda x: int(x, 0), parser.add_argument('block_size', type=lambda x: int(x, 0),
help="Size of a block in bytes.") help="Size of a block in bytes.")
parser.add_argument('block1', nargs='?', default=0, parser.add_argument('block1', nargs='?', default=0,
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Optional first block address for finding the superblock.") help="Optional first block address for finding the superblock.")
parser.add_argument('block2', nargs='?', default=1, parser.add_argument('block2', nargs='?', default=1,
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Optional second block address for finding the superblock.") help="Optional second block address for finding the superblock.")
parser.add_argument('-l', '--log', action='store_true', parser.add_argument('-l', '--log', action='store_true',
help="Show tags in log.") help="Show tags in log.")
parser.add_argument('-a', '--all', action='store_true', parser.add_argument('-a', '--all', action='store_true',
help="Show all tags in log, included tags in corrupted commits.") help="Show all tags in log, included tags in corrupted commits.")
parser.add_argument('-T', '--no-truncate', action='store_true', parser.add_argument('-T', '--no-truncate', action='store_true',
help="Show the full contents of files/attrs/tags.") help="Show the full contents of files/attrs/tags.")
sys.exit(main(parser.parse_args())) sys.exit(main(parser.parse_args()))
scripts/stack.py
+221 -216
View File
@@ -18,7 +18,6 @@ import os
import re import re
# integer fields # integer fields
class RInt(co.namedtuple('RInt', 'x')): class RInt(co.namedtuple('RInt', 'x')):
__slots__ = () __slots__ = ()
@@ -107,14 +106,14 @@ class StackResult(co.namedtuple('StackResult', [
frame=0, limit=0, frame=0, limit=0,
children=[]): children=[]):
return super().__new__(cls, file, function, return super().__new__(cls, file, function,
RInt(frame), RInt(limit), RInt(frame), RInt(limit),
children) children)
def __add__(self, other): def __add__(self, other):
return StackResult(self.file, self.function, return StackResult(self.file, self.function,
self.frame + other.frame, self.frame + other.frame,
max(self.limit, other.limit), max(self.limit, other.limit),
self.children + other.children) self.children + other.children)
def openio(path, mode='r', buffering=-1): def openio(path, mode='r', buffering=-1):
@@ -163,7 +162,7 @@ def collect(ci_paths, *,
# collect into functions # collect into functions
callgraph = co.defaultdict(lambda: (None, None, 0, set())) callgraph = co.defaultdict(lambda: (None, None, 0, set()))
f_pattern = re.compile( f_pattern = re.compile(
r'([^\\]*)\\n([^:]*)[^\\]*\\n([0-9]+) bytes \((.*)\)') r'([^\\]*)\\n([^:]*)[^\\]*\\n([0-9]+) bytes \((.*)\)')
for path in ci_paths: for path in ci_paths:
with open(path) as f: with open(path) as f:
vcg = parse_vcg(f.read()) vcg = parse_vcg(f.read())
@@ -179,12 +178,12 @@ def collect(ci_paths, *,
if (not args.get('quiet') if (not args.get('quiet')
and 'static' not in type and 'static' not in type
and 'bounded' not in type): and 'bounded' not in type):
print("warning: " print("warning: found non-static stack "
"found non-static stack for %s (%s, %s)" % ( "for %s (%s, %s)" % (
function, type, size)) function, type, size))
_, _, _, targets = callgraph[info['title']] _, _, _, targets = callgraph[info['title']]
callgraph[info['title']] = ( callgraph[info['title']] = (
file, function, int(size), targets) file, function, int(size), targets)
elif k == 'edge': elif k == 'edge':
info = dict(info) info = dict(info)
_, _, _, targets = callgraph[info['sourcename']] _, _, _, targets = callgraph[info['sourcename']]
@@ -199,8 +198,7 @@ def collect(ci_paths, *,
continue continue
# ignore filtered sources # ignore filtered sources
if sources is not None: if sources is not None:
if not any( if not any(os.path.abspath(s_file) == os.path.abspath(s)
os.path.abspath(s_file) == os.path.abspath(s)
for s in sources): for s in sources):
continue continue
else: else:
@@ -268,9 +266,9 @@ def collect(ci_paths, *,
# in the case of recursion # in the case of recursion
for source, (_, _, _, targets) in callgraph.items(): for source, (_, _, _, targets) in callgraph.items():
results[source].children.extend( results[source].children.extend(
results[target] results[target]
for target in targets for target in targets
if target in results) if target in results)
return list(results.values()) return list(results.values())
@@ -282,7 +280,7 @@ def fold(Result, results, by=None, defines=[]):
for k in it.chain(by or [], (k for k, _ in defines)): for k in it.chain(by or [], (k for k, _ in defines)):
if k not in Result._by and k not in Result._fields: if k not in Result._by and k not in Result._fields:
print("error: could not find field %r?" % k, print("error: could not find field %r?" % k,
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
# filter by matching defines # filter by matching defines
@@ -338,74 +336,78 @@ def table(Result, results, diff_results=None, *,
return [] return []
r = max(results_, r = max(results_,
key=lambda r: tuple( key=lambda r: tuple(
tuple( tuple((getattr(r, k),)
(getattr(r, k),) if getattr(r, k, None) is not None
if getattr(r, k, None) is not None else ()
else () for k in (
for k in ([k] if k else [ [k] if k else [
k for k in Result._sort if k in fields]) k for k in Result._sort
if k in fields) if k in fields])
for k in it.chain(hot, [None]))) if k in fields)
for k in it.chain(hot, [None])))
# found a cycle? # found a cycle?
if tuple(getattr(r, k) for k in Result._by) in seen: if tuple(getattr(r, k) for k in Result._by) in seen:
return [] return []
return [r._replace(children=[])] + rec_hot( return [r._replace(children=[])] + rec_hot(
r.children, r.children,
seen | {tuple(getattr(r, k) for k in Result._by)}) seen | {tuple(getattr(r, k) for k in Result._by)})
results = [r._replace(children=rec_hot(r.children)) for r in results] results = [r._replace(children=rec_hot(r.children)) for r in results]
# organize by name # organize by name
table = { table = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in results} for r in results}
diff_table = { diff_table = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in diff_results or []} for r in diff_results or []}
names = [name names = [name
for name in table.keys() | diff_table.keys() for name in table.keys() | diff_table.keys()
if diff_results is None if diff_results is None
or all_ or all_
or any( or any(
types[k].ratio( types[k].ratio(
getattr(table.get(name), k, None), getattr(table.get(name), k, None),
getattr(diff_table.get(name), k, None)) getattr(diff_table.get(name), k, None))
for k in fields)] for k in fields)]
# sort again, now with diff info, note that python's sort is stable # sort again, now with diff info, note that python's sort is stable
names.sort() names.sort()
if diff_results is not None: if diff_results is not None:
names.sort(key=lambda n: tuple( names.sort(
types[k].ratio( key=lambda n: tuple(
getattr(table.get(n), k, None), types[k].ratio(
getattr(diff_table.get(n), k, None)) getattr(table.get(n), k, None),
for k in fields), getattr(diff_table.get(n), k, None))
reverse=True) for k in fields),
reverse=True)
if sort: if sort:
for k, reverse in reversed(sort): for k, reverse in reversed(sort):
names.sort( names.sort(
key=lambda n: tuple( key=lambda n: tuple(
(getattr(table[n], k),) (getattr(table[n], k),)
if getattr(table.get(n), k, None) is not None else () if getattr(table.get(n), k, None) is not None
for k in ([k] if k else [ else ()
k for k in Result._sort if k in fields])), for k in (
reverse=reverse ^ (not k or k in Result._fields)) [k] if k else [
k for k in Result._sort
if k in fields])),
reverse=reverse ^ (not k or k in Result._fields))
# build up our lines # build up our lines
lines = [] lines = []
# header # header
header = [ header = ['%s%s' % (
'%s%s' % ( ','.join(by),
','.join(by), ' (%d added, %d removed)' % (
' (%d added, %d removed)' % ( sum(1 for n in table if n not in diff_table),
sum(1 for n in table if n not in diff_table), sum(1 for n in diff_table if n not in table))
sum(1 for n in diff_table if n not in table)) if diff_results is not None and not percent else '')
if diff_results is not None and not percent else '')
if not summary else ''] if not summary else '']
if diff_results is None: if diff_results is None:
for k in fields: for k in fields:
@@ -428,43 +430,43 @@ def table(Result, results, diff_results=None, *,
if diff_results is None: if diff_results is None:
for k in fields: for k in fields:
entry.append( entry.append(
(getattr(r, k).table(), (getattr(r, k).table(),
getattr(getattr(r, k), 'notes', lambda: [])()) getattr(getattr(r, k), 'notes', lambda: [])())
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none) else types[k].none)
elif percent: elif percent:
for k in fields: for k in fields:
entry.append( entry.append(
(getattr(r, k).table() (getattr(r, k).table()
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none, else types[k].none,
(lambda t: ['+∞%'] if t == +mt.inf (lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)])( else ['%+.1f%%' % (100*t)])(
types[k].ratio( types[k].ratio(
getattr(r, k, None), getattr(r, k, None),
getattr(diff_r, k, None))))) getattr(diff_r, k, None)))))
else: else:
for k in fields: for k in fields:
entry.append(getattr(diff_r, k).table() entry.append(getattr(diff_r, k).table()
if getattr(diff_r, k, None) is not None if getattr(diff_r, k, None) is not None
else types[k].none) else types[k].none)
for k in fields: for k in fields:
entry.append(getattr(r, k).table() entry.append(getattr(r, k).table()
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none) else types[k].none)
for k in fields: for k in fields:
entry.append( entry.append(
(types[k].diff( (types[k].diff(
getattr(r, k, None),
getattr(diff_r, k, None)),
(lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)] if t
else [])(
types[k].ratio(
getattr(r, k, None), getattr(r, k, None),
getattr(diff_r, k, None))))) getattr(diff_r, k, None)),
(lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)] if t
else [])(
types[k].ratio(
getattr(r, k, None),
getattr(diff_r, k, None)))))
return entry return entry
# recursive entry helper # recursive entry helper
@@ -473,8 +475,8 @@ def table(Result, results, diff_results=None, *,
# build the children table at each layer # build the children table at each layer
results_ = fold(Result, results_, by=by) results_ = fold(Result, results_, by=by)
table_ = { table_ = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in results_} for r in results_}
names_ = list(table_.keys()) names_ = list(table_.keys())
# sort the children layer # sort the children layer
@@ -482,13 +484,16 @@ def table(Result, results, diff_results=None, *,
if sort: if sort:
for k, reverse in reversed(sort): for k, reverse in reversed(sort):
names_.sort( names_.sort(
key=lambda n: tuple( key=lambda n: tuple(
(getattr(table_[n], k),) (getattr(table_[n], k),)
if getattr(table_.get(n), k, None) is not None if getattr(table_.get(n), k, None)
else () is not None
for k in ([k] if k else [ else ()
k for k in Result._sort if k in fields])), for k in (
reverse=reverse ^ (not k or k in Result._fields)) [k] if k else [
k for k in Result._sort
if k in fields])),
reverse=reverse ^ (not k or k in Result._fields))
for i, name in enumerate(names_): for i, name in enumerate(names_):
r = table_[name] r = table_[name]
@@ -509,14 +514,13 @@ def table(Result, results, diff_results=None, *,
# recurse? # recurse?
if depth_ > 1: if depth_ > 1:
recurse( recurse(r.children,
r.children, depth_-1,
depth_-1, seen | {name},
seen | {name}, (prefixes[2+is_last] + "|-> ",
(prefixes[2+is_last] + "|-> ", prefixes[2+is_last] + "'-> ",
prefixes[2+is_last] + "'-> ", prefixes[2+is_last] + "| ",
prefixes[2+is_last] + "| ", prefixes[2+is_last] + " "))
prefixes[2+is_last] + " "))
# entries # entries
if not summary: if not summary:
@@ -530,14 +534,13 @@ def table(Result, results, diff_results=None, *,
# recursive entries # recursive entries
if name in table and depth > 1: if name in table and depth > 1:
recurse( recurse(table[name].children,
table[name].children, depth-1,
depth-1, {name},
{name}, ("|-> ",
("|-> ", "'-> ",
"'-> ", "| ",
"| ", " "))
" "))
# total # total
r = next(iter(fold(Result, results, by=[])), None) r = next(iter(fold(Result, results, by=[])), None)
@@ -549,8 +552,8 @@ def table(Result, results, diff_results=None, *,
# homogenize # homogenize
lines = [ lines = [
[x if isinstance(x, tuple) else (x, []) for x in line] [x if isinstance(x, tuple) else (x, []) for x in line]
for line in lines] for line in lines]
# find the best widths, note that column 0 contains the names and is # find the best widths, note that column 0 contains the names and is
# handled a bit differently # handled a bit differently
@@ -564,11 +567,11 @@ def table(Result, results, diff_results=None, *,
# print our table # print our table
for line in lines: for line in lines:
print('%-*s %s' % ( print('%-*s %s' % (
widths[0], line[0][0], widths[0], line[0][0],
' '.join('%*s%-*s' % ( ' '.join('%*s%-*s' % (
widths[i], x[0], widths[i], x[0],
notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '')
for i, x in enumerate(line[1:], 1)))) for i, x in enumerate(line[1:], 1))))
def main(ci_paths, def main(ci_paths,
@@ -600,10 +603,10 @@ def main(ci_paths,
continue continue
try: try:
results.append(StackResult( results.append(StackResult(
**{k: r[k] for k in StackResult._by **{k: r[k] for k in StackResult._by
if k in r and r[k].strip()}, if k in r and r[k].strip()},
**{k: r[k] for k in StackResult._fields **{k: r[k] for k in StackResult._fields
if k in r and r[k].strip()})) if k in r and r[k].strip()}))
except TypeError: except TypeError:
pass pass
@@ -615,25 +618,27 @@ def main(ci_paths,
if sort: if sort:
for k, reverse in reversed(sort): for k, reverse in reversed(sort):
results.sort( results.sort(
key=lambda r: tuple( key=lambda r: tuple(
(getattr(r, k),) if getattr(r, k) is not None else () (getattr(r, k),) if getattr(r, k) is not None else ()
for k in ([k] if k else StackResult._sort)), for k in ([k] if k else StackResult._sort)),
reverse=reverse ^ (not k or k in StackResult._fields)) reverse=reverse ^ (not k or k in StackResult._fields))
# write results to CSV # write results to CSV
if args.get('output'): if args.get('output'):
with openio(args['output'], 'w') as f: with openio(args['output'], 'w') as f:
writer = csv.DictWriter(f, writer = csv.DictWriter(f,
(by if by is not None else StackResult._by) (by if by is not None else StackResult._by)
+ [k for k in ( + [k for k in (
fields if fields is not None else StackResult._fields)]) fields if fields is not None
else StackResult._fields)])
writer.writeheader() writer.writeheader()
for r in results: for r in results:
writer.writerow( writer.writerow(
{k: getattr(r, k) for k in ( {k: getattr(r, k) for k in (
by if by is not None else StackResult._by)} by if by is not None else StackResult._by)}
| {k: getattr(r, k) for k in ( | {k: getattr(r, k) for k in (
fields if fields is not None else StackResult._fields)}) fields if fields is not None
else StackResult._fields)})
# find previous results? # find previous results?
if args.get('diff'): if args.get('diff'):
@@ -651,10 +656,10 @@ def main(ci_paths,
continue continue
try: try:
diff_results.append(StackResult( diff_results.append(StackResult(
**{k: r[k] for k in StackResult._by **{k: r[k] for k in StackResult._by
if k in r and r[k].strip()}, if k in r and r[k].strip()},
**{k: r[k] for k in StackResult._fields **{k: r[k] for k in StackResult._fields
if k in r and r[k].strip()})) if k in r and r[k].strip()}))
except TypeError: except TypeError:
raise raise
except FileNotFoundError: except FileNotFoundError:
@@ -666,11 +671,11 @@ def main(ci_paths,
# print table # print table
if not args.get('quiet'): if not args.get('quiet'):
table(StackResult, results, table(StackResult, results,
diff_results if args.get('diff') else None, diff_results if args.get('diff') else None,
by=by if by is not None else ['function'], by=by if by is not None else ['function'],
fields=fields, fields=fields,
sort=sort, sort=sort,
**args) **args)
# error on recursion # error on recursion
if args.get('error_on_recursion') and any( if args.get('error_on_recursion') and any(
@@ -682,103 +687,103 @@ if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Find stack usage at the function level.", description="Find stack usage at the function level.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'ci_paths', 'ci_paths',
nargs='*', nargs='*',
help="Input *.ci files.") help="Input *.ci files.")
parser.add_argument( parser.add_argument(
'-v', '--verbose', '-v', '--verbose',
action='store_true', action='store_true',
help="Output commands that run behind the scenes.") help="Output commands that run behind the scenes.")
parser.add_argument( parser.add_argument(
'-q', '--quiet', '-q', '--quiet',
action='store_true', action='store_true',
help="Don't show anything, useful with -o.") help="Don't show anything, useful with -o.")
parser.add_argument( parser.add_argument(
'-o', '--output', '-o', '--output',
help="Specify CSV file to store results.") help="Specify CSV file to store results.")
parser.add_argument( parser.add_argument(
'-u', '--use', '-u', '--use',
help="Don't parse anything, use this CSV file.") help="Don't parse anything, use this CSV file.")
parser.add_argument( parser.add_argument(
'-d', '--diff', '-d', '--diff',
help="Specify CSV file to diff against.") help="Specify CSV file to diff against.")
parser.add_argument( parser.add_argument(
'-a', '--all', '-a', '--all',
action='store_true', action='store_true',
help="Show all, not just the ones that changed.") help="Show all, not just the ones that changed.")
parser.add_argument( parser.add_argument(
'-p', '--percent', '-p', '--percent',
action='store_true', action='store_true',
help="Only show percentage change, not a full diff.") help="Only show percentage change, not a full diff.")
parser.add_argument( parser.add_argument(
'-b', '--by', '-b', '--by',
action='append', action='append',
choices=StackResult._by, choices=StackResult._by,
help="Group by this field.") help="Group by this field.")
parser.add_argument( parser.add_argument(
'-f', '--field', '-f', '--field',
dest='fields', dest='fields',
action='append', action='append',
choices=StackResult._fields, choices=StackResult._fields,
help="Show this field.") help="Show this field.")
parser.add_argument( parser.add_argument(
'-D', '--define', '-D', '--define',
dest='defines', dest='defines',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs: ( lambda k, vs: (
k.strip(), k.strip(),
{v.strip() for v in vs.split(',')}) {v.strip() for v in vs.split(',')})
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Only include results where this field is this value.") help="Only include results where this field is this value.")
class AppendSort(argparse.Action): class AppendSort(argparse.Action):
def __call__(self, parser, namespace, value, option): def __call__(self, parser, namespace, value, option):
if namespace.sort is None: if namespace.sort is None:
namespace.sort = [] namespace.sort = []
namespace.sort.append((value, True if option == '-S' else False)) namespace.sort.append((value, True if option == '-S' else False))
parser.add_argument( parser.add_argument(
'-s', '--sort', '-s', '--sort',
nargs='?', nargs='?',
action=AppendSort, action=AppendSort,
help="Sort by this field.") help="Sort by this field.")
parser.add_argument( parser.add_argument(
'-S', '--reverse-sort', '-S', '--reverse-sort',
nargs='?', nargs='?',
action=AppendSort, action=AppendSort,
help="Sort by this field, but backwards.") help="Sort by this field, but backwards.")
parser.add_argument( parser.add_argument(
'-Y', '--summary', '-Y', '--summary',
action='store_true', action='store_true',
help="Only show the total.") help="Only show the total.")
parser.add_argument( parser.add_argument(
'-F', '--source', '-F', '--source',
dest='sources', dest='sources',
action='append', action='append',
help="Only consider definitions in this file. Defaults to anything " help="Only consider definitions in this file. Defaults to "
"in the current directory.") "anything in the current directory.")
parser.add_argument( parser.add_argument(
'--everything', '--everything',
action='store_true', action='store_true',
help="Include builtin and libc specific symbols.") help="Include builtin and libc specific symbols.")
parser.add_argument( parser.add_argument(
'-z', '--depth', '-z', '--depth',
nargs='?', nargs='?',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
const=0, const=0,
help="Depth of function calls to show. 0 shows all calls unless we " help="Depth of function calls to show. 0 shows all calls unless "
"find a cycle. Defaults to 0.") "we find a cycle. Defaults to 0.")
parser.add_argument( parser.add_argument(
'-t', '--hot', '-t', '--hot',
nargs='?', nargs='?',
action='append', action='append',
help="Show only the hot path for each function call.") help="Show only the hot path for each function call.")
parser.add_argument( parser.add_argument(
'-e', '--error-on-recursion', '-e', '--error-on-recursion',
action='store_true', action='store_true',
help="Error if any functions are recursive.") help="Error if any functions are recursive.")
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/structs.py
+197 -192
View File
@@ -23,7 +23,6 @@ import subprocess as sp
OBJDUMP_PATH = ['objdump'] OBJDUMP_PATH = ['objdump']
# integer fields # integer fields
class RInt(co.namedtuple('RInt', 'x')): class RInt(co.namedtuple('RInt', 'x')):
__slots__ = () __slots__ = ()
@@ -100,7 +99,9 @@ class RInt(co.namedtuple('RInt', 'x')):
return self.__class__(self.x * other.x) return self.__class__(self.x * other.x)
# struct size results # struct size results
class StructResult(co.namedtuple('StructResult', ['file', 'struct', 'size'])): class StructResult(co.namedtuple('StructResult', [
'file', 'struct',
'size'])):
_by = ['file', 'struct'] _by = ['file', 'struct']
_fields = ['size'] _fields = ['size']
_sort = ['size'] _sort = ['size']
@@ -109,11 +110,11 @@ class StructResult(co.namedtuple('StructResult', ['file', 'struct', 'size'])):
__slots__ = () __slots__ = ()
def __new__(cls, file='', struct='', size=0): def __new__(cls, file='', struct='', size=0):
return super().__new__(cls, file, struct, return super().__new__(cls, file, struct,
RInt(size)) RInt(size))
def __add__(self, other): def __add__(self, other):
return StructResult(self.file, self.struct, return StructResult(self.file, self.struct,
self.size + other.size) self.size + other.size)
def openio(path, mode='r', buffering=-1): def openio(path, mode='r', buffering=-1):
@@ -133,15 +134,15 @@ def collect(obj_paths, *,
internal=False, internal=False,
**args): **args):
line_pattern = re.compile( line_pattern = re.compile(
'^\s+(?P<no>[0-9]+)' '^\s+(?P<no>[0-9]+)'
'(?:\s+(?P<dir>[0-9]+))?' '(?:\s+(?P<dir>[0-9]+))?'
'\s+.*' '\s+.*'
'\s+(?P<path>[^\s]+)$') '\s+(?P<path>[^\s]+)$')
info_pattern = re.compile( info_pattern = re.compile(
'^(?:.*(?P<tag>DW_TAG_[a-z_]+).*' '^(?:.*(?P<tag>DW_TAG_[a-z_]+).*'
'|.*DW_AT_name.*:\s*(?P<name>[^:\s]+)\s*' '|.*DW_AT_name.*:\s*(?P<name>[^:\s]+)\s*'
'|.*DW_AT_decl_file.*:\s*(?P<file>[0-9]+)\s*' '|.*DW_AT_decl_file.*:\s*(?P<file>[0-9]+)\s*'
'|.*DW_AT_byte_size.*:\s*(?P<size>[0-9]+)\s*)$') '|.*DW_AT_byte_size.*:\s*(?P<size>[0-9]+)\s*)$')
results = [] results = []
for path in obj_paths: for path in obj_paths:
@@ -153,11 +154,11 @@ def collect(obj_paths, *,
if args.get('verbose'): if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd)) print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd, proc = sp.Popen(cmd,
stdout=sp.PIPE, stdout=sp.PIPE,
stderr=None if args.get('verbose') else sp.DEVNULL, stderr=None if args.get('verbose') else sp.DEVNULL,
universal_newlines=True, universal_newlines=True,
errors='replace', errors='replace',
close_fds=False) close_fds=False)
for line in proc.stdout: for line in proc.stdout:
# note that files contain references to dirs, which we # note that files contain references to dirs, which we
# dereference as soon as we see them as each file table follows a # dereference as soon as we see them as each file table follows a
@@ -172,8 +173,8 @@ def collect(obj_paths, *,
dir = int(m.group('dir')) dir = int(m.group('dir'))
if dir in dirs: if dir in dirs:
files[int(m.group('no'))] = os.path.join( files[int(m.group('no'))] = os.path.join(
dirs[dir], dirs[dir],
m.group('path')) m.group('path'))
else: else:
files[int(m.group('no'))] = m.group('path') files[int(m.group('no'))] = m.group('path')
proc.wait() proc.wait()
@@ -199,11 +200,11 @@ def collect(obj_paths, *,
if args.get('verbose'): if args.get('verbose'):
print(' '.join(shlex.quote(c) for c in cmd)) print(' '.join(shlex.quote(c) for c in cmd))
proc = sp.Popen(cmd, proc = sp.Popen(cmd,
stdout=sp.PIPE, stdout=sp.PIPE,
stderr=None if args.get('verbose') else sp.DEVNULL, stderr=None if args.get('verbose') else sp.DEVNULL,
universal_newlines=True, universal_newlines=True,
errors='replace', errors='replace',
close_fds=False) close_fds=False)
for i, line in enumerate(proc.stdout): for i, line in enumerate(proc.stdout):
# state machine here to find structs # state machine here to find structs
m = info_pattern.match(line) m = info_pattern.match(line)
@@ -211,7 +212,7 @@ def collect(obj_paths, *,
if m.group('tag'): if m.group('tag'):
append() append()
is_struct = (m.group('tag') == 'DW_TAG_structure_type' is_struct = (m.group('tag') == 'DW_TAG_structure_type'
or m.group('tag') == 'DW_TAG_union_type') or m.group('tag') == 'DW_TAG_union_type')
s_name = None s_name = None
s_file = None s_file = None
s_size = None s_size = None
@@ -233,15 +234,14 @@ def collect(obj_paths, *,
for r in results_: for r in results_:
# ignore filtered sources # ignore filtered sources
if sources is not None: if sources is not None:
if not any( if not any(os.path.abspath(r.file) == os.path.abspath(s)
os.path.abspath(r.file) == os.path.abspath(s)
for s in sources): for s in sources):
continue continue
else: else:
# default to only cwd # default to only cwd
if not everything and not os.path.commonpath([ if not everything and not os.path.commonpath([
os.getcwd(), os.getcwd(),
os.path.abspath(r.file)]) == os.getcwd(): os.path.abspath(r.file)]) == os.getcwd():
continue continue
# limit to .h files unless --internal # limit to .h files unless --internal
@@ -250,8 +250,8 @@ def collect(obj_paths, *,
# simplify path # simplify path
if os.path.commonpath([ if os.path.commonpath([
os.getcwd(), os.getcwd(),
os.path.abspath(r.file)]) == os.getcwd(): os.path.abspath(r.file)]) == os.getcwd():
file = os.path.relpath(r.file) file = os.path.relpath(r.file)
else: else:
file = os.path.abspath(r.file) file = os.path.abspath(r.file)
@@ -268,7 +268,7 @@ def fold(Result, results, by=None, defines=[]):
for k in it.chain(by or [], (k for k, _ in defines)): for k in it.chain(by or [], (k for k, _ in defines)):
if k not in Result._by and k not in Result._fields: if k not in Result._by and k not in Result._fields:
print("error: could not find field %r?" % k, print("error: could not find field %r?" % k,
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
# filter by matching defines # filter by matching defines
@@ -317,52 +317,55 @@ def table(Result, results, diff_results=None, *,
# organize by name # organize by name
table = { table = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in results} for r in results}
diff_table = { diff_table = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in diff_results or []} for r in diff_results or []}
names = [name names = [name
for name in table.keys() | diff_table.keys() for name in table.keys() | diff_table.keys()
if diff_results is None if diff_results is None
or all_ or all_
or any( or any(
types[k].ratio( types[k].ratio(
getattr(table.get(name), k, None), getattr(table.get(name), k, None),
getattr(diff_table.get(name), k, None)) getattr(diff_table.get(name), k, None))
for k in fields)] for k in fields)]
# sort again, now with diff info, note that python's sort is stable # sort again, now with diff info, note that python's sort is stable
names.sort() names.sort()
if diff_results is not None: if diff_results is not None:
names.sort(key=lambda n: tuple( names.sort(
types[k].ratio( key=lambda n: tuple(
getattr(table.get(n), k, None), types[k].ratio(
getattr(diff_table.get(n), k, None)) getattr(table.get(n), k, None),
for k in fields), getattr(diff_table.get(n), k, None))
reverse=True) for k in fields),
reverse=True)
if sort: if sort:
for k, reverse in reversed(sort): for k, reverse in reversed(sort):
names.sort( names.sort(
key=lambda n: tuple( key=lambda n: tuple(
(getattr(table[n], k),) (getattr(table[n], k),)
if getattr(table.get(n), k, None) is not None else () if getattr(table.get(n), k, None) is not None
for k in ([k] if k else [ else ()
k for k in Result._sort if k in fields])), for k in (
reverse=reverse ^ (not k or k in Result._fields)) [k] if k else [
k for k in Result._sort
if k in fields])),
reverse=reverse ^ (not k or k in Result._fields))
# build up our lines # build up our lines
lines = [] lines = []
# header # header
header = [ header = ['%s%s' % (
'%s%s' % ( ','.join(by),
','.join(by), ' (%d added, %d removed)' % (
' (%d added, %d removed)' % ( sum(1 for n in table if n not in diff_table),
sum(1 for n in table if n not in diff_table), sum(1 for n in diff_table if n not in table))
sum(1 for n in diff_table if n not in table)) if diff_results is not None and not percent else '')
if diff_results is not None and not percent else '')
if not summary else ''] if not summary else '']
if diff_results is None: if diff_results is None:
for k in fields: for k in fields:
@@ -385,43 +388,43 @@ def table(Result, results, diff_results=None, *,
if diff_results is None: if diff_results is None:
for k in fields: for k in fields:
entry.append( entry.append(
(getattr(r, k).table(), (getattr(r, k).table(),
getattr(getattr(r, k), 'notes', lambda: [])()) getattr(getattr(r, k), 'notes', lambda: [])())
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none) else types[k].none)
elif percent: elif percent:
for k in fields: for k in fields:
entry.append( entry.append(
(getattr(r, k).table() (getattr(r, k).table()
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none, else types[k].none,
(lambda t: ['+∞%'] if t == +mt.inf (lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)])( else ['%+.1f%%' % (100*t)])(
types[k].ratio( types[k].ratio(
getattr(r, k, None), getattr(r, k, None),
getattr(diff_r, k, None))))) getattr(diff_r, k, None)))))
else: else:
for k in fields: for k in fields:
entry.append(getattr(diff_r, k).table() entry.append(getattr(diff_r, k).table()
if getattr(diff_r, k, None) is not None if getattr(diff_r, k, None) is not None
else types[k].none) else types[k].none)
for k in fields: for k in fields:
entry.append(getattr(r, k).table() entry.append(getattr(r, k).table()
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none) else types[k].none)
for k in fields: for k in fields:
entry.append( entry.append(
(types[k].diff( (types[k].diff(
getattr(r, k, None),
getattr(diff_r, k, None)),
(lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)] if t
else [])(
types[k].ratio(
getattr(r, k, None), getattr(r, k, None),
getattr(diff_r, k, None))))) getattr(diff_r, k, None)),
(lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)] if t
else [])(
types[k].ratio(
getattr(r, k, None),
getattr(diff_r, k, None)))))
return entry return entry
# entries # entries
@@ -444,8 +447,8 @@ def table(Result, results, diff_results=None, *,
# homogenize # homogenize
lines = [ lines = [
[x if isinstance(x, tuple) else (x, []) for x in line] [x if isinstance(x, tuple) else (x, []) for x in line]
for line in lines] for line in lines]
# find the best widths, note that column 0 contains the names and is # find the best widths, note that column 0 contains the names and is
# handled a bit differently # handled a bit differently
@@ -459,11 +462,11 @@ def table(Result, results, diff_results=None, *,
# print our table # print our table
for line in lines: for line in lines:
print('%-*s %s' % ( print('%-*s %s' % (
widths[0], line[0][0], widths[0], line[0][0],
' '.join('%*s%-*s' % ( ' '.join('%*s%-*s' % (
widths[i], x[0], widths[i], x[0],
notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '')
for i, x in enumerate(line[1:], 1)))) for i, x in enumerate(line[1:], 1))))
def main(obj_paths, *, def main(obj_paths, *,
@@ -489,11 +492,11 @@ def main(obj_paths, *,
continue continue
try: try:
results.append(StructResult( results.append(StructResult(
**{k: r[k] for k in StructResult._by **{k: r[k] for k in StructResult._by
if k in r and r[k].strip()}, if k in r and r[k].strip()},
**{k: r[k] **{k: r[k]
for k in StructResult._fields for k in StructResult._fields
if k in r and r[k].strip()})) if k in r and r[k].strip()}))
except TypeError: except TypeError:
pass pass
@@ -505,25 +508,27 @@ def main(obj_paths, *,
if sort: if sort:
for k, reverse in reversed(sort): for k, reverse in reversed(sort):
results.sort( results.sort(
key=lambda r: tuple( key=lambda r: tuple(
(getattr(r, k),) if getattr(r, k) is not None else () (getattr(r, k),) if getattr(r, k) is not None else ()
for k in ([k] if k else StructResult._sort)), for k in ([k] if k else StructResult._sort)),
reverse=reverse ^ (not k or k in StructResult._fields)) reverse=reverse ^ (not k or k in StructResult._fields))
# write results to CSV # write results to CSV
if args.get('output'): if args.get('output'):
with openio(args['output'], 'w') as f: with openio(args['output'], 'w') as f:
writer = csv.DictWriter(f, writer = csv.DictWriter(f,
(by if by is not None else StructResult._by) (by if by is not None else StructResult._by)
+ [k for k in ( + [k for k in (
fields if fields is not None else StructResult._fields)]) fields if fields is not None
else StructResult._fields)])
writer.writeheader() writer.writeheader()
for r in results: for r in results:
writer.writerow( writer.writerow(
{k: getattr(r, k) for k in ( {k: getattr(r, k) for k in (
by if by is not None else StructResult._by)} by if by is not None else StructResult._by)}
| {k: getattr(r, k) for k in ( | {k: getattr(r, k) for k in (
fields if fields is not None else StructResult._fields)}) fields if fields is not None
else StructResult._fields)})
# find previous results? # find previous results?
if args.get('diff'): if args.get('diff'):
@@ -541,11 +546,11 @@ def main(obj_paths, *,
continue continue
try: try:
diff_results.append(StructResult( diff_results.append(StructResult(
**{k: r[k] for k in StructResult._by **{k: r[k] for k in StructResult._by
if k in r and r[k].strip()}, if k in r and r[k].strip()},
**{k: r[k] **{k: r[k]
for k in StructResult._fields for k in StructResult._fields
if k in r and r[k].strip()})) if k in r and r[k].strip()}))
except TypeError: except TypeError:
pass pass
except FileNotFoundError: except FileNotFoundError:
@@ -557,108 +562,108 @@ def main(obj_paths, *,
# print table # print table
if not args.get('quiet'): if not args.get('quiet'):
table(StructResult, results, table(StructResult, results,
diff_results if args.get('diff') else None, diff_results if args.get('diff') else None,
by=by if by is not None else ['struct'], by=by if by is not None else ['struct'],
fields=fields, fields=fields,
sort=sort, sort=sort,
**args) **args)
if __name__ == "__main__": if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Find struct sizes.", description="Find struct sizes.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'obj_paths', 'obj_paths',
nargs='*', nargs='*',
help="Input *.o files.") help="Input *.o files.")
parser.add_argument( parser.add_argument(
'-v', '--verbose', '-v', '--verbose',
action='store_true', action='store_true',
help="Output commands that run behind the scenes.") help="Output commands that run behind the scenes.")
parser.add_argument( parser.add_argument(
'-q', '--quiet', '-q', '--quiet',
action='store_true', action='store_true',
help="Don't show anything, useful with -o.") help="Don't show anything, useful with -o.")
parser.add_argument( parser.add_argument(
'-o', '--output', '-o', '--output',
help="Specify CSV file to store results.") help="Specify CSV file to store results.")
parser.add_argument( parser.add_argument(
'-u', '--use', '-u', '--use',
help="Don't parse anything, use this CSV file.") help="Don't parse anything, use this CSV file.")
parser.add_argument( parser.add_argument(
'-d', '--diff', '-d', '--diff',
help="Specify CSV file to diff against.") help="Specify CSV file to diff against.")
parser.add_argument( parser.add_argument(
'-a', '--all', '-a', '--all',
action='store_true', action='store_true',
help="Show all, not just the ones that changed.") help="Show all, not just the ones that changed.")
parser.add_argument( parser.add_argument(
'-p', '--percent', '-p', '--percent',
action='store_true', action='store_true',
help="Only show percentage change, not a full diff.") help="Only show percentage change, not a full diff.")
parser.add_argument( parser.add_argument(
'-b', '--by', '-b', '--by',
action='append', action='append',
choices=StructResult._by, choices=StructResult._by,
help="Group by this field.") help="Group by this field.")
parser.add_argument( parser.add_argument(
'-f', '--field', '-f', '--field',
dest='fields', dest='fields',
action='append', action='append',
choices=StructResult._fields, choices=StructResult._fields,
help="Show this field.") help="Show this field.")
parser.add_argument( parser.add_argument(
'-D', '--define', '-D', '--define',
dest='defines', dest='defines',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs: ( lambda k, vs: (
k.strip(), k.strip(),
{v.strip() for v in vs.split(',')}) {v.strip() for v in vs.split(',')})
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Only include results where this field is this value.") help="Only include results where this field is this value.")
class AppendSort(argparse.Action): class AppendSort(argparse.Action):
def __call__(self, parser, namespace, value, option): def __call__(self, parser, namespace, value, option):
if namespace.sort is None: if namespace.sort is None:
namespace.sort = [] namespace.sort = []
namespace.sort.append((value, True if option == '-S' else False)) namespace.sort.append((value, True if option == '-S' else False))
parser.add_argument( parser.add_argument(
'-s', '--sort', '-s', '--sort',
nargs='?', nargs='?',
action=AppendSort, action=AppendSort,
help="Sort by this field.") help="Sort by this field.")
parser.add_argument( parser.add_argument(
'-S', '--reverse-sort', '-S', '--reverse-sort',
nargs='?', nargs='?',
action=AppendSort, action=AppendSort,
help="Sort by this field, but backwards.") help="Sort by this field, but backwards.")
parser.add_argument( parser.add_argument(
'-Y', '--summary', '-Y', '--summary',
action='store_true', action='store_true',
help="Only show the total.") help="Only show the total.")
parser.add_argument( parser.add_argument(
'-F', '--source', '-F', '--source',
dest='sources', dest='sources',
action='append', action='append',
help="Only consider definitions in this file. Defaults to anything " help="Only consider definitions in this file. Defaults to "
"in the current directory.") "anything in the current directory.")
parser.add_argument( parser.add_argument(
'--everything', '--everything',
action='store_true', action='store_true',
help="Include builtin and libc specific symbols.") help="Include builtin and libc specific symbols.")
parser.add_argument( parser.add_argument(
'--internal', '--internal',
action='store_true', action='store_true',
help="Also show structs in .c files.") help="Also show structs in .c files.")
parser.add_argument( parser.add_argument(
'--objdump-path', '--objdump-path',
type=lambda x: x.split(), type=lambda x: x.split(),
default=OBJDUMP_PATH, default=OBJDUMP_PATH,
help="Path to the objdump executable, may include flags. " help="Path to the objdump executable, may include flags. "
"Defaults to %r." % OBJDUMP_PATH) "Defaults to %r." % OBJDUMP_PATH)
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/summary.py
+221 -217
View File
@@ -32,16 +32,16 @@ OPS = {
'max': max, 'max': max,
'avg': lambda xs: RFloat(sum(float(x) for x in xs) / len(xs)), 'avg': lambda xs: RFloat(sum(float(x) for x in xs) / len(xs)),
'stddev': lambda xs: ( 'stddev': lambda xs: (
lambda avg: RFloat( lambda avg: RFloat(
mt.sqrt(sum((float(x) - avg)**2 for x in xs) / len(xs))) mt.sqrt(sum((float(x) - avg)**2 for x in xs) / len(xs)))
)(sum(float(x) for x in xs) / len(xs)), )(sum(float(x) for x in xs) / len(xs)),
'gmean': lambda xs: RFloat(mt.prod(float(x) for x in xs)**(1/len(xs))), 'gmean': lambda xs: RFloat(mt.prod(float(x) for x in xs)**(1/len(xs))),
'gstddev': lambda xs: ( 'gstddev': lambda xs: (
lambda gmean: RFloat( lambda gmean: RFloat(
mt.exp(mt.sqrt( mt.exp(mt.sqrt(
sum(mt.log(float(x)/gmean)**2 for x in xs) sum(mt.log(float(x)/gmean)**2 for x in xs)
/ len(xs))) / len(xs)))
if gmean else mt.inf) if gmean else mt.inf)
)(mt.prod(float(x) for x in xs)**(1/len(xs))), )(mt.prod(float(x) for x in xs)**(1/len(xs))),
} }
@@ -196,15 +196,15 @@ class RFrac(co.namedtuple('RFrac', 'a,b')):
def notes(self): def notes(self):
t = self.a.x/self.b.x if self.b.x else 1.0 t = self.a.x/self.b.x if self.b.x else 1.0
return ['∞%' if t == +mt.inf return ['∞%' if t == +mt.inf
else '-∞%' if t == -mt.inf else '-∞%' if t == -mt.inf
else '%.1f%%' % (100*t)] else '%.1f%%' % (100*t)]
def diff(self, other): def diff(self, other):
new_a, new_b = self if self else (RInt(0), RInt(0)) new_a, new_b = self if self else (RInt(0), RInt(0))
old_a, old_b = other if other else (RInt(0), RInt(0)) old_a, old_b = other if other else (RInt(0), RInt(0))
return '%11s' % ('%s/%s' % ( return '%11s' % ('%s/%s' % (
new_a.diff(old_a).strip(), new_a.diff(old_a).strip(),
new_b.diff(old_b).strip())) new_b.diff(old_b).strip()))
def ratio(self, other): def ratio(self, other):
new_a, new_b = self if self else (RInt(0), RInt(0)) new_a, new_b = self if self else (RInt(0), RInt(0))
@@ -271,8 +271,8 @@ def collect(csv_paths, renames=[], defines=[]):
with openio(path) as f: with openio(path) as f:
reader = csv.DictReader(f, restval='') reader = csv.DictReader(f, restval='')
fields.extend( fields.extend(
k for k in reader.fieldnames k for k in reader.fieldnames
if k not in fields) if k not in fields)
for r in reader: for r in reader:
# apply any renames # apply any renames
if renames: if renames:
@@ -302,19 +302,17 @@ def infer(fields_, results,
defines=[]): defines=[]):
# if by not specified, guess it's anything not in fields/renames/defines # if by not specified, guess it's anything not in fields/renames/defines
if by is None: if by is None:
by = [ by = [k for k in fields_
k for k in fields_ if k not in (fields or [])
if k not in (fields or []) and not any(k == old_k for _, old_k in renames)
and not any(k == old_k for _, old_k in renames) and not any(k == k_ for k_, _ in defines)]
and not any(k == k_ for k_, _ in defines)]
# if fields not specified, guess it's anything not in by/renames/defines # if fields not specified, guess it's anything not in by/renames/defines
if fields is None: if fields is None:
fields = [ fields = [k for k in fields_
k for k in fields_ if k not in (by or [])
if k not in (by or []) and not any(k == old_k for _, old_k in renames)
and not any(k == old_k for _, old_k in renames) and not any(k == k_ for k_, _ in defines)]
and not any(k == k_ for k_, _ in defines)]
# deduplicate by/fields # deduplicate by/fields
by = list(co.OrderedDict.fromkeys(by).keys()) by = list(co.OrderedDict.fromkeys(by).keys())
@@ -338,7 +336,7 @@ def infer(fields_, results,
break break
else: else:
print("error: no type matches field %r?" % k, print("error: no type matches field %r?" % k,
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
types = types_ types = types_
@@ -351,11 +349,11 @@ def infer(fields_, results,
# create result class # create result class
def __new__(cls, **r): def __new__(cls, **r):
return cls.__mro__[1].__new__(cls, return cls.__mro__[1].__new__(cls,
**{k: r.get(k, '') for k in by}, **{k: r.get(k, '') for k in by},
**{k: r[k] if k in r and isinstance(r[k], tuple) **{k: r[k] if k in r and isinstance(r[k], tuple)
else ([types[k](r[k])], 1) if k in r else ([types[k](r[k])], 1) if k in r
else ([], 0) else ([], 0)
for k in fields}) for k in fields})
def __add__(self, other): def __add__(self, other):
# reuse lists if possible # reuse lists if possible
@@ -367,11 +365,11 @@ def infer(fields_, results,
return (a[0][:a[1]] + b[0][:b[1]], a[1] + b[1]) return (a[0][:a[1]] + b[0][:b[1]], a[1] + b[1])
return self.__class__( return self.__class__(
**{k: getattr(self, k) for k in by}, **{k: getattr(self, k) for k in by},
**{k: extend( **{k: extend(
object.__getattribute__(self, k), object.__getattribute__(self, k),
object.__getattribute__(other, k)) object.__getattribute__(other, k))
for k in fields}) for k in fields})
def __getattribute__(self, k): def __getattribute__(self, k):
if k in fields: if k in fields:
@@ -401,7 +399,7 @@ def fold(Result, results, by=None, defines=[]):
for k in it.chain(by or [], (k for k, _ in defines)): for k in it.chain(by or [], (k for k, _ in defines)):
if k not in Result._by and k not in Result._fields: if k not in Result._by and k not in Result._fields:
print("error: could not find field %r?" % k, print("error: could not find field %r?" % k,
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
# filter by matching defines # filter by matching defines
@@ -450,52 +448,55 @@ def table(Result, results, diff_results=None, *,
# organize by name # organize by name
table = { table = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in results} for r in results}
diff_table = { diff_table = {
','.join(str(getattr(r, k) or '') for k in by): r ','.join(str(getattr(r, k) or '') for k in by): r
for r in diff_results or []} for r in diff_results or []}
names = [name names = [name
for name in table.keys() | diff_table.keys() for name in table.keys() | diff_table.keys()
if diff_results is None if diff_results is None
or all_ or all_
or any( or any(
types[k].ratio( types[k].ratio(
getattr(table.get(name), k, None), getattr(table.get(name), k, None),
getattr(diff_table.get(name), k, None)) getattr(diff_table.get(name), k, None))
for k in fields)] for k in fields)]
# sort again, now with diff info, note that python's sort is stable # sort again, now with diff info, note that python's sort is stable
names.sort() names.sort()
if diff_results is not None: if diff_results is not None:
names.sort(key=lambda n: tuple( names.sort(
types[k].ratio( key=lambda n: tuple(
getattr(table.get(n), k, None), types[k].ratio(
getattr(diff_table.get(n), k, None)) getattr(table.get(n), k, None),
for k in fields), getattr(diff_table.get(n), k, None))
reverse=True) for k in fields),
reverse=True)
if sort: if sort:
for k, reverse in reversed(sort): for k, reverse in reversed(sort):
names.sort( names.sort(
key=lambda n: tuple( key=lambda n: tuple(
(getattr(table[n], k),) (getattr(table[n], k),)
if getattr(table.get(n), k, None) is not None else () if getattr(table.get(n), k, None) is not None
for k in ([k] if k else [ else ()
k for k in Result._sort if k in fields])), for k in (
reverse=reverse ^ (not k or k in Result._fields)) [k] if k else [
k for k in Result._sort
if k in fields])),
reverse=reverse ^ (not k or k in Result._fields))
# build up our lines # build up our lines
lines = [] lines = []
# header # header
header = [ header = ['%s%s' % (
'%s%s' % ( ','.join(by),
','.join(by), ' (%d added, %d removed)' % (
' (%d added, %d removed)' % ( sum(1 for n in table if n not in diff_table),
sum(1 for n in table if n not in diff_table), sum(1 for n in diff_table if n not in table))
sum(1 for n in diff_table if n not in table)) if diff_results is not None and not percent else '')
if diff_results is not None and not percent else '')
if not summary else ''] if not summary else '']
if diff_results is None: if diff_results is None:
for k in fields: for k in fields:
@@ -518,43 +519,43 @@ def table(Result, results, diff_results=None, *,
if diff_results is None: if diff_results is None:
for k in fields: for k in fields:
entry.append( entry.append(
(getattr(r, k).table(), (getattr(r, k).table(),
getattr(getattr(r, k), 'notes', lambda: [])()) getattr(getattr(r, k), 'notes', lambda: [])())
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none) else types[k].none)
elif percent: elif percent:
for k in fields: for k in fields:
entry.append( entry.append(
(getattr(r, k).table() (getattr(r, k).table()
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none, else types[k].none,
(lambda t: ['+∞%'] if t == +mt.inf (lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)])( else ['%+.1f%%' % (100*t)])(
types[k].ratio( types[k].ratio(
getattr(r, k, None), getattr(r, k, None),
getattr(diff_r, k, None))))) getattr(diff_r, k, None)))))
else: else:
for k in fields: for k in fields:
entry.append(getattr(diff_r, k).table() entry.append(getattr(diff_r, k).table()
if getattr(diff_r, k, None) is not None if getattr(diff_r, k, None) is not None
else types[k].none) else types[k].none)
for k in fields: for k in fields:
entry.append(getattr(r, k).table() entry.append(getattr(r, k).table()
if getattr(r, k, None) is not None if getattr(r, k, None) is not None
else types[k].none) else types[k].none)
for k in fields: for k in fields:
entry.append( entry.append(
(types[k].diff( (types[k].diff(
getattr(r, k, None),
getattr(diff_r, k, None)),
(lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)] if t
else [])(
types[k].ratio(
getattr(r, k, None), getattr(r, k, None),
getattr(diff_r, k, None))))) getattr(diff_r, k, None)),
(lambda t: ['+∞%'] if t == +mt.inf
else ['-∞%'] if t == -mt.inf
else ['%+.1f%%' % (100*t)] if t
else [])(
types[k].ratio(
getattr(r, k, None),
getattr(diff_r, k, None)))))
return entry return entry
# entries # entries
@@ -577,8 +578,8 @@ def table(Result, results, diff_results=None, *,
# homogenize # homogenize
lines = [ lines = [
[x if isinstance(x, tuple) else (x, []) for x in line] [x if isinstance(x, tuple) else (x, []) for x in line]
for line in lines] for line in lines]
# find the best widths, note that column 0 contains the names and is # find the best widths, note that column 0 contains the names and is
# handled a bit differently # handled a bit differently
@@ -592,11 +593,11 @@ def table(Result, results, diff_results=None, *,
# print our table # print our table
for line in lines: for line in lines:
print('%-*s %s' % ( print('%-*s %s' % (
widths[0], line[0][0], widths[0], line[0][0],
' '.join('%*s%-*s' % ( ' '.join('%*s%-*s' % (
widths[i], x[0], widths[i], x[0],
notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '') notes[i], ' (%s)' % ', '.join(x[1]) if x[1] else '')
for i, x in enumerate(line[1:], 1)))) for i, x in enumerate(line[1:], 1))))
def main(csv_paths, *, def main(csv_paths, *,
@@ -607,8 +608,8 @@ def main(csv_paths, *,
**args): **args):
# separate out renames # separate out renames
renames = list(it.chain.from_iterable( renames = list(it.chain.from_iterable(
((k, v) for v in vs) ((k, v) for v in vs)
for k, vs in it.chain(by or [], fields or []))) for k, vs in it.chain(by or [], fields or [])))
if by is not None: if by is not None:
by = [k for k, _ in by] by = [k for k, _ in by]
if fields is not None: if fields is not None:
@@ -620,7 +621,7 @@ def main(csv_paths, *,
for k in args.get(t, []): for k in args.get(t, []):
if k in types: if k in types:
print("error: conflicting type for field %r?" % k, print("error: conflicting type for field %r?" % k,
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
types[k] = TYPES[t] types[k] = TYPES[t]
# rename types? # rename types?
@@ -637,7 +638,7 @@ def main(csv_paths, *,
for k in args.get(o, []): for k in args.get(o, []):
if k in ops: if k in ops:
print("error: conflicting op for field %r?" % k, print("error: conflicting op for field %r?" % k,
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
ops[k] = OPS[o] ops[k] = OPS[o]
# rename ops? # rename ops?
@@ -650,7 +651,7 @@ def main(csv_paths, *,
if by is None and fields is None: if by is None and fields is None:
print("error: needs --by or --fields to figure out fields", print("error: needs --by or --fields to figure out fields",
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
# use is just an alias # use is just an alias
@@ -662,12 +663,12 @@ def main(csv_paths, *,
# homogenize # homogenize
Result = infer(fields_, results, Result = infer(fields_, results,
by=by, by=by,
fields=fields, fields=fields,
types=types, types=types,
ops=ops, ops=ops,
renames=renames, renames=renames,
defines=defines) defines=defines)
results_ = [] results_ = []
for r in results: for r in results:
if not any(k in r and r[k].strip() if not any(k in r and r[k].strip()
@@ -675,8 +676,8 @@ def main(csv_paths, *,
continue continue
try: try:
results_.append(Result(**{ results_.append(Result(**{
k: r[k] for k in Result._by + Result._fields k: r[k] for k in Result._by + Result._fields
if k in r and r[k].strip()})) if k in r and r[k].strip()}))
except TypeError: except TypeError:
pass pass
results = results_ results = results_
@@ -689,10 +690,10 @@ def main(csv_paths, *,
if sort: if sort:
for k, reverse in reversed(sort): for k, reverse in reversed(sort):
results.sort( results.sort(
key=lambda r: tuple( key=lambda r: tuple(
(getattr(r, k),) if getattr(r, k) is not None else () (getattr(r, k),) if getattr(r, k) is not None else ()
for k in ([k] if k else Result._sort)), for k in ([k] if k else Result._sort)),
reverse=reverse ^ (not k or k in Result._fields)) reverse=reverse ^ (not k or k in Result._fields))
# write results to CSV # write results to CSV
if args.get('output'): if args.get('output'):
@@ -702,7 +703,8 @@ def main(csv_paths, *,
for r in results: for r in results:
# note we need to go through getattr to resolve lazy fields # note we need to go through getattr to resolve lazy fields
writer.writerow({ writer.writerow({
k: getattr(r, k) for k in Result._by + Result._fields}) k: getattr(r, k)
for k in Result._by + Result._fields})
# find previous results? # find previous results?
if args.get('diff'): if args.get('diff'):
@@ -714,8 +716,8 @@ def main(csv_paths, *,
continue continue
try: try:
diff_results_.append(Result(**{ diff_results_.append(Result(**{
k: r[k] for k in Result._by + Result._fields k: r[k] for k in Result._by + Result._fields
if k in r and r[k].strip()})) if k in r and r[k].strip()}))
except TypeError: except TypeError:
pass pass
diff_results = diff_results_ diff_results = diff_results_
@@ -726,139 +728,141 @@ def main(csv_paths, *,
# print table # print table
if not args.get('quiet'): if not args.get('quiet'):
table(Result, results, table(Result, results,
diff_results if args.get('diff') else None, diff_results if args.get('diff') else None,
by=by, by=by,
fields=fields, fields=fields,
sort=sort, sort=sort,
**args) **args)
if __name__ == "__main__": if __name__ == "__main__":
import argparse import argparse
import sys import sys
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Summarize measurements in CSV files.", description="Summarize measurements in CSV files.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'csv_paths', 'csv_paths',
nargs='*', nargs='*',
help="Input *.csv files.") help="Input *.csv files.")
parser.add_argument( parser.add_argument(
'-q', '--quiet', '-q', '--quiet',
action='store_true', action='store_true',
help="Don't show anything, useful with -o.") help="Don't show anything, useful with -o.")
parser.add_argument( parser.add_argument(
'-o', '--output', '-o', '--output',
help="Specify CSV file to store results.") help="Specify CSV file to store results.")
parser.add_argument( parser.add_argument(
'-u', '--use', '-u', '--use',
help="Don't parse anything, use this CSV file.") help="Don't parse anything, use this CSV file.")
parser.add_argument( parser.add_argument(
'-d', '--diff', '-d', '--diff',
help="Specify CSV file to diff against.") help="Specify CSV file to diff against.")
parser.add_argument( parser.add_argument(
'-a', '--all', '-a', '--all',
action='store_true', action='store_true',
help="Show all, not just the ones that changed.") help="Show all, not just the ones that changed.")
parser.add_argument( parser.add_argument(
'-p', '--percent', '-p', '--percent',
action='store_true', action='store_true',
help="Only show percentage change, not a full diff.") help="Only show percentage change, not a full diff.")
parser.add_argument( parser.add_argument(
'-b', '--by', '-b', '--by',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs=None: ( lambda k, vs=None: (
k.strip(), k.strip(),
tuple(v.strip() for v in vs.split(',')) tuple(v.strip() for v in vs.split(','))
if vs is not None else ()) if vs is not None else ())
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Group by this field. Can rename fields with new_name=old_name.") help="Group by this field. Can rename fields with "
"new_name=old_name.")
parser.add_argument( parser.add_argument(
'-f', '--field', '-f', '--field',
dest='fields', dest='fields',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs=None: ( lambda k, vs=None: (
k.strip(), k.strip(),
tuple(v.strip() for v in vs.split(',')) tuple(v.strip() for v in vs.split(','))
if vs is not None else ()) if vs is not None else ())
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Show this field. Can rename fields with new_name=old_name.") help="Show this field. Can rename fields with "
"new_name=old_name.")
parser.add_argument( parser.add_argument(
'-D', '--define', '-D', '--define',
dest='defines', dest='defines',
action='append', action='append',
type=lambda x: ( type=lambda x: (
lambda k, vs: ( lambda k, vs: (
k.strip(), k.strip(),
{v.strip() for v in vs.split(',')}) {v.strip() for v in vs.split(',')})
)(*x.split('=', 1)), )(*x.split('=', 1)),
help="Only include results where this field is this value. May include " help="Only include results where this field is this value. May "
"comma-separated options.") "include comma-separated options.")
class AppendSort(argparse.Action): class AppendSort(argparse.Action):
def __call__(self, parser, namespace, value, option): def __call__(self, parser, namespace, value, option):
if namespace.sort is None: if namespace.sort is None:
namespace.sort = [] namespace.sort = []
namespace.sort.append((value, True if option == '-S' else False)) namespace.sort.append((value, True if option == '-S' else False))
parser.add_argument( parser.add_argument(
'-s', '--sort', '-s', '--sort',
nargs='?', nargs='?',
action=AppendSort, action=AppendSort,
help="Sort by this field.") help="Sort by this field.")
parser.add_argument( parser.add_argument(
'-S', '--reverse-sort', '-S', '--reverse-sort',
nargs='?', nargs='?',
action=AppendSort, action=AppendSort,
help="Sort by this field, but backwards.") help="Sort by this field, but backwards.")
parser.add_argument( parser.add_argument(
'-Y', '--summary', '-Y', '--summary',
action='store_true', action='store_true',
help="Only show the total.") help="Only show the total.")
parser.add_argument( parser.add_argument(
'--int', '--int',
action='append', action='append',
help="Treat these fields as ints.") help="Treat these fields as ints.")
parser.add_argument( parser.add_argument(
'--float', '--float',
action='append', action='append',
help="Treat these fields as floats.") help="Treat these fields as floats.")
parser.add_argument( parser.add_argument(
'--frac', '--frac',
action='append', action='append',
help="Treat these fields as fractions.") help="Treat these fields as fractions.")
parser.add_argument( parser.add_argument(
'--sum', '--sum',
action='append', action='append',
help="Add these fields (the default).") help="Add these fields (the default).")
parser.add_argument( parser.add_argument(
'--prod', '--prod',
action='append', action='append',
help="Multiply these fields.") help="Multiply these fields.")
parser.add_argument( parser.add_argument(
'--min', '--min',
action='append', action='append',
help="Take the minimum of these fields.") help="Take the minimum of these fields.")
parser.add_argument( parser.add_argument(
'--max', '--max',
action='append', action='append',
help="Take the maximum of these fields.") help="Take the maximum of these fields.")
parser.add_argument( parser.add_argument(
'--avg', '--mean', '--avg', '--mean',
action='append', action='append',
help="Average these fields.") help="Average these fields.")
parser.add_argument( parser.add_argument(
'--stddev', '--stddev',
action='append', action='append',
help="Find the standard deviation of these fields.") help="Find the standard deviation of these fields.")
parser.add_argument( parser.add_argument(
'--gmean', '--gmean',
action='append', action='append',
help="Find the geometric mean of these fields.") help="Find the geometric mean of these fields.")
parser.add_argument( parser.add_argument(
'--gstddev', '--gstddev',
action='append', action='append',
help="Find the geometric standard deviation of these fields.") help="Find the geometric standard deviation of these fields.")
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/tailpipe.py
+27 -26
View File
@@ -75,8 +75,8 @@ class RingIO:
# pad to fill any existing canvas, but truncate to terminal size # pad to fill any existing canvas, but truncate to terminal size
h = shutil.get_terminal_size((80, 5))[1] h = shutil.get_terminal_size((80, 5))[1]
lines.extend('' for _ in range( lines.extend('' for _ in range(
len(lines), len(lines),
min(RingIO.canvas_lines, h))) min(RingIO.canvas_lines, h)))
while len(lines) > h: while len(lines) > h:
if self.head: if self.head:
lines.pop() lines.pop()
@@ -142,7 +142,7 @@ def main(path='-', *,
time.sleep(sleep or 0.1) time.sleep(sleep or 0.1)
except FileNotFoundError as e: except FileNotFoundError as e:
print("error: file not found %r" % path, print("error: file not found %r" % path,
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
except KeyboardInterrupt: except KeyboardInterrupt:
pass pass
@@ -157,32 +157,33 @@ if __name__ == "__main__":
import sys import sys
import argparse import argparse
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Efficiently displays the last n lines of a file/pipe.", description="Efficiently displays the last n lines of a "
allow_abbrev=False) "file/pipe.",
allow_abbrev=False)
parser.add_argument( parser.add_argument(
'path', 'path',
nargs='?', nargs='?',
help="Path to read from.") help="Path to read from.")
parser.add_argument( parser.add_argument(
'-n', '--lines', '-n', '--lines',
nargs='?', nargs='?',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
const=0, const=0,
help="Show this many lines of history. 0 uses the terminal height. " help="Show this many lines of history. 0 uses the terminal "
"Defaults to 5.") "height. Defaults to 5.")
parser.add_argument( parser.add_argument(
'-z', '--cat', '-z', '--cat',
action='store_true', action='store_true',
help="Pipe directly to stdout.") help="Pipe directly to stdout.")
parser.add_argument( parser.add_argument(
'-s', '--sleep', '-s', '--sleep',
type=float, type=float,
help="Seconds to sleep between reads. Defaults to 0.01.") help="Seconds to sleep between reads. Defaults to 0.01.")
parser.add_argument( parser.add_argument(
'-k', '--keep-open', '-k', '--keep-open',
action='store_true', action='store_true',
help="Reopen the pipe on EOF, useful when multiple " help="Reopen the pipe on EOF, useful when multiple "
"processes are writing.") "processes are writing.")
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/teepipe.py
+14 -14
View File
@@ -45,7 +45,7 @@ def main(in_path, out_paths, *, keep_open=False):
pass pass
except FileNotFoundError as e: except FileNotFoundError as e:
print("error: file not found %r" % in_path, print("error: file not found %r" % in_path,
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
except KeyboardInterrupt: except KeyboardInterrupt:
pass pass
@@ -55,20 +55,20 @@ if __name__ == "__main__":
import sys import sys
import argparse import argparse
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="tee, but for pipes.", description="tee, but for pipes.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'in_path', 'in_path',
help="Path to read from.") help="Path to read from.")
parser.add_argument( parser.add_argument(
'out_paths', 'out_paths',
nargs='+', nargs='+',
help="Path to write to.") help="Path to write to.")
parser.add_argument( parser.add_argument(
'-k', '--keep-open', '-k', '--keep-open',
action='store_true', action='store_true',
help="Reopen the pipe on EOF, useful when multiple " help="Reopen the pipe on EOF, useful when multiple "
"processes are writing.") "processes are writing.")
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/test.py
+543 -534
View File
File diff suppressed because it is too large Load Diff
scripts/tracebd.py
+272 -268
View File
@@ -29,14 +29,14 @@ WEAR_COLORS = ['90', '', '', '', '', '', '', '35', '35', '1;31']
CHARS_DOTS = " .':" CHARS_DOTS = " .':"
CHARS_BRAILLE = ( CHARS_BRAILLE = (
'⠀⢀⡀⣀⠠⢠⡠⣠⠄⢄⡄⣄⠤⢤⡤⣤' '⠐⢐⡐⣐⠰⢰⡰⣰⠔⢔⡔⣔⠴⢴⡴⣴' '⠀⢀⡀⣀⠠⢠⡠⣠⠄⢄⡄⣄⠤⢤⡤⣤' '⠐⢐⡐⣐⠰⢰⡰⣰⠔⢔⡔⣔⠴⢴⡴⣴'
'⠂⢂⡂⣂⠢⢢⡢⣢⠆⢆⡆⣆⠦⢦⡦⣦' '⠒⢒⡒⣒⠲⢲⡲⣲⠖⢖⡖⣖⠶⢶⡶⣶' '⠂⢂⡂⣂⠢⢢⡢⣢⠆⢆⡆⣆⠦⢦⡦⣦' '⠒⢒⡒⣒⠲⢲⡲⣲⠖⢖⡖⣖⠶⢶⡶⣶'
'⠈⢈⡈⣈⠨⢨⡨⣨⠌⢌⡌⣌⠬⢬⡬⣬' '⠘⢘⡘⣘⠸⢸⡸⣸⠜⢜⡜⣜⠼⢼⡼⣼' '⠈⢈⡈⣈⠨⢨⡨⣨⠌⢌⡌⣌⠬⢬⡬⣬' '⠘⢘⡘⣘⠸⢸⡸⣸⠜⢜⡜⣜⠼⢼⡼⣼'
'⠊⢊⡊⣊⠪⢪⡪⣪⠎⢎⡎⣎⠮⢮⡮⣮' '⠚⢚⡚⣚⠺⢺⡺⣺⠞⢞⡞⣞⠾⢾⡾⣾' '⠊⢊⡊⣊⠪⢪⡪⣪⠎⢎⡎⣎⠮⢮⡮⣮' '⠚⢚⡚⣚⠺⢺⡺⣺⠞⢞⡞⣞⠾⢾⡾⣾'
'⠁⢁⡁⣁⠡⢡⡡⣡⠅⢅⡅⣅⠥⢥⡥⣥' '⠑⢑⡑⣑⠱⢱⡱⣱⠕⢕⡕⣕⠵⢵⡵⣵' '⠁⢁⡁⣁⠡⢡⡡⣡⠅⢅⡅⣅⠥⢥⡥⣥' '⠑⢑⡑⣑⠱⢱⡱⣱⠕⢕⡕⣕⠵⢵⡵⣵'
'⠃⢃⡃⣃⠣⢣⡣⣣⠇⢇⡇⣇⠧⢧⡧⣧' '⠓⢓⡓⣓⠳⢳⡳⣳⠗⢗⡗⣗⠷⢷⡷⣷' '⠃⢃⡃⣃⠣⢣⡣⣣⠇⢇⡇⣇⠧⢧⡧⣧' '⠓⢓⡓⣓⠳⢳⡳⣳⠗⢗⡗⣗⠷⢷⡷⣷'
'⠉⢉⡉⣉⠩⢩⡩⣩⠍⢍⡍⣍⠭⢭⡭⣭' '⠙⢙⡙⣙⠹⢹⡹⣹⠝⢝⡝⣝⠽⢽⡽⣽' '⠉⢉⡉⣉⠩⢩⡩⣩⠍⢍⡍⣍⠭⢭⡭⣭' '⠙⢙⡙⣙⠹⢹⡹⣹⠝⢝⡝⣝⠽⢽⡽⣽'
'⠋⢋⡋⣋⠫⢫⡫⣫⠏⢏⡏⣏⠯⢯⡯⣯' '⠛⢛⡛⣛⠻⢻⡻⣻⠟⢟⡟⣟⠿⢿⡿⣿') '⠋⢋⡋⣋⠫⢫⡫⣫⠏⢏⡏⣏⠯⢯⡯⣯' '⠛⢛⡛⣛⠻⢻⡻⣻⠟⢟⡟⣟⠿⢿⡿⣿')
def openio(path, mode='r', buffering=-1): def openio(path, mode='r', buffering=-1):
@@ -156,8 +156,8 @@ class RingIO:
# pad to fill any existing canvas, but truncate to terminal size # pad to fill any existing canvas, but truncate to terminal size
h = shutil.get_terminal_size((80, 5))[1] h = shutil.get_terminal_size((80, 5))[1]
lines.extend('' for _ in range( lines.extend('' for _ in range(
len(lines), len(lines),
min(RingIO.canvas_lines, h))) min(RingIO.canvas_lines, h)))
while len(lines) > h: while len(lines) > h:
if self.head: if self.head:
lines.pop() lines.pop()
@@ -238,8 +238,8 @@ def hilbert_curve(width, height):
yield from hilbert_(x, y, b_x_, b_y_, a_x_, a_y_) yield from hilbert_(x, y, b_x_, b_y_, a_x_, a_y_)
yield from hilbert_(x+b_x_, y+b_y_, a_x, a_y, b_x-b_x_, b_y-b_y_) yield from hilbert_(x+b_x_, y+b_y_, a_x, a_y, b_x-b_x_, b_y-b_y_)
yield from hilbert_( yield from hilbert_(
x+(a_x-a_dx)+(b_x_-b_dx), y+(a_y-a_dy)+(b_y_-b_dy), x+(a_x-a_dx)+(b_x_-b_dx), y+(a_y-a_dy)+(b_y_-b_dy),
-b_x_, -b_y_, -(a_x-a_x_), -(a_y-a_y_)) -b_x_, -b_y_, -(a_x-a_x_), -(a_y-a_y_))
if width >= height: if width >= height:
curve = hilbert_(0, 0, +width, 0, 0, +height) curve = hilbert_(0, 0, +width, 0, 0, +height)
@@ -276,11 +276,11 @@ class Pixel(int):
proged=False, proged=False,
erased=False): erased=False):
return super().__new__(cls, return super().__new__(cls,
state state
| (wear << 3) | (wear << 3)
| (1 if readed else 0) | (1 if readed else 0)
| (2 if proged else 0) | (2 if proged else 0)
| (4 if erased else 0)) | (4 if erased else 0))
@property @property
def wear(self): def wear(self):
@@ -312,8 +312,8 @@ class Pixel(int):
def __or__(self, other): def __or__(self, other):
return Pixel( return Pixel(
(int(self) | int(other)) & 7, (int(self) | int(other)) & 7,
wear=max(self.wear, other.wear)) wear=max(self.wear, other.wear))
def worn(self, max_wear, *, def worn(self, max_wear, *,
block_cycles=None, block_cycles=None,
@@ -363,12 +363,11 @@ class Pixel(int):
f = [colors[3]] f = [colors[3]]
if wear: if wear:
w = min( w = min(self.worn(
self.worn( max_wear,
max_wear, block_cycles=block_cycles,
block_cycles=block_cycles, wear_chars=wear_chars),
wear_chars=wear_chars), 1)
1)
c = wear_chars[int(w * (len(wear_chars)-1))] c = wear_chars[int(w * (len(wear_chars)-1))]
f.append(wear_colors[int(w * (len(wear_colors)-1))]) f.append(wear_colors[int(w * (len(wear_colors)-1))])
@@ -390,8 +389,8 @@ class Pixel(int):
# apply colors # apply colors
if f and color: if f and color:
c = '%s%s\x1b[m' % ( c = '%s%s\x1b[m' % (
''.join('\x1b[%sm' % f_ for f_ in f), ''.join('\x1b[%sm' % f_ for f_ in f),
c) c)
return c return c
@@ -457,25 +456,25 @@ class Bmap:
block -= self._block_window.start block -= self._block_window.start
size = (max(self._off_window.start, size = (max(self._off_window.start,
min(self._off_window.stop, off+size)) min(self._off_window.stop, off+size))
- max(self._off_window.start, - max(self._off_window.start,
min(self._off_window.stop, off))) min(self._off_window.stop, off)))
off = (max(self._off_window.start, off = (max(self._off_window.start,
min(self._off_window.stop, off)) min(self._off_window.stop, off))
- self._off_window.start) - self._off_window.start)
if size == 0: if size == 0:
return return
# map to our block space # map to our block space
range_ = range( range_ = range(
block*len(self._off_window) + off, block*len(self._off_window) + off,
block*len(self._off_window) + off+size) block*len(self._off_window) + off+size)
range_ = range( range_ = range(
(range_.start*len(self.pixels)) // self._window, (range_.start*len(self.pixels)) // self._window,
(range_.stop*len(self.pixels)) // self._window) (range_.stop*len(self.pixels)) // self._window)
range_ = range( range_ = range(
range_.start, range_.start,
max(range_.stop, range_.start+1)) max(range_.stop, range_.start+1))
# apply the op # apply the op
for i in range_: for i in range_:
@@ -499,9 +498,9 @@ class Bmap:
width=None, width=None,
height=None): height=None):
block_size = (block_size if block_size is not None block_size = (block_size if block_size is not None
else self.block_size) else self.block_size)
block_count = (block_count if block_count is not None block_count = (block_count if block_count is not None
else self.block_count) else self.block_count)
width = width if width is not None else self.width width = width if width is not None else self.width
height = height if height is not None else self.height height = height if height is not None else self.height
@@ -519,17 +518,17 @@ class Bmap:
for x in range(width*height): for x in range(width*height):
# map into our old bd space # map into our old bd space
range_ = range( range_ = range(
(x*self._window) // (width*height), (x*self._window) // (width*height),
((x+1)*self._window) // (width*height)) ((x+1)*self._window) // (width*height))
range_ = range( range_ = range(
range_.start, range_.start,
max(range_.stop, range_.start+1)) max(range_.stop, range_.start+1))
# aggregate state # aggregate state
pixels.append(ft.reduce( pixels.append(ft.reduce(
Pixel.__or__, Pixel.__or__,
self.pixels[range_.start:range_.stop], self.pixels[range_.start:range_.stop],
Pixel())) Pixel()))
self.width = width self.width = width
self.height = height self.height = height
@@ -580,10 +579,10 @@ class Bmap:
return None return None
grid = list(it.chain.from_iterable( grid = list(it.chain.from_iterable(
# did we resize? # did we resize?
it.islice(it.chain(h, it.repeat(Pixel())), it.islice(it.chain(h, it.repeat(Pixel())),
self.width*self.height) self.width*self.height)
for h in self.history)) for h in self.history))
self.history = [] self.history = []
line = [] line = []
@@ -603,14 +602,14 @@ class Bmap:
byte_p |= 1 << i byte_p |= 1 << i
line.append(best_p.draw( line.append(best_p.draw(
max_wear, max_wear,
CHARS_BRAILLE[byte_p], CHARS_BRAILLE[byte_p],
braille=True, braille=True,
read=read, read=read,
prog=prog, prog=prog,
erase=erase, erase=erase,
wear=wear, wear=wear,
**args)) **args))
elif dots: elif dots:
# encode into a byte # encode into a byte
for x in range(self.width): for x in range(self.width):
@@ -627,23 +626,23 @@ class Bmap:
byte_p |= 1 << i byte_p |= 1 << i
line.append(best_p.draw( line.append(best_p.draw(
max_wear, max_wear,
CHARS_DOTS[byte_p], CHARS_DOTS[byte_p],
dots=True, dots=True,
read=read, read=read,
prog=prog, prog=prog,
erase=erase, erase=erase,
wear=wear, wear=wear,
**args)) **args))
else: else:
for x in range(self.width): for x in range(self.width):
line.append(grid[x + row*self.width].draw( line.append(grid[x + row*self.width].draw(
max_wear, max_wear,
read=read, read=read,
prog=prog, prog=prog,
erase=erase, erase=erase,
wear=wear, wear=wear,
**args)) **args))
return ''.join(line) return ''.join(line)
@@ -725,7 +724,7 @@ def main(path='-', *,
if any(isinstance(b, list) and len(b) > 1 for b in block): if any(isinstance(b, list) and len(b) > 1 for b in block):
print("error: more than one block address?", print("error: more than one block address?",
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
if isinstance(block[0], list): if isinstance(block[0], list):
block = (block[0][0], *block[1:]) block = (block[0][0], *block[1:])
@@ -765,10 +764,10 @@ def main(path='-', *,
# create our block device representation # create our block device representation
bmap = Bmap( bmap = Bmap(
block_size=block_size if block_size is not None else 1, block_size=block_size if block_size is not None else 1,
block_count=block_count if block_count is not None else 1, block_count=block_count if block_count is not None else 1,
block_window=block_window, block_window=block_window,
off_window=off_window) off_window=off_window)
def resize(): def resize():
nonlocal bmap nonlocal bmap
@@ -791,12 +790,13 @@ def main(path='-', *,
# terminal size changed? # terminal size changed?
if width_ != bmap.width or height_ != bmap.height: if width_ != bmap.width or height_ != bmap.height:
bmap.resize( bmap.resize(
# scale if we're printing with dots or braille # scale if we're printing with dots or braille
width=2*width_ if braille else width_, width=2*width_ if braille else width_,
height=max(1, height=max(
4*height_ if braille 1,
else 2*height_ if dots 4*height_ if braille
else height_)) else 2*height_ if dots
else height_))
resize() resize()
# keep track of some extra info # keep track of some extra info
@@ -806,30 +806,32 @@ def main(path='-', *,
# parse a line of trace output # parse a line of trace output
pattern = re.compile( pattern = re.compile(
'^(?P<file>[^:]*):(?P<line>[0-9]+):trace:.*?bd_(?:' '^(?P<file>[^:]*):(?P<line>[0-9]+):trace:.*?bd_(?:'
'(?P<create>create\w*)\(' '(?P<create>create\w*)\('
'(?:' '(?:'
'block_size=(?P<block_size>\w+)' 'block_size=(?P<block_size>\w+)'
'|' 'block_count=(?P<block_count>\w+)' '|' 'block_count=(?P<block_count>\w+)'
'|' '.*?' ')*' '\)' '|' '.*?' ')*'
'|' '(?P<read>read)\(' '\)'
'\s*(?P<read_ctx>\w+)' '\s*,' '|' '(?P<read>read)\('
'\s*(?P<read_block>\w+)' '\s*,' '\s*(?P<read_ctx>\w+)' '\s*,'
'\s*(?P<read_off>\w+)' '\s*,' '\s*(?P<read_block>\w+)' '\s*,'
'\s*(?P<read_buffer>\w+)' '\s*,' '\s*(?P<read_off>\w+)' '\s*,'
'\s*(?P<read_size>\w+)' '\s*\)' '\s*(?P<read_buffer>\w+)' '\s*,'
'|' '(?P<prog>prog)\(' '\s*(?P<read_size>\w+)' '\s*\)'
'\s*(?P<prog_ctx>\w+)' '\s*,' '|' '(?P<prog>prog)\('
'\s*(?P<prog_block>\w+)' '\s*,' '\s*(?P<prog_ctx>\w+)' '\s*,'
'\s*(?P<prog_off>\w+)' '\s*,' '\s*(?P<prog_block>\w+)' '\s*,'
'\s*(?P<prog_buffer>\w+)' '\s*,' '\s*(?P<prog_off>\w+)' '\s*,'
'\s*(?P<prog_size>\w+)' '\s*\)' '\s*(?P<prog_buffer>\w+)' '\s*,'
'|' '(?P<erase>erase)\(' '\s*(?P<prog_size>\w+)' '\s*\)'
'\s*(?P<erase_ctx>\w+)' '\s*,' '|' '(?P<erase>erase)\('
'\s*(?P<erase_block>\w+)' '\s*(?P<erase_ctx>\w+)' '\s*,'
'\s*\(\s*(?P<erase_size>\w+)\s*\)' '\s*\)' '\s*(?P<erase_block>\w+)'
'|' '(?P<sync>sync)\(' '\s*\(\s*(?P<erase_size>\w+)\s*\)' '\s*\)'
'\s*(?P<sync_ctx>\w+)' '\s*\)' ')\s*$') '|' '(?P<sync>sync)\('
'\s*(?P<sync_ctx>\w+)' '\s*\)'
')\s*$')
def parse(line): def parse(line):
nonlocal bmap nonlocal bmap
nonlocal readed nonlocal readed
@@ -852,21 +854,21 @@ def main(path='-', *,
if reset: if reset:
bmap = Bmap( bmap = Bmap(
block_size=block_size_, block_size=block_size_,
block_count=block_count_, block_count=block_count_,
block_window=bmap.block_window, block_window=bmap.block_window,
off_window=bmap.off_window, off_window=bmap.off_window,
width=bmap.width, width=bmap.width,
height=bmap.height) height=bmap.height)
elif ((block_size is None elif ((block_size is None
and block_size_ != bmap.block_size) and block_size_ != bmap.block_size)
or (block_count is None or (block_count is None
and block_count_ != bmap.block_count)): and block_count_ != bmap.block_count)):
bmap.resize( bmap.resize(
block_size=block_size if block_size is not None block_size=block_size if block_size is not None
else block_size_, else block_size_,
block_count=block_count if block_count is not None block_count=block_count if block_count is not None
else block_count_) else block_count_)
return True return True
elif m.group('read') and read: elif m.group('read') and read:
@@ -877,10 +879,10 @@ def main(path='-', *,
if ((block_size is None and off+size > bmap.block_size) if ((block_size is None and off+size > bmap.block_size)
or (block_count is None and block >= bmap.block_count)): or (block_count is None and block >= bmap.block_count)):
bmap.resize( bmap.resize(
block_size=block_size if block_size is not None block_size=block_size if block_size is not None
else max(off+size, bmap.block_size), else max(off+size, bmap.block_size),
block_count=block_count if block_count is not None block_count=block_count if block_count is not None
else max(block+1, bmap.block_count)) else max(block+1, bmap.block_count))
bmap.read(block, off, size) bmap.read(block, off, size)
readed += size readed += size
@@ -894,10 +896,10 @@ def main(path='-', *,
if ((block_size is None and off+size > bmap.block_size) if ((block_size is None and off+size > bmap.block_size)
or (block_count is None and block >= bmap.block_count)): or (block_count is None and block >= bmap.block_count)):
bmap.resize( bmap.resize(
block_size=block_size if block_size is not None block_size=block_size if block_size is not None
else max(off+size, bmap.block_size), else max(off+size, bmap.block_size),
block_count=block_count if block_count is not None block_count=block_count if block_count is not None
else max(block+1, bmap.block_count)) else max(block+1, bmap.block_count))
bmap.prog(block, off, size) bmap.prog(block, off, size)
proged += size proged += size
@@ -910,10 +912,10 @@ def main(path='-', *,
if ((block_size is None and size > bmap.block_size) if ((block_size is None and size > bmap.block_size)
or (block_count is None and block >= bmap.block_count)): or (block_count is None and block >= bmap.block_count)):
bmap.resize( bmap.resize(
block_size=block_size if block_size is not None block_size=block_size if block_size is not None
else max(size, bmap.block_size), else max(size, bmap.block_size),
block_count=block_count if block_count is not None block_count=block_count if block_count is not None
else max(block+1, bmap.block_count)) else max(block+1, bmap.block_count))
bmap.erase(block, size) bmap.erase(block, size)
erased += size erased += size
@@ -936,20 +938,20 @@ def main(path='-', *,
# don't forget we've scaled this for braille/dots! # don't forget we've scaled this for braille/dots!
for row in range( for row in range(
mt.ceil(bmap.height/4) if braille mt.ceil(bmap.height/4) if braille
else mt.ceil(bmap.height/2) if dots else mt.ceil(bmap.height/2) if dots
else bmap.height): else bmap.height):
line = bmap.draw(row, line = bmap.draw(row,
read=read, read=read,
prog=prog, prog=prog,
erase=erase, erase=erase,
wear=wear, wear=wear,
block_cycles=block_cycles, block_cycles=block_cycles,
color=color, color=color,
dots=dots, dots=dots,
braille=braille, braille=braille,
hilbert=hilbert, hilbert=hilbert,
lebesgue=lebesgue, lebesgue=lebesgue,
**args) **args)
if line: if line:
f.writeln(line) f.writeln(line)
@@ -965,9 +967,9 @@ def main(path='-', *,
# what we have # what we have
if wear: if wear:
mean = (sum(p.wear for p in bmap.pixels) mean = (sum(p.wear for p in bmap.pixels)
/ max(len(bmap.pixels), 1)) / max(len(bmap.pixels), 1))
stddev = mt.sqrt(sum((p.wear - mean)**2 for p in bmap.pixels) stddev = mt.sqrt(sum((p.wear - mean)**2 for p in bmap.pixels)
/ max(len(bmap.pixels), 1)) / max(len(bmap.pixels), 1))
worst = max((p.wear for p in bmap.pixels), default=0) worst = max((p.wear for p in bmap.pixels), default=0)
# a bit of a hack here, but this forces our header to always be # a bit of a hack here, but this forces our header to always be
@@ -975,17 +977,17 @@ def main(path='-', *,
if len(f.lines) == 0: if len(f.lines) == 0:
f.lines.append('') f.lines.append('')
f.lines[0] = 'bd %dx%d%s%s%s%s' % ( f.lines[0] = 'bd %dx%d%s%s%s%s' % (
bmap.block_size, bmap.block_count, bmap.block_size, bmap.block_count,
', %6s read' % ('%.1f%%' % (100*readed / max(total, 1))) ', %6s read' % ('%.1f%%' % (100*readed / max(total, 1)))
if read else '', if read else '',
', %6s prog' % ('%.1f%%' % (100*proged / max(total, 1))) ', %6s prog' % ('%.1f%%' % (100*proged / max(total, 1)))
if prog else '', if prog else '',
', %6s erase' % ('%.1f%%' % (100*erased / max(total, 1))) ', %6s erase' % ('%.1f%%' % (100*erased / max(total, 1)))
if erase else '', if erase else '',
', %13s wear' % ('%.1fσ (%.1f%%)' % ( ', %13s wear' % ('%.1fσ (%.1f%%)' % (
worst / max(stddev, 1), worst / max(stddev, 1),
100*stddev / max(worst, 1))) 100*stddev / max(worst, 1)))
if wear else '') if wear else '')
bmap.clear() bmap.clear()
readed = 0 readed = 0
@@ -1040,7 +1042,7 @@ def main(path='-', *,
time.sleep(sleep or 0.1) time.sleep(sleep or 0.1)
except FileNotFoundError as e: except FileNotFoundError as e:
print("error: file not found %r" % path, print("error: file not found %r" % path,
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
except KeyboardInterrupt: except KeyboardInterrupt:
pass pass
@@ -1056,145 +1058,147 @@ if __name__ == "__main__":
import sys import sys
import argparse import argparse
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Render operations on block devices based on " description="Render operations on block devices based on "
"trace output.", "trace output.",
allow_abbrev=False) allow_abbrev=False)
parser.add_argument( parser.add_argument(
'path', 'path',
nargs='?', nargs='?',
help="Path to read from.") help="Path to read from.")
parser.add_argument( parser.add_argument(
'-b', '--block-size', '-b', '--block-size',
type=bdgeom, type=bdgeom,
help="Block size/geometry in bytes.") help="Block size/geometry in bytes.")
parser.add_argument( parser.add_argument(
'--block-count', '--block-count',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Block count in blocks.") help="Block count in blocks.")
parser.add_argument( parser.add_argument(
'-c', '--block-cycles', '-c', '--block-cycles',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Assumed maximum number of erase cycles when measuring wear.") help="Assumed maximum number of erase cycles when measuring "
"wear.")
parser.add_argument( parser.add_argument(
'-@', '--block', '-@', '--block',
nargs='?', nargs='?',
type=lambda x: tuple( type=lambda x: tuple(
rbydaddr(x) if x.strip() else None rbydaddr(x) if x.strip() else None
for x in x.split(',')), for x in x.split(',')),
help="Optional block to show, may be a range.") help="Optional block to show, may be a range.")
parser.add_argument( parser.add_argument(
'--off', '--off',
type=lambda x: tuple( type=lambda x: tuple(
int(x, 0) if x.strip() else None int(x, 0) if x.strip() else None
for x in x.split(',')), for x in x.split(',')),
help="Show a specific offset, may be a range.") help="Show a specific offset, may be a range.")
parser.add_argument( parser.add_argument(
'--size', '--size',
type=lambda x: tuple( type=lambda x: tuple(
int(x, 0) if x.strip() else None int(x, 0) if x.strip() else None
for x in x.split(',')), for x in x.split(',')),
help="Show this many bytes, may be a range.") help="Show this many bytes, may be a range.")
parser.add_argument( parser.add_argument(
'-r', '--read', '-r', '--read',
action='store_true', action='store_true',
help="Render reads.") help="Render reads.")
parser.add_argument( parser.add_argument(
'-p', '--prog', '-p', '--prog',
action='store_true', action='store_true',
help="Render progs.") help="Render progs.")
parser.add_argument( parser.add_argument(
'-e', '--erase', '-e', '--erase',
action='store_true', action='store_true',
help="Render erases.") help="Render erases.")
parser.add_argument( parser.add_argument(
'-w', '--wear', '-w', '--wear',
action='store_true', action='store_true',
help="Render wear.") help="Render wear.")
parser.add_argument( parser.add_argument(
'-R', '--reset', '-R', '--reset',
action='store_true', action='store_true',
help="Reset wear on block device initialization.") help="Reset wear on block device initialization.")
parser.add_argument( parser.add_argument(
'-N', '--no-header', '-N', '--no-header',
action='store_true', action='store_true',
help="Don't show the header.") help="Don't show the header.")
parser.add_argument( parser.add_argument(
'--color', '--color',
choices=['never', 'always', 'auto'], choices=['never', 'always', 'auto'],
default='auto', default='auto',
help="When to use terminal colors. Defaults to 'auto'.") help="When to use terminal colors. Defaults to 'auto'.")
parser.add_argument( parser.add_argument(
'-:', '--dots', '-:', '--dots',
action='store_true', action='store_true',
help="Use 1x2 ascii dot characters.") help="Use 1x2 ascii dot characters.")
parser.add_argument( parser.add_argument(
'-⣿', '--braille', '-⣿', '--braille',
action='store_true', action='store_true',
help="Use 2x4 unicode braille characters. Note that braille characters " help="Use 2x4 unicode braille characters. Note that braille "
"sometimes suffer from inconsistent widths.") "characters sometimes suffer from inconsistent widths.")
parser.add_argument( parser.add_argument(
'--chars', '--chars',
help="Characters to use for read, prog, erase, noop operations.") help="Characters to use for read, prog, erase, noop operations.")
parser.add_argument( parser.add_argument(
'--wear-chars', '--wear-chars',
help="Characters to use for showing wear.") help="Characters to use for showing wear.")
parser.add_argument( parser.add_argument(
'--colors', '--colors',
type=lambda x: [x.strip() for x in x.split(',')], type=lambda x: [x.strip() for x in x.split(',')],
help="Colors to use for read, prog, erase, noop operations.") help="Colors to use for read, prog, erase, noop operations.")
parser.add_argument( parser.add_argument(
'--wear-colors', '--wear-colors',
type=lambda x: [x.strip() for x in x.split(',')], type=lambda x: [x.strip() for x in x.split(',')],
help="Colors to use for showing wear.") help="Colors to use for showing wear.")
parser.add_argument( parser.add_argument(
'-W', '--width', '-W', '--width',
nargs='?', nargs='?',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
const=0, const=0,
help="Width in columns. 0 uses the terminal width. Defaults to " help="Width in columns. 0 uses the terminal width. Defaults to "
"min(terminal, 80).") "min(terminal, 80).")
parser.add_argument( parser.add_argument(
'-H', '--height', '-H', '--height',
nargs='?', nargs='?',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
const=0, const=0,
help="Height in rows. 0 uses the terminal height. Defaults to 1.") help="Height in rows. 0 uses the terminal height. Defaults to 1.")
parser.add_argument( parser.add_argument(
'-n', '--lines', '-n', '--lines',
nargs='?', nargs='?',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
const=0, const=0,
help="Show this many lines of history. 0 uses the terminal height. " help="Show this many lines of history. 0 uses the terminal "
"Defaults to 5.") "height. Defaults to 5.")
parser.add_argument( parser.add_argument(
'-^', '--head', '-^', '--head',
action='store_true', action='store_true',
help="Show the first n lines.") help="Show the first n lines.")
parser.add_argument( parser.add_argument(
'-z', '--cat', '-z', '--cat',
action='store_true', action='store_true',
help="Pipe directly to stdout.") help="Pipe directly to stdout.")
parser.add_argument( parser.add_argument(
'-U', '--hilbert', '-U', '--hilbert',
action='store_true', action='store_true',
help="Render as a space-filling Hilbert curve.") help="Render as a space-filling Hilbert curve.")
parser.add_argument( parser.add_argument(
'-Z', '--lebesgue', '-Z', '--lebesgue',
action='store_true', action='store_true',
help="Render as a space-filling Z-curve.") help="Render as a space-filling Z-curve.")
parser.add_argument( parser.add_argument(
'-S', '--coalesce', '-S', '--coalesce',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Number of operations to coalesce together.") help="Number of operations to coalesce together.")
parser.add_argument( parser.add_argument(
'-s', '--sleep', '-s', '--sleep',
type=float, type=float,
help="Time in seconds to sleep between reads, coalescing operations.") help="Time in seconds to sleep between reads, coalescing "
"operations.")
parser.add_argument( parser.add_argument(
'-k', '--keep-open', '-k', '--keep-open',
action='store_true', action='store_true',
help="Reopen the pipe on EOF, useful when multiple " help="Reopen the pipe on EOF, useful when multiple "
"processes are writing.") "processes are writing.")
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items() for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None})) if v is not None}))
scripts/watch.py
+55 -54
View File
@@ -49,13 +49,13 @@ else:
# wait for interesting events # wait for interesting events
flags = (inotify_simple.flags.ATTRIB flags = (inotify_simple.flags.ATTRIB
| inotify_simple.flags.CREATE | inotify_simple.flags.CREATE
| inotify_simple.flags.DELETE | inotify_simple.flags.DELETE
| inotify_simple.flags.DELETE_SELF | inotify_simple.flags.DELETE_SELF
| inotify_simple.flags.MODIFY | inotify_simple.flags.MODIFY
| inotify_simple.flags.MOVED_FROM | inotify_simple.flags.MOVED_FROM
| inotify_simple.flags.MOVED_TO | inotify_simple.flags.MOVED_TO
| inotify_simple.flags.MOVE_SELF) | inotify_simple.flags.MOVE_SELF)
# recurse into directories # recurse into directories
for path in paths: for path in paths:
@@ -113,8 +113,8 @@ class RingIO:
# pad to fill any existing canvas, but truncate to terminal size # pad to fill any existing canvas, but truncate to terminal size
h = shutil.get_terminal_size((80, 5))[1] h = shutil.get_terminal_size((80, 5))[1]
lines.extend('' for _ in range( lines.extend('' for _ in range(
len(lines), len(lines),
min(RingIO.canvas_lines, h))) min(RingIO.canvas_lines, h)))
while len(lines) > h: while len(lines) > h:
if self.head: if self.head:
lines.pop() lines.pop()
@@ -154,7 +154,7 @@ def main(command, *,
exit_on_error=False): exit_on_error=False):
if not command: if not command:
print('usage: %s [options] command' % sys.argv[0], print('usage: %s [options] command' % sys.argv[0],
file=sys.stderr) file=sys.stderr)
sys.exit(-1) sys.exit(-1)
# if we have keep_open_paths, assume user wanted keep_open # if we have keep_open_paths, assume user wanted keep_open
@@ -199,12 +199,12 @@ def main(command, *,
if lines: if lines:
h = lines h = lines
fcntl.ioctl(spty, termios.TIOCSWINSZ, fcntl.ioctl(spty, termios.TIOCSWINSZ,
struct.pack('HHHH', h, w, 0, 0)) struct.pack('HHHH', h, w, 0, 0))
proc = sp.Popen(command, proc = sp.Popen(command,
stdout=spty, stdout=spty,
stderr=spty, stderr=spty,
close_fds=False) close_fds=False)
os.close(spty) os.close(spty)
mpty = os.fdopen(mpty, 'r', 1) mpty = os.fdopen(mpty, 'r', 1)
@@ -261,54 +261,55 @@ if __name__ == "__main__":
import sys import sys
import argparse import argparse
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Traditional watch command, but with higher resolution " description="Traditional watch command, but with higher "
"updates and a bit different options/output format.", "resolution updates and a bit different options/output "
allow_abbrev=False) "format.",
allow_abbrev=False)
parser.add_argument( parser.add_argument(
'command', 'command',
nargs=argparse.REMAINDER, nargs=argparse.REMAINDER,
help="Command to run.") help="Command to run.")
parser.add_argument( parser.add_argument(
'-n', '--lines', '-n', '--lines',
nargs='?', nargs='?',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
const=0, const=0,
help="Show this many lines of history. 0 uses the terminal height. " help="Show this many lines of history. 0 uses the terminal "
"Defaults to 0.") "height. Defaults to 0.")
parser.add_argument( parser.add_argument(
'-^', '--head', '-^', '--head',
action='store_true', action='store_true',
help="Show the first n lines.") help="Show the first n lines.")
parser.add_argument( parser.add_argument(
'-z', '--cat', '-z', '--cat',
action='store_true', action='store_true',
help="Pipe directly to stdout.") help="Pipe directly to stdout.")
parser.add_argument( parser.add_argument(
'-s', '--sleep', '-s', '--sleep',
type=float, type=float,
help="Seconds to sleep between runs. Defaults to 0.1.") help="Seconds to sleep between runs. Defaults to 0.1.")
parser.add_argument( parser.add_argument(
'-k', '--keep-open', '-k', '--keep-open',
action='store_true', action='store_true',
help="Try to use inotify to wait for changes.") help="Try to use inotify to wait for changes.")
parser.add_argument( parser.add_argument(
'-K', '--keep-open-path', '-K', '--keep-open-path',
dest='keep_open_paths', dest='keep_open_paths',
action='append', action='append',
help="Use this path for inotify. Defaults to guessing. Implies " help="Use this path for inotify. Defaults to guessing. Implies "
"--keep-open.") "--keep-open.")
parser.add_argument( parser.add_argument(
'-b', '--buffer', '-b', '--buffer',
action='store_true', action='store_true',
help="Wait until command finishes to show the output.") help="Wait until command finishes to show the output.")
parser.add_argument( parser.add_argument(
'-i', '--ignore-errors', '-i', '--ignore-errors',
action='store_true', action='store_true',
help="Only show output after successful runs. Implies --buffer.") help="Only show output after successful runs. Implies --buffer.")
parser.add_argument( parser.add_argument(
'-e', '--exit-on-error', '-e', '--exit-on-error',
action='store_true', action='store_true',
help="Exit on error.") help="Exit on error.")
sys.exit(main(**{k: v sys.exit(main(**{k: v
for k, v in vars(parser.parse_args()).items() for k, v in vars(parser.parse_args()).items()
if v is not None})) if v is not None}))