From b5c3b97ae186ad40439be6d1d45728450bc9a1c3 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 14 Apr 2025 02:45:50 -0500 Subject: [PATCH] scripts: Reworked dbgtag.py, added -i/--input, included hex in output This just gives dbgtag.py a few more bells and whistles that may be useful: - Can now parse multiple tags from hex: $ ./scripts/dbgtag.py -x 71 01 01 01 12 02 02 02 71 01 01 01 altrgt 0x101 w1 -1 12 02 02 02 shrubdir w2 2 Note this _does_ skip attached data, which risks some confusion but not skipping attached data will probably end up printing a bunch of garbage for most use cases: $ ./scripts/dbgtag.py -x 01 01 01 04 02 02 02 02 03 03 03 03 01 01 01 04 gdelta 0x01 w1 4 03 03 03 03 struct 0x03 w3 3 - Included hex in output. This is helpful for learning about the tag encoding and also helps identify tags when parsing multiple tags. I considered also included offsets, which might help with understanding attached data, but decided it would be too noisy. At some point you should probably jump to dbgrbyd.py anyways... - Added -i/--input to read tags from a file. This is roughly the same as -x/--hex, but allows piping from other scripts: $ ./scripts/dbgcat.py disk -b4096 0 -n4,8 | ./scripts/dbgtag.py -i- 80 03 00 08 magic 8 Note this reads the entire file in before processing. We'd need to fit everything into RAM anyways to figure out padding. --- scripts/crc32c.py | 10 +++-- scripts/dbgbmap.py | 2 +- scripts/dbgbmapd3.py | 2 +- scripts/dbgbtree.py | 2 +- scripts/dbglfs.py | 2 +- scripts/dbgmtree.py | 2 +- scripts/dbgrbyd.py | 2 +- scripts/dbgtag.py | 93 ++++++++++++++++++++++++++++++++------------ scripts/parity.py | 10 +++-- 9 files changed, 88 insertions(+), 37 deletions(-) diff --git a/scripts/crc32c.py b/scripts/crc32c.py index 1584e5e9..5ae309f1 100755 --- a/scripts/crc32c.py +++ b/scripts/crc32c.py @@ -38,14 +38,18 @@ def crc32c(data, crc=0): return 0xffffffff ^ crc -def main(paths, **args): +def main(paths, *, + hex=False, + string=False): + hex_ = hex; del hex + # interpret as sequence of hex bytes - if args.get('hex'): + if hex_: bytes_ = [b for path in paths for b in path.split()] print('%08x' % crc32c(bytes(int(b, 16) for b in bytes_))) # interpret as strings - elif args.get('string'): + elif string: for path in paths: print('%08x %s' % (crc32c(path.encode('utf8')), path)) diff --git a/scripts/dbgbmap.py b/scripts/dbgbmap.py index 2c0a6ea4..d538647e 100755 --- a/scripts/dbgbmap.py +++ b/scripts/dbgbmap.py @@ -214,7 +214,7 @@ def fromleb128(data): def fromtag(data): data = data.ljust(4, b'\0') - tag = (data[0] << 8) | data[1] + tag = struct.unpack('>H', data[:2])[0] weight, d = fromleb128(data[2:]) size, d_ = fromleb128(data[2+d:]) return tag>>15, tag&0x7fff, weight, size, 2+d+d_ diff --git a/scripts/dbgbmapd3.py b/scripts/dbgbmapd3.py index 714c1bf0..15944b6a 100755 --- a/scripts/dbgbmapd3.py +++ b/scripts/dbgbmapd3.py @@ -244,7 +244,7 @@ def fromleb128(data): def fromtag(data): data = data.ljust(4, b'\0') - tag = (data[0] << 8) | data[1] + tag = struct.unpack('>H', data[:2])[0] weight, d = fromleb128(data[2:]) size, d_ = fromleb128(data[2+d:]) return tag>>15, tag&0x7fff, weight, size, 2+d+d_ diff --git a/scripts/dbgbtree.py b/scripts/dbgbtree.py index 1692c6a4..6606f58e 100755 --- a/scripts/dbgbtree.py +++ b/scripts/dbgbtree.py @@ -152,7 +152,7 @@ def fromleb128(data): def fromtag(data): data = data.ljust(4, b'\0') - tag = (data[0] << 8) | data[1] + tag = struct.unpack('>H', data[:2])[0] weight, d = fromleb128(data[2:]) size, d_ = fromleb128(data[2+d:]) return tag>>15, tag&0x7fff, weight, size, 2+d+d_ diff --git a/scripts/dbglfs.py b/scripts/dbglfs.py index 557eaedb..b8ea43b9 100755 --- a/scripts/dbglfs.py +++ b/scripts/dbglfs.py @@ -171,7 +171,7 @@ def fromleb128(data): def fromtag(data): data = data.ljust(4, b'\0') - tag = (data[0] << 8) | data[1] + tag = struct.unpack('>H', data[:2])[0] weight, d = fromleb128(data[2:]) size, d_ = fromleb128(data[2+d:]) return tag>>15, tag&0x7fff, weight, size, 2+d+d_ diff --git a/scripts/dbgmtree.py b/scripts/dbgmtree.py index 35f0e685..9f759938 100755 --- a/scripts/dbgmtree.py +++ b/scripts/dbgmtree.py @@ -152,7 +152,7 @@ def fromleb128(data): def fromtag(data): data = data.ljust(4, b'\0') - tag = (data[0] << 8) | data[1] + tag = struct.unpack('>H', data[:2])[0] weight, d = fromleb128(data[2:]) size, d_ = fromleb128(data[2+d:]) return tag>>15, tag&0x7fff, weight, size, 2+d+d_ diff --git a/scripts/dbgrbyd.py b/scripts/dbgrbyd.py index 439d43dc..4e075997 100755 --- a/scripts/dbgrbyd.py +++ b/scripts/dbgrbyd.py @@ -162,7 +162,7 @@ def fromleb128(data): def fromtag(data): data = data.ljust(4, b'\0') - tag = (data[0] << 8) | data[1] + tag = struct.unpack('>H', data[:2])[0] weight, d = fromleb128(data[2:]) size, d_ = fromleb128(data[2+d:]) return tag>>15, tag&0x7fff, weight, size, 2+d+d_ diff --git a/scripts/dbgtag.py b/scripts/dbgtag.py index 652859e1..a3bef20c 100755 --- a/scripts/dbgtag.py +++ b/scripts/dbgtag.py @@ -53,6 +53,17 @@ TAG_ECKSUM = 0x3200 ## 0x3200 v-11 --1- ---- ---- TAG_GCKSUMDELTA = 0x3300 ## 0x3300 v-11 --11 ---- ---- +# open with '-' for stdin/stdout +def openio(path, mode='r', buffering=-1): + import os + if path == '-': + if 'r' in mode: + return os.fdopen(os.dup(sys.stdin.fileno()), mode, buffering) + else: + return os.fdopen(os.dup(sys.stdout.fileno()), mode, buffering) + else: + return open(path, mode, buffering) + def fromleb128(data): word = 0 for i, b in enumerate(data): @@ -62,6 +73,13 @@ def fromleb128(data): return word, i+1 return word, len(data) +def fromtag(data): + data = data.ljust(4, b'\0') + tag = struct.unpack('>H', data[:2])[0] + weight, d = fromleb128(data[2:]) + size, d_ = fromleb128(data[2+d:]) + return tag>>15, tag&0x7fff, weight, size, 2+d+d_ + # human readable tag repr def tagrepr(tag, weight=None, size=None, *, global_=False, @@ -213,42 +231,64 @@ def list_tags(): w[0], 'LFSR_'+n, c)) -def dbg_tag(data): - if isinstance(data, int): - tag = data - weight = None - size = None - else: - data = data.ljust(2, b'\0') - tag = (data[0] << 8) | data[1] - weight, d = fromleb128(data[2:]) if 2 < len(data) else (None, 2) - size, d_ = fromleb128(data[2+d:]) if d < len(data) else (None, d) +def dbg_tags(data): + lines = [] + # interpret as ints? + if not isinstance(data, bytes): + for tag in data: + lines.append(( + ' '.join('%02x' % b for b in struct.pack('>H', tag)), + tagrepr(tag))) - print(tagrepr(tag, weight, size)) + # interpret as bytes? + else: + j = 0 + while j < len(data): + v, tag, w, size, d = fromtag(data[j:]) + lines.append(( + ' '.join('%02x' % b for b in data[j:j+d]), + tagrepr(tag, w, size))) + j += d + + # skip attached data if there is any + if not tag & TAG_ALT: + j += size + + # figure out widths + w = [0] + for l in lines: + w[0] = max(w[0], len(l[0])) + + # then print results + for l in lines: + print('%-*s %s' % ( + w[0], l[0], + l[1])) def main(tags, *, list=False, - block_size=None, - block_count=None, - off=None, - **args): + hex=False, + input=None): list_ = list; del list + hex_ = hex; del hex # list all known tags if list_: list_tags() - # try to decode these tags - else: - # interpret as a sequence of hex bytes - if args.get('hex'): - bytes_ = [b for tag in tags for b in tag.split()] - dbg_tag(bytes(int(b, 16) for b in bytes_)) + # interpret as a sequence of hex bytes + elif hex_: + bytes_ = [b for tag in tags for b in tag.split()] + dbg_tags(bytes(int(b, 16) for b in bytes_)) - # default to interpreting as ints - else: - for tag in tags: - dbg_tag(int(tag, 0)) + # parse tags in a file + elif input: + with openio(input, 'rb') as f: + dbg_tags(f.read()) + + # default to interpreting as ints + else: + dbg_tags(int(tag, 0) for tag in tags) if __name__ == "__main__": @@ -269,6 +309,9 @@ if __name__ == "__main__": '-x', '--hex', action='store_true', help="Interpret as a sequence of hex bytes.") + parser.add_argument( + '-i', '--input', + help="Read tags from this file. Can use - for stdin.") sys.exit(main(**{k: v for k, v in vars(parser.parse_intermixed_args()).items() if v is not None})) diff --git a/scripts/parity.py b/scripts/parity.py index 09d89265..78fd0702 100755 --- a/scripts/parity.py +++ b/scripts/parity.py @@ -46,14 +46,18 @@ def parity(x): return popc(x) & 1 -def main(paths, **args): +def main(paths, *, + hex=False, + string=False): + hex_ = hex; del hex + # interpret as sequence of hex bytes - if args.get('hex'): + if hex_: bytes_ = [b for path in paths for b in path.split()] print('%01x' % parity(crc32c(bytes(int(b, 16) for b in bytes_)))) # interpret as strings - elif args.get('string'): + elif string: for path in paths: print('%01x %s' % (parity(crc32c(path.encode('utf8'))), path))