From 21049321a55ddd599f8c4e4d2d2199d39e65f1ef Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 18 Mar 2025 02:29:45 -0500 Subject: [PATCH] scripts: Reworked dbgrbyd.py, adopted Rbyd class This reworks dbgrbyd.py to use the Rbyd class (well, a rewrite of the Rbyd class) as an abstraction of littlefs's rbyd disk structure in Python. Duplicating common classes/functions across these scripts has proven useful for sharing code without preventing these scripts from being standalone (a problem for _actual_ code sharing, relative imports, etc). And, because of how these scripts were written, dbgrbyd.py humorously ended up the only script not sharing the Rbyd class. I'm also trying to make the actual Rbyd abstraction a bit more concrete now that the filesystem's design has had some time to mature. This means more classes for things like Rattrs that reduce the sheer number of tuples that were flying around. New classes: - Rattr - rbyd attrs, tag + weight + data, this includes all relevant offsets which is useful for rendering hexdumps/etc. - Ralt - rbyd alt pointers, useful for building tree representations. - Rbyd - rbyd abstraction, including lookup/traversal methods Note also that while the Rbyd class replaces most of the dbg_tree logic, dbg_log is still pretty low-level and abstractionless. --- Eventually I hope to have well defined classes for Btrees, Mdirs, Files, etc, to make it easier to write more interesting debug scripts such as dbgbmap.py. Separating Btree, Mdirs, etc also means we shouldn't need the hacky btree_lookup/tree_lookup methods in every script anymore. Having those in dbgrbyd.py would've been a bit weird. --- scripts/dbgrbyd.py | 974 ++++++++++++++++++++++++++++----------------- 1 file changed, 609 insertions(+), 365 deletions(-) diff --git a/scripts/dbgrbyd.py b/scripts/dbgrbyd.py index 7b14cef4..6c1886e9 100755 --- a/scripts/dbgrbyd.py +++ b/scripts/dbgrbyd.py @@ -88,11 +88,12 @@ def bdgeom(s): else: return int(s, b) +# TODO sync across scripts # parse some rbyd addr encodings -# 0xa -> (0xa,) -# 0xa.c -> ((0xa, 0xc),) -# 0x{a,b} -> (0xa, 0xb) -# 0x{a,b}.c -> ((0xa, 0xc), (0xb, 0xc)) +# 0xa -> [0xa] +# 0xa.c -> [(0xa, 0xc)] +# 0x{a,b} -> [0xa, 0xb] +# 0x{a,b}.c -> [(0xa, 0xc), (0xb, 0xc)] def rbydaddr(s): s = s.strip() b = 10 @@ -123,7 +124,7 @@ def rbydaddr(s): else: addr.append(int(s, b)) - return tuple(addr) + return addr def crc32c(data, crc=0): crc ^= 0xffffffff @@ -168,11 +169,12 @@ def xxd(data, width=16): b if b >= ' ' and b <= '~' else '.' for b in map(chr, data[i:i+width]))) -def tagrepr(tag, w=None, size=None, off=None): +# TODO sync across scripts +def tagrepr(tag, weight=None, size=None, off=None): if (tag & 0x6fff) == TAG_NULL: return '%snull%s%s' % ( 'shrub' if tag & TAG_SHRUB else '', - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %d' % size if size else '') elif (tag & 0x6f00) == TAG_CONFIG: return '%s%s%s%s' % ( @@ -186,14 +188,14 @@ def tagrepr(tag, w=None, size=None, off=None): else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT else 'config 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_GDELTA: return '%s%s%s%s' % ( 'shrub' if tag & TAG_SHRUB else '', 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA else 'gdelta 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_NAME: return '%s%s%s%s' % ( @@ -204,7 +206,7 @@ def tagrepr(tag, w=None, size=None, off=None): else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK else 'stickynote' if (tag & 0xfff) == TAG_STICKYNOTE else 'name 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_STRUCT: return '%s%s%s%s' % ( @@ -219,21 +221,21 @@ def tagrepr(tag, w=None, size=None, off=None): else 'did' if (tag & 0xfff) == TAG_DID else 'branch' if (tag & 0xfff) == TAG_BRANCH else 'struct 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') elif (tag & 0x6e00) == TAG_ATTR: return '%s%sattr 0x%02x%s%s' % ( 'shrub' if tag & TAG_SHRUB else '', 's' if tag & 0x100 else 'u', ((tag & 0x100) >> 1) ^ (tag & 0xff), - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') elif tag & TAG_ALT: return 'alt%s%s 0x%03x%s%s' % ( 'r' if tag & TAG_R else 'b', 'gt' if tag & TAG_GT else 'le', tag & 0x0fff, - ' w%d' % w if w is not None else '', + ' w%d' % weight if weight is not None else '', ' 0x%x' % (0xffffffff & (off-size)) if size and off is not None else ' -%d' % size if size @@ -242,38 +244,559 @@ def tagrepr(tag, w=None, size=None, off=None): return 'cksum%s%s%s%s' % ( 'p' if not tag & 0xfe and tag & TAG_P else '', ' 0x%02x' % (tag & 0xff) if tag & 0xfe else '', - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') elif (tag & 0x7f00) == TAG_NOTE: return 'note%s%s%s' % ( ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') elif (tag & 0x7f00) == TAG_ECKSUM: return 'ecksum%s%s%s' % ( ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') elif (tag & 0x7f00) == TAG_GCKSUMDELTA: return 'gcksumdelta%s%s%s' % ( ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') else: return '0x%04x%s%s' % ( tag, - ' w%d' % w if w is not None else '', + ' w%d' % weight if weight is not None else '', ' %d' % size if size is not None else '') +# tagged data in an rbyd +class Rattr: + def __init__(self, tag, weight, block, toff, off, data): + self.tag = tag + self.weight = weight + self.block = block + self.toff = toff + self.off = off + self.data = data -def dbg_log(data, block_size, rev, eoff, weight, *, + @property + def size(self): + return len(self.data) + + def __repr__(self): + return '<%s %s>' % (self.__class__.__name__, self.tagrepr()) + + def tagrepr(self): + return tagrepr(self.tag, self.weight, self.size) + + def __bool__(self): + return bool(self.data) + + def __len__(self): + return len(self.data) + + def __getitem__(self, key): + return self.data[key] + + def __iter__(self): + return iter(self.data) + +class Ralt: + def __init__(self, tag, weight, block, toff, off, jump, color=None): + self.tag = tag + self.weight = weight + self.block = block + self.toff = toff + self.off = off + self.jump = jump + + if color is not None: + self.color = color + else: + self.color = 'r' if tag & TAG_R else 'b' + + @property + def joff(self): + return self.toff - self.jump + + def __repr__(self): + return '<%s %s>' % (self.__class__.__name__, self.tagrepr()) + + def tagrepr(self): + return tagrepr(self.tag, self.weight, self.jump, self.toff) + +# tree branches are an abstract thing for tree rendering +class TreeBranch: + def __init__(self, a, b, depth, color): + # note a and b are context specific + self.a = a + self.b = b + self.depth = depth + self.color = color + + def __repr__(self): + return '%s(%s, %s, %s, %s)' % ( + self.__class__.__name__, + self.a, + self.b, + self.depth, + self.color) + +# our core rbyd type +class Rbyd: + def __init__(self, data, blocks, trunk, weight, rev, eoff, cksum, *, + gcksumdelta=None, + corrupt=False): + if isinstance(blocks, int): + blocks = [blocks] + + self.data = data + self.blocks = list(blocks) + self.trunk = trunk + self.weight = weight + self.rev = rev + self.eoff = eoff + self.cksum = cksum + self.gcksumdelta = gcksumdelta + self.corrupt = corrupt + + @property + def block(self): + return self.blocks[0] + + def addr(self): + if len(self.blocks) == 1: + return '0x%x.%x' % (self.block, self.trunk) + else: + return '0x{%s}.%x' % ( + ','.join('%x' % block for block in self.blocks), + self.trunk) + + def __repr__(self): + return '<%s %s>' % (self.__class__.__name__, self.addr()) + + def __bool__(self): + return not self.corrupt + + def __eq__(self, other): + return (self.blocks, self.trunk) == (other.blocks, other.trunk) + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.blocks, self.trunk)) + + @classmethod + def fetch(cls, data, block, trunk=None, cksum=None): + # multiple blocks? + if isinstance(block, list): + # fetch all blocks + rbyds = [cls.fetch(data, block, trunk, cksum) for block in block] + # determine most recent revision + i = 0 + for i_, rbyd in enumerate(rbyds): + # compare with sequence arithmetic + if rbyd and ( + not rbyds[i] + or not ((rbyd.rev - rbyds[i].rev) & 0x80000000) + or (rbyd.rev == rbyds[i].rev + and rbyd.trunk > rbyds[i].trunk)): + i = i_ + # keep track of the other blocks + rbyd = rbyds[i] + rbyd.blocks += tuple( + rbyds[(i+1+j) % len(rbyds)].block + for j in range(len(rbyds)-1)) + return rbyd + + # block may encode a trunk + block, trunk = ( + block[0] if isinstance(block, tuple) + else block, + trunk if trunk is not None + else block[1] if isinstance(block, tuple) + else None) + + # data can be either disk + block_size tuple or data + # + # note preread data can be useful for avoiding race conditions + # with cksums and shrubs + if isinstance(data, tuple): + f, block_size, *_ = data + # seek to the block + f.seek(block * block_size) + data = f.read(block_size) + + # fetch the rbyd + rev = fromle32(data[0:4]) + cksum_ = 0 + cksum__ = crc32c(data[0:4]) + cksum___ = cksum__ + perturb = False + eoff = 0 + eoff_ = None + j_ = 4 + trunk_ = 0 + trunk__ = 0 + trunk___ = 0 + weight = 0 + weight_ = 0 + weight__ = 0 + gcksumdelta = None + gcksumdelta_ = None + while j_ < len(data) and (not trunk or eoff <= trunk): + # read next tag + v, tag, w, size, d = fromtag(data[j_:]) + if v != parity(cksum___): + break + cksum___ ^= 0x00000080 if v else 0 + cksum___ = crc32c(data[j_:j_+d], cksum___) + j_ += d + if not tag & TAG_ALT and j_ + size > len(data): + break + + # take care of cksums + if not tag & TAG_ALT: + if (tag & 0xff00) != TAG_CKSUM: + cksum___ = crc32c(data[j_:j_+size], cksum___) + + # found a gcksumdelta? + if (tag & 0xff00) == TAG_GCKSUMDELTA: + gcksumdelta_ = Rattr(tag, w, + block, j_-d, d, data[j_:j_+size]) + + # found a cksum? + else: + # check cksum + cksum____ = fromle32(data[j_:j_+4]) + if cksum___ != cksum____: + break + # commit what we have + eoff = eoff_ if eoff_ else j_ + size + cksum_ = cksum__ + trunk_ = trunk__ + weight = weight_ + gcksumdelta = gcksumdelta_ + gcksumdelta_ = None + # update perturb bit + perturb = tag & TAG_P + # revert to data cksum and perturb + cksum___ = cksum__ ^ (0xfca42daf if perturb else 0) + + # evaluate trunks + if (tag & 0xf000) != TAG_CKSUM: + if not (trunk and j_-d > trunk and not trunk___): + # new trunk? + if not trunk___: + trunk___ = j_-d + weight__ = 0 + + # keep track of weight + weight__ += w + + # end of trunk? + if not tag & TAG_ALT: + # update trunk/weight unless we found a shrub or an + # explicit trunk (which may be a shrub) is requested + if not tag & TAG_SHRUB or trunk___ == trunk: + trunk__ = trunk___ + weight_ = weight__ + # keep track of eoff for best matching trunk + if trunk and j_ + size > trunk: + eoff_ = j_ + size + eoff = eoff_ + cksum_ = cksum___ ^ ( + 0xfca42daf if perturb else 0) + trunk_ = trunk__ + weight = weight_ + gcksumdelta = gcksumdelta_ + trunk___ = 0 + + # update canonical checksum, xoring out any perturb state + cksum__ = cksum___ ^ (0xfca42daf if perturb else 0) + + if not tag & TAG_ALT: + j_ += size + + # cksum mismatch? + if cksum is not None and cksum_ != cksum: + return cls(data, block, trunk or 0, 0, rev, 0, cksum_, + corrupt=True) + + return cls(data, block, trunk_, weight, rev, eoff, cksum_, + gcksumdelta=gcksumdelta, + corrupt=not trunk_) + + def lookupnext(self, rid, tag=None, *, + path=False): + if not self: + return None, None, None, *(([],) if path else ()) + + tag = max(tag or 0, 0x1) + lower = 0 + upper = self.weight + path_ = [] + + # descend down tree + j = self.trunk + while True: + _, alt, w, jump, d = fromtag(self.data[j:]) + + # found an alt? + if alt & TAG_ALT: + # follow? + if ((rid, tag & 0xfff) > (upper-w-1, alt & 0xfff) + if alt & TAG_GT + else ((rid, tag & 0xfff) + <= (lower+w-1, alt & 0xfff))): + lower += upper-lower-w if alt & TAG_GT else 0 + upper -= upper-lower-w if not alt & TAG_GT else 0 + j = j - jump + + if path: + # figure out which color + if alt & TAG_R: + _, nalt, _, _, _ = fromtag(self.data[j+jump+d:]) + if nalt & TAG_R: + color = 'y' + else: + color = 'r' + else: + color = 'b' + + path_.append(( + Ralt(alt, w, self.block, j+jump, j+jump+d, + jump, color), + True)) + + # stay on path + else: + lower += w if not alt & TAG_GT else 0 + upper -= w if alt & TAG_GT else 0 + j = j + d + + if path: + # figure out which color + if alt & TAG_R: + _, nalt, _, _, _ = fromtag(self.data[j:]) + if nalt & TAG_R: + color = 'y' + else: + color = 'r' + else: + color = 'b' + + path_.append(( + Ralt(alt, w, self.block, j-d, j, + jump, color), + False)) + + # found tag + else: + rid_ = upper-1 + tag_ = alt + w_ = upper-lower + + if not tag_ or (rid_, tag_) < (rid, tag): + return None, None, None, *(([],) if path else ()) + + return (rid_, tag_, + Rattr(tag_, w_, self.block, j, j+d, + self.data[j+d:j+d+jump]), + *((path_,) if path else ())) + + def lookup(self, rid, tag=None, mask=None, *, + path=False): + if tag is None: + tag, mask = 0, 0xffff + + rid_, tag_, rattr_, *path_ = self.lookupnext(rid, tag & ~(mask or 0), + path=path) + if (rid_ is None + or rid_ != rid + or (tag_ & ~(mask or 0)) != (tag & ~(mask or 0))): + if mask is not None: + return None, None, *path_ + elif path: + return None, *path_ + else: + return None + + if mask is not None: + return tag_, rattr_, *path_ + elif path: + return rattr_, *path_ + else: + return rattr_ + + def __getitem__(self, key): + if not isinstance(key, tuple): + key = (key,) + + return self.lookup(*key) + + def __contains__(self, key): + if not isinstance(key, tuple): + key = (key,) + + v = self.lookup(*key) + if isinstance(v, tuple): + return v[0] is not None + else: + return v is not None + + def __iter__(self): + rid, tag = -1, 0 + while True: + rid, tag, rattr = self.lookupnext(rid, tag+0x1) + # found end of tree? + if rid is None: + break + + yield rid, tag, rattr + + # lookup by name + def namelookup(self, did, name): + # binary search + best = (False, None, None, None) + lower = 0 + upper = self.weight + while lower < upper: + rid, tag, rattr = self.lookupnext(lower + (upper-1-lower)//2) + if rid is None: + break + + # treat vestigial names as a catch-all + if ((tag == TAG_NAME and rid-(rattr.weight-1) == 0) + or (tag & 0xff00) != TAG_NAME): + did_ = 0 + name_ = b'' + else: + did_, d = fromleb128(rattr[:]) + name_ = rattr[d:] + + # bisect search space + if (did_, name_) > (did, name): + upper = rid-(rattr.weight-1) + elif (did_, name_) < (did, name): + lower = rid + 1 + # keep track of best match + best = (False, rid, tag, rattr) + else: + # found a match + return True, rid, tag, rattr + + return best + + # create tree representation for debugging + def tree(self, *, + rbyd=False): + trunks = co.defaultdict(lambda: (-1, 0)) + alts = co.defaultdict(lambda: {}) + + rid, tag = -1, 0 + while True: + rid, tag, rattr, path = self.lookupnext(rid, tag+0x1, + path=True) + # found end of tree? + if rid is None: + break + + # keep track of trunks/alts + trunks[rattr.toff] = (rid, tag) + + for ralt, followed in path: + if followed: + alts[ralt.toff] |= {'f': ralt.joff, 'c': ralt.color} + else: + alts[ralt.toff] |= {'nf': ralt.off, 'c': ralt.color} + + if rbyd: + # treat unreachable alts as converging paths + for j_, alt in alts.items(): + if 'f' not in alt: + alt['f'] = alt['nf'] + elif 'nf' not in alt: + alt['nf'] = alt['f'] + + else: + # prune any alts with unreachable edges + pruned = {} + for j, alt in alts.items(): + if 'f' not in alt: + pruned[j] = alt['nf'] + elif 'nf' not in alt: + pruned[j] = alt['f'] + for j in pruned.keys(): + del alts[j] + + for j, alt in alts.items(): + while alt['f'] in pruned: + alt['f'] = pruned[alt['f']] + while alt['nf'] in pruned: + alt['nf'] = pruned[alt['nf']] + + # find the trunk and depth of each alt + def rec_trunk(j): + if j not in alts: + return trunks[j] + else: + if 'nft' not in alts[j]: + alts[j]['nft'] = rec_trunk(alts[j]['nf']) + return alts[j]['nft'] + + for j in alts.keys(): + rec_trunk(j) + for j, alt in alts.items(): + if alt['f'] in alts: + alt['ft'] = alts[alt['f']]['nft'] + else: + alt['ft'] = trunks[alt['f']] + + def rec_height(j): + if j not in alts: + return 0 + else: + if 'h' not in alts[j]: + alts[j]['h'] = max( + rec_height(alts[j]['f']), + rec_height(alts[j]['nf'])) + 1 + return alts[j]['h'] + + for j in alts.keys(): + rec_height(j) + + t_depth = max((alt['h']+1 for alt in alts.values()), default=0) + + # convert to more general tree representation + tree = set() + for j, alt in alts.items(): + # note all non-trunk edges should be colored black + tree.add(TreeBranch( + alt['nft'], + alt['nft'], + depth=t_depth-1 - alt['h'], + color=alt['c'])) + if alt['ft'] != alt['nft']: + tree.add(TreeBranch( + alt['nft'], + alt['ft'], + depth=t_depth-1 - alt['h'], + color='b')) + + return tree + + +def dbg_log(rbyd, *, + block_size, color=False, **args): + data = rbyd.data + # preprocess jumps if args.get('jumps'): jumps = [] j_ = 4 - while j_ < (block_size if args.get('all') else eoff): + while j_ < (block_size if args.get('all') else rbyd.eoff): j = j_ v, tag, w, size, d = fromtag(data[j_:]) j_ += d @@ -370,7 +893,7 @@ def dbg_log(data, block_size, rev, eoff, weight, *, weight_ = 0 trunk_ = 0 j_ = 4 - while j_ < (block_size if args.get('all') else eoff): + while j_ < (block_size if args.get('all') else rbyd.eoff): j = j_ v, tag, w, size, d = fromtag(data[j_:]) j_ += d @@ -498,7 +1021,7 @@ def dbg_log(data, block_size, rev, eoff, weight, *, weight__ = 0 trunk_ = 0 j_ = 4 - while j_ < (block_size if args.get('all') else eoff): + while j_ < (block_size if args.get('all') else rbyd.eoff): j = j_ v, tag, w, size, d = fromtag(data[j_:]) j_ += d @@ -536,7 +1059,7 @@ def dbg_log(data, block_size, rev, eoff, weight, *, lower_, upper_ = 0, 0 trunk_ = 0 j_ = 4 - while j_ < (block_size if args.get('all') else eoff): + while j_ < (block_size if args.get('all') else rbyd.eoff): notes = [] # read next tag @@ -586,13 +1109,13 @@ def dbg_log(data, block_size, rev, eoff, weight, *, # show human-readable tag representation print('%s%08x:%s %*s%s%*s %-*s%s%s%s' % ( - '\x1b[90m' if color and j >= eoff else '', + '\x1b[90m' if color and j >= rbyd.eoff else '', j, - '\x1b[m' if color and j >= eoff else '', + '\x1b[m' if color and j >= rbyd.eoff else '', lifetime_width, lifetimerepr(j) if args.get('lifetimes') else '', - '\x1b[90m' if color and j >= eoff else '', + '\x1b[90m' if color and j >= rbyd.eoff else '', 2*w_width+1, '' if (tag & 0xe000) != 0x0000 else '%d-%d' % (rid-(w-1), rid) if w > 1 else rid, @@ -604,7 +1127,7 @@ def dbg_log(data, block_size, rev, eoff, weight, *, and not tag & TAG_ALT else ''), ' (%s)' % ', '.join(notes) if notes else '', - '\x1b[m' if color and j >= eoff else '', + '\x1b[m' if color and j >= rbyd.eoff else '', ' %s' % jumprepr(j) if args.get('jumps') and not notes else '')) @@ -613,189 +1136,40 @@ def dbg_log(data, block_size, rev, eoff, weight, *, if args.get('raw'): for o, line in enumerate(xxd(data[j:j+d])): print('%s%8s: %*s%*s %s%s' % ( - '\x1b[90m' if color and j >= eoff else '', + '\x1b[90m' if color and j >= rbyd.eoff else '', '%04x' % (j + o*16), lifetime_width, '', 2*w_width+1, '', line, - '\x1b[m' if color and j >= eoff else '')) + '\x1b[m' if color and j >= rbyd.eoff else '')) if args.get('raw') or args.get('no_truncate'): if not tag & TAG_ALT: for o, line in enumerate(xxd(data[j+d:j+d+size])): print('%s%8s: %*s%*s %s%s' % ( - '\x1b[90m' if color and j >= eoff else '', + '\x1b[90m' if color and j >= rbyd.eoff else '', '%04x' % (j+d + o*16), lifetime_width, '', 2*w_width+1, '', line, - '\x1b[m' if color and j >= eoff else '')) + '\x1b[m' if color and j >= rbyd.eoff else '')) -def dbg_tree(data, block_size, rev, trunk, weight, *, +def dbg_tree(rbyd, *, + block_size, color=False, **args): - if not trunk: + if not rbyd: return - # lookup a tag, returning also the search path for decoration - # purposes - def lookup(rid, tag): - tag = max(tag, 0x1) - lower = 0 - upper = weight - path = [] - - # descend down tree - j = trunk - while True: - _, alt, w, jump, d = fromtag(data[j:]) - - # found an alt? - if alt & TAG_ALT: - # follow? - if ((rid, tag & 0xfff) > (upper-w-1, alt & 0xfff) - if alt & TAG_GT - else ((rid, tag & 0xfff) <= (lower+w-1, alt & 0xfff))): - lower += upper-lower-w if alt & TAG_GT else 0 - upper -= upper-lower-w if not alt & TAG_GT else 0 - j = j - jump - - # figure out which color - if alt & TAG_R: - _, nalt, _, _, _ = fromtag(data[j+jump+d:]) - if nalt & TAG_R: - path.append((j+jump, j, True, 'y')) - else: - path.append((j+jump, j, True, 'r')) - else: - path.append((j+jump, j, True, 'b')) - - # stay on path - else: - lower += w if not alt & TAG_GT else 0 - upper -= w if alt & TAG_GT else 0 - j = j + d - - # figure out which color - if alt & TAG_R: - _, nalt, _, _, _ = fromtag(data[j:]) - if nalt & TAG_R: - path.append((j-d, j, False, 'y')) - else: - path.append((j-d, j, False, 'r')) - else: - path.append((j-d, j, False, 'b')) - - # found tag - else: - rid_ = upper-1 - tag_ = alt - w_ = upper-lower - - done = not tag_ or (rid_, tag_) < (rid, tag) - - return done, rid_, tag_, w_, j, d, jump, path + data = rbyd.data # precompute tree t_width = 0 - if args.get('tree') or args.get('rbyd'): - trunks = co.defaultdict(lambda: (-1, 0)) - alts = co.defaultdict(lambda: {}) - - rid, tag = -1, 0 - while True: - done, rid, tag, w, j, d, size, path = lookup(rid, tag+0x1) - # found end of tree? - if done: - break - - # keep track of trunks/alts - trunks[j] = (rid, tag) - - for j_, j__, followed, c in path: - if followed: - alts[j_] |= {'f': j__, 'c': c} - else: - alts[j_] |= {'nf': j__, 'c': c} - - if args.get('rbyd'): - # treat unreachable alts as converging paths - for j_, alt in alts.items(): - if 'f' not in alt: - alt['f'] = alt['nf'] - elif 'nf' not in alt: - alt['nf'] = alt['f'] - - else: - # prune any alts with unreachable edges - pruned = {} - for j_, alt in alts.items(): - if 'f' not in alt: - pruned[j_] = alt['nf'] - elif 'nf' not in alt: - pruned[j_] = alt['f'] - for j_ in pruned.keys(): - del alts[j_] - - for j_, alt in alts.items(): - while alt['f'] in pruned: - alt['f'] = pruned[alt['f']] - while alt['nf'] in pruned: - alt['nf'] = pruned[alt['nf']] - - # find the trunk and depth of each alt - def rec_trunk(j_): - if j_ not in alts: - return trunks[j_] - else: - if 'nft' not in alts[j_]: - alts[j_]['nft'] = rec_trunk(alts[j_]['nf']) - return alts[j_]['nft'] - - for j_ in alts.keys(): - rec_trunk(j_) - for j_, alt in alts.items(): - if alt['f'] in alts: - alt['ft'] = alts[alt['f']]['nft'] - else: - alt['ft'] = trunks[alt['f']] - - def rec_height(j_): - if j_ not in alts: - return 0 - else: - if 'h' not in alts[j_]: - alts[j_]['h'] = max( - rec_height(alts[j_]['f']), - rec_height(alts[j_]['nf'])) + 1 - return alts[j_]['h'] - - for j_ in alts.keys(): - rec_height(j_) - - t_depth = max((alt['h']+1 for alt in alts.values()), default=0) - - # convert to more general tree representation - TBranch = co.namedtuple('TBranch', 'a, b, d, c') - tree = set() - for j, alt in alts.items(): - # note all non-trunk edges should be black - tree.add(TBranch( - a=alt['nft'], - b=alt['nft'], - d=t_depth-1 - alt['h'], - c=alt['c'], - )) - if alt['ft'] != alt['nft']: - tree.add(TBranch( - a=alt['nft'], - b=alt['ft'], - d=t_depth-1 - alt['h'], - c='b', - )) + if args.get('tree') or args.get('tree_rbyd'): + tree = rbyd.tree(rbyd=args.get('tree_rbyd')) # find the max depth from the tree - t_depth = max((branch.d+1 for branch in tree), default=0) + t_depth = max((b.depth+1 for b in tree), default=0) if t_depth > 0: t_width = 2*t_depth + 2 @@ -804,28 +1178,28 @@ def dbg_tree(data, block_size, rev, trunk, weight, *, return '' def branchrepr(x, d, was): - for branch in tree: - if branch.d == d and branch.b == x: - if any(branch.d == d and branch.a == x - for branch in tree): - return '+-', branch.c, branch.c - elif any(branch.d == d - and x > min(branch.a, branch.b) - and x < max(branch.a, branch.b) - for branch in tree): - return '|-', branch.c, branch.c - elif branch.a < branch.b: - return '\'-', branch.c, branch.c + for b in tree: + if b.depth == d and b.b == x: + if any(b.depth == d and b.a == x + for b in tree): + return '+-', b.color, b.color + elif any(b.depth == d + and x > min(b.a, b.b) + and x < max(b.a, b.b) + for b in tree): + return '|-', b.color, b.color + elif b.a < b.b: + return '\'-', b.color, b.color else: - return '.-', branch.c, branch.c - for branch in tree: - if branch.d == d and branch.a == x: - return '+ ', branch.c, None - for branch in tree: - if (branch.d == d - and x > min(branch.a, branch.b) - and x < max(branch.a, branch.b)): - return '| ', branch.c, was + return '.-', b.color, b.color + for b in tree: + if b.depth == d and b.a == x: + return '+ ', b.color, None + for b in tree: + if (b.depth == d + and x > min(b.a, b.b) + and x < max(b.a, b.b)): + return '| ', b.color, was if was: return '--', was, was return ' ', None, None @@ -848,26 +1222,21 @@ def dbg_tree(data, block_size, rev, trunk, weight, *, # dynamically size the id field - w_width = mt.ceil(mt.log10(max(1, weight)+1)) - - rid, tag = -1, 0 - for i in it.count(): - done, rid, tag, w, j, d, size, path = lookup(rid, tag+0x1) - # found end of tree? - if done: - break + w_width = mt.ceil(mt.log10(max(1, rbyd.weight)+1)) + for i, (rid, tag, rattr) in enumerate(rbyd): # show human-readable tag representation print('%08x: %s%*s %-*s %s' % ( - j, + rattr.toff, treerepr(rid, tag) - if args.get('tree') or args.get('rbyd') + if args.get('tree') or args.get('tree_rbyd') else '', - 2*w_width+1, '%d-%d' % (rid-(w-1), rid) if w > 1 - else rid if w > 0 or i == 0 + 2*w_width+1, '%d-%d' % (rid-(rattr.weight-1), rid) + if rattr.weight > 1 + else rid if rattr.weight > 0 or i == 0 else '', - 21+w_width, tagrepr(tag, w, size, j), - next(xxd(data[j+d:j+d+min(size, 8)], 8), '') + 21+w_width, rattr.tagrepr(), + next(xxd(rattr[:8], 8), '') if not args.get('raw') and not args.get('no_truncate') and not tag & TAG_ALT @@ -875,17 +1244,17 @@ def dbg_tree(data, block_size, rev, trunk, weight, *, # show on-disk encoding of tags if args.get('raw'): - for o, line in enumerate(xxd(data[j:j+d])): + for o, line in enumerate(xxd(data[rattr.toff:rattr.off])): print('%8s: %*s%*s %s' % ( - '%04x' % (j + o*16), + '%04x' % (rattr.toff + o*16), t_width, '', 2*w_width+1, '', line)) if args.get('raw') or args.get('no_truncate'): if not tag & TAG_ALT: - for o, line in enumerate(xxd(data[j+d:j+d+size])): + for o, line in enumerate(xxd(rattr[:])): print('%8s: %*s%*s %s' % ( - '%04x' % (j+d + o*16), + '%04x' % (rattr.off + o*16), t_width, '', 2*w_width+1, '', line)) @@ -922,154 +1291,28 @@ def main(disk, blocks=None, *, f.seek(0, os.SEEK_END) block_size = f.tell() - # blocks may also encode trunks - blocks, trunks = ( - [block[0] if isinstance(block, tuple) else block - for block in blocks], - [trunk if trunk is not None - else block[1] if isinstance(block, tuple) - else None - for block in blocks]) - - # read each block - datas = [] - for block in blocks: - f.seek(block * block_size) - datas.append(f.read(block_size)) - - # first figure out which block as the most recent revision - def fetch(data, trunk): - rev = fromle32(data[0:4]) - cksum = 0 - cksum_ = crc32c(data[0:4]) - cksum__ = cksum_ - perturb = False - eoff = 0 - eoff_ = None - j_ = 4 - trunk_ = 0 - trunk__ = 0 - trunk___ = 0 - weight = 0 - weight_ = 0 - weight__ = 0 - while j_ < len(data) and (not trunk or eoff <= trunk): - # read next tag - v, tag, w, size, d = fromtag(data[j_:]) - if v != parity(cksum__): - break - cksum__ ^= 0x00000080 if v else 0 - cksum__ = crc32c(data[j_:j_+d], cksum__) - j_ += d - if not tag & TAG_ALT and j_ + size > len(data): - break - - # take care of cksums - if not tag & TAG_ALT: - if (tag & 0xff00) != TAG_CKSUM: - cksum__ = crc32c(data[j_:j_+size], cksum__) - # found a cksum? - else: - # check cksum - cksum___ = fromle32(data[j_:j_+4]) - if cksum__ != cksum___: - break - # commit what we have - eoff = eoff_ if eoff_ else j_ + size - cksum = cksum_ - trunk_ = trunk__ - weight = weight_ - # update perturb bit - perturb = tag & TAG_P - # revert to data cksum and perturb - cksum__ = cksum_ ^ (0xfca42daf if perturb else 0) - - # evaluate trunks - if (tag & 0xf000) != TAG_CKSUM: - if not (trunk and j_-d > trunk and not trunk___): - # new trunk? - if not trunk___: - trunk___ = j_-d - weight__ = 0 - - # keep track of weight - weight__ += w - - # end of trunk? - if not tag & TAG_ALT: - # update trunk/weight unless we found a shrub or an - # explicit trunk (which may be a shrub) is requested - if not tag & TAG_SHRUB or trunk___ == trunk: - trunk__ = trunk___ - weight_ = weight__ - # keep track of eoff for best matching trunk - if trunk and j_ + size > trunk: - eoff_ = j_ + size - eoff = eoff_ - cksum = cksum__ ^ ( - 0xfca42daf if perturb else 0) - trunk_ = trunk__ - weight = weight_ - trunk___ = 0 - - # update canonical checksum, xoring out any perturb state - cksum_ = cksum__ ^ (0xfca42daf if perturb else 0) - - if not tag & TAG_ALT: - j_ += size - - return rev, eoff, trunk_, weight, cksum - - revs, eoffs, trunks_, weights, cksums = [], [], [], [], [] - i = 0 - for i_, (data, trunk_) in enumerate(zip(datas, trunks)): - rev, eoff, trunk_, weight, cksum = fetch(data, trunk_) - revs.append(rev) - eoffs.append(eoff) - trunks_.append(trunk_) - weights.append(weight) - cksums.append(cksum) - - # compare with sequence arithmetic - if trunk_ and ( - not trunks_[i] - or not ((rev - revs[i]) & 0x80000000) - or (rev == revs[i] and trunk_ > trunks_[i])): - i = i_ - - # print contents of the winning metadata block - block, data, rev, eoff, trunk_, weight, cksum = ( - blocks[i], - datas[i], - revs[i], - eoffs[i], - trunks_[i], - weights[i], - cksums[i]) + # fetch the rbyd + rbyd = Rbyd.fetch((f, block_size), blocks) print('rbyd %s w%d, rev %08x, size %d, cksum %08x' % ( - '0x%x.%x' % (block, trunk_) - if len(blocks) == 1 - else '0x{%x,%s}.%x' % ( - block, - ','.join('%x' % blocks[(i+1+j) % len(blocks)] - for j in range(len(blocks)-1)), - trunk_), - weight, - rev, - eoff, - cksum)) + rbyd.addr(), + rbyd.weight, + rbyd.rev, + rbyd.eoff, + rbyd.cksum)) if args.get('log'): - dbg_log(data, block_size, rev, eoff, weight, + dbg_log(rbyd, + block_size=block_size, color=color, **args) else: - dbg_tree(data, block_size, rev, trunk_, weight, + dbg_tree(rbyd, + block_size=block_size, color=color, **args) - if args.get('error_on_corrupt') and eoff == 0: + if args.get('error_on_corrupt') and not rbyd: sys.exit(2) @@ -1124,8 +1367,9 @@ if __name__ == "__main__": '-t', '--tree', action='store_true', help="Show the rbyd tree.") + # TODO adopt this rename in all scripts parser.add_argument( - '-R', '--rbyd', + '-R', '--tree-rbyd', action='store_true', help="Show the full rbyd tree.") parser.add_argument(