From af0c3967b44dafa47d55193ed98737aed5729614 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 21 May 2023 02:27:09 -0500 Subject: [PATCH] Adopted new tree renderer in dbgmtree, implemented mtree rendering In addition to plugging in the rbyd and btree renderers in dbgbtree.py, this required wiring in rbyd trees in the mdirs and mroots. A bit tricky, but with a more-or-less straightforward implementation thanks to the common edge description used for the tree renderer. For example, a relatively small mtree: $ ./scripts/dbgmtree.py disk -B4096 -t -i mroot 0x{0,1}.45, rev 1, weight 0 mdir ids tag ... {0000,0001}: .---------> -1 magic 8 ... | .-------> config 21 ... +-+-+ btree 7 ... 0006.000a: | .-+ 0 mdir w1 2 ... {0002,0003}: | | '-> 0.0 inlined w1 1024 ... 0006.000a: '-+-+ 1 mdir w1 2 ... {0004,0005}: '-> 1.0 inlined w1 1024 ... --- scripts/dbgbtree.py | 8 +- scripts/dbgmtree.py | 1159 +++++++++++++++++++++++++++++++++++-------- 2 files changed, 964 insertions(+), 203 deletions(-) diff --git a/scripts/dbgbtree.py b/scripts/dbgbtree.py index cfebe87d..0af4db31 100755 --- a/scripts/dbgbtree.py +++ b/scripts/dbgbtree.py @@ -367,7 +367,7 @@ class Rbyd: id = -1 while True: - done, id, tag, w, j, d, data = self.lookup(id, tag+0x10) + done, id, tag, w, j, d, data, _ = self.lookup(id, tag+0x10) if done: break @@ -628,7 +628,7 @@ def main(disk, roots=None, *, 'c': branch['c'], }) - d_ += max(rdepth, 1) + d_ += max(bdepths.get(d, 0), 1) leaf = (bid-(w-1), d, rid-(w-1), TAG_BTREE) # remap branches to leaves if we aren't showing inner branches @@ -679,7 +679,6 @@ def main(disk, roots=None, *, tree = [] root = None branches = {} - leaves = {} bid = -1 while True: done, bid, w, rbyd, rid, tags, path = btree_lookup( @@ -741,7 +740,6 @@ def main(disk, roots=None, *, t_width = 2*t_depth + 2 def treerepr(bid, w, bd, rid, tag): -# print('%-32s' % repr((bid-(w-1), bd, rid, tag)), end='') if t_depth == 0: return '' @@ -873,6 +871,7 @@ def main(disk, roots=None, *, if done: break + # print inner btree entries if requested if args.get('inner'): changed = False for (x, px) in it.zip_longest( @@ -897,7 +896,6 @@ def main(disk, roots=None, *, '\x1b[31m' if color else '', '(corrupted rbyd %s)' % rbyd.addr(), '\x1b[m' if color else '')) - prbyd = rbyd corrupted = True continue diff --git a/scripts/dbgmtree.py b/scripts/dbgmtree.py index 04d4c8e9..824205d1 100755 --- a/scripts/dbgmtree.py +++ b/scripts/dbgmtree.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import bisect +import collections as co import itertools as it import math as m import os @@ -304,10 +305,11 @@ class Rbyd: def lookup(self, id, tag): if not self: - return True, 0, -1, 0, 0, 0, b'' + return True, 0, -1, 0, 0, 0, b'', [] lower = -1 upper = self.weight + path = [] # descend down tree j = self.trunk @@ -323,11 +325,33 @@ class Rbyd: lower += upper-lower-1-weight_ if alt & 0x4 else 0 upper -= upper-lower-1-weight_ if not alt & 0x4 else 0 j = j - jump + + # figure out which color + if alt & 0x2: + _, nalt, _, _, _ = fromtag(self.data[j+jump+d:]) + if nalt & 0x2: + path.append((j+jump, j, True, 'y')) + else: + path.append((j+jump, j, True, 'r')) + else: + path.append((j+jump, j, True, 'b')) + # stay on path else: lower += weight_ if not alt & 0x4 else 0 upper -= weight_ if alt & 0x4 else 0 j = j + d + + # figure out which color + if alt & 0x2: + _, nalt, _, _, _ = fromtag(self.data[j:]) + if nalt & 0x2: + path.append((j-d, j, False, 'y')) + else: + path.append((j-d, j, False, 'r')) + else: + path.append((j-d, j, False, 'b')) + # found tag else: id_ = upper-1 @@ -336,8 +360,7 @@ class Rbyd: done = (id_, tag_) < (id, tag) or tag_ & 2 - return (done, id_, tag_, w_, - j, d, self.data[j+d:j+d+jump]) + return done, id_, tag_, w_, j, d, self.data[j+d:j+d+jump], path def __bool__(self): return bool(self.trunk) @@ -353,12 +376,345 @@ class Rbyd: id = -1 while True: - done, id, tag, w, j, d, data = self.lookup(id, tag+0x10) + done, id, tag, w, j, d, data, _ = self.lookup(id, tag+0x10) if done: break yield id, tag, w, j, d, data + # create tree representation for debugging + def tree(self): + trunks = co.defaultdict(lambda: (-1, 0)) + alts = co.defaultdict(lambda: {}) + + id, tag = -1, 0 + while True: + done, id, tag, w, j, d, data, path = self.lookup(id, tag+0x10) + # found end of tree? + if done: + break + + # keep track of trunks/alts + trunks[j] = (id, tag) + + for j_, j__, followed, c in path: + if followed: + alts[j_] |= {'f': j__, 'c': c} + else: + alts[j_] |= {'nf': j__, 'c': c} + + # prune any alts with unreachable edges + pruned = {} + for j_, alt in alts.items(): + if 'f' not in alt: + pruned[j_] = alt['nf'] + elif 'nf' not in alt: + pruned[j_] = alt['f'] + for j_ in pruned.keys(): + del alts[j_] + + for j_, alt in alts.items(): + while alt['f'] in pruned: + alt['f'] = pruned[alt['f']] + while alt['nf'] in pruned: + alt['nf'] = pruned[alt['nf']] + + # find the trunk and depth of each alt, assuming pruned alts + # didn't exist + def rec_trunk(j_): + if j_ not in alts: + return trunks[j_] + else: + if 'nft' not in alts[j_]: + alts[j_]['nft'] = rec_trunk(alts[j_]['nf']) + return alts[j_]['nft'] + + for j_ in alts.keys(): + rec_trunk(j_) + for j_, alt in alts.items(): + if alt['f'] in alts: + alt['ft'] = alts[alt['f']]['nft'] + else: + alt['ft'] = trunks[alt['f']] + + def rec_height(j_): + if j_ not in alts: + return 0 + else: + if 'h' not in alts[j_]: + alts[j_]['h'] = max( + rec_height(alts[j_]['f']), + rec_height(alts[j_]['nf'])) + 1 + return alts[j_]['h'] + + for j_ in alts.keys(): + rec_height(j_) + + t_depth = max((alt['h']+1 for alt in alts.values()), default=0) + + # convert to more general tree representation + tree = [] + for j, alt in alts.items(): + # note all non-trunk edges should be black + tree.append({ + 'a': alt['nft'], + 'b': alt['nft'], + 'd': t_depth-1 - alt['h'], + 'c': alt['c'], + }) + tree.append({ + 'a': alt['nft'], + 'b': alt['ft'], + 'd': t_depth-1 - alt['h'], + 'c': 'b', + }) + + return tree, t_depth + + # btree lookup with this rbyd as the root + def btree_lookup(self, f, block_size, bid, depth=None): + rbyd = self + rid = bid + depth_ = 1 + path = [] + + # corrupted? return a corrupted block once + if not rbyd: + return bid > 0, bid, 0, rbyd, -1, [], path + + while True: + # collect all tags, normally you don't need to do this + # but we are debugging here + name = None + tags = [] + branch = None + rid_ = rid + tag = 0 + w = 0 + for i in it.count(): + done, rid__, tag, w_, j, d, data, _ = rbyd.lookup( + rid_, tag+0x10) + if done or (i != 0 and rid__ != rid_): + break + + # first tag indicates the branch's weight + if i == 0: + rid_, w = rid__, w_ + + # catch any branches + if tag == TAG_BTREE: + branch = (tag, j, d, data) + + tags.append((tag, j, d, data)) + + # keep track of path + path.append((bid + (rid_-rid), w, rbyd, rid_, tags)) + + # descend down branch? + if branch is not None and ( + not depth or depth_ < depth): + tag, j, d, data = branch + w_, trunk, block, crc = frombtree(data) + rbyd = Rbyd.fetch(f, block_size, block, trunk) + + # corrupted? bail here so we can keep traversing the tree + if not rbyd: + return False, bid + (rid_-rid), w, rbyd, -1, [], path + + rid -= (rid_-(w-1)) + depth_ += 1 + else: + return not tags, bid + (rid_-rid), w, rbyd, rid_, tags, path + + # btree rbyd-tree generation for debugging + def btree_tree(self, f, block_size, depth=None, *, + inner=False): + # find the max depth of each layer to nicely align trees + # TODO memoize + bdepths = {} + bid = -1 + while True: + done, bid, w, rbyd, rid, tags, path = self.btree_lookup( + f, block_size, bid+1, depth=depth) + if done: + break + + for d, (bid, w, rbyd, rid, tags) in enumerate(path): + _, rdepth = rbyd.tree() + bdepths[d] = max(bdepths.get(d, 0), rdepth) + + # find all branches + tree = [] + root = None + branches = {} + bid = -1 + while True: + done, bid, w, rbyd, rid, tags, path = self.btree_lookup( + f, block_size, bid+1, depth=depth) + if done: + break + + d_ = 0 + leaf = None + for d, (bid, w, rbyd, rid, tags) in enumerate(path): + if not tags: + continue + + # map rbyd tree into B-tree space + rtree, rdepth = rbyd.tree() + + # note we adjust our bid/rids to be left-leaning, + # this allows a global order and make tree rendering quite + # a bit easier + for i in range(len(rtree)): + a_rid, a_tag = rtree[i]['a'] + b_rid, b_tag = rtree[i]['b'] + _, _, _, a_w, _, _, _, _ = rbyd.lookup(a_rid, 0) + _, _, _, b_w, _, _, _, _ = rbyd.lookup(b_rid, 0) + rtree[i] = { + 'a': (a_rid-(a_w-1), a_tag), + 'b': (b_rid-(b_w-1), b_tag), + 'd': rtree[i]['d'], + 'c': rtree[i]['c'], + } + + # connect our branch to the rbyd's root + if leaf is not None: + root = min(rtree, + key=lambda branch: branch['d'], + default=None) + + if root is not None: + r_rid, r_tag = root['a'] + else: + r_rid, r_tag = rid-(w-1), tags[0][0] + tree.append({ + 'a': leaf, + 'b': (bid-rid+r_rid, d, r_rid, r_tag), + 'd': d_-1, + 'c': 'b', + }) + + for branch in rtree: + # map rbyd branches into our btree space + a_rid, a_tag = branch['a'] + b_rid, b_tag = branch['b'] + tree.append({ + 'a': (bid-rid+a_rid, d, a_rid, a_tag), + 'b': (bid-rid+b_rid, d, b_rid, b_tag), + 'd': branch['d'] + d_ + bdepths.get(d, 0)-rdepth, + 'c': branch['c'], + }) + + d_ += max(bdepths.get(d, 0), 1) + leaf = (bid-(w-1), d, rid-(w-1), TAG_BTREE) + + # remap branches to leaves if we aren't showing inner branches + if not inner: + # step through each layer backwards + b_depth = max((branch['a'][1]+1 for branch in tree), default=0) + + # keep track of the original tree to find the original bids, + # unfortunately because we store the bids in the branches we + # overwrite these + tree_ = tree.copy() + + for bd in reversed(range(b_depth-1)): + # find leaf-roots at this level + roots = {} + for branch_, branch in zip(tree_, tree): + bid = branch['b'][0] - branch['b'][2] + # choose the highest node as the root + if (branch_['b'][1] == b_depth-1 + and (bid not in roots + or branch_['d'] < roots[bid]['d'])): + roots[bid] = branch_ + + # remap branches to leaf-roots + tree__ = [] + for branch_ in tree_: + if branch_['a'][1] == bd and branch_['a'][0] in roots: + branch_ = { + 'a': roots[branch_['a'][0]]['b'], + 'b': branch_['b'], + 'd': branch_['d'], + 'c': branch_['c'], + } + if branch_['b'][1] == bd and branch_['b'][0] in roots: + branch_ = { + 'a': branch_['a'], + 'b': roots[branch_['b'][0]]['b'], + 'd': branch_['d'], + 'c': branch_['c'], + } + tree__.append(branch_) + tree_ = tree__ + tree = tree_ + + return tree, max((branch['d']+1 for branch in tree), default=0) + + # btree B-tree generation for debugging + def btree_btree(self, f, block_size, depth=None, *, + inner=False): + # find all branches + tree = [] + root = None + branches = {} + bid = -1 + while True: + done, bid, w, rbyd, rid, tags, path = self.btree_lookup( + f, block_size, bid+1, depth=depth) + if done: + break + + # if we're not showing inner nodes, prefer names higher in + # the tree since this avoids showing vestigial names + name = None + if not inner: + name = None + for bid_, w_, rbyd_, rid_, tags_ in reversed(path): + for tag_, j_, d_, data_ in tags_: + if tag_ & 0xf00f == TAG_NAME: + name = (tag_, j_, d_, data_) + + if rid_-(w_-1) != 0: + break + + a = root + for d, (bid, w, rbyd, rid, tags) in enumerate(path): + if not tags: + continue + + b = (bid-(w-1), d, rid-(w-1), + (name if name else tags[0])[0]) + + # remap branches to leaves if we aren't showing + # inner branches + if not inner: + if b not in branches: + bid, w, rbyd, rid, tags = path[-1] + if not tags: + continue + branches[b] = ( + bid-(w-1), len(path)-1, rid-(w-1), + (name if name else tags[0])[0]) + b = branches[b] + + # found entry point? + if root is None: + root = b + a = root + + tree.append({ + 'a': a, + 'b': b, + 'd': d, + 'c': 'b', + }) + a = b + + return tree, max((branch['d']+1 for branch in tree), default=0) + def main(disk, mroots=None, *, block_size=None, @@ -384,71 +740,6 @@ def main(disk, mroots=None, *, f.seek(0, os.SEEK_END) block_size = f.tell() - # look up an id, while keeping track of the search path - def btree_lookup(btree, id, depth=None): - rbyd = btree - rid = id - depth_ = 1 - path = [] - - # corrupted? return a corrupted block once - if not rbyd: - return (id > 0, id, 0, rbyd, -1, - (0, 0, 0, b''), - (0, 0, 0, b''), - path) - - while True: - # first lookup id/name - (done, rid_, name_tag, w, - name_j, name_d, name) = rbyd.lookup(rid, 0) - if done: - return (True, id, 0, rbyd, -1, - (0, 0, 0, b''), - (0, 0, 0, b''), - path) - - if name_tag & 0xf00f == TAG_NAME: - # then lookup struct - (done, _, struct_tag, _, - struct_j, struct_d, struct_) = rbyd.lookup( - rid_, TAG_STRUCT) - if done: - return (True, id, 0, rbyd, -1, - (0, 0, 0, b''), - (0, 0, 0, b''), - path) - else: - tag = name_tag - struct_tag, struct_j, struct_d, struct_ = ( - name_tag, name_j, name_d, name) - name_tag, name_j, name_d, name = 0, 0, 0, b'' - - path.append((id + (rid_-rid), w, rbyd, rid_, - (name_tag, name_j, name_d, name), - (struct_tag, struct_j, struct_d, struct_))) - - # is it another branch? continue down tree - if struct_tag == TAG_BTREE and ( - not depth or depth_ < depth): - w, trunk, block, crc = frombtree(struct_) - rbyd = Rbyd.fetch(f, block_size, block, trunk) - - # corrupted? bail here so we can keep traversing the tree - if not rbyd: - return (False, id + (rid_-rid), w, rbyd, -1, - (0, 0, 0, b''), - (0, 0, 0, b''), - path) - - rid -= (rid_-(w-1)) - depth_ += 1 - else: - return (False, id + (rid_-rid), w, rbyd, rid_, - (name_tag, name_j, name_d, name), - (struct_tag, struct_j, struct_d, struct_), - path) - # before we print, we need to do a pass for a few things: # - find the actual mroot # - find the total weight @@ -457,19 +748,35 @@ def main(disk, mroots=None, *, mroot = Rbyd.fetch(f, block_size, mroots) while True: + # corrupted? + if not mroot: + break + rweight = max(rweight, mroot.weight) # fetch the next mroot - done, rid, tag, w, j, d, data = mroot.lookup(-1, TAG_MROOT) + done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MROOT) if not (not done and rid == -1 and tag == TAG_MROOT): break blocks = frommdir(data) mroot = Rbyd.fetch(f, block_size, blocks) + # fetch the mdir, if there is one + mdir = None + done, rid, tag, w, j, _, data, _ = mroot.lookup(-1, TAG_MDIR) + if not done and rid == -1 and tag == TAG_MDIR: + blocks = frommdir(data) + mdir = Rbyd.fetch(f, block_size, blocks) + + # corrupted? + if mdir: + rweight = max(rweight, mdir.weight) + # fetch the actual mtree, if there is one - done, rid, tag, w, j, d, data = mroot.lookup(-1, TAG_BTREE) - if not done and rid == -1 or tag == TAG_BTREE: + mtree = None + done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_BTREE) + if not done and rid == -1 and tag == TAG_BTREE: w, trunk, block, crc = frombtree(data) mtree = Rbyd.fetch(f, block_size, block, trunk) @@ -478,11 +785,8 @@ def main(disk, mroots=None, *, # traverse entries mid = -1 while True: - (done, mid, w, rbyd, rid, - (name_tag, name_j, name_d, name), - (struct_tag, struct_j, struct_d, struct_), - path) = (btree_lookup(mtree, mid+1, - depth=args.get('depth'))) + done, mid, w, rbyd, rid, tags, path = mtree.btree_lookup( + f, block_size, mid+1, depth=args.get('depth')) if done: break @@ -490,27 +794,469 @@ def main(disk, mroots=None, *, if not rbyd: continue - if struct_tag != TAG_MDIR: - continue + mdir__ = None + if not args.get('depth') or len(path) < args.get('depth'): + mdir__ = next(((tag, j, d, data) + for tag, j, d, data in tags + if tag == TAG_MDIR), + None) - # fetch the mdir - blocks = frommdir(struct_) - mdir = Rbyd.fetch(f, block_size, blocks) + if mdir__: + # fetch the mdir + _, _, _, data = mdir__ + blocks = frommdir(data) + mdir_ = Rbyd.fetch(f, block_size, blocks) - rweight = max(rweight, mroot.weight) + # corrupted? + if mdir_: + rweight = max(rweight, mdir_.weight) + + # precompute rbyd-tree if requested + t_width = 0 + if args.get('tree'): + # compute mroot chain "tree", prefix our actual mtree with this + tree = [] + d_ = 0 + mroot_ = Rbyd.fetch(f, block_size, mroots) + for d in it.count(): + # corrupted? + if not mroot_: + break + + # compute the mroots rbyd-tree + rtree, rdepth = mroot_.tree() + + # connect branch to our root + if d > 0: + root = min(rtree, + key=lambda branch: branch['d'], + default=None) + + if root: + r_rid, r_tag = root['a'] + else: + _, r_rid, r_tag, _, _, _, _, _ = mroot_.lookup(-1, 0x10) + tree.append({ + 'a': (-1, d-1, 0, -1, TAG_MROOT), + 'b': (-1, d, 0, r_rid, r_tag), + 'd': d_-1, + 'c': 'b', + }) + + # map the tree into our metadata space + for branch in rtree: + a_rid, a_tag = branch['a'] + b_rid, b_tag = branch['b'] + tree.append({ + 'a': (-1, d, 0, a_rid, a_tag), + 'b': (-1, d, 0, b_rid, b_tag), + 'd': d_ + branch['d'], + 'c': branch['c'], + }) + d_ += rdepth + + # fetch the next mroot + done, rid, tag, w, j, _, data, _ = mroot_.lookup(-1, TAG_MROOT) + if not (not done and rid == -1 and tag == TAG_MROOT): + break + + blocks = frommdir(data) + mroot_ = Rbyd.fetch(f, block_size, blocks) + + # compute mdir's rbyd-tree if there is one + if mdir: + rtree, rdepth = mdir.tree() + + # connect branch to our root + root = min(rtree, + key=lambda branch: branch['d'], + default=None) + + if root: + r_rid, r_tag = root['a'] + else: + _, r_rid, r_tag, _, _, _, _, _ = mdir.lookup(-1, 0x10) + tree.append({ + 'a': (-1, d, 0, -1, TAG_MDIR), + 'b': (0, 0, 0, r_rid, r_tag), + 'd': d_-1, + 'c': 'b', + }) + + # map the tree into our metadata space + for branch in rtree: + a_rid, a_tag = branch['a'] + b_rid, b_tag = branch['b'] + tree.append({ + 'a': (0, 0, 0, a_rid, a_tag), + 'b': (0, 0, 0, b_rid, b_tag), + 'd': d_ + branch['d'], + 'c': branch['c'], + }) + + # compute the mtree's rbyd-tree if there is one + if mtree: + tree_, tdepth = mtree.btree_tree( + f, block_size, + depth=args.get('depth'), + inner=args.get('inner')) + + # connect a branch to the root of the tree + root = min(tree_, key=lambda branch: branch['d'], default=None) + if root: + r_bid, r_bd, r_rid, r_tag = root['a'] + tree.append({ + 'a': (-1, d, 0, -1, TAG_BTREE), + 'b': (r_bid, r_bd, r_rid, 0, r_tag), + 'd': d_-1, + 'c': 'b', + }) + + # map the tree into our metadata space + for branch in tree_: + a_bid, a_bd, a_rid, a_tag = branch['a'] + b_bid, b_bd, b_rid, b_tag = branch['b'] + tree.append({ + 'a': (a_bid, a_bd, a_rid, 0, a_tag), + 'b': (b_bid, b_bd, b_rid, 0, b_tag), + 'd': d_ + branch['d'], + 'c': branch['c'], + }) + + # find the max depth of each mdir to nicely align trees + # TODO memoize + mdepth = 0 + mid = -1 + while True: + done, mid, w, rbyd, rid, tags, path = mtree.btree_lookup( + f, block_size, mid+1, depth=args.get('depth')) + if done: + break + + # corrupted? + if not rbyd: + continue + + mdir__ = None + if not args.get('depth') or len(path) < args.get('depth'): + mdir__ = next(((tag, j, d, data) + for tag, j, d, data in tags + if tag == TAG_MDIR), + None) + + if mdir__: + # fetch the mdir + _, _, _, data = mdir__ + blocks = frommdir(data) + mdir_ = Rbyd.fetch(f, block_size, blocks) + + rtree, rdepth = mdir_.tree() + mdepth = max(mdepth, rdepth) + + # compute the rbyd-tree for each mdir + mid = -1 + while True: + done, mid, w, rbyd, rid, tags, path = mtree.btree_lookup( + f, block_size, mid+1, depth=args.get('depth')) + if done: + break + + # corrupted? + if not rbyd: + continue + + mdir__ = None + if not args.get('depth') or len(path) < args.get('depth'): + mdir__ = next(((tag, j, d, data) + for tag, j, d, data in tags + if tag == TAG_MDIR), + None) + + if mdir__: + # fetch the mdir + _, _, _, data = mdir__ + blocks = frommdir(data) + mdir_ = Rbyd.fetch(f, block_size, blocks) + + rtree, rdepth = mdir_.tree() + + # connect the root to the mtree + branch = max( + (branch for branch in tree + if branch['b'][0] == mid-(w-1)), + key=lambda branch: branch['d'], + default=None) + if branch: + root = min(rtree, + key=lambda branch: branch['d'], + default=None) + if root: + r_rid, r_tag = root['a'] + else: + _, r_rid, r_tag, _, _, _, _, _ = ( + mdir_.lookup(-1, 0x10)) + tree.append({ + 'a': branch['b'], + 'b': (mid-(w-1), len(path), 0, r_rid, r_tag), + 'd': d_ + tdepth, + 'c': 'b', + }) + + # map the tree into our metadata space + for branch in rtree: + a_rid, a_tag = branch['a'] + b_rid, b_tag = branch['b'] + tree.append({ + 'a': (mid-(w-1), len(path), 0, a_rid, a_tag), + 'b': (mid-(w-1), len(path), 0, b_rid, b_tag), + 'd': (d_ + tdepth + 1 + + branch['d'] + mdepth-rdepth), + 'c': branch['c'], + }) + + # remap branches to leaves if we aren't showing inner branches + if not args.get('inner'): + # step through each layer backwards + b_depth = max((branch['a'][1]+1 for branch in tree), + default=0) + + # keep track of the original tree to find the original bids, + # unfortunately because we store the bids in the branches we + # overwrite these + tree_ = tree.copy() + + for bd in reversed(range(b_depth-1)): + # find leaf-roots at this level + roots = {} + for branch_, branch in zip(tree_, tree): + bid = branch['b'][0] - branch['b'][2] + # choose the highest node as the root + if (branch_['b'][1] == b_depth-1 + and (bid not in roots + or branch_['d'] < roots[bid]['d'])): + roots[bid] = branch_ + + # remap branches to leaf-roots + tree__ = [] + for branch_ in tree_: + # note we ignore mroot branches, we don't collapse + # normally these + if (branch_['a'][0] != -1 + and branch_['a'][1] == bd + and branch_['a'][0] in roots): + branch_ = { + 'a': roots[branch_['a'][0]]['b'], + 'b': branch_['b'], + 'd': branch_['d'], + 'c': branch_['c'], + } + if (branch_['b'][0] != -1 + and branch_['b'][1] == bd + and branch_['b'][0] in roots): + branch_ = { + 'a': branch_['a'], + 'b': roots[branch_['b'][0]]['b'], + 'd': branch_['d'], + 'c': branch_['c'], + } + tree__.append(branch_) + tree_ = tree__ + tree = tree_ + + # precompute B-tree if requested + elif args.get('btree'): + # compute mroot chain "tree", prefix our actual mtree with this + tree = [] + mroot_ = Rbyd.fetch(f, block_size, mroots) + for d in it.count(): + # corrupted? + if not mroot_: + break + + # connect branch to our first tag + if d > 0: + done, rid, tag, w, j, _, data, _ = mroot_.lookup(-1, 0x10) + if not done: + tree.append({ + 'a': (-1, d-1, 0, -1, TAG_MROOT), + 'b': (-1, d, 0, rid, tag), + 'd': 0, + 'c': 'b', + }) + + # fetch the next mroot + done, rid, tag, w, j, _, data, _ = mroot_.lookup(-1, TAG_MROOT) + if not (not done and rid == -1 and tag == TAG_MROOT): + break + + blocks = frommdir(data) + mroot_ = Rbyd.fetch(f, block_size, blocks) + + # create a branch to our mdir if there is one + if mdir: + # connect branch to our first tag + done, rid, tag, w, j, _, data, _ = mdir.lookup(-1, 0x10) + if not done: + tree.append({ + 'a': (-1, d, 0, -1, TAG_MDIR), + 'b': (0, 0, 0, rid, tag), + 'd': 0, + 'c': 'b', + }) + + # compute the mtree's B-tree if there is one + if mtree: + tree_, tdepth = mtree.btree_btree( + f, block_size, + depth=args.get('depth'), + inner=args.get('inner')) + + # connect a branch to the root of the tree + root = min(tree_, key=lambda branch: branch['d'], default=None) + if root: + r_bid, r_bd, r_rid, r_tag = root['a'] + tree.append({ + 'a': (-1, d, 0, -1, TAG_BTREE), + 'b': (r_bid, r_bd, r_rid, 0, r_tag), + 'd': 0, + 'c': 'b', + }) + + # map the tree into our metadata space + for branch in tree_: + a_bid, a_bd, a_rid, a_tag = branch['a'] + b_bid, b_bd, b_rid, b_tag = branch['b'] + tree.append({ + 'a': (a_bid, a_bd, a_rid, 0, a_tag), + 'b': (b_bid, b_bd, b_rid, 0, b_tag), + 'd': 1 + branch['d'], + 'c': branch['c'], + }) + + # remap branches to leaves if we aren't showing inner branches + if not args.get('inner'): + mid = -1 + while True: + done, mid, w, rbyd, rid, tags, path = ( + mtree.btree_lookup( + f, block_size, mid+1, depth=args.get('depth'))) + if done: + break + + # corrupted? + if not rbyd: + continue + + mdir__ = None + if (not args.get('depth') + or len(path) < args.get('depth')): + mdir__ = next(((tag, j, d, data) + for tag, j, d, data in tags + if tag == TAG_MDIR), + None) + + if mdir__: + # fetch the mdir + _, _, _, data = mdir__ + blocks = frommdir(data) + mdir_ = Rbyd.fetch(f, block_size, blocks) + + # find the first entry in the mdir, map branches + # to this entry + done, rid, tag, _, j, d, data, _ = ( + mdir_.lookup(-1, 0x10)) + + tree_ = [] + for branch in tree: + if branch['a'][0] == mid-(w-1): + a_bid, a_bd, _, _, _ = branch['a'] + branch = { + 'a': (a_bid, a_bd+1, 0, rid, tag), + 'b': branch['b'], + 'd': branch['d'], + 'c': branch['c'], + } + if branch['b'][0] == mid-(w-1): + b_bid, b_bd, _, _, _ = branch['b'] + branch = { + 'a': branch['a'], + 'b': (b_bid, b_bd+1, 0, rid, tag), + 'd': branch['d'], + 'c': branch['c'], + } + tree_.append(branch) + tree = tree_ + + # common tree renderer + if args.get('tree') or args.get('btree'): + # find the max depth from the tree + t_depth = max((branch['d']+1 for branch in tree), default=0) + if t_depth > 0: + t_width = 2*t_depth + 2 + + def treerepr(mid, w, md, mrid, rid, tag): + if t_depth == 0: + return '' + + def branchrepr(x, d, was): + for branch in tree: + if branch['d'] == d and branch['b'] == x: + if any(branch['d'] == d and branch['a'] == x + for branch in tree): + return '+-', branch['c'], branch['c'] + elif any(branch['d'] == d + and x > min(branch['a'], branch['b']) + and x < max(branch['a'], branch['b']) + for branch in tree): + return '|-', branch['c'], branch['c'] + elif branch['a'] < branch['b']: + return '\'-', branch['c'], branch['c'] + else: + return '.-', branch['c'], branch['c'] + for branch in tree: + if branch['d'] == d and branch['a'] == x: + return '+ ', branch['c'], None + for branch in tree: + if (branch['d'] == d + and x > min(branch['a'], branch['b']) + and x < max(branch['a'], branch['b'])): + return '| ', branch['c'], was + if was: + return '--', was, was + return ' ', None, None + + trunk = [] + was = None + for d in range(t_depth): + t, c, was = branchrepr( + (mid-max(w-1, 0), md, mrid-max(w-1, 0), rid, tag), + d, was) + + trunk.append('%s%s%s%s' % ( + '\x1b[33m' if color and c == 'y' + else '\x1b[31m' if color and c == 'r' + else '\x1b[90m' if color and c == 'b' + else '', + t, + ('>' if was else ' ') if d == t_depth-1 else '', + '\x1b[m' if color and c else '')) + + return '%s ' % ''.join(trunk) - def dbg_mdir(mdir, mid): - for i, (id, tag, w, j, d, data) in enumerate(mdir): + def dbg_mdir(mdir, mid, md): + for i, (rid, tag, w, j, d, data) in enumerate(mdir): # show human-readable tag representation - print('%12s %-57s' % ( + print('%12s %s%-57s' % ( '{%s}:' % ','.join('%04x' % block for block in it.chain([mdir.block], mdir.other_blocks)) if i == 0 else '', + treerepr(mid, 1, md, 0, rid, tag) + if args.get('tree') or args.get('btree') else '', '%*s %-22s%s' % ( - w_width, '%d.%d-%d' % (mid, id-(w-1), id) - if w > 1 else '%d.%d' % (mid, id) + w_width, '%d.%d-%d' % (mid, rid-(w-1), rid) + if w > 1 else '%d.%d' % (mid, rid) if w > 0 else '%d' % mid if i == 0 else '', tagrepr(tag, w, len(data), j), @@ -550,93 +1296,65 @@ def main(disk, mroots=None, *, line)) # prbyd here means the last rendered rbyd, we update - # in dbg_entry to always print interleaved addresses + # in dbg_branch to always print interleaved addresses prbyd = None - def dbg_entry(id, w, rbyd, rid, - name_tag, name_j, name_d, name, - struct_tag, struct_j, struct_d, struct_, - depth=None): + def dbg_branch(bid, w, rbyd, rid, tags, bd): nonlocal prbyd # show human-readable representation - if name_tag: - print('%12s %*s %-22s %s' % ( + for i, (tag, j, d, data) in enumerate(tags): + print('%12s %s%*s %-22s %s' % ( '%04x.%04x:' % (rbyd.block, rbyd.trunk) if prbyd is None or rbyd != prbyd else '', - w_width, '%d-%d' % (id-(w-1), id) if w > 1 - else id if w > 0 + treerepr(bid, w, bd, rid, 0, tag) + if args.get('tree') or args.get('btree') else '', + w_width, '' if i != 0 + else '%d-%d' % (bid-(w-1), bid) if w > 1 + else bid if w > 0 else '', - tagrepr(name_tag, w, len(name), None), + tagrepr(tag, w if i == 0 else 0, len(data), None), + # note we render names a bit different here ''.join( b if b >= ' ' and b <= '~' else '.' - for b in map(chr, name)))) + for b in map(chr, data)) + if tag & 0xf00f == TAG_NAME + else next(xxd(data, 8), '') + if not args.get('no_truncate') + else '')) prbyd = rbyd - print('%12s %*s %-22s %s' % ( - '%04x.%04x:' % (rbyd.block, rbyd.trunk) - if prbyd is None or rbyd != prbyd - else '', - w_width, '' if name_tag - else '%d-%d' % (id-(w-1), id) if w > 1 - else id if w > 0 - else '', - tagrepr(struct_tag, - w if not name_tag else 0, - len(struct_), None), - next(xxd(struct_, 8), '') - if not args.get('no_truncate') else '')) - prbyd = rbyd # show in-device representation if args.get('device'): - if name_tag: - print('%11s %*s %-22s%s' % ( + for i, (tag, j, d, data) in enumerate(tags): + print('%11s %*s%*s %-22s%s' % ( '', + t_width, '', w_width, '', - '%04x %08x %07x' % (name_tag, w, len(name)), + '%04x %08x %07x' % (tag, w if i == 0 else 0, len(data)), ' %s' % ' '.join( '%08x' % fromle32( - rbyd.data[name_j+name_d+i*4 - : name_j+name_d + min(i*4+4,len(name))]) - for i in range(min(m.ceil(len(name)/4), 3)))[:23])) - print('%11s %*s %-22s%s' % ( - '', - w_width, '', - '%04x %08x %07x' % ( - struct_tag, w if not name_tag else 0, len(struct_)), - ' %s' % ' '.join( - '%08x' % fromle32( - rbyd.data[struct_j+struct_d+i*4 - : struct_j+struct_d + min(i*4+4,len(struct_))]) - for i in range(min(m.ceil(len(struct_)/4), 3)))[:23])) + rbyd.data[j+d+i*4 : j+d + min(i*4+4,len(data))]) + for i in range(min(m.ceil(len(data)/4), 3)))[:23])) # show on-disk encoding of tags/data - if args.get('raw'): - for o, line in enumerate(xxd( - rbyd.data[name_j:name_j+name_d])): - print('%11s: %*s %s' % ( - '%04x' % (name_j + o*16), - w_width, '', - line)) - if args.get('raw'): - for o, line in enumerate(xxd(name)): - print('%11s: %*s %s' % ( - '%04x' % (name_j+name_d + o*16), - w_width, '', - line)) - if args.get('raw'): - for o, line in enumerate(xxd( - rbyd.data[struct_j:struct_j+struct_d])): - print('%11s: %*s %s' % ( - '%04x' % (struct_j + o*16), - w_width, '', - line)) - if args.get('raw') or args.get('no_truncate'): - for o, line in enumerate(xxd(struct_)): - print('%11s: %*s %s' % ( - '%04x' % (struct_j+struct_d + o*16), - w_width, '', - line)) + for i, (tag, j, d, data) in enumerate(tags): + if args.get('raw'): + for o, line in enumerate(xxd(rbyd.data[j:j+d])): + print('%11s: %*s%*s %s' % ( + '%04x' % (j + o*16), + t_width, '', + w_width, '', + line)) + # note we don't render name tags with no_truncate + if args.get('raw') or ( + args.get('no_truncate') and tag & 0xf00f != TAG_NAME): + for o, line in enumerate(xxd(data)): + print('%11s: %*s%*s %s' % ( + '%04x' % (j+d + o*16), + t_width, '', + w_width, '', + line)) # print the actual mroot @@ -647,8 +1365,9 @@ def main(disk, mroots=None, *, w_width = (m.ceil(m.log10(max(1, mweight)+1)) + 2*m.ceil(m.log10(max(1, rweight)+1)) + 2) - print('%-11s %-*s %-22s %s' % ( + print('%-11s %*s%-*s %-22s %s' % ( 'mdir', + t_width, '', w_width, 'ids', 'tag', 'data (truncated)' @@ -659,7 +1378,7 @@ def main(disk, mroots=None, *, ppath = [] corrupted = False mroot = Rbyd.fetch(f, block_size, mroots) - while True: + for d in it.count(): # corrupted? if not mroot: print('{%s}: %s%s%s' % ( @@ -670,97 +1389,133 @@ def main(disk, mroots=None, *, '(corrupted mroot %s)' % mroot.addr(), '\x1b[m' if color else '')) corrupted = True + break else: # show the mdir - dbg_mdir(mroot, -1) + dbg_mdir(mroot, -1, d) # fetch the next mroot - done, rid, tag, w, j, d, data = mroot.lookup(-1, TAG_MROOT) + done, rid, tag, w, j, _, data, _ = mroot.lookup(-1, TAG_MROOT) if not (not done and rid == -1 and tag == TAG_MROOT): break blocks = frommdir(data) mroot = Rbyd.fetch(f, block_size, blocks) + # show the mdir, if there is one + done, rid, tag, w, j, _, data, _ = mroot.lookup(-1, TAG_MDIR) + if not done and rid == -1 and tag == TAG_MDIR: + blocks = frommdir(data) + mdir = Rbyd.fetch(f, block_size, blocks) + + # corrupted? + if not mdir: + print('{%s}: %s%s%s' % ( + ','.join('%04x' % block + for block in it.chain([mdir.block], + mdir.other_blocks)), + '\x1b[31m' if color else '', + '(corrupted mdir %s)' % mdir.addr(), + '\x1b[m' if color else '')) + corrupted = True + else: + # show the mdir + dbg_mdir(mdir, 0, 0) + # fetch the actual mtree, if there is one - done, rid, tag, w, j, d, data = mroot.lookup(-1, TAG_BTREE) - if not done and rid == -1 or tag == TAG_BTREE: + done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_BTREE) + if not done and rid == -1 and tag == TAG_BTREE: w, trunk, block, crc = frombtree(data) mtree = Rbyd.fetch(f, block_size, block, trunk) # traverse entries mid = -1 while True: - (done, mid, w, rbyd, rid, - (name_tag, name_j, name_d, name), - (struct_tag, struct_j, struct_d, struct_), - path) = (btree_lookup(mtree, mid+1, - depth=args.get('depth'))) + done, mid, w, rbyd, rid, tags, path = mtree.btree_lookup( + f, block_size, mid+1, depth=args.get('depth')) if done: break # print inner btree entries if requested if args.get('inner'): changed = False - for i, (x, px) in enumerate( - it.zip_longest(path[:-1], ppath[:-1])): + for (x, px) in it.zip_longest( + enumerate(path[:-1]), + enumerate(ppath[:-1])): if x is None: break + if not (changed or px is None or x != px): + continue + changed = True - (id_, w_, rbyd_, rid_, - (name_tag_, name_j_, name_d_, name_), - (struct_tag_, struct_j_, struct_d_, struct__)) = x - - if args.get('inner'): - if not (changed or px is None or x != px): - continue - changed = True - - # show the inner entry - dbg_entry(id_, w_, rbyd_, rid_, - name_tag_, name_j_, name_d_, name_, - struct_tag_, struct_j_, struct_d_, struct__, - i) + # show the inner entry + d, (mid_, w_, rbyd_, rid_, tags_) = x + dbg_branch(mid_, w_, rbyd_, rid_, tags_, d) ppath = path # corrupted? try to keep printing the tree if not rbyd: - print('%11s: %s%s%s' % ( + print('%11s: %*s%s%s%s' % ( '%04x.%04x' % (rbyd.block, rbyd.trunk), + t_width, '', '\x1b[31m' if color else '', '(corrupted rbyd %s)' % rbyd.addr(), '\x1b[m' if color else '')) prbyd = rbyd - ppath = path corrupted = True continue - # print btree entries in certain cases - if args.get('inner') or struct_tag != TAG_MDIR: - dbg_entry(mid, w, rbyd, rid, - name_tag, name_j, name_d, name, - struct_tag, struct_j, struct_d, struct_) + # if we're not showing inner nodes, prefer names higher in the + # tree since this avoids showing vestigial names + if not args.get('inner'): + name = None + for mid_, w_, rbyd_, rid_, tags_ in reversed(path): + for tag_, j_, d_, data_ in tags_: + if tag_ & 0xf00f == TAG_NAME: + name = (tag_, j_, d_, data_) - if struct_tag != TAG_MDIR: + if rid_-(w_-1) != 0: + break + + if name is not None: + tags = [name] + [(tag, j, d, data) + for tag, j, d, data in tags + if tag & 0xf00f != TAG_NAME] + + # find mdir in the tags + mdir__ = None + if not args.get('depth') or len(path) < args.get('depth'): + mdir__ = next(((tag, j, d, data) + for tag, j, d, data in tags + if tag == TAG_MDIR), + None) + + # print btree entries in certain cases + if args.get('inner') or not mdir__: + dbg_branch(mid, w, rbyd, rid, tags, len(path)-1) + + if not mdir__: continue # fetch the mdir - blocks = frommdir(struct_) - mdir = Rbyd.fetch(f, block_size, blocks) + _, _, _, data = mdir__ + blocks = frommdir(data) + mdir_ = Rbyd.fetch(f, block_size, blocks) # corrupted? - if not mdir: - print('{%s}: %s%s%s' % ( + if not mdir_: + print('{%s}: %*s%s%s%s' % ( ','.join('%04x' % block - for block in it.chain([mdir.block], - mdir.other_blocks)), + for block in it.chain([mdir_.block], + mdir_.other_blocks)), + t_width, '', '\x1b[31m' if color else '', - '(corrupted mdir %s)' % mdir.addr(), + '(corrupted mdir %s)' % mdir_.addr(), '\x1b[m' if color else '')) corrupted = True else: # show the mdir - dbg_mdir(mdir, mid) + dbg_mdir(mdir_, mid, len(path)) # force next btree entry to be shown prbyd = None @@ -808,6 +1563,14 @@ if __name__ == "__main__": '-i', '--inner', action='store_true', help="Show inner branches.") + parser.add_argument( + '-t', '--tree', + action='store_true', + help="Show the underlying rbyd trees.") + parser.add_argument( + '-b', '--btree', + action='store_true', + help="Show the underlying B-tree.") parser.add_argument( '-Z', '--depth', type=lambda x: int(x, 0),