diff --git a/scripts/dbgbtree.py b/scripts/dbgbtree.py index cfebe87d..0af4db31 100755 --- a/scripts/dbgbtree.py +++ b/scripts/dbgbtree.py @@ -367,7 +367,7 @@ class Rbyd: id = -1 while True: - done, id, tag, w, j, d, data = self.lookup(id, tag+0x10) + done, id, tag, w, j, d, data, _ = self.lookup(id, tag+0x10) if done: break @@ -628,7 +628,7 @@ def main(disk, roots=None, *, 'c': branch['c'], }) - d_ += max(rdepth, 1) + d_ += max(bdepths.get(d, 0), 1) leaf = (bid-(w-1), d, rid-(w-1), TAG_BTREE) # remap branches to leaves if we aren't showing inner branches @@ -679,7 +679,6 @@ def main(disk, roots=None, *, tree = [] root = None branches = {} - leaves = {} bid = -1 while True: done, bid, w, rbyd, rid, tags, path = btree_lookup( @@ -741,7 +740,6 @@ def main(disk, roots=None, *, t_width = 2*t_depth + 2 def treerepr(bid, w, bd, rid, tag): -# print('%-32s' % repr((bid-(w-1), bd, rid, tag)), end='') if t_depth == 0: return '' @@ -873,6 +871,7 @@ def main(disk, roots=None, *, if done: break + # print inner btree entries if requested if args.get('inner'): changed = False for (x, px) in it.zip_longest( @@ -897,7 +896,6 @@ def main(disk, roots=None, *, '\x1b[31m' if color else '', '(corrupted rbyd %s)' % rbyd.addr(), '\x1b[m' if color else '')) - prbyd = rbyd corrupted = True continue diff --git a/scripts/dbgmtree.py b/scripts/dbgmtree.py index 04d4c8e9..824205d1 100755 --- a/scripts/dbgmtree.py +++ b/scripts/dbgmtree.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import bisect +import collections as co import itertools as it import math as m import os @@ -304,10 +305,11 @@ class Rbyd: def lookup(self, id, tag): if not self: - return True, 0, -1, 0, 0, 0, b'' + return True, 0, -1, 0, 0, 0, b'', [] lower = -1 upper = self.weight + path = [] # descend down tree j = self.trunk @@ -323,11 +325,33 @@ class Rbyd: lower += upper-lower-1-weight_ if alt & 0x4 else 0 upper -= upper-lower-1-weight_ if not alt & 0x4 else 0 j = j - jump + + # figure out which color + if alt & 0x2: + _, nalt, _, _, _ = fromtag(self.data[j+jump+d:]) + if nalt & 0x2: + path.append((j+jump, j, True, 'y')) + else: + path.append((j+jump, j, True, 'r')) + else: + path.append((j+jump, j, True, 'b')) + # stay on path else: lower += weight_ if not alt & 0x4 else 0 upper -= weight_ if alt & 0x4 else 0 j = j + d + + # figure out which color + if alt & 0x2: + _, nalt, _, _, _ = fromtag(self.data[j:]) + if nalt & 0x2: + path.append((j-d, j, False, 'y')) + else: + path.append((j-d, j, False, 'r')) + else: + path.append((j-d, j, False, 'b')) + # found tag else: id_ = upper-1 @@ -336,8 +360,7 @@ class Rbyd: done = (id_, tag_) < (id, tag) or tag_ & 2 - return (done, id_, tag_, w_, - j, d, self.data[j+d:j+d+jump]) + return done, id_, tag_, w_, j, d, self.data[j+d:j+d+jump], path def __bool__(self): return bool(self.trunk) @@ -353,12 +376,345 @@ class Rbyd: id = -1 while True: - done, id, tag, w, j, d, data = self.lookup(id, tag+0x10) + done, id, tag, w, j, d, data, _ = self.lookup(id, tag+0x10) if done: break yield id, tag, w, j, d, data + # create tree representation for debugging + def tree(self): + trunks = co.defaultdict(lambda: (-1, 0)) + alts = co.defaultdict(lambda: {}) + + id, tag = -1, 0 + while True: + done, id, tag, w, j, d, data, path = self.lookup(id, tag+0x10) + # found end of tree? + if done: + break + + # keep track of trunks/alts + trunks[j] = (id, tag) + + for j_, j__, followed, c in path: + if followed: + alts[j_] |= {'f': j__, 'c': c} + else: + alts[j_] |= {'nf': j__, 'c': c} + + # prune any alts with unreachable edges + pruned = {} + for j_, alt in alts.items(): + if 'f' not in alt: + pruned[j_] = alt['nf'] + elif 'nf' not in alt: + pruned[j_] = alt['f'] + for j_ in pruned.keys(): + del alts[j_] + + for j_, alt in alts.items(): + while alt['f'] in pruned: + alt['f'] = pruned[alt['f']] + while alt['nf'] in pruned: + alt['nf'] = pruned[alt['nf']] + + # find the trunk and depth of each alt, assuming pruned alts + # didn't exist + def rec_trunk(j_): + if j_ not in alts: + return trunks[j_] + else: + if 'nft' not in alts[j_]: + alts[j_]['nft'] = rec_trunk(alts[j_]['nf']) + return alts[j_]['nft'] + + for j_ in alts.keys(): + rec_trunk(j_) + for j_, alt in alts.items(): + if alt['f'] in alts: + alt['ft'] = alts[alt['f']]['nft'] + else: + alt['ft'] = trunks[alt['f']] + + def rec_height(j_): + if j_ not in alts: + return 0 + else: + if 'h' not in alts[j_]: + alts[j_]['h'] = max( + rec_height(alts[j_]['f']), + rec_height(alts[j_]['nf'])) + 1 + return alts[j_]['h'] + + for j_ in alts.keys(): + rec_height(j_) + + t_depth = max((alt['h']+1 for alt in alts.values()), default=0) + + # convert to more general tree representation + tree = [] + for j, alt in alts.items(): + # note all non-trunk edges should be black + tree.append({ + 'a': alt['nft'], + 'b': alt['nft'], + 'd': t_depth-1 - alt['h'], + 'c': alt['c'], + }) + tree.append({ + 'a': alt['nft'], + 'b': alt['ft'], + 'd': t_depth-1 - alt['h'], + 'c': 'b', + }) + + return tree, t_depth + + # btree lookup with this rbyd as the root + def btree_lookup(self, f, block_size, bid, depth=None): + rbyd = self + rid = bid + depth_ = 1 + path = [] + + # corrupted? return a corrupted block once + if not rbyd: + return bid > 0, bid, 0, rbyd, -1, [], path + + while True: + # collect all tags, normally you don't need to do this + # but we are debugging here + name = None + tags = [] + branch = None + rid_ = rid + tag = 0 + w = 0 + for i in it.count(): + done, rid__, tag, w_, j, d, data, _ = rbyd.lookup( + rid_, tag+0x10) + if done or (i != 0 and rid__ != rid_): + break + + # first tag indicates the branch's weight + if i == 0: + rid_, w = rid__, w_ + + # catch any branches + if tag == TAG_BTREE: + branch = (tag, j, d, data) + + tags.append((tag, j, d, data)) + + # keep track of path + path.append((bid + (rid_-rid), w, rbyd, rid_, tags)) + + # descend down branch? + if branch is not None and ( + not depth or depth_ < depth): + tag, j, d, data = branch + w_, trunk, block, crc = frombtree(data) + rbyd = Rbyd.fetch(f, block_size, block, trunk) + + # corrupted? bail here so we can keep traversing the tree + if not rbyd: + return False, bid + (rid_-rid), w, rbyd, -1, [], path + + rid -= (rid_-(w-1)) + depth_ += 1 + else: + return not tags, bid + (rid_-rid), w, rbyd, rid_, tags, path + + # btree rbyd-tree generation for debugging + def btree_tree(self, f, block_size, depth=None, *, + inner=False): + # find the max depth of each layer to nicely align trees + # TODO memoize + bdepths = {} + bid = -1 + while True: + done, bid, w, rbyd, rid, tags, path = self.btree_lookup( + f, block_size, bid+1, depth=depth) + if done: + break + + for d, (bid, w, rbyd, rid, tags) in enumerate(path): + _, rdepth = rbyd.tree() + bdepths[d] = max(bdepths.get(d, 0), rdepth) + + # find all branches + tree = [] + root = None + branches = {} + bid = -1 + while True: + done, bid, w, rbyd, rid, tags, path = self.btree_lookup( + f, block_size, bid+1, depth=depth) + if done: + break + + d_ = 0 + leaf = None + for d, (bid, w, rbyd, rid, tags) in enumerate(path): + if not tags: + continue + + # map rbyd tree into B-tree space + rtree, rdepth = rbyd.tree() + + # note we adjust our bid/rids to be left-leaning, + # this allows a global order and make tree rendering quite + # a bit easier + for i in range(len(rtree)): + a_rid, a_tag = rtree[i]['a'] + b_rid, b_tag = rtree[i]['b'] + _, _, _, a_w, _, _, _, _ = rbyd.lookup(a_rid, 0) + _, _, _, b_w, _, _, _, _ = rbyd.lookup(b_rid, 0) + rtree[i] = { + 'a': (a_rid-(a_w-1), a_tag), + 'b': (b_rid-(b_w-1), b_tag), + 'd': rtree[i]['d'], + 'c': rtree[i]['c'], + } + + # connect our branch to the rbyd's root + if leaf is not None: + root = min(rtree, + key=lambda branch: branch['d'], + default=None) + + if root is not None: + r_rid, r_tag = root['a'] + else: + r_rid, r_tag = rid-(w-1), tags[0][0] + tree.append({ + 'a': leaf, + 'b': (bid-rid+r_rid, d, r_rid, r_tag), + 'd': d_-1, + 'c': 'b', + }) + + for branch in rtree: + # map rbyd branches into our btree space + a_rid, a_tag = branch['a'] + b_rid, b_tag = branch['b'] + tree.append({ + 'a': (bid-rid+a_rid, d, a_rid, a_tag), + 'b': (bid-rid+b_rid, d, b_rid, b_tag), + 'd': branch['d'] + d_ + bdepths.get(d, 0)-rdepth, + 'c': branch['c'], + }) + + d_ += max(bdepths.get(d, 0), 1) + leaf = (bid-(w-1), d, rid-(w-1), TAG_BTREE) + + # remap branches to leaves if we aren't showing inner branches + if not inner: + # step through each layer backwards + b_depth = max((branch['a'][1]+1 for branch in tree), default=0) + + # keep track of the original tree to find the original bids, + # unfortunately because we store the bids in the branches we + # overwrite these + tree_ = tree.copy() + + for bd in reversed(range(b_depth-1)): + # find leaf-roots at this level + roots = {} + for branch_, branch in zip(tree_, tree): + bid = branch['b'][0] - branch['b'][2] + # choose the highest node as the root + if (branch_['b'][1] == b_depth-1 + and (bid not in roots + or branch_['d'] < roots[bid]['d'])): + roots[bid] = branch_ + + # remap branches to leaf-roots + tree__ = [] + for branch_ in tree_: + if branch_['a'][1] == bd and branch_['a'][0] in roots: + branch_ = { + 'a': roots[branch_['a'][0]]['b'], + 'b': branch_['b'], + 'd': branch_['d'], + 'c': branch_['c'], + } + if branch_['b'][1] == bd and branch_['b'][0] in roots: + branch_ = { + 'a': branch_['a'], + 'b': roots[branch_['b'][0]]['b'], + 'd': branch_['d'], + 'c': branch_['c'], + } + tree__.append(branch_) + tree_ = tree__ + tree = tree_ + + return tree, max((branch['d']+1 for branch in tree), default=0) + + # btree B-tree generation for debugging + def btree_btree(self, f, block_size, depth=None, *, + inner=False): + # find all branches + tree = [] + root = None + branches = {} + bid = -1 + while True: + done, bid, w, rbyd, rid, tags, path = self.btree_lookup( + f, block_size, bid+1, depth=depth) + if done: + break + + # if we're not showing inner nodes, prefer names higher in + # the tree since this avoids showing vestigial names + name = None + if not inner: + name = None + for bid_, w_, rbyd_, rid_, tags_ in reversed(path): + for tag_, j_, d_, data_ in tags_: + if tag_ & 0xf00f == TAG_NAME: + name = (tag_, j_, d_, data_) + + if rid_-(w_-1) != 0: + break + + a = root + for d, (bid, w, rbyd, rid, tags) in enumerate(path): + if not tags: + continue + + b = (bid-(w-1), d, rid-(w-1), + (name if name else tags[0])[0]) + + # remap branches to leaves if we aren't showing + # inner branches + if not inner: + if b not in branches: + bid, w, rbyd, rid, tags = path[-1] + if not tags: + continue + branches[b] = ( + bid-(w-1), len(path)-1, rid-(w-1), + (name if name else tags[0])[0]) + b = branches[b] + + # found entry point? + if root is None: + root = b + a = root + + tree.append({ + 'a': a, + 'b': b, + 'd': d, + 'c': 'b', + }) + a = b + + return tree, max((branch['d']+1 for branch in tree), default=0) + def main(disk, mroots=None, *, block_size=None, @@ -384,71 +740,6 @@ def main(disk, mroots=None, *, f.seek(0, os.SEEK_END) block_size = f.tell() - # look up an id, while keeping track of the search path - def btree_lookup(btree, id, depth=None): - rbyd = btree - rid = id - depth_ = 1 - path = [] - - # corrupted? return a corrupted block once - if not rbyd: - return (id > 0, id, 0, rbyd, -1, - (0, 0, 0, b''), - (0, 0, 0, b''), - path) - - while True: - # first lookup id/name - (done, rid_, name_tag, w, - name_j, name_d, name) = rbyd.lookup(rid, 0) - if done: - return (True, id, 0, rbyd, -1, - (0, 0, 0, b''), - (0, 0, 0, b''), - path) - - if name_tag & 0xf00f == TAG_NAME: - # then lookup struct - (done, _, struct_tag, _, - struct_j, struct_d, struct_) = rbyd.lookup( - rid_, TAG_STRUCT) - if done: - return (True, id, 0, rbyd, -1, - (0, 0, 0, b''), - (0, 0, 0, b''), - path) - else: - tag = name_tag - struct_tag, struct_j, struct_d, struct_ = ( - name_tag, name_j, name_d, name) - name_tag, name_j, name_d, name = 0, 0, 0, b'' - - path.append((id + (rid_-rid), w, rbyd, rid_, - (name_tag, name_j, name_d, name), - (struct_tag, struct_j, struct_d, struct_))) - - # is it another branch? continue down tree - if struct_tag == TAG_BTREE and ( - not depth or depth_ < depth): - w, trunk, block, crc = frombtree(struct_) - rbyd = Rbyd.fetch(f, block_size, block, trunk) - - # corrupted? bail here so we can keep traversing the tree - if not rbyd: - return (False, id + (rid_-rid), w, rbyd, -1, - (0, 0, 0, b''), - (0, 0, 0, b''), - path) - - rid -= (rid_-(w-1)) - depth_ += 1 - else: - return (False, id + (rid_-rid), w, rbyd, rid_, - (name_tag, name_j, name_d, name), - (struct_tag, struct_j, struct_d, struct_), - path) - # before we print, we need to do a pass for a few things: # - find the actual mroot # - find the total weight @@ -457,19 +748,35 @@ def main(disk, mroots=None, *, mroot = Rbyd.fetch(f, block_size, mroots) while True: + # corrupted? + if not mroot: + break + rweight = max(rweight, mroot.weight) # fetch the next mroot - done, rid, tag, w, j, d, data = mroot.lookup(-1, TAG_MROOT) + done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MROOT) if not (not done and rid == -1 and tag == TAG_MROOT): break blocks = frommdir(data) mroot = Rbyd.fetch(f, block_size, blocks) + # fetch the mdir, if there is one + mdir = None + done, rid, tag, w, j, _, data, _ = mroot.lookup(-1, TAG_MDIR) + if not done and rid == -1 and tag == TAG_MDIR: + blocks = frommdir(data) + mdir = Rbyd.fetch(f, block_size, blocks) + + # corrupted? + if mdir: + rweight = max(rweight, mdir.weight) + # fetch the actual mtree, if there is one - done, rid, tag, w, j, d, data = mroot.lookup(-1, TAG_BTREE) - if not done and rid == -1 or tag == TAG_BTREE: + mtree = None + done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_BTREE) + if not done and rid == -1 and tag == TAG_BTREE: w, trunk, block, crc = frombtree(data) mtree = Rbyd.fetch(f, block_size, block, trunk) @@ -478,11 +785,8 @@ def main(disk, mroots=None, *, # traverse entries mid = -1 while True: - (done, mid, w, rbyd, rid, - (name_tag, name_j, name_d, name), - (struct_tag, struct_j, struct_d, struct_), - path) = (btree_lookup(mtree, mid+1, - depth=args.get('depth'))) + done, mid, w, rbyd, rid, tags, path = mtree.btree_lookup( + f, block_size, mid+1, depth=args.get('depth')) if done: break @@ -490,27 +794,469 @@ def main(disk, mroots=None, *, if not rbyd: continue - if struct_tag != TAG_MDIR: - continue + mdir__ = None + if not args.get('depth') or len(path) < args.get('depth'): + mdir__ = next(((tag, j, d, data) + for tag, j, d, data in tags + if tag == TAG_MDIR), + None) - # fetch the mdir - blocks = frommdir(struct_) - mdir = Rbyd.fetch(f, block_size, blocks) + if mdir__: + # fetch the mdir + _, _, _, data = mdir__ + blocks = frommdir(data) + mdir_ = Rbyd.fetch(f, block_size, blocks) - rweight = max(rweight, mroot.weight) + # corrupted? + if mdir_: + rweight = max(rweight, mdir_.weight) + + # precompute rbyd-tree if requested + t_width = 0 + if args.get('tree'): + # compute mroot chain "tree", prefix our actual mtree with this + tree = [] + d_ = 0 + mroot_ = Rbyd.fetch(f, block_size, mroots) + for d in it.count(): + # corrupted? + if not mroot_: + break + + # compute the mroots rbyd-tree + rtree, rdepth = mroot_.tree() + + # connect branch to our root + if d > 0: + root = min(rtree, + key=lambda branch: branch['d'], + default=None) + + if root: + r_rid, r_tag = root['a'] + else: + _, r_rid, r_tag, _, _, _, _, _ = mroot_.lookup(-1, 0x10) + tree.append({ + 'a': (-1, d-1, 0, -1, TAG_MROOT), + 'b': (-1, d, 0, r_rid, r_tag), + 'd': d_-1, + 'c': 'b', + }) + + # map the tree into our metadata space + for branch in rtree: + a_rid, a_tag = branch['a'] + b_rid, b_tag = branch['b'] + tree.append({ + 'a': (-1, d, 0, a_rid, a_tag), + 'b': (-1, d, 0, b_rid, b_tag), + 'd': d_ + branch['d'], + 'c': branch['c'], + }) + d_ += rdepth + + # fetch the next mroot + done, rid, tag, w, j, _, data, _ = mroot_.lookup(-1, TAG_MROOT) + if not (not done and rid == -1 and tag == TAG_MROOT): + break + + blocks = frommdir(data) + mroot_ = Rbyd.fetch(f, block_size, blocks) + + # compute mdir's rbyd-tree if there is one + if mdir: + rtree, rdepth = mdir.tree() + + # connect branch to our root + root = min(rtree, + key=lambda branch: branch['d'], + default=None) + + if root: + r_rid, r_tag = root['a'] + else: + _, r_rid, r_tag, _, _, _, _, _ = mdir.lookup(-1, 0x10) + tree.append({ + 'a': (-1, d, 0, -1, TAG_MDIR), + 'b': (0, 0, 0, r_rid, r_tag), + 'd': d_-1, + 'c': 'b', + }) + + # map the tree into our metadata space + for branch in rtree: + a_rid, a_tag = branch['a'] + b_rid, b_tag = branch['b'] + tree.append({ + 'a': (0, 0, 0, a_rid, a_tag), + 'b': (0, 0, 0, b_rid, b_tag), + 'd': d_ + branch['d'], + 'c': branch['c'], + }) + + # compute the mtree's rbyd-tree if there is one + if mtree: + tree_, tdepth = mtree.btree_tree( + f, block_size, + depth=args.get('depth'), + inner=args.get('inner')) + + # connect a branch to the root of the tree + root = min(tree_, key=lambda branch: branch['d'], default=None) + if root: + r_bid, r_bd, r_rid, r_tag = root['a'] + tree.append({ + 'a': (-1, d, 0, -1, TAG_BTREE), + 'b': (r_bid, r_bd, r_rid, 0, r_tag), + 'd': d_-1, + 'c': 'b', + }) + + # map the tree into our metadata space + for branch in tree_: + a_bid, a_bd, a_rid, a_tag = branch['a'] + b_bid, b_bd, b_rid, b_tag = branch['b'] + tree.append({ + 'a': (a_bid, a_bd, a_rid, 0, a_tag), + 'b': (b_bid, b_bd, b_rid, 0, b_tag), + 'd': d_ + branch['d'], + 'c': branch['c'], + }) + + # find the max depth of each mdir to nicely align trees + # TODO memoize + mdepth = 0 + mid = -1 + while True: + done, mid, w, rbyd, rid, tags, path = mtree.btree_lookup( + f, block_size, mid+1, depth=args.get('depth')) + if done: + break + + # corrupted? + if not rbyd: + continue + + mdir__ = None + if not args.get('depth') or len(path) < args.get('depth'): + mdir__ = next(((tag, j, d, data) + for tag, j, d, data in tags + if tag == TAG_MDIR), + None) + + if mdir__: + # fetch the mdir + _, _, _, data = mdir__ + blocks = frommdir(data) + mdir_ = Rbyd.fetch(f, block_size, blocks) + + rtree, rdepth = mdir_.tree() + mdepth = max(mdepth, rdepth) + + # compute the rbyd-tree for each mdir + mid = -1 + while True: + done, mid, w, rbyd, rid, tags, path = mtree.btree_lookup( + f, block_size, mid+1, depth=args.get('depth')) + if done: + break + + # corrupted? + if not rbyd: + continue + + mdir__ = None + if not args.get('depth') or len(path) < args.get('depth'): + mdir__ = next(((tag, j, d, data) + for tag, j, d, data in tags + if tag == TAG_MDIR), + None) + + if mdir__: + # fetch the mdir + _, _, _, data = mdir__ + blocks = frommdir(data) + mdir_ = Rbyd.fetch(f, block_size, blocks) + + rtree, rdepth = mdir_.tree() + + # connect the root to the mtree + branch = max( + (branch for branch in tree + if branch['b'][0] == mid-(w-1)), + key=lambda branch: branch['d'], + default=None) + if branch: + root = min(rtree, + key=lambda branch: branch['d'], + default=None) + if root: + r_rid, r_tag = root['a'] + else: + _, r_rid, r_tag, _, _, _, _, _ = ( + mdir_.lookup(-1, 0x10)) + tree.append({ + 'a': branch['b'], + 'b': (mid-(w-1), len(path), 0, r_rid, r_tag), + 'd': d_ + tdepth, + 'c': 'b', + }) + + # map the tree into our metadata space + for branch in rtree: + a_rid, a_tag = branch['a'] + b_rid, b_tag = branch['b'] + tree.append({ + 'a': (mid-(w-1), len(path), 0, a_rid, a_tag), + 'b': (mid-(w-1), len(path), 0, b_rid, b_tag), + 'd': (d_ + tdepth + 1 + + branch['d'] + mdepth-rdepth), + 'c': branch['c'], + }) + + # remap branches to leaves if we aren't showing inner branches + if not args.get('inner'): + # step through each layer backwards + b_depth = max((branch['a'][1]+1 for branch in tree), + default=0) + + # keep track of the original tree to find the original bids, + # unfortunately because we store the bids in the branches we + # overwrite these + tree_ = tree.copy() + + for bd in reversed(range(b_depth-1)): + # find leaf-roots at this level + roots = {} + for branch_, branch in zip(tree_, tree): + bid = branch['b'][0] - branch['b'][2] + # choose the highest node as the root + if (branch_['b'][1] == b_depth-1 + and (bid not in roots + or branch_['d'] < roots[bid]['d'])): + roots[bid] = branch_ + + # remap branches to leaf-roots + tree__ = [] + for branch_ in tree_: + # note we ignore mroot branches, we don't collapse + # normally these + if (branch_['a'][0] != -1 + and branch_['a'][1] == bd + and branch_['a'][0] in roots): + branch_ = { + 'a': roots[branch_['a'][0]]['b'], + 'b': branch_['b'], + 'd': branch_['d'], + 'c': branch_['c'], + } + if (branch_['b'][0] != -1 + and branch_['b'][1] == bd + and branch_['b'][0] in roots): + branch_ = { + 'a': branch_['a'], + 'b': roots[branch_['b'][0]]['b'], + 'd': branch_['d'], + 'c': branch_['c'], + } + tree__.append(branch_) + tree_ = tree__ + tree = tree_ + + # precompute B-tree if requested + elif args.get('btree'): + # compute mroot chain "tree", prefix our actual mtree with this + tree = [] + mroot_ = Rbyd.fetch(f, block_size, mroots) + for d in it.count(): + # corrupted? + if not mroot_: + break + + # connect branch to our first tag + if d > 0: + done, rid, tag, w, j, _, data, _ = mroot_.lookup(-1, 0x10) + if not done: + tree.append({ + 'a': (-1, d-1, 0, -1, TAG_MROOT), + 'b': (-1, d, 0, rid, tag), + 'd': 0, + 'c': 'b', + }) + + # fetch the next mroot + done, rid, tag, w, j, _, data, _ = mroot_.lookup(-1, TAG_MROOT) + if not (not done and rid == -1 and tag == TAG_MROOT): + break + + blocks = frommdir(data) + mroot_ = Rbyd.fetch(f, block_size, blocks) + + # create a branch to our mdir if there is one + if mdir: + # connect branch to our first tag + done, rid, tag, w, j, _, data, _ = mdir.lookup(-1, 0x10) + if not done: + tree.append({ + 'a': (-1, d, 0, -1, TAG_MDIR), + 'b': (0, 0, 0, rid, tag), + 'd': 0, + 'c': 'b', + }) + + # compute the mtree's B-tree if there is one + if mtree: + tree_, tdepth = mtree.btree_btree( + f, block_size, + depth=args.get('depth'), + inner=args.get('inner')) + + # connect a branch to the root of the tree + root = min(tree_, key=lambda branch: branch['d'], default=None) + if root: + r_bid, r_bd, r_rid, r_tag = root['a'] + tree.append({ + 'a': (-1, d, 0, -1, TAG_BTREE), + 'b': (r_bid, r_bd, r_rid, 0, r_tag), + 'd': 0, + 'c': 'b', + }) + + # map the tree into our metadata space + for branch in tree_: + a_bid, a_bd, a_rid, a_tag = branch['a'] + b_bid, b_bd, b_rid, b_tag = branch['b'] + tree.append({ + 'a': (a_bid, a_bd, a_rid, 0, a_tag), + 'b': (b_bid, b_bd, b_rid, 0, b_tag), + 'd': 1 + branch['d'], + 'c': branch['c'], + }) + + # remap branches to leaves if we aren't showing inner branches + if not args.get('inner'): + mid = -1 + while True: + done, mid, w, rbyd, rid, tags, path = ( + mtree.btree_lookup( + f, block_size, mid+1, depth=args.get('depth'))) + if done: + break + + # corrupted? + if not rbyd: + continue + + mdir__ = None + if (not args.get('depth') + or len(path) < args.get('depth')): + mdir__ = next(((tag, j, d, data) + for tag, j, d, data in tags + if tag == TAG_MDIR), + None) + + if mdir__: + # fetch the mdir + _, _, _, data = mdir__ + blocks = frommdir(data) + mdir_ = Rbyd.fetch(f, block_size, blocks) + + # find the first entry in the mdir, map branches + # to this entry + done, rid, tag, _, j, d, data, _ = ( + mdir_.lookup(-1, 0x10)) + + tree_ = [] + for branch in tree: + if branch['a'][0] == mid-(w-1): + a_bid, a_bd, _, _, _ = branch['a'] + branch = { + 'a': (a_bid, a_bd+1, 0, rid, tag), + 'b': branch['b'], + 'd': branch['d'], + 'c': branch['c'], + } + if branch['b'][0] == mid-(w-1): + b_bid, b_bd, _, _, _ = branch['b'] + branch = { + 'a': branch['a'], + 'b': (b_bid, b_bd+1, 0, rid, tag), + 'd': branch['d'], + 'c': branch['c'], + } + tree_.append(branch) + tree = tree_ + + # common tree renderer + if args.get('tree') or args.get('btree'): + # find the max depth from the tree + t_depth = max((branch['d']+1 for branch in tree), default=0) + if t_depth > 0: + t_width = 2*t_depth + 2 + + def treerepr(mid, w, md, mrid, rid, tag): + if t_depth == 0: + return '' + + def branchrepr(x, d, was): + for branch in tree: + if branch['d'] == d and branch['b'] == x: + if any(branch['d'] == d and branch['a'] == x + for branch in tree): + return '+-', branch['c'], branch['c'] + elif any(branch['d'] == d + and x > min(branch['a'], branch['b']) + and x < max(branch['a'], branch['b']) + for branch in tree): + return '|-', branch['c'], branch['c'] + elif branch['a'] < branch['b']: + return '\'-', branch['c'], branch['c'] + else: + return '.-', branch['c'], branch['c'] + for branch in tree: + if branch['d'] == d and branch['a'] == x: + return '+ ', branch['c'], None + for branch in tree: + if (branch['d'] == d + and x > min(branch['a'], branch['b']) + and x < max(branch['a'], branch['b'])): + return '| ', branch['c'], was + if was: + return '--', was, was + return ' ', None, None + + trunk = [] + was = None + for d in range(t_depth): + t, c, was = branchrepr( + (mid-max(w-1, 0), md, mrid-max(w-1, 0), rid, tag), + d, was) + + trunk.append('%s%s%s%s' % ( + '\x1b[33m' if color and c == 'y' + else '\x1b[31m' if color and c == 'r' + else '\x1b[90m' if color and c == 'b' + else '', + t, + ('>' if was else ' ') if d == t_depth-1 else '', + '\x1b[m' if color and c else '')) + + return '%s ' % ''.join(trunk) - def dbg_mdir(mdir, mid): - for i, (id, tag, w, j, d, data) in enumerate(mdir): + def dbg_mdir(mdir, mid, md): + for i, (rid, tag, w, j, d, data) in enumerate(mdir): # show human-readable tag representation - print('%12s %-57s' % ( + print('%12s %s%-57s' % ( '{%s}:' % ','.join('%04x' % block for block in it.chain([mdir.block], mdir.other_blocks)) if i == 0 else '', + treerepr(mid, 1, md, 0, rid, tag) + if args.get('tree') or args.get('btree') else '', '%*s %-22s%s' % ( - w_width, '%d.%d-%d' % (mid, id-(w-1), id) - if w > 1 else '%d.%d' % (mid, id) + w_width, '%d.%d-%d' % (mid, rid-(w-1), rid) + if w > 1 else '%d.%d' % (mid, rid) if w > 0 else '%d' % mid if i == 0 else '', tagrepr(tag, w, len(data), j), @@ -550,93 +1296,65 @@ def main(disk, mroots=None, *, line)) # prbyd here means the last rendered rbyd, we update - # in dbg_entry to always print interleaved addresses + # in dbg_branch to always print interleaved addresses prbyd = None - def dbg_entry(id, w, rbyd, rid, - name_tag, name_j, name_d, name, - struct_tag, struct_j, struct_d, struct_, - depth=None): + def dbg_branch(bid, w, rbyd, rid, tags, bd): nonlocal prbyd # show human-readable representation - if name_tag: - print('%12s %*s %-22s %s' % ( + for i, (tag, j, d, data) in enumerate(tags): + print('%12s %s%*s %-22s %s' % ( '%04x.%04x:' % (rbyd.block, rbyd.trunk) if prbyd is None or rbyd != prbyd else '', - w_width, '%d-%d' % (id-(w-1), id) if w > 1 - else id if w > 0 + treerepr(bid, w, bd, rid, 0, tag) + if args.get('tree') or args.get('btree') else '', + w_width, '' if i != 0 + else '%d-%d' % (bid-(w-1), bid) if w > 1 + else bid if w > 0 else '', - tagrepr(name_tag, w, len(name), None), + tagrepr(tag, w if i == 0 else 0, len(data), None), + # note we render names a bit different here ''.join( b if b >= ' ' and b <= '~' else '.' - for b in map(chr, name)))) + for b in map(chr, data)) + if tag & 0xf00f == TAG_NAME + else next(xxd(data, 8), '') + if not args.get('no_truncate') + else '')) prbyd = rbyd - print('%12s %*s %-22s %s' % ( - '%04x.%04x:' % (rbyd.block, rbyd.trunk) - if prbyd is None or rbyd != prbyd - else '', - w_width, '' if name_tag - else '%d-%d' % (id-(w-1), id) if w > 1 - else id if w > 0 - else '', - tagrepr(struct_tag, - w if not name_tag else 0, - len(struct_), None), - next(xxd(struct_, 8), '') - if not args.get('no_truncate') else '')) - prbyd = rbyd # show in-device representation if args.get('device'): - if name_tag: - print('%11s %*s %-22s%s' % ( + for i, (tag, j, d, data) in enumerate(tags): + print('%11s %*s%*s %-22s%s' % ( '', + t_width, '', w_width, '', - '%04x %08x %07x' % (name_tag, w, len(name)), + '%04x %08x %07x' % (tag, w if i == 0 else 0, len(data)), ' %s' % ' '.join( '%08x' % fromle32( - rbyd.data[name_j+name_d+i*4 - : name_j+name_d + min(i*4+4,len(name))]) - for i in range(min(m.ceil(len(name)/4), 3)))[:23])) - print('%11s %*s %-22s%s' % ( - '', - w_width, '', - '%04x %08x %07x' % ( - struct_tag, w if not name_tag else 0, len(struct_)), - ' %s' % ' '.join( - '%08x' % fromle32( - rbyd.data[struct_j+struct_d+i*4 - : struct_j+struct_d + min(i*4+4,len(struct_))]) - for i in range(min(m.ceil(len(struct_)/4), 3)))[:23])) + rbyd.data[j+d+i*4 : j+d + min(i*4+4,len(data))]) + for i in range(min(m.ceil(len(data)/4), 3)))[:23])) # show on-disk encoding of tags/data - if args.get('raw'): - for o, line in enumerate(xxd( - rbyd.data[name_j:name_j+name_d])): - print('%11s: %*s %s' % ( - '%04x' % (name_j + o*16), - w_width, '', - line)) - if args.get('raw'): - for o, line in enumerate(xxd(name)): - print('%11s: %*s %s' % ( - '%04x' % (name_j+name_d + o*16), - w_width, '', - line)) - if args.get('raw'): - for o, line in enumerate(xxd( - rbyd.data[struct_j:struct_j+struct_d])): - print('%11s: %*s %s' % ( - '%04x' % (struct_j + o*16), - w_width, '', - line)) - if args.get('raw') or args.get('no_truncate'): - for o, line in enumerate(xxd(struct_)): - print('%11s: %*s %s' % ( - '%04x' % (struct_j+struct_d + o*16), - w_width, '', - line)) + for i, (tag, j, d, data) in enumerate(tags): + if args.get('raw'): + for o, line in enumerate(xxd(rbyd.data[j:j+d])): + print('%11s: %*s%*s %s' % ( + '%04x' % (j + o*16), + t_width, '', + w_width, '', + line)) + # note we don't render name tags with no_truncate + if args.get('raw') or ( + args.get('no_truncate') and tag & 0xf00f != TAG_NAME): + for o, line in enumerate(xxd(data)): + print('%11s: %*s%*s %s' % ( + '%04x' % (j+d + o*16), + t_width, '', + w_width, '', + line)) # print the actual mroot @@ -647,8 +1365,9 @@ def main(disk, mroots=None, *, w_width = (m.ceil(m.log10(max(1, mweight)+1)) + 2*m.ceil(m.log10(max(1, rweight)+1)) + 2) - print('%-11s %-*s %-22s %s' % ( + print('%-11s %*s%-*s %-22s %s' % ( 'mdir', + t_width, '', w_width, 'ids', 'tag', 'data (truncated)' @@ -659,7 +1378,7 @@ def main(disk, mroots=None, *, ppath = [] corrupted = False mroot = Rbyd.fetch(f, block_size, mroots) - while True: + for d in it.count(): # corrupted? if not mroot: print('{%s}: %s%s%s' % ( @@ -670,97 +1389,133 @@ def main(disk, mroots=None, *, '(corrupted mroot %s)' % mroot.addr(), '\x1b[m' if color else '')) corrupted = True + break else: # show the mdir - dbg_mdir(mroot, -1) + dbg_mdir(mroot, -1, d) # fetch the next mroot - done, rid, tag, w, j, d, data = mroot.lookup(-1, TAG_MROOT) + done, rid, tag, w, j, _, data, _ = mroot.lookup(-1, TAG_MROOT) if not (not done and rid == -1 and tag == TAG_MROOT): break blocks = frommdir(data) mroot = Rbyd.fetch(f, block_size, blocks) + # show the mdir, if there is one + done, rid, tag, w, j, _, data, _ = mroot.lookup(-1, TAG_MDIR) + if not done and rid == -1 and tag == TAG_MDIR: + blocks = frommdir(data) + mdir = Rbyd.fetch(f, block_size, blocks) + + # corrupted? + if not mdir: + print('{%s}: %s%s%s' % ( + ','.join('%04x' % block + for block in it.chain([mdir.block], + mdir.other_blocks)), + '\x1b[31m' if color else '', + '(corrupted mdir %s)' % mdir.addr(), + '\x1b[m' if color else '')) + corrupted = True + else: + # show the mdir + dbg_mdir(mdir, 0, 0) + # fetch the actual mtree, if there is one - done, rid, tag, w, j, d, data = mroot.lookup(-1, TAG_BTREE) - if not done and rid == -1 or tag == TAG_BTREE: + done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_BTREE) + if not done and rid == -1 and tag == TAG_BTREE: w, trunk, block, crc = frombtree(data) mtree = Rbyd.fetch(f, block_size, block, trunk) # traverse entries mid = -1 while True: - (done, mid, w, rbyd, rid, - (name_tag, name_j, name_d, name), - (struct_tag, struct_j, struct_d, struct_), - path) = (btree_lookup(mtree, mid+1, - depth=args.get('depth'))) + done, mid, w, rbyd, rid, tags, path = mtree.btree_lookup( + f, block_size, mid+1, depth=args.get('depth')) if done: break # print inner btree entries if requested if args.get('inner'): changed = False - for i, (x, px) in enumerate( - it.zip_longest(path[:-1], ppath[:-1])): + for (x, px) in it.zip_longest( + enumerate(path[:-1]), + enumerate(ppath[:-1])): if x is None: break + if not (changed or px is None or x != px): + continue + changed = True - (id_, w_, rbyd_, rid_, - (name_tag_, name_j_, name_d_, name_), - (struct_tag_, struct_j_, struct_d_, struct__)) = x - - if args.get('inner'): - if not (changed or px is None or x != px): - continue - changed = True - - # show the inner entry - dbg_entry(id_, w_, rbyd_, rid_, - name_tag_, name_j_, name_d_, name_, - struct_tag_, struct_j_, struct_d_, struct__, - i) + # show the inner entry + d, (mid_, w_, rbyd_, rid_, tags_) = x + dbg_branch(mid_, w_, rbyd_, rid_, tags_, d) ppath = path # corrupted? try to keep printing the tree if not rbyd: - print('%11s: %s%s%s' % ( + print('%11s: %*s%s%s%s' % ( '%04x.%04x' % (rbyd.block, rbyd.trunk), + t_width, '', '\x1b[31m' if color else '', '(corrupted rbyd %s)' % rbyd.addr(), '\x1b[m' if color else '')) prbyd = rbyd - ppath = path corrupted = True continue - # print btree entries in certain cases - if args.get('inner') or struct_tag != TAG_MDIR: - dbg_entry(mid, w, rbyd, rid, - name_tag, name_j, name_d, name, - struct_tag, struct_j, struct_d, struct_) + # if we're not showing inner nodes, prefer names higher in the + # tree since this avoids showing vestigial names + if not args.get('inner'): + name = None + for mid_, w_, rbyd_, rid_, tags_ in reversed(path): + for tag_, j_, d_, data_ in tags_: + if tag_ & 0xf00f == TAG_NAME: + name = (tag_, j_, d_, data_) - if struct_tag != TAG_MDIR: + if rid_-(w_-1) != 0: + break + + if name is not None: + tags = [name] + [(tag, j, d, data) + for tag, j, d, data in tags + if tag & 0xf00f != TAG_NAME] + + # find mdir in the tags + mdir__ = None + if not args.get('depth') or len(path) < args.get('depth'): + mdir__ = next(((tag, j, d, data) + for tag, j, d, data in tags + if tag == TAG_MDIR), + None) + + # print btree entries in certain cases + if args.get('inner') or not mdir__: + dbg_branch(mid, w, rbyd, rid, tags, len(path)-1) + + if not mdir__: continue # fetch the mdir - blocks = frommdir(struct_) - mdir = Rbyd.fetch(f, block_size, blocks) + _, _, _, data = mdir__ + blocks = frommdir(data) + mdir_ = Rbyd.fetch(f, block_size, blocks) # corrupted? - if not mdir: - print('{%s}: %s%s%s' % ( + if not mdir_: + print('{%s}: %*s%s%s%s' % ( ','.join('%04x' % block - for block in it.chain([mdir.block], - mdir.other_blocks)), + for block in it.chain([mdir_.block], + mdir_.other_blocks)), + t_width, '', '\x1b[31m' if color else '', - '(corrupted mdir %s)' % mdir.addr(), + '(corrupted mdir %s)' % mdir_.addr(), '\x1b[m' if color else '')) corrupted = True else: # show the mdir - dbg_mdir(mdir, mid) + dbg_mdir(mdir_, mid, len(path)) # force next btree entry to be shown prbyd = None @@ -808,6 +1563,14 @@ if __name__ == "__main__": '-i', '--inner', action='store_true', help="Show inner branches.") + parser.add_argument( + '-t', '--tree', + action='store_true', + help="Show the underlying rbyd trees.") + parser.add_argument( + '-b', '--btree', + action='store_true', + help="Show the underlying B-tree.") parser.add_argument( '-Z', '--depth', type=lambda x: int(x, 0),