From c60fa69ce13b5cd1aba5bd034176eeeadbb63038 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 21 May 2023 14:59:05 -0500 Subject: [PATCH] Optimized dbg*.py tree generation/rendering by deduplicating edges Optimizing a script? This might sound premature, but the tree rendering was, uh, quite slow for any decently sized (>1024) btree. The main reason is that tree generation is quite hacky in places, repeatedly spitting out multiple copies of the inner node's rbyd trees for example. Rather than rewrite the tree generation implementation to be smarter, this just changes all edge representations to namedtuples (which may reduce memory pressure a bit), and collects them into a Python set. This has the effect of deduplicating generated edges efficiently, and improved the rendering performance significantly. --- I also considered memoizing rbyd tree, but dropped the idea since the current renderer performs well enough. --- scripts/dbgbtree.py | 190 ++++++++--------- scripts/dbgmtree.py | 492 ++++++++++++++++++++++---------------------- scripts/dbgrbyd.py | 61 +++--- 3 files changed, 378 insertions(+), 365 deletions(-) diff --git a/scripts/dbgbtree.py b/scripts/dbgbtree.py index 0af4db31..6342b44c 100755 --- a/scripts/dbgbtree.py +++ b/scripts/dbgbtree.py @@ -177,6 +177,10 @@ def tagrepr(tag, w, size, off=None): else: return '0x%04x w%d %d' % (tag, w, size) +# this type is used for tree representations +TBranch = co.namedtuple('TBranch', 'a, b, d, c') + +# our core rbyd type class Rbyd: def __init__(self, block, data, rev, off, trunk, weight): self.block = block @@ -444,21 +448,21 @@ class Rbyd: t_depth = max((alt['h']+1 for alt in alts.values()), default=0) # convert to more general tree representation - tree = [] + tree = set() for j, alt in alts.items(): # note all non-trunk edges should be black - tree.append({ - 'a': alt['nft'], - 'b': alt['nft'], - 'd': t_depth-1 - alt['h'], - 'c': alt['c'], - }) - tree.append({ - 'a': alt['nft'], - 'b': alt['ft'], - 'd': t_depth-1 - alt['h'], - 'c': 'b', - }) + tree.add(TBranch( + a=alt['nft'], + b=alt['nft'], + d=t_depth-1 - alt['h'], + c=alt['c'], + )) + tree.add(TBranch( + a=alt['nft'], + b=alt['ft'], + d=t_depth-1 - alt['h'], + c='b', + )) return tree, t_depth @@ -566,7 +570,7 @@ def main(disk, roots=None, *, bdepths[d] = max(bdepths.get(d, 0), rdepth) # find all branches - tree = [] + tree = set() root = None branches = {} bid = -1 @@ -588,45 +592,47 @@ def main(disk, roots=None, *, # note we adjust our bid/rids to be left-leaning, # this allows a global order and make tree rendering quite # a bit easier - for i in range(len(rtree)): - a_rid, a_tag = rtree[i]['a'] - b_rid, b_tag = rtree[i]['b'] + rtree_ = set() + for branch in rtree: + a_rid, a_tag = branch.a + b_rid, b_tag = branch.b _, _, _, a_w, _, _, _, _ = rbyd.lookup(a_rid, 0) _, _, _, b_w, _, _, _, _ = rbyd.lookup(b_rid, 0) - rtree[i] = { - 'a': (a_rid-(a_w-1), a_tag), - 'b': (b_rid-(b_w-1), b_tag), - 'd': rtree[i]['d'], - 'c': rtree[i]['c'], - } + rtree_.add(TBranch( + a=(a_rid-(a_w-1), a_tag), + b=(b_rid-(b_w-1), b_tag), + d=branch.d, + c=branch.c, + )) + rtree = rtree_ # connect our branch to the rbyd's root if leaf is not None: root = min(rtree, - key=lambda branch: branch['d'], + key=lambda branch: branch.d, default=None) if root is not None: - r_rid, r_tag = root['a'] + r_rid, r_tag = root.a else: r_rid, r_tag = rid-(w-1), tags[0][0] - tree.append({ - 'a': leaf, - 'b': (bid-rid+r_rid, d, r_rid, r_tag), - 'd': d_-1, - 'c': 'b', - }) + tree.add(TBranch( + a=leaf, + b=(bid-rid+r_rid, d, r_rid, r_tag), + d=d_-1, + c='b', + )) for branch in rtree: # map rbyd branches into our btree space - a_rid, a_tag = branch['a'] - b_rid, b_tag = branch['b'] - tree.append({ - 'a': (bid-rid+a_rid, d, a_rid, a_tag), - 'b': (bid-rid+b_rid, d, b_rid, b_tag), - 'd': branch['d'] + d_ + bdepths.get(d, 0)-rdepth, - 'c': branch['c'], - }) + a_rid, a_tag = branch.a + b_rid, b_tag = branch.b + tree.add(TBranch( + a=(bid-rid+a_rid, d, a_rid, a_tag), + b=(bid-rid+b_rid, d, b_rid, b_tag), + d=branch.d + d_ + bdepths.get(d, 0)-rdepth, + c=branch.c, + )) d_ += max(bdepths.get(d, 0), 1) leaf = (bid-(w-1), d, rid-(w-1), TAG_BTREE) @@ -634,49 +640,49 @@ def main(disk, roots=None, *, # remap branches to leaves if we aren't showing inner branches if not args.get('inner'): # step through each layer backwards - b_depth = max((branch['a'][1]+1 for branch in tree), default=0) + b_depth = max((branch.a[1]+1 for branch in tree), default=0) - # keep track of the original tree to find the original bids, - # unfortunately because we store the bids in the branches we - # overwrite these - tree_ = tree.copy() + # keep track of the original bids, unfortunately because we + # store the bids in the branches we overwrite these + tree = {(branch.b[0] - branch.b[2], branch) for branch in tree} for bd in reversed(range(b_depth-1)): # find leaf-roots at this level roots = {} - for branch_, branch in zip(tree_, tree): - bid = branch['b'][0] - branch['b'][2] + for bid, branch in tree: # choose the highest node as the root - if (branch_['b'][1] == b_depth-1 + if (branch.b[1] == b_depth-1 and (bid not in roots - or branch_['d'] < roots[bid]['d'])): - roots[bid] = branch_ + or branch.d < roots[bid].d)): + roots[bid] = branch # remap branches to leaf-roots - tree__ = [] - for branch_ in tree_: - if branch_['a'][1] == bd and branch_['a'][0] in roots: - branch_ = { - 'a': roots[branch_['a'][0]]['b'], - 'b': branch_['b'], - 'd': branch_['d'], - 'c': branch_['c'], - } - if branch_['b'][1] == bd and branch_['b'][0] in roots: - branch_ = { - 'a': branch_['a'], - 'b': roots[branch_['b'][0]]['b'], - 'd': branch_['d'], - 'c': branch_['c'], - } - tree__.append(branch_) - tree_ = tree__ - tree = tree_ + tree_ = set() + for bid, branch in tree: + if branch.a[1] == bd and branch.a[0] in roots: + branch = TBranch( + a=roots[branch.a[0]].b, + b=branch.b, + d=branch.d, + c=branch.c, + ) + if branch.b[1] == bd and branch.b[0] in roots: + branch = TBranch( + a=branch.a, + b=roots[branch.b[0]].b, + d=branch.d, + c=branch.c, + ) + tree_.add((bid, branch)) + tree = tree_ + + # strip out bids + tree = {branch for _, branch in tree} # precompute B-trees if requested elif args.get('btree'): # find all branches - tree = [] + tree = set() root = None branches = {} bid = -1 @@ -724,18 +730,18 @@ def main(disk, roots=None, *, root = b a = root - tree.append({ - 'a': a, - 'b': b, - 'd': d, - 'c': 'b', - }) + tree.add(TBranch( + a=a, + b=b, + d=d, + c='b', + )) a = b # common tree renderer if args.get('tree') or args.get('btree'): # find the max depth from the tree - t_depth = max((branch['d']+1 for branch in tree), default=0) + t_depth = max((branch.d+1 for branch in tree), default=0) if t_depth > 0: t_width = 2*t_depth + 2 @@ -745,27 +751,27 @@ def main(disk, roots=None, *, def branchrepr(x, d, was): for branch in tree: - if branch['d'] == d and branch['b'] == x: - if any(branch['d'] == d and branch['a'] == x + if branch.d == d and branch.b == x: + if any(branch.d == d and branch.a == x for branch in tree): - return '+-', branch['c'], branch['c'] - elif any(branch['d'] == d - and x > min(branch['a'], branch['b']) - and x < max(branch['a'], branch['b']) + return '+-', branch.c, branch.c + elif any(branch.d == d + and x > min(branch.a, branch.b) + and x < max(branch.a, branch.b) for branch in tree): - return '|-', branch['c'], branch['c'] - elif branch['a'] < branch['b']: - return '\'-', branch['c'], branch['c'] + return '|-', branch.c, branch.c + elif branch.a < branch.b: + return '\'-', branch.c, branch.c else: - return '.-', branch['c'], branch['c'] + return '.-', branch.c, branch.c for branch in tree: - if branch['d'] == d and branch['a'] == x: - return '+ ', branch['c'], None + if branch.d == d and branch.a == x: + return '+ ', branch.c, None for branch in tree: - if (branch['d'] == d - and x > min(branch['a'], branch['b']) - and x < max(branch['a'], branch['b'])): - return '| ', branch['c'], was + if (branch.d == d + and x > min(branch.a, branch.b) + and x < max(branch.a, branch.b)): + return '| ', branch.c, was if was: return '--', was, was return ' ', None, None diff --git a/scripts/dbgmtree.py b/scripts/dbgmtree.py index 824205d1..a1c9daf7 100755 --- a/scripts/dbgmtree.py +++ b/scripts/dbgmtree.py @@ -186,6 +186,10 @@ def tagrepr(tag, w, size, off=None): else: return '0x%04x w%d %d' % (tag, w, size) +# this type is used for tree representations +TBranch = co.namedtuple('TBranch', 'a, b, d, c') + +# our core rbyd type class Rbyd: def __init__(self, block, data, rev, off, trunk, weight): self.block = block @@ -453,21 +457,21 @@ class Rbyd: t_depth = max((alt['h']+1 for alt in alts.values()), default=0) # convert to more general tree representation - tree = [] + tree = set() for j, alt in alts.items(): # note all non-trunk edges should be black - tree.append({ - 'a': alt['nft'], - 'b': alt['nft'], - 'd': t_depth-1 - alt['h'], - 'c': alt['c'], - }) - tree.append({ - 'a': alt['nft'], - 'b': alt['ft'], - 'd': t_depth-1 - alt['h'], - 'c': 'b', - }) + tree.add(TBranch( + a=alt['nft'], + b=alt['nft'], + d=t_depth-1 - alt['h'], + c=alt['c'], + )) + tree.add(TBranch( + a=alt['nft'], + b=alt['ft'], + d=t_depth-1 - alt['h'], + c='b', + )) return tree, t_depth @@ -544,7 +548,7 @@ class Rbyd: bdepths[d] = max(bdepths.get(d, 0), rdepth) # find all branches - tree = [] + tree = set() root = None branches = {} bid = -1 @@ -566,45 +570,47 @@ class Rbyd: # note we adjust our bid/rids to be left-leaning, # this allows a global order and make tree rendering quite # a bit easier - for i in range(len(rtree)): - a_rid, a_tag = rtree[i]['a'] - b_rid, b_tag = rtree[i]['b'] + rtree_ = set() + for branch in rtree: + a_rid, a_tag = branch.a + b_rid, b_tag = branch.b _, _, _, a_w, _, _, _, _ = rbyd.lookup(a_rid, 0) _, _, _, b_w, _, _, _, _ = rbyd.lookup(b_rid, 0) - rtree[i] = { - 'a': (a_rid-(a_w-1), a_tag), - 'b': (b_rid-(b_w-1), b_tag), - 'd': rtree[i]['d'], - 'c': rtree[i]['c'], - } + rtree_.add(TBranch( + a=(a_rid-(a_w-1), a_tag), + b=(b_rid-(b_w-1), b_tag), + d=branch.d, + c=branch.c, + )) + rtree = rtree_ # connect our branch to the rbyd's root if leaf is not None: root = min(rtree, - key=lambda branch: branch['d'], + key=lambda branch: branch.d, default=None) if root is not None: - r_rid, r_tag = root['a'] + r_rid, r_tag = root.a else: r_rid, r_tag = rid-(w-1), tags[0][0] - tree.append({ - 'a': leaf, - 'b': (bid-rid+r_rid, d, r_rid, r_tag), - 'd': d_-1, - 'c': 'b', - }) + tree.add(TBranch( + a=leaf, + b=(bid-rid+r_rid, d, r_rid, r_tag), + d=d_-1, + c='b', + )) for branch in rtree: # map rbyd branches into our btree space - a_rid, a_tag = branch['a'] - b_rid, b_tag = branch['b'] - tree.append({ - 'a': (bid-rid+a_rid, d, a_rid, a_tag), - 'b': (bid-rid+b_rid, d, b_rid, b_tag), - 'd': branch['d'] + d_ + bdepths.get(d, 0)-rdepth, - 'c': branch['c'], - }) + a_rid, a_tag = branch.a + b_rid, b_tag = branch.b + tree.add(TBranch( + a=(bid-rid+a_rid, d, a_rid, a_tag), + b=(bid-rid+b_rid, d, b_rid, b_tag), + d=branch.d + d_ + bdepths.get(d, 0)-rdepth, + c=branch.c, + )) d_ += max(bdepths.get(d, 0), 1) leaf = (bid-(w-1), d, rid-(w-1), TAG_BTREE) @@ -612,52 +618,52 @@ class Rbyd: # remap branches to leaves if we aren't showing inner branches if not inner: # step through each layer backwards - b_depth = max((branch['a'][1]+1 for branch in tree), default=0) + b_depth = max((branch.a[1]+1 for branch in tree), default=0) - # keep track of the original tree to find the original bids, - # unfortunately because we store the bids in the branches we - # overwrite these - tree_ = tree.copy() + # keep track of the original bids, unfortunately because we + # store the bids in the branches we overwrite these + tree = {(branch.b[0] - branch.b[2], branch) for branch in tree} for bd in reversed(range(b_depth-1)): # find leaf-roots at this level roots = {} - for branch_, branch in zip(tree_, tree): - bid = branch['b'][0] - branch['b'][2] + for bid, branch in tree: # choose the highest node as the root - if (branch_['b'][1] == b_depth-1 + if (branch.b[1] == b_depth-1 and (bid not in roots - or branch_['d'] < roots[bid]['d'])): - roots[bid] = branch_ + or branch.d < roots[bid].d)): + roots[bid] = branch # remap branches to leaf-roots - tree__ = [] - for branch_ in tree_: - if branch_['a'][1] == bd and branch_['a'][0] in roots: - branch_ = { - 'a': roots[branch_['a'][0]]['b'], - 'b': branch_['b'], - 'd': branch_['d'], - 'c': branch_['c'], - } - if branch_['b'][1] == bd and branch_['b'][0] in roots: - branch_ = { - 'a': branch_['a'], - 'b': roots[branch_['b'][0]]['b'], - 'd': branch_['d'], - 'c': branch_['c'], - } - tree__.append(branch_) - tree_ = tree__ - tree = tree_ + tree_ = set() + for bid, branch in tree: + if branch.a[1] == bd and branch.a[0] in roots: + branch = TBranch( + a=roots[branch.a[0]].b, + b=branch.b, + d=branch.d, + c=branch.c, + ) + if branch.b[1] == bd and branch.b[0] in roots: + branch = TBranch( + a=branch.a, + b=roots[branch.b[0]].b, + d=branch.d, + c=branch.c, + ) + tree_.add((bid, branch)) + tree = tree_ - return tree, max((branch['d']+1 for branch in tree), default=0) + # strip out bids + tree = {branch for _, branch in tree} + + return tree, max((branch.d+1 for branch in tree), default=0) # btree B-tree generation for debugging def btree_btree(self, f, block_size, depth=None, *, inner=False): # find all branches - tree = [] + tree = set() root = None branches = {} bid = -1 @@ -705,15 +711,15 @@ class Rbyd: root = b a = root - tree.append({ - 'a': a, - 'b': b, - 'd': d, - 'c': 'b', - }) + tree.add(TBranch( + a=a, + b=b, + d=d, + c='b', + )) a = b - return tree, max((branch['d']+1 for branch in tree), default=0) + return tree, max((branch.d+1 for branch in tree), default=0) def main(disk, mroots=None, *, @@ -815,7 +821,7 @@ def main(disk, mroots=None, *, t_width = 0 if args.get('tree'): # compute mroot chain "tree", prefix our actual mtree with this - tree = [] + tree = set() d_ = 0 mroot_ = Rbyd.fetch(f, block_size, mroots) for d in it.count(): @@ -829,30 +835,30 @@ def main(disk, mroots=None, *, # connect branch to our root if d > 0: root = min(rtree, - key=lambda branch: branch['d'], + key=lambda branch: branch.d, default=None) if root: - r_rid, r_tag = root['a'] + r_rid, r_tag = root.a else: _, r_rid, r_tag, _, _, _, _, _ = mroot_.lookup(-1, 0x10) - tree.append({ - 'a': (-1, d-1, 0, -1, TAG_MROOT), - 'b': (-1, d, 0, r_rid, r_tag), - 'd': d_-1, - 'c': 'b', - }) + tree.add(TBranch( + a=(-1, d-1, 0, -1, TAG_MROOT), + b=(-1, d, 0, r_rid, r_tag), + d=d_-1, + c='b', + )) # map the tree into our metadata space for branch in rtree: - a_rid, a_tag = branch['a'] - b_rid, b_tag = branch['b'] - tree.append({ - 'a': (-1, d, 0, a_rid, a_tag), - 'b': (-1, d, 0, b_rid, b_tag), - 'd': d_ + branch['d'], - 'c': branch['c'], - }) + a_rid, a_tag = branch.a + b_rid, b_tag = branch.b + tree.add(TBranch( + a=(-1, d, 0, a_rid, a_tag), + b=(-1, d, 0, b_rid, b_tag), + d=d_ + branch.d, + c=branch.c, + )) d_ += rdepth # fetch the next mroot @@ -869,30 +875,30 @@ def main(disk, mroots=None, *, # connect branch to our root root = min(rtree, - key=lambda branch: branch['d'], + key=lambda branch: branch.d, default=None) if root: - r_rid, r_tag = root['a'] + r_rid, r_tag = root.a else: _, r_rid, r_tag, _, _, _, _, _ = mdir.lookup(-1, 0x10) - tree.append({ - 'a': (-1, d, 0, -1, TAG_MDIR), - 'b': (0, 0, 0, r_rid, r_tag), - 'd': d_-1, - 'c': 'b', - }) + tree.add(TBranch( + a=(-1, d, 0, -1, TAG_MDIR), + b=(0, 0, 0, r_rid, r_tag), + d=d_-1, + c='b', + )) # map the tree into our metadata space for branch in rtree: - a_rid, a_tag = branch['a'] - b_rid, b_tag = branch['b'] - tree.append({ - 'a': (0, 0, 0, a_rid, a_tag), - 'b': (0, 0, 0, b_rid, b_tag), - 'd': d_ + branch['d'], - 'c': branch['c'], - }) + a_rid, a_tag = branch.a + b_rid, b_tag = branch.b + tree.add(TBranch( + a=(0, 0, 0, a_rid, a_tag), + b=(0, 0, 0, b_rid, b_tag), + d=d_ + branch.d, + c=branch.c, + )) # compute the mtree's rbyd-tree if there is one if mtree: @@ -902,26 +908,26 @@ def main(disk, mroots=None, *, inner=args.get('inner')) # connect a branch to the root of the tree - root = min(tree_, key=lambda branch: branch['d'], default=None) + root = min(tree_, key=lambda branch: branch.d, default=None) if root: - r_bid, r_bd, r_rid, r_tag = root['a'] - tree.append({ - 'a': (-1, d, 0, -1, TAG_BTREE), - 'b': (r_bid, r_bd, r_rid, 0, r_tag), - 'd': d_-1, - 'c': 'b', - }) + r_bid, r_bd, r_rid, r_tag = root.a + tree.add(TBranch( + a=(-1, d, 0, -1, TAG_BTREE), + b=(r_bid, r_bd, r_rid, 0, r_tag), + d=d_-1, + c='b', + )) # map the tree into our metadata space for branch in tree_: - a_bid, a_bd, a_rid, a_tag = branch['a'] - b_bid, b_bd, b_rid, b_tag = branch['b'] - tree.append({ - 'a': (a_bid, a_bd, a_rid, 0, a_tag), - 'b': (b_bid, b_bd, b_rid, 0, b_tag), - 'd': d_ + branch['d'], - 'c': branch['c'], - }) + a_bid, a_bd, a_rid, a_tag = branch.a + b_bid, b_bd, b_rid, b_tag = branch.b + tree.add(TBranch( + a=(a_bid, a_bd, a_rid, 0, a_tag), + b=(b_bid, b_bd, b_rid, 0, b_tag), + d=d_ + branch.d, + c=branch.c, + )) # find the max depth of each mdir to nicely align trees # TODO memoize @@ -983,90 +989,90 @@ def main(disk, mroots=None, *, # connect the root to the mtree branch = max( (branch for branch in tree - if branch['b'][0] == mid-(w-1)), - key=lambda branch: branch['d'], + if branch.b[0] == mid-(w-1)), + key=lambda branch: branch.d, default=None) if branch: root = min(rtree, - key=lambda branch: branch['d'], + key=lambda branch: branch.d, default=None) if root: - r_rid, r_tag = root['a'] + r_rid, r_tag = root.a else: _, r_rid, r_tag, _, _, _, _, _ = ( mdir_.lookup(-1, 0x10)) - tree.append({ - 'a': branch['b'], - 'b': (mid-(w-1), len(path), 0, r_rid, r_tag), - 'd': d_ + tdepth, - 'c': 'b', - }) + tree.add(TBranch( + a=branch.b, + b=(mid-(w-1), len(path), 0, r_rid, r_tag), + d=d_ + tdepth, + c='b', + )) # map the tree into our metadata space for branch in rtree: - a_rid, a_tag = branch['a'] - b_rid, b_tag = branch['b'] - tree.append({ - 'a': (mid-(w-1), len(path), 0, a_rid, a_tag), - 'b': (mid-(w-1), len(path), 0, b_rid, b_tag), - 'd': (d_ + tdepth + 1 - + branch['d'] + mdepth-rdepth), - 'c': branch['c'], - }) + a_rid, a_tag = branch.a + b_rid, b_tag = branch.b + tree.add(TBranch( + a=(mid-(w-1), len(path), 0, a_rid, a_tag), + b=(mid-(w-1), len(path), 0, b_rid, b_tag), + d=(d_ + tdepth + 1 + + branch.d + mdepth-rdepth), + c=branch.c, + )) # remap branches to leaves if we aren't showing inner branches if not args.get('inner'): # step through each layer backwards - b_depth = max((branch['a'][1]+1 for branch in tree), - default=0) + b_depth = max((branch.a[1]+1 for branch in tree), default=0) - # keep track of the original tree to find the original bids, - # unfortunately because we store the bids in the branches we - # overwrite these - tree_ = tree.copy() + # keep track of the original bids, unfortunately because we + # store the bids in the branches we overwrite these + tree = {(branch.b[0] - branch.b[2], branch) + for branch in tree} for bd in reversed(range(b_depth-1)): # find leaf-roots at this level roots = {} - for branch_, branch in zip(tree_, tree): - bid = branch['b'][0] - branch['b'][2] + for bid, branch in tree: # choose the highest node as the root - if (branch_['b'][1] == b_depth-1 + if (branch.b[1] == b_depth-1 and (bid not in roots - or branch_['d'] < roots[bid]['d'])): - roots[bid] = branch_ + or branch.d < roots[bid].d)): + roots[bid] = branch # remap branches to leaf-roots - tree__ = [] - for branch_ in tree_: + tree_ = set() + for bid, branch in tree: # note we ignore mroot branches, we don't collapse # normally these - if (branch_['a'][0] != -1 - and branch_['a'][1] == bd - and branch_['a'][0] in roots): - branch_ = { - 'a': roots[branch_['a'][0]]['b'], - 'b': branch_['b'], - 'd': branch_['d'], - 'c': branch_['c'], - } - if (branch_['b'][0] != -1 - and branch_['b'][1] == bd - and branch_['b'][0] in roots): - branch_ = { - 'a': branch_['a'], - 'b': roots[branch_['b'][0]]['b'], - 'd': branch_['d'], - 'c': branch_['c'], - } - tree__.append(branch_) - tree_ = tree__ - tree = tree_ + if (branch.a[0] != -1 + and branch.a[1] == bd + and branch.a[0] in roots): + branch = TBranch( + a=roots[branch.a[0]].b, + b=branch.b, + d=branch.d, + c=branch.c, + ) + if (branch.b[0] != -1 + and branch.b[1] == bd + and branch.b[0] in roots): + branch = TBranch( + a=branch.a, + b=roots[branch.b[0]].b, + d=branch.d, + c=branch.c, + ) + tree_.add((bid, branch)) + tree = tree_ + + # strip out bids + tree = {branch for _, branch in tree} # precompute B-tree if requested elif args.get('btree'): # compute mroot chain "tree", prefix our actual mtree with this - tree = [] + tree = set() mroot_ = Rbyd.fetch(f, block_size, mroots) for d in it.count(): # corrupted? @@ -1077,12 +1083,12 @@ def main(disk, mroots=None, *, if d > 0: done, rid, tag, w, j, _, data, _ = mroot_.lookup(-1, 0x10) if not done: - tree.append({ - 'a': (-1, d-1, 0, -1, TAG_MROOT), - 'b': (-1, d, 0, rid, tag), - 'd': 0, - 'c': 'b', - }) + tree.add(TBranch( + a=(-1, d-1, 0, -1, TAG_MROOT), + b=(-1, d, 0, rid, tag), + d=0, + c='b', + )) # fetch the next mroot done, rid, tag, w, j, _, data, _ = mroot_.lookup(-1, TAG_MROOT) @@ -1097,12 +1103,12 @@ def main(disk, mroots=None, *, # connect branch to our first tag done, rid, tag, w, j, _, data, _ = mdir.lookup(-1, 0x10) if not done: - tree.append({ - 'a': (-1, d, 0, -1, TAG_MDIR), - 'b': (0, 0, 0, rid, tag), - 'd': 0, - 'c': 'b', - }) + tree.add(TBranch( + a=(-1, d, 0, -1, TAG_MDIR), + b=(0, 0, 0, rid, tag), + d=0, + c='b', + )) # compute the mtree's B-tree if there is one if mtree: @@ -1112,26 +1118,26 @@ def main(disk, mroots=None, *, inner=args.get('inner')) # connect a branch to the root of the tree - root = min(tree_, key=lambda branch: branch['d'], default=None) + root = min(tree_, key=lambda branch: branch.d, default=None) if root: - r_bid, r_bd, r_rid, r_tag = root['a'] - tree.append({ - 'a': (-1, d, 0, -1, TAG_BTREE), - 'b': (r_bid, r_bd, r_rid, 0, r_tag), - 'd': 0, - 'c': 'b', - }) + r_bid, r_bd, r_rid, r_tag = root.a + tree.add(TBranch( + a=(-1, d, 0, -1, TAG_BTREE), + b=(r_bid, r_bd, r_rid, 0, r_tag), + d=0, + c='b', + )) # map the tree into our metadata space for branch in tree_: - a_bid, a_bd, a_rid, a_tag = branch['a'] - b_bid, b_bd, b_rid, b_tag = branch['b'] - tree.append({ - 'a': (a_bid, a_bd, a_rid, 0, a_tag), - 'b': (b_bid, b_bd, b_rid, 0, b_tag), - 'd': 1 + branch['d'], - 'c': branch['c'], - }) + a_bid, a_bd, a_rid, a_tag = branch.a + b_bid, b_bd, b_rid, b_tag = branch.b + tree.add(TBranch( + a=(a_bid, a_bd, a_rid, 0, a_tag), + b=(b_bid, b_bd, b_rid, 0, b_tag), + d=1 + branch.d, + c=branch.c, + )) # remap branches to leaves if we aren't showing inner branches if not args.get('inner'): @@ -1166,31 +1172,31 @@ def main(disk, mroots=None, *, done, rid, tag, _, j, d, data, _ = ( mdir_.lookup(-1, 0x10)) - tree_ = [] + tree_ = set() for branch in tree: - if branch['a'][0] == mid-(w-1): - a_bid, a_bd, _, _, _ = branch['a'] - branch = { - 'a': (a_bid, a_bd+1, 0, rid, tag), - 'b': branch['b'], - 'd': branch['d'], - 'c': branch['c'], - } - if branch['b'][0] == mid-(w-1): - b_bid, b_bd, _, _, _ = branch['b'] - branch = { - 'a': branch['a'], - 'b': (b_bid, b_bd+1, 0, rid, tag), - 'd': branch['d'], - 'c': branch['c'], - } - tree_.append(branch) + if branch.a[0] == mid-(w-1): + a_bid, a_bd, _, _, _ = branch.a + branch = TBranch( + a=(a_bid, a_bd+1, 0, rid, tag), + b=branch.b, + d=branch.d, + c=branch.c, + ) + if branch.b[0] == mid-(w-1): + b_bid, b_bd, _, _, _ = branch.b + branch = TBranch( + a=branch.a, + b=(b_bid, b_bd+1, 0, rid, tag), + d=branch.d, + c=branch.c, + ) + tree_.add(branch) tree = tree_ # common tree renderer if args.get('tree') or args.get('btree'): # find the max depth from the tree - t_depth = max((branch['d']+1 for branch in tree), default=0) + t_depth = max((branch.d+1 for branch in tree), default=0) if t_depth > 0: t_width = 2*t_depth + 2 @@ -1200,27 +1206,27 @@ def main(disk, mroots=None, *, def branchrepr(x, d, was): for branch in tree: - if branch['d'] == d and branch['b'] == x: - if any(branch['d'] == d and branch['a'] == x + if branch.d == d and branch.b == x: + if any(branch.d == d and branch.a == x for branch in tree): - return '+-', branch['c'], branch['c'] - elif any(branch['d'] == d - and x > min(branch['a'], branch['b']) - and x < max(branch['a'], branch['b']) + return '+-', branch.c, branch.c + elif any(branch.d == d + and x > min(branch.a, branch.b) + and x < max(branch.a, branch.b) for branch in tree): - return '|-', branch['c'], branch['c'] - elif branch['a'] < branch['b']: - return '\'-', branch['c'], branch['c'] + return '|-', branch.c, branch.c + elif branch.a < branch.b: + return '\'-', branch.c, branch.c else: - return '.-', branch['c'], branch['c'] + return '.-', branch.c, branch.c for branch in tree: - if branch['d'] == d and branch['a'] == x: - return '+ ', branch['c'], None + if branch.d == d and branch.a == x: + return '+ ', branch.c, None for branch in tree: - if (branch['d'] == d - and x > min(branch['a'], branch['b']) - and x < max(branch['a'], branch['b'])): - return '| ', branch['c'], was + if (branch.d == d + and x > min(branch.a, branch.b) + and x < max(branch.a, branch.b)): + return '| ', branch.c, was if was: return '--', was, was return ' ', None, None diff --git a/scripts/dbgrbyd.py b/scripts/dbgrbyd.py index 11056c8a..96a1789e 100755 --- a/scripts/dbgrbyd.py +++ b/scripts/dbgrbyd.py @@ -660,24 +660,25 @@ def dbg_tree(data, block_size, rev, trunk, weight, *, t_depth = max((alt['h']+1 for alt in alts.values()), default=0) # convert to more general tree representation - tree = [] + TBranch = co.namedtuple('TBranch', 'a, b, d, c') + tree = set() for j, alt in alts.items(): # note all non-trunk edges should be black - tree.append({ - 'a': alt['nft'], - 'b': alt['nft'], - 'd': t_depth-1 - alt['h'], - 'c': alt['c'], - }) - tree.append({ - 'a': alt['nft'], - 'b': alt['ft'], - 'd': t_depth-1 - alt['h'], - 'c': 'b', - }) + tree.add(TBranch( + a=alt['nft'], + b=alt['nft'], + d=t_depth-1 - alt['h'], + c=alt['c'], + )) + tree.add(TBranch( + a=alt['nft'], + b=alt['ft'], + d=t_depth-1 - alt['h'], + c='b', + )) # find the max depth from the tree - t_depth = max((branch['d']+1 for branch in tree), default=0) + t_depth = max((branch.d+1 for branch in tree), default=0) if t_depth > 0: t_width = 2*t_depth + 2 @@ -687,27 +688,27 @@ def dbg_tree(data, block_size, rev, trunk, weight, *, def branchrepr(x, d, was): for branch in tree: - if branch['d'] == d and branch['b'] == x: - if any(branch['d'] == d and branch['a'] == x + if branch.d == d and branch.b == x: + if any(branch.d == d and branch.a == x for branch in tree): - return '+-', branch['c'], branch['c'] - elif any(branch['d'] == d - and x > min(branch['a'], branch['b']) - and x < max(branch['a'], branch['b']) + return '+-', branch.c, branch.c + elif any(branch.d == d + and x > min(branch.a, branch.b) + and x < max(branch.a, branch.b) for branch in tree): - return '|-', branch['c'], branch['c'] - elif branch['a'] < branch['b']: - return '\'-', branch['c'], branch['c'] + return '|-', branch.c, branch.c + elif branch.a < branch.b: + return '\'-', branch.c, branch.c else: - return '.-', branch['c'], branch['c'] + return '.-', branch.c, branch.c for branch in tree: - if branch['d'] == d and branch['a'] == x: - return '+ ', branch['c'], None + if branch.d == d and branch.a == x: + return '+ ', branch.c, None for branch in tree: - if (branch['d'] == d - and x > min(branch['a'], branch['b']) - and x < max(branch['a'], branch['b'])): - return '| ', branch['c'], was + if (branch.d == d + and x > min(branch.a, branch.b) + and x < max(branch.a, branch.b)): + return '| ', branch.c, was if was: return '--', was, was return ' ', None, None