diff --git a/scripts/dbgbtree.py b/scripts/dbgbtree.py index f3b8fc25..523650b5 100755 --- a/scripts/dbgbtree.py +++ b/scripts/dbgbtree.py @@ -292,119 +292,6 @@ def tagrepr(tag, weight=None, size=None, *, ' w%d' % weight if weight is not None else '', ' %d' % size if size is not None else '') -# tree branches are an abstract thing for tree rendering -class TreeBranch(co.namedtuple('TreeBranch', ['a', 'b', 'depth', 'color'])): - __slots__ = () - def __new__(cls, a, b, depth=0, color='b'): - # a and b are context specific - return super().__new__(cls, a, b, depth, color) - - def __repr__(self): - return '%s(%s, %s, %s, %s)' % ( - self.__class__.__name__, - self.a, - self.b, - self.depth, - self.color) - - # don't include color in branch comparisons, or else our tree - # renderings can end up with inconsistent colors between runs - def __eq__(self, other): - return (self.a, self.b, self.depth) == (other.a, other.b, other.depth) - - def __ne__(self, other): - return (self.a, self.b, self.depth) != (other.a, other.b, other.depth) - - def __hash__(self): - return hash((self.a, self.b, self.depth)) - - # also order by depth first, which can be useful for reproducibly - # prioritizing branches when simplifying trees - def __lt__(self, other): - return (self.depth, self.a, self.b) < (other.depth, other.a, other.b) - - def __le__(self, other): - return (self.depth, self.a, self.b) <= (other.depth, other.a, other.b) - - def __gt__(self, other): - return (self.depth, self.a, self.b) > (other.depth, other.a, other.b) - - def __ge__(self, other): - return (self.depth, self.a, self.b) >= (other.depth, other.a, other.b) - - # apply a function to a/b while trying to avoid copies - def map(self, filter_, map_=None): - if map_ is None: - filter_, map_ = None, filter_ - - a = self.a - if filter_ is None or filter_(a): - a = map_(a) - - b = self.b - if filter_ is None or filter_(b): - b = map_(b) - - if a != self.a or b != self.b: - return self.__class__( - a if a != self.a else self.a, - b if b != self.b else self.b, - self.depth, - self.color) - else: - return self - -# render some nice ascii trees -def treerepr(tree, x, depth=None, color=False): - # find the max depth from the tree - if depth is None: - depth = max((t.depth+1 for t in tree), default=0) - if depth == 0: - return '' - - def branchrepr(tree, x, d, was): - for t in tree: - if t.depth == d and t.b == x: - if any(t.depth == d and t.a == x - for t in tree): - return '+-', t.color, t.color - elif any(t.depth == d - and x > min(t.a, t.b) - and x < max(t.a, t.b) - for t in tree): - return '|-', t.color, t.color - elif t.a < t.b: - return '\'-', t.color, t.color - else: - return '.-', t.color, t.color - for t in tree: - if t.depth == d and t.a == x: - return '+ ', t.color, None - for t in tree: - if (t.depth == d - and x > min(t.a, t.b) - and x < max(t.a, t.b)): - return '| ', t.color, was - if was: - return '--', was, was - return ' ', None, None - - trunk = [] - was = None - for d in range(depth): - t, c, was = branchrepr(tree, x, d, was) - - trunk.append('%s%s%s%s' % ( - '\x1b[33m' if color and c == 'y' - else '\x1b[31m' if color and c == 'r' - else '\x1b[90m' if color and c == 'b' - else '', - t, - ('>' if was else ' ') if d == depth-1 else '', - '\x1b[m' if color and c else '')) - - return '%s ' % ''.join(trunk) - # compute the difference between two paths, returning everything # in a after the paths diverge, as well as the relevant index def pathdelta(a, b): @@ -1008,115 +895,6 @@ class Rbyd: return best - # create an rbyd tree for debugging - def _tree_rtree(self, **args): - trunks = co.defaultdict(lambda: (-1, 0)) - alts = co.defaultdict(lambda: {}) - - for rid, rattr, path in self.rattrs(path=True): - # keep track of trunks/alts - trunks[rattr.toff] = (rid, rattr.tag) - - for ralt in path: - if ralt.followed: - alts[ralt.toff] |= {'f': ralt.joff, 'c': ralt.color} - else: - alts[ralt.toff] |= {'nf': ralt.off, 'c': ralt.color} - - if args.get('tree_rbyd'): - # treat unreachable alts as converging paths - for j_, alt in alts.items(): - if 'f' not in alt: - alt['f'] = alt['nf'] - elif 'nf' not in alt: - alt['nf'] = alt['f'] - - else: - # prune any alts with unreachable edges - pruned = {} - for j, alt in alts.items(): - if 'f' not in alt: - pruned[j] = alt['nf'] - elif 'nf' not in alt: - pruned[j] = alt['f'] - for j in pruned.keys(): - del alts[j] - - for j, alt in alts.items(): - while alt['f'] in pruned: - alt['f'] = pruned[alt['f']] - while alt['nf'] in pruned: - alt['nf'] = pruned[alt['nf']] - - # find the trunk and depth of each alt - def rec_trunk(j): - if j not in alts: - return trunks[j] - else: - if 'nft' not in alts[j]: - alts[j]['nft'] = rec_trunk(alts[j]['nf']) - return alts[j]['nft'] - - for j in alts.keys(): - rec_trunk(j) - for j, alt in alts.items(): - if alt['f'] in alts: - alt['ft'] = alts[alt['f']]['nft'] - else: - alt['ft'] = trunks[alt['f']] - - def rec_height(j): - if j not in alts: - return 0 - else: - if 'h' not in alts[j]: - alts[j]['h'] = max( - rec_height(alts[j]['f']), - rec_height(alts[j]['nf'])) + 1 - return alts[j]['h'] - - for j in alts.keys(): - rec_height(j) - - t_depth = max((alt['h']+1 for alt in alts.values()), default=0) - - # convert to more general tree representation - tree = set() - for j, alt in alts.items(): - # note all non-trunk edges should be colored black - tree.add(TreeBranch( - alt['nft'], - alt['nft'], - t_depth-1 - alt['h'], - alt['c'])) - if alt['ft'] != alt['nft']: - tree.add(TreeBranch( - alt['nft'], - alt['ft'], - t_depth-1 - alt['h'], - 'b')) - - return tree - - # create a btree tree for debugging - def _tree_btree(self, **args): - # for rbyds this is just a pointer to ever rid - tree = set() - root = None - for rid, name in self.rids(): - b = (rid, name.tag) - if root is None: - root = b - tree.add(TreeBranch(root, b)) - return tree - - # create tree representation for debugging - def tree(self, **args): - if args.get('tree_btree'): - return self._tree_btree(**args) - else: - return self._tree_rtree(**args) - # our rbyd btree type class Btree: @@ -1501,8 +1279,248 @@ class Btree: else: return bid, name - # create an rbyd tree for debugging - def _tree_rtree(self, *, + + +# tree renderer +class TreeArt: + # tree branches are an abstract thing for tree rendering + class Branch(co.namedtuple('Branch', ['a', 'b', 'z', 'color'])): + __slots__ = () + def __new__(cls, a, b, z=0, color='b'): + # a and b are context specific + return super().__new__(cls, a, b, z, color) + + def __repr__(self): + return '%s(%s, %s, %s, %s)' % ( + self.__class__.__name__, + self.a, + self.b, + self.z, + self.color) + + # don't include color in branch comparisons, or else our tree + # renderings can end up with inconsistent colors between runs + def __eq__(self, other): + return (self.a, self.b, self.z) == (other.a, other.b, other.z) + + def __ne__(self, other): + return (self.a, self.b, self.z) != (other.a, other.b, other.z) + + def __hash__(self): + return hash((self.a, self.b, self.z)) + + # also order by z first, which can be useful for reproducibly + # prioritizing branches when simplifying trees + def __lt__(self, other): + return (self.z, self.a, self.b) < (other.z, other.a, other.b) + + def __le__(self, other): + return (self.z, self.a, self.b) <= (other.z, other.a, other.b) + + def __gt__(self, other): + return (self.z, self.a, self.b) > (other.z, other.a, other.b) + + def __ge__(self, other): + return (self.z, self.a, self.b) >= (other.z, other.a, other.b) + + # apply a function to a/b while trying to avoid copies + def map(self, filter_, map_=None): + if map_ is None: + filter_, map_ = None, filter_ + + a = self.a + if filter_ is None or filter_(a): + a = map_(a) + + b = self.b + if filter_ is None or filter_(b): + b = map_(b) + + if a != self.a or b != self.b: + return self.__class__( + a if a != self.a else self.a, + b if b != self.b else self.b, + self.z, + self.color) + else: + return self + + def __init__(self, tree): + self.tree = tree + self.depth = max((t.z+1 for t in tree), default=0) + if self.depth > 0: + self.width = 2*self.depth + 2 + else: + self.width = 0 + + def __iter__(self): + return iter(self.tree) + + # render an rbyd rbyd tree for debugging + @classmethod + def _fromrbydrtree(cls, rbyd, **args): + trunks = co.defaultdict(lambda: (-1, 0)) + alts = co.defaultdict(lambda: {}) + + for rid, rattr, path in rbyd.rattrs(path=True): + # keep track of trunks/alts + trunks[rattr.toff] = (rid, rattr.tag) + + for ralt in path: + if ralt.followed: + alts[ralt.toff] |= {'f': ralt.joff, 'c': ralt.color} + else: + alts[ralt.toff] |= {'nf': ralt.off, 'c': ralt.color} + + if args.get('tree_rbyd'): + # treat unreachable alts as converging paths + for j_, alt in alts.items(): + if 'f' not in alt: + alt['f'] = alt['nf'] + elif 'nf' not in alt: + alt['nf'] = alt['f'] + + else: + # prune any alts with unreachable edges + pruned = {} + for j, alt in alts.items(): + if 'f' not in alt: + pruned[j] = alt['nf'] + elif 'nf' not in alt: + pruned[j] = alt['f'] + for j in pruned.keys(): + del alts[j] + + for j, alt in alts.items(): + while alt['f'] in pruned: + alt['f'] = pruned[alt['f']] + while alt['nf'] in pruned: + alt['nf'] = pruned[alt['nf']] + + # find the trunk and depth of each alt + def rec_trunk(j): + if j not in alts: + return trunks[j] + else: + if 'nft' not in alts[j]: + alts[j]['nft'] = rec_trunk(alts[j]['nf']) + return alts[j]['nft'] + + for j in alts.keys(): + rec_trunk(j) + for j, alt in alts.items(): + if alt['f'] in alts: + alt['ft'] = alts[alt['f']]['nft'] + else: + alt['ft'] = trunks[alt['f']] + + def rec_height(j): + if j not in alts: + return 0 + else: + if 'h' not in alts[j]: + alts[j]['h'] = max( + rec_height(alts[j]['f']), + rec_height(alts[j]['nf'])) + 1 + return alts[j]['h'] + + for j in alts.keys(): + rec_height(j) + + t_depth = max((alt['h']+1 for alt in alts.values()), default=0) + + # convert to more general tree representation + tree = set() + for j, alt in alts.items(): + # note all non-trunk edges should be colored black + tree.add(cls.Branch( + alt['nft'], + alt['nft'], + t_depth-1 - alt['h'], + alt['c'])) + if alt['ft'] != alt['nft']: + tree.add(cls.Branch( + alt['nft'], + alt['ft'], + t_depth-1 - alt['h'], + 'b')) + + return cls(tree) + + # render an rbyd btree tree for debugging + @classmethod + def _fromrbydbtree(cls, rbyd, **args): + # for rbyds this is just a pointer to every rid + tree = set() + root = None + for rid, name in rbyd.rids(): + b = (rid, name.tag) + if root is None: + root = b + tree.add(cls.Branch(root, b)) + return cls(tree) + + # render an rbyd tree for debugging + @classmethod + def fromrbyd(cls, rbyd, **args): + if args.get('tree_btree'): + return cls._fromrbydbtree(rbyd, **args) + else: + return cls._fromrbydrtree(rbyd, **args) + + # render some nice ascii trees + def repr(self, x, color=False): + if self.depth == 0: + return '' + + def branchrepr(tree, x, d, was): + for t in tree: + if t.z == d and t.b == x: + if any(t.z == d and t.a == x + for t in tree): + return '+-', t.color, t.color + elif any(t.z == d + and x > min(t.a, t.b) + and x < max(t.a, t.b) + for t in tree): + return '|-', t.color, t.color + elif t.a < t.b: + return '\'-', t.color, t.color + else: + return '.-', t.color, t.color + for t in tree: + if t.z == d and t.a == x: + return '+ ', t.color, None + for t in tree: + if (t.z == d + and x > min(t.a, t.b) + and x < max(t.a, t.b)): + return '| ', t.color, was + if was: + return '--', was, was + return ' ', None, None + + trunk = [] + was = None + for d in range(self.depth): + t, c, was = branchrepr(self.tree, x, d, was) + + trunk.append('%s%s%s%s' % ( + '\x1b[33m' if color and c == 'y' + else '\x1b[31m' if color and c == 'r' + else '\x1b[90m' if color and c == 'b' + else '', + t, + ('>' if was else ' ') if d == self.depth-1 else '', + '\x1b[m' if color and c else '')) + + return '%s ' % ''.join(trunk) + + # some more renderers + + # render a btree rbyd tree for debugging + @classmethod + def _frombtreertree(cls, btree, *, depth=None, inner=False, **args): @@ -1510,18 +1528,17 @@ class Btree: # to nicely align trees rtrees = {} rdepths = {} - for bid, rbyd, path in self.traverse(path=True, depth=depth): + for bid, rbyd, path in btree.traverse(path=True, depth=depth): if not rbyd: continue - rtrees[rbyd] = rbyd.tree(**args) - rdepths[len(path)] = max( - rdepths.get(len(path), 0), - max((t.depth+1 for t in rtrees[rbyd]), default=0)) + rtree = cls.fromrbyd(rbyd, **args) + rtrees[rbyd] = rtree + rdepths[len(path)] = max(rdepths.get(len(path), 0), rtree.depth) # map rbyd branches into our btree space tree = set() - for bid, rbyd, path in self.traverse(path=True, depth=depth): + for bid, rbyd, path in btree.traverse(path=True, depth=depth): if not rbyd: continue @@ -1531,7 +1548,7 @@ class Btree: continue rtree = rtrees[rbyd] - rdepth = max((t.depth+1 for t in rtree), default=0) + rz = max((t.z+1 for t in rtree), default=0) d = sum(rdepths[d]+1 for d in range(len(path))) # map into our btree space @@ -1542,10 +1559,10 @@ class Btree: b_rid, b_tag = t.b _, (_, a_w, _) = rbyd.lookupnext(a_rid) _, (_, b_w, _) = rbyd.lookupnext(b_rid) - tree.add(TreeBranch( + tree.add(cls.Branch( (bid-(rbyd.weight-1)+a_rid-(a_w-1), len(path), a_tag), (bid-(rbyd.weight-1)+b_rid-(b_w-1), len(path), b_tag), - d + rdepths[len(path)]-rdepth + t.depth, + d + rdepths[len(path)]-rz + t.z, t.color)) # connect rbyd branches to rbyd roots @@ -1554,12 +1571,12 @@ class Btree: l_branch = l_rbyd.lookup(l_rid, TAG_BRANCH, 0x3) if rtree: - r_rid, r_tag = min(rtree, key=lambda t: t.depth).a + r_rid, r_tag = min(rtree, key=lambda t: t.z).a _, (_, r_w, _) = rbyd.lookupnext(r_rid) else: r_rid, (r_tag, r_w, _) = rbyd.lookupnext(-1) - tree.add(TreeBranch( + tree.add(cls.Branch( (l_bid-(l_name.weight-1), len(path)-1, l_branch.tag), (bid-(rbyd.weight-1)+r_rid-(r_w-1), len(path), r_tag), d-1)) @@ -1594,10 +1611,11 @@ class Btree: lambda x: roots[x[0]].a) for t in tree} - return tree + return cls(tree) - # create a btree tree for debugging - def _tree_btree(self, *, + # render a btree btree tree for debugging + @classmethod + def _frombtreebtree(cls, btree, *, depth=None, inner=False, **args): @@ -1605,7 +1623,7 @@ class Btree: tree = set() root = None branches = {} - for bid, name, path in self.bids( + for bid, name, path in btree.bids( path=True, depth=depth): # create branch for each jump in path @@ -1631,17 +1649,18 @@ class Btree: if root is None: root, a = b, b - tree.add(TreeBranch(a, b, d)) + tree.add(cls.Branch(a, b, d)) a = b - return tree + return cls(tree) - # create tree representation for debugging - def tree(self, **args): + # render a btree tree for debugging + @classmethod + def frombtree(cls, btree, **args): if args.get('tree_btree'): - return self._tree_btree(**args) + return cls._frombtreebtree(btree, **args) else: - return self._tree_rtree(**args) + return cls._frombtreertree(btree, **args) @@ -1703,12 +1722,8 @@ def main(disk, roots=None, *, if (args.get('tree') or args.get('tree_rbyd') or args.get('tree_btree')): - tree = btree.tree(**args) - - # find the max depth from the tree - t_depth = max((t.depth+1 for t in tree), default=0) - if t_depth > 0: - t_width = 2*t_depth + 2 + treeart = TreeArt.frombtree(btree, **args) + t_width = treeart.width # dynamically size the id field w_width = mt.ceil(mt.log10(max(1, btree.weight)+1)) @@ -1725,8 +1740,7 @@ def main(disk, roots=None, *, '%04x.%04x:' % (rbyd.block, rbyd.trunk) if prbyd is None or rbyd != prbyd else '', - treerepr(tree, (bid-(name.weight-1), d, rattr.tag), - t_depth, color) + treeart.repr((bid-(name.weight-1), d, rattr.tag), color) if args.get('tree') or args.get('tree_rbyd') or args.get('tree_btree') diff --git a/scripts/dbglfs.py b/scripts/dbglfs.py index f2c5de2b..86ddd059 100755 --- a/scripts/dbglfs.py +++ b/scripts/dbglfs.py @@ -341,119 +341,6 @@ def tagrepr(tag, weight=None, size=None, *, ' w%d' % weight if weight is not None else '', ' %d' % size if size is not None else '') -# tree branches are an abstract thing for tree rendering -class TreeBranch(co.namedtuple('TreeBranch', ['a', 'b', 'depth', 'color'])): - __slots__ = () - def __new__(cls, a, b, depth=0, color='b'): - # a and b are context specific - return super().__new__(cls, a, b, depth, color) - - def __repr__(self): - return '%s(%s, %s, %s, %s)' % ( - self.__class__.__name__, - self.a, - self.b, - self.depth, - self.color) - - # don't include color in branch comparisons, or else our tree - # renderings can end up with inconsistent colors between runs - def __eq__(self, other): - return (self.a, self.b, self.depth) == (other.a, other.b, other.depth) - - def __ne__(self, other): - return (self.a, self.b, self.depth) != (other.a, other.b, other.depth) - - def __hash__(self): - return hash((self.a, self.b, self.depth)) - - # also order by depth first, which can be useful for reproducibly - # prioritizing branches when simplifying trees - def __lt__(self, other): - return (self.depth, self.a, self.b) < (other.depth, other.a, other.b) - - def __le__(self, other): - return (self.depth, self.a, self.b) <= (other.depth, other.a, other.b) - - def __gt__(self, other): - return (self.depth, self.a, self.b) > (other.depth, other.a, other.b) - - def __ge__(self, other): - return (self.depth, self.a, self.b) >= (other.depth, other.a, other.b) - - # apply a function to a/b while trying to avoid copies - def map(self, filter_, map_=None): - if map_ is None: - filter_, map_ = None, filter_ - - a = self.a - if filter_ is None or filter_(a): - a = map_(a) - - b = self.b - if filter_ is None or filter_(b): - b = map_(b) - - if a != self.a or b != self.b: - return self.__class__( - a if a != self.a else self.a, - b if b != self.b else self.b, - self.depth, - self.color) - else: - return self - -# render some nice ascii trees -def treerepr(tree, x, depth=None, color=False): - # find the max depth from the tree - if depth is None: - depth = max((t.depth+1 for t in tree), default=0) - if depth == 0: - return '' - - def branchrepr(tree, x, d, was): - for t in tree: - if t.depth == d and t.b == x: - if any(t.depth == d and t.a == x - for t in tree): - return '+-', t.color, t.color - elif any(t.depth == d - and x > min(t.a, t.b) - and x < max(t.a, t.b) - for t in tree): - return '|-', t.color, t.color - elif t.a < t.b: - return '\'-', t.color, t.color - else: - return '.-', t.color, t.color - for t in tree: - if t.depth == d and t.a == x: - return '+ ', t.color, None - for t in tree: - if (t.depth == d - and x > min(t.a, t.b) - and x < max(t.a, t.b)): - return '| ', t.color, was - if was: - return '--', was, was - return ' ', None, None - - trunk = [] - was = None - for d in range(depth): - t, c, was = branchrepr(tree, x, d, was) - - trunk.append('%s%s%s%s' % ( - '\x1b[33m' if color and c == 'y' - else '\x1b[31m' if color and c == 'r' - else '\x1b[90m' if color and c == 'b' - else '', - t, - ('>' if was else ' ') if d == depth-1 else '', - '\x1b[m' if color and c else '')) - - return '%s ' % ''.join(trunk) - # compute the difference between two paths, returning everything # in a after the paths diverge, as well as the relevant index def pathdelta(a, b): @@ -1057,115 +944,6 @@ class Rbyd: return best - # create an rbyd tree for debugging - def _tree_rtree(self, **args): - trunks = co.defaultdict(lambda: (-1, 0)) - alts = co.defaultdict(lambda: {}) - - for rid, rattr, path in self.rattrs(path=True): - # keep track of trunks/alts - trunks[rattr.toff] = (rid, rattr.tag) - - for ralt in path: - if ralt.followed: - alts[ralt.toff] |= {'f': ralt.joff, 'c': ralt.color} - else: - alts[ralt.toff] |= {'nf': ralt.off, 'c': ralt.color} - - if args.get('tree_rbyd'): - # treat unreachable alts as converging paths - for j_, alt in alts.items(): - if 'f' not in alt: - alt['f'] = alt['nf'] - elif 'nf' not in alt: - alt['nf'] = alt['f'] - - else: - # prune any alts with unreachable edges - pruned = {} - for j, alt in alts.items(): - if 'f' not in alt: - pruned[j] = alt['nf'] - elif 'nf' not in alt: - pruned[j] = alt['f'] - for j in pruned.keys(): - del alts[j] - - for j, alt in alts.items(): - while alt['f'] in pruned: - alt['f'] = pruned[alt['f']] - while alt['nf'] in pruned: - alt['nf'] = pruned[alt['nf']] - - # find the trunk and depth of each alt - def rec_trunk(j): - if j not in alts: - return trunks[j] - else: - if 'nft' not in alts[j]: - alts[j]['nft'] = rec_trunk(alts[j]['nf']) - return alts[j]['nft'] - - for j in alts.keys(): - rec_trunk(j) - for j, alt in alts.items(): - if alt['f'] in alts: - alt['ft'] = alts[alt['f']]['nft'] - else: - alt['ft'] = trunks[alt['f']] - - def rec_height(j): - if j not in alts: - return 0 - else: - if 'h' not in alts[j]: - alts[j]['h'] = max( - rec_height(alts[j]['f']), - rec_height(alts[j]['nf'])) + 1 - return alts[j]['h'] - - for j in alts.keys(): - rec_height(j) - - t_depth = max((alt['h']+1 for alt in alts.values()), default=0) - - # convert to more general tree representation - tree = set() - for j, alt in alts.items(): - # note all non-trunk edges should be colored black - tree.add(TreeBranch( - alt['nft'], - alt['nft'], - t_depth-1 - alt['h'], - alt['c'])) - if alt['ft'] != alt['nft']: - tree.add(TreeBranch( - alt['nft'], - alt['ft'], - t_depth-1 - alt['h'], - 'b')) - - return tree - - # create a btree tree for debugging - def _tree_btree(self, **args): - # for rbyds this is just a pointer to ever rid - tree = set() - root = None - for rid, name in self.rids(): - b = (rid, name.tag) - if root is None: - root = b - tree.add(TreeBranch(root, b)) - return tree - - # create tree representation for debugging - def tree(self, **args): - if args.get('tree_btree'): - return self._tree_btree(**args) - else: - return self._tree_rtree(**args) - # our rbyd btree type class Btree: @@ -1550,148 +1328,6 @@ class Btree: else: return bid, name - # create an rbyd tree for debugging - def _tree_rtree(self, *, - depth=None, - inner=False, - **args): - # precompute rbyd trees so we know the max depth at each layer - # to nicely align trees - rtrees = {} - rdepths = {} - for bid, rbyd, path in self.traverse(path=True, depth=depth): - if not rbyd: - continue - - rtrees[rbyd] = rbyd.tree(**args) - rdepths[len(path)] = max( - rdepths.get(len(path), 0), - max((t.depth+1 for t in rtrees[rbyd]), default=0)) - - # map rbyd branches into our btree space - tree = set() - for bid, rbyd, path in self.traverse(path=True, depth=depth): - if not rbyd: - continue - - # yes we can find new rbyds if disk is being mutated, just - # ignore these - if rbyd not in rtrees: - continue - - rtree = rtrees[rbyd] - rdepth = max((t.depth+1 for t in rtree), default=0) - d = sum(rdepths[d]+1 for d in range(len(path))) - - # map into our btree space - for t in rtree: - # note we adjust our bid to be left-leaning, this allows - # a global order and makes tree rendering quite a bit easier - a_rid, a_tag = t.a - b_rid, b_tag = t.b - _, (_, a_w, _) = rbyd.lookupnext(a_rid) - _, (_, b_w, _) = rbyd.lookupnext(b_rid) - tree.add(TreeBranch( - (bid-(rbyd.weight-1)+a_rid-(a_w-1), len(path), a_tag), - (bid-(rbyd.weight-1)+b_rid-(b_w-1), len(path), b_tag), - d + rdepths[len(path)]-rdepth + t.depth, - t.color)) - - # connect rbyd branches to rbyd roots - if path: - l_bid, l_rbyd, l_rid, l_name = path[-1] - l_branch = l_rbyd.lookup(l_rid, TAG_BRANCH, 0x3) - - if rtree: - r_rid, r_tag = min(rtree, key=lambda t: t.depth).a - _, (_, r_w, _) = rbyd.lookupnext(r_rid) - else: - r_rid, (r_tag, r_w, _) = rbyd.lookupnext(-1) - - tree.add(TreeBranch( - (l_bid-(l_name.weight-1), len(path)-1, l_branch.tag), - (bid-(rbyd.weight-1)+r_rid-(r_w-1), len(path), r_tag), - d-1)) - - # remap branches to leaves if we aren't showing inner branches - if not inner: - # step through each btree layer backwards - b_depth = max((t.a[1]+1 for t in tree), default=0) - - for d in reversed(range(b_depth-1)): - # find bid ranges at this level - bids = set() - for t in tree: - if t.b[1] == d: - bids.add(t.b[0]) - bids = sorted(bids) - - # find the best root for each bid range - roots = {} - for i in range(len(bids)): - for t in tree: - if (t.a[1] > d - and t.a[0] >= bids[i] - and (i == len(bids)-1 or t.a[0] < bids[i+1]) - and (bids[i] not in roots - or t < roots[bids[i]])): - roots[bids[i]] = t - - # remap branches to leaf-roots - tree = {t.map( - lambda x: x[1] == d and x[0] in roots, - lambda x: roots[x[0]].a) - for t in tree} - - return tree - - # create a btree tree for debugging - def _tree_btree(self, *, - depth=None, - inner=False, - **args): - # find all branches - tree = set() - root = None - branches = {} - for bid, name, path in self.bids( - path=True, - depth=depth): - # create branch for each jump in path - # - # note we adjust our bid to be left-leaning, this allows - # a global order and makes tree rendering quite a bit easier - a = root - for d, (bid_, rbyd_, rid_, name_) in enumerate(path): - # map into our btree space - bid__ = bid_-(name_.weight-1) - b = (bid__, d, name_.tag) - - # remap branches to leaves if we aren't showing inner - # branches - if not inner: - if b not in branches: - bid_, rbyd_, rid_, name_ = path[-1] - bid__ = bid_-(name_.weight-1) - branches[b] = (bid__, len(path)-1, name_.tag) - b = branches[b] - - # render the root path on first rid, this is arbitrary - if root is None: - root, a = b, b - - tree.add(TreeBranch(a, b, d)) - a = b - - return tree - - # create tree representation for debugging - def tree(self, **args): - if args.get('tree_btree'): - return self._tree_btree(**args) - else: - return self._tree_rtree(**args) - # a metadata id, this includes mbits for convenience class Mid: @@ -1932,24 +1568,6 @@ class Mdir: return Mid(self.mid, rid), name_ - # create tree representation for debugging - def tree(self, **args): - tree = self.rbyd.tree(**args) - - # map to mid - tree_ = set() - for t in tree: - a_rid, a_tag = t.a - b_rid, b_tag = t.b - tree_.add(TreeBranch( - (Mid(self.mid, a_rid), a_tag), - (Mid(self.mid, b_rid), b_tag), - t.depth, - t.color)) - tree = tree_ - - return tree - # the mtree, the skeletal structure of littlefs class Mtree: def __init__(self, bd, mrootchain, mtree, *, @@ -2612,195 +2230,6 @@ class Mtree: return mid_, mdir, name_, path_+[(mid_, mdir, name_)] else: return mid_, mdir, name_ - - # create an rbyd tree for debugging - def _tree_rtree(self, *, - depth=None, - inner=False, - **args): - # precompute rbyd trees so we know the max depth at each layer - # to nicely align trees - rtrees = {} - rdepths = {} - for mdir, path in self.traverse(path=True, depth=depth): - if isinstance(mdir, Mdir): - if not mdir: - continue - rbyd = mdir.rbyd - else: - bid, rbyd = mdir - if not rbyd: - continue - - rtrees[rbyd] = rbyd.tree(**args) - rdepths[len(path)] = max( - rdepths.get(len(path), 0), - max((t.depth+1 for t in rtrees[rbyd]), default=0)) - - # map rbyd branches into our mtree space - tree = set() - branches = {} - for mdir, path in self.traverse(path=True, depth=depth): - if isinstance(mdir, Mdir): - if not mdir: - continue - rbyd = mdir.rbyd - else: - bid, rbyd = mdir - if not rbyd: - continue - - # yes we can find new rbyds if disk is being mutated, just - # ignore these - if rbyd not in rtrees: - continue - - rtree = rtrees[rbyd] - rdepth = max((t.depth+1 for t in rtree), default=0) - d = sum(rdepths[d]+1 for d, p in enumerate(path)) - - # map into our mtree space - for t in rtree: - # note we adjust our mid/bid to be left-leaning, this allows - # a global order and makes tree rendering quite a bit easier - # - # we also need to give btree nodes mrid=-1 so they come - # before and mrid=-1 mdir attrs - a_rid, a_tag = t.a - b_rid, b_tag = t.b - _, (_, a_w, _) = rbyd.lookupnext(a_rid) - _, (_, b_w, _) = rbyd.lookupnext(b_rid) - if isinstance(mdir, Mdir): - a_mid = self.mid(mdir.mid, a_rid) - b_mid = self.mid(mdir.mid, b_rid) - else: - a_mid = self.mid(bid-(rbyd.weight-1)+a_rid-(a_w-1), -1) - b_mid = self.mid(bid-(rbyd.weight-1)+b_rid-(b_w-1), -1) - - tree.add(TreeBranch( - (a_mid, len(path), a_tag), - (b_mid, len(path), b_tag), - d + rdepths[len(path)]-rdepth + t.depth, - t.color)) - - # connect rbyd branches to rbyd roots - if path: - # figure out branch mid/attr - if isinstance(path[-1][1], Mdir): - l_mid, l_mdir, l_name = path[-1] - l_branch = (l_mdir.lookup(l_mid, TAG_MROOT, 0x3) - or l_mdir.lookup(l_mid, TAG_MTREE, 0x3)) - else: - l_bid, l_rbyd, l_rid, l_name = path[-1] - l_mid = self.mid(l_bid-(l_name.weight-1), -1) - l_branch = (l_rbyd.lookup(l_rid, TAG_BRANCH, 0x3) - or l_rbyd.lookup(l_rid, TAG_MDIR, 0x3)) - - # figure out root mid/rattr - if rtree: - r_rid, r_tag = min(rtree, key=lambda t: t.depth).a - _, (_, r_w, _) = rbyd.lookupnext(r_rid) - else: - r_rid, (r_tag, r_w, _) = rbyd.lookupnext(-1) - - if isinstance(mdir, Mdir): - r_mid = self.mid(mdir.mid, r_rid) - else: - r_mid = self.mid(bid-(rbyd.weight-1)+r_rid-(r_w-1), -1) - - tree.add(TreeBranch( - (l_mid, len(path)-1, l_branch.tag), - (r_mid, len(path), r_tag), - d-1)) - - # remap branches to leaves if we aren't showing inner branches - if not inner: - # step through each btree layer backwards - b_depth = max((t.a[1]+1 for t in tree), default=0) - - for d in reversed(range(len(self.mrootchain), b_depth-1)): - # find mid ranges at this level - mids = set() - for t in tree: - if t.b[1] == d: - mids.add(t.b[0]) - mids = sorted(mids) - - # find the best root for each mid range - roots = {} - for i in range(len(mids)): - for t in tree: - if (t.a[1] > d - and t.a[0] >= mids[i] - and (i == len(mids)-1 or t.a[0] < mids[i+1]) - and (mids[i] not in roots - or t < roots[mids[i]])): - roots[mids[i]] = t - - # remap branches to leaf-roots - tree = {t.map( - lambda x: x[1] == d and x[0] in roots, - lambda x: roots[x[0]].a) - for t in tree} - - return tree - - # create a btree tree for debugging - def _tree_btree(self, *, - depth=None, - inner=False, - **args): - tree = set() - root = None - branches = {} - for mid, mdir, name, path in self.mids( - mdirs_only=False, - path=True, - depth=depth): - # create branch for each jump in path - # - # note we adjust our mid/bid to be left-leaning, this allows - # a global order and makes tree rendering quite a bit easier - # - # we also need to give btree nodes mrid=-1 so they come - # before and mrid=-1 mdir attrs - a = root - for d, p in enumerate(path): - # map into our mtree space - if isinstance(p[1], Mdir): - mid_, mdir_, name_ = p - else: - bid_, rbyd_, rid_, name_ = p - mid_ = self.mid(bid_-(name_.weight-1), -1) - b = (mid_, d, name_.tag) - - # remap branches to leaves if we aren't showing inner - # branches - if not inner: - if b not in branches: - if isinstance(path[-1][1], Mdir): - mid_, mdir_, name_ = path[-1] - else: - bid_, rbyd_, rid_, name_ = path[-1] - mid_ = self.mid(bid_-(name_.weight-1), -1) - branches[b] = (mid_, len(path)-1, name_.tag) - b = branches[b] - - # render the root path on first rid, this is arbitrary - if root is None: - root, a = b, b - - tree.add(TreeBranch(a, b, d)) - a = b - - return tree - - # create tree representation for debugging - def tree(self, **args): - if args.get('tree_btree'): - return self._tree_btree(**args) - else: - return self._tree_rtree(**args) # in-btree block pointers @@ -3976,31 +3405,6 @@ class Lfs: depth=None): return b''.join(self.bytes()) - def tree(self, **args): - tree = self.bshrub.tree(**args) - - # find max depth - t_depth = max((t.depth+1 for t in tree), default=0) - - # connect bptr tags to bptrs - tree = set(tree) - bptrs = {} - for pos, data, path in self.datas( - path=True, - depth=args.get('depth')): - if isinstance(data, Bptr): - a = (pos, len(path)-1, data.tag) - b = (pos, len(path), data.tag) - bptrs[a] = b - tree.add(TreeBranch(a, b, t_depth)) - - # if we're not showing inner branches, nudge bptr tags to - # their bptrs - if not args.get('inner'): - tree = {t.map(lambda x: bptrs.get(x, x)) for t in tree} - - return tree - # bleh, with that out of the way, here are our known file types # regular files @@ -4109,6 +3513,414 @@ class Lfs: +# tree renderer +class TreeArt: + # tree branches are an abstract thing for tree rendering + class Branch(co.namedtuple('Branch', ['a', 'b', 'z', 'color'])): + __slots__ = () + def __new__(cls, a, b, z=0, color='b'): + # a and b are context specific + return super().__new__(cls, a, b, z, color) + + def __repr__(self): + return '%s(%s, %s, %s, %s)' % ( + self.__class__.__name__, + self.a, + self.b, + self.z, + self.color) + + # don't include color in branch comparisons, or else our tree + # renderings can end up with inconsistent colors between runs + def __eq__(self, other): + return (self.a, self.b, self.z) == (other.a, other.b, other.z) + + def __ne__(self, other): + return (self.a, self.b, self.z) != (other.a, other.b, other.z) + + def __hash__(self): + return hash((self.a, self.b, self.z)) + + # also order by z first, which can be useful for reproducibly + # prioritizing branches when simplifying trees + def __lt__(self, other): + return (self.z, self.a, self.b) < (other.z, other.a, other.b) + + def __le__(self, other): + return (self.z, self.a, self.b) <= (other.z, other.a, other.b) + + def __gt__(self, other): + return (self.z, self.a, self.b) > (other.z, other.a, other.b) + + def __ge__(self, other): + return (self.z, self.a, self.b) >= (other.z, other.a, other.b) + + # apply a function to a/b while trying to avoid copies + def map(self, filter_, map_=None): + if map_ is None: + filter_, map_ = None, filter_ + + a = self.a + if filter_ is None or filter_(a): + a = map_(a) + + b = self.b + if filter_ is None or filter_(b): + b = map_(b) + + if a != self.a or b != self.b: + return self.__class__( + a if a != self.a else self.a, + b if b != self.b else self.b, + self.z, + self.color) + else: + return self + + def __init__(self, tree): + self.tree = tree + self.depth = max((t.z+1 for t in tree), default=0) + if self.depth > 0: + self.width = 2*self.depth + 2 + else: + self.width = 0 + + def __iter__(self): + return iter(self.tree) + + # render an rbyd rbyd tree for debugging + @classmethod + def _fromrbydrtree(cls, rbyd, **args): + trunks = co.defaultdict(lambda: (-1, 0)) + alts = co.defaultdict(lambda: {}) + + for rid, rattr, path in rbyd.rattrs(path=True): + # keep track of trunks/alts + trunks[rattr.toff] = (rid, rattr.tag) + + for ralt in path: + if ralt.followed: + alts[ralt.toff] |= {'f': ralt.joff, 'c': ralt.color} + else: + alts[ralt.toff] |= {'nf': ralt.off, 'c': ralt.color} + + if args.get('tree_rbyd'): + # treat unreachable alts as converging paths + for j_, alt in alts.items(): + if 'f' not in alt: + alt['f'] = alt['nf'] + elif 'nf' not in alt: + alt['nf'] = alt['f'] + + else: + # prune any alts with unreachable edges + pruned = {} + for j, alt in alts.items(): + if 'f' not in alt: + pruned[j] = alt['nf'] + elif 'nf' not in alt: + pruned[j] = alt['f'] + for j in pruned.keys(): + del alts[j] + + for j, alt in alts.items(): + while alt['f'] in pruned: + alt['f'] = pruned[alt['f']] + while alt['nf'] in pruned: + alt['nf'] = pruned[alt['nf']] + + # find the trunk and depth of each alt + def rec_trunk(j): + if j not in alts: + return trunks[j] + else: + if 'nft' not in alts[j]: + alts[j]['nft'] = rec_trunk(alts[j]['nf']) + return alts[j]['nft'] + + for j in alts.keys(): + rec_trunk(j) + for j, alt in alts.items(): + if alt['f'] in alts: + alt['ft'] = alts[alt['f']]['nft'] + else: + alt['ft'] = trunks[alt['f']] + + def rec_height(j): + if j not in alts: + return 0 + else: + if 'h' not in alts[j]: + alts[j]['h'] = max( + rec_height(alts[j]['f']), + rec_height(alts[j]['nf'])) + 1 + return alts[j]['h'] + + for j in alts.keys(): + rec_height(j) + + t_depth = max((alt['h']+1 for alt in alts.values()), default=0) + + # convert to more general tree representation + tree = set() + for j, alt in alts.items(): + # note all non-trunk edges should be colored black + tree.add(cls.Branch( + alt['nft'], + alt['nft'], + t_depth-1 - alt['h'], + alt['c'])) + if alt['ft'] != alt['nft']: + tree.add(cls.Branch( + alt['nft'], + alt['ft'], + t_depth-1 - alt['h'], + 'b')) + + return cls(tree) + + # render an rbyd btree tree for debugging + @classmethod + def _fromrbydbtree(cls, rbyd, **args): + # for rbyds this is just a pointer to every rid + tree = set() + root = None + for rid, name in rbyd.rids(): + b = (rid, name.tag) + if root is None: + root = b + tree.add(cls.Branch(root, b)) + return cls(tree) + + # render an rbyd tree for debugging + @classmethod + def fromrbyd(cls, rbyd, **args): + if args.get('tree_btree'): + return cls._fromrbydbtree(rbyd, **args) + else: + return cls._fromrbydrtree(rbyd, **args) + + # render some nice ascii trees + def repr(self, x, color=False): + if self.depth == 0: + return '' + + def branchrepr(tree, x, d, was): + for t in tree: + if t.z == d and t.b == x: + if any(t.z == d and t.a == x + for t in tree): + return '+-', t.color, t.color + elif any(t.z == d + and x > min(t.a, t.b) + and x < max(t.a, t.b) + for t in tree): + return '|-', t.color, t.color + elif t.a < t.b: + return '\'-', t.color, t.color + else: + return '.-', t.color, t.color + for t in tree: + if t.z == d and t.a == x: + return '+ ', t.color, None + for t in tree: + if (t.z == d + and x > min(t.a, t.b) + and x < max(t.a, t.b)): + return '| ', t.color, was + if was: + return '--', was, was + return ' ', None, None + + trunk = [] + was = None + for d in range(self.depth): + t, c, was = branchrepr(self.tree, x, d, was) + + trunk.append('%s%s%s%s' % ( + '\x1b[33m' if color and c == 'y' + else '\x1b[31m' if color and c == 'r' + else '\x1b[90m' if color and c == 'b' + else '', + t, + ('>' if was else ' ') if d == self.depth-1 else '', + '\x1b[m' if color and c else '')) + + return '%s ' % ''.join(trunk) + + # some more renderers + + # render a btree rbyd tree for debugging + @classmethod + def _frombtreertree(cls, btree, *, + depth=None, + inner=False, + **args): + # precompute rbyd trees so we know the max depth at each layer + # to nicely align trees + rtrees = {} + rdepths = {} + for bid, rbyd, path in btree.traverse(path=True, depth=depth): + if not rbyd: + continue + + rtree = cls.fromrbyd(rbyd, **args) + rtrees[rbyd] = rtree + rdepths[len(path)] = max(rdepths.get(len(path), 0), rtree.depth) + + # map rbyd branches into our btree space + tree = set() + for bid, rbyd, path in btree.traverse(path=True, depth=depth): + if not rbyd: + continue + + # yes we can find new rbyds if disk is being mutated, just + # ignore these + if rbyd not in rtrees: + continue + + rtree = rtrees[rbyd] + rz = max((t.z+1 for t in rtree), default=0) + d = sum(rdepths[d]+1 for d in range(len(path))) + + # map into our btree space + for t in rtree: + # note we adjust our bid to be left-leaning, this allows + # a global order and makes tree rendering quite a bit easier + a_rid, a_tag = t.a + b_rid, b_tag = t.b + _, (_, a_w, _) = rbyd.lookupnext(a_rid) + _, (_, b_w, _) = rbyd.lookupnext(b_rid) + tree.add(cls.Branch( + (bid-(rbyd.weight-1)+a_rid-(a_w-1), len(path), a_tag), + (bid-(rbyd.weight-1)+b_rid-(b_w-1), len(path), b_tag), + d + rdepths[len(path)]-rz + t.z, + t.color)) + + # connect rbyd branches to rbyd roots + if path: + l_bid, l_rbyd, l_rid, l_name = path[-1] + l_branch = l_rbyd.lookup(l_rid, TAG_BRANCH, 0x3) + + if rtree: + r_rid, r_tag = min(rtree, key=lambda t: t.z).a + _, (_, r_w, _) = rbyd.lookupnext(r_rid) + else: + r_rid, (r_tag, r_w, _) = rbyd.lookupnext(-1) + + tree.add(cls.Branch( + (l_bid-(l_name.weight-1), len(path)-1, l_branch.tag), + (bid-(rbyd.weight-1)+r_rid-(r_w-1), len(path), r_tag), + d-1)) + + # remap branches to leaves if we aren't showing inner branches + if not inner: + # step through each btree layer backwards + b_depth = max((t.a[1]+1 for t in tree), default=0) + + for d in reversed(range(b_depth-1)): + # find bid ranges at this level + bids = set() + for t in tree: + if t.b[1] == d: + bids.add(t.b[0]) + bids = sorted(bids) + + # find the best root for each bid range + roots = {} + for i in range(len(bids)): + for t in tree: + if (t.a[1] > d + and t.a[0] >= bids[i] + and (i == len(bids)-1 or t.a[0] < bids[i+1]) + and (bids[i] not in roots + or t < roots[bids[i]])): + roots[bids[i]] = t + + # remap branches to leaf-roots + tree = {t.map( + lambda x: x[1] == d and x[0] in roots, + lambda x: roots[x[0]].a) + for t in tree} + + return cls(tree) + + # render a btree btree tree for debugging + @classmethod + def _frombtreebtree(cls, btree, *, + depth=None, + inner=False, + **args): + # find all branches + tree = set() + root = None + branches = {} + for bid, name, path in btree.bids( + path=True, + depth=depth): + # create branch for each jump in path + # + # note we adjust our bid to be left-leaning, this allows + # a global order and makes tree rendering quite a bit easier + a = root + for d, (bid_, rbyd_, rid_, name_) in enumerate(path): + # map into our btree space + bid__ = bid_-(name_.weight-1) + b = (bid__, d, name_.tag) + + # remap branches to leaves if we aren't showing inner + # branches + if not inner: + if b not in branches: + bid_, rbyd_, rid_, name_ = path[-1] + bid__ = bid_-(name_.weight-1) + branches[b] = (bid__, len(path)-1, name_.tag) + b = branches[b] + + # render the root path on first rid, this is arbitrary + if root is None: + root, a = b, b + + tree.add(cls.Branch(a, b, d)) + a = b + + return cls(tree) + + # render a btree tree for debugging + @classmethod + def frombtree(cls, btree, **args): + if args.get('tree_btree'): + return cls._frombtreebtree(btree, **args) + else: + return cls._frombtreertree(btree, **args) + + # render a file tree for debugging + @classmethod + def fromfile(cls, file, **args): + tree = cls.frombtree(file.bshrub, **args) + t_depth = tree.depth + + # connect bptr tags to bptrs + tree = set(tree) + bptrs = {} + for pos, data, path in file.datas( + path=True, + depth=args.get('depth')): + if isinstance(data, Bptr): + a = (pos, len(path)-1, data.tag) + b = (pos, len(path), data.tag) + bptrs[a] = b + tree.add(cls.Branch(a, b, t_depth)) + + # if we're not showing inner branches, nudge bptr tags to + # their bptrs + if not args.get('inner'): + tree = {t.map(lambda x: bptrs.get(x, x)) for t in tree} + + return cls(tree) + + + # show the littlefs config def dbg_config(lfs, color=False, @@ -4379,12 +4191,8 @@ def dbg_files(lfs, paths, if (args.get('tree') or args.get('tree_rbyd') or args.get('tree_btree')): - tree = file.tree(**args) - - # find the max depth from the tree - bt_depth = max((t.depth+1 for t in tree), default=0) - if bt_depth > 0: - bt_width = 2*bt_depth + 2 + treeart = TreeArt.fromfile(file, **args) + bt_width = treeart.width # dynamically size the id field bw_width = mt.ceil(mt.log10(max(1, file.size)+1)) @@ -4399,8 +4207,9 @@ def dbg_files(lfs, paths, if pblock is None or rbyd.block != pblock else '', 2*w_width+1, '', - treerepr(tree, (bid-(name.weight-1), d, rattr.tag), - bt_depth, color) + treeart.repr( + (bid-(name.weight-1), d, rattr.tag), + color) if args.get('tree') or args.get('tree_rbyd') or args.get('tree_btree') @@ -4451,8 +4260,7 @@ def dbg_files(lfs, paths, else '', '\x1b[0m' if color and notes else '', 2*w_width+1, '', - treerepr(tree, (pos, d, bptr.tag), - bt_depth, color) + treeart.repr((pos, d, bptr.tag), color) if args.get('tree') or args.get('tree_rbyd') or args.get('tree_btree') diff --git a/scripts/dbgmtree.py b/scripts/dbgmtree.py index 58e2ab62..a296c19f 100755 --- a/scripts/dbgmtree.py +++ b/scripts/dbgmtree.py @@ -307,119 +307,6 @@ def tagrepr(tag, weight=None, size=None, *, ' w%d' % weight if weight is not None else '', ' %d' % size if size is not None else '') -# tree branches are an abstract thing for tree rendering -class TreeBranch(co.namedtuple('TreeBranch', ['a', 'b', 'depth', 'color'])): - __slots__ = () - def __new__(cls, a, b, depth=0, color='b'): - # a and b are context specific - return super().__new__(cls, a, b, depth, color) - - def __repr__(self): - return '%s(%s, %s, %s, %s)' % ( - self.__class__.__name__, - self.a, - self.b, - self.depth, - self.color) - - # don't include color in branch comparisons, or else our tree - # renderings can end up with inconsistent colors between runs - def __eq__(self, other): - return (self.a, self.b, self.depth) == (other.a, other.b, other.depth) - - def __ne__(self, other): - return (self.a, self.b, self.depth) != (other.a, other.b, other.depth) - - def __hash__(self): - return hash((self.a, self.b, self.depth)) - - # also order by depth first, which can be useful for reproducibly - # prioritizing branches when simplifying trees - def __lt__(self, other): - return (self.depth, self.a, self.b) < (other.depth, other.a, other.b) - - def __le__(self, other): - return (self.depth, self.a, self.b) <= (other.depth, other.a, other.b) - - def __gt__(self, other): - return (self.depth, self.a, self.b) > (other.depth, other.a, other.b) - - def __ge__(self, other): - return (self.depth, self.a, self.b) >= (other.depth, other.a, other.b) - - # apply a function to a/b while trying to avoid copies - def map(self, filter_, map_=None): - if map_ is None: - filter_, map_ = None, filter_ - - a = self.a - if filter_ is None or filter_(a): - a = map_(a) - - b = self.b - if filter_ is None or filter_(b): - b = map_(b) - - if a != self.a or b != self.b: - return self.__class__( - a if a != self.a else self.a, - b if b != self.b else self.b, - self.depth, - self.color) - else: - return self - -# render some nice ascii trees -def treerepr(tree, x, depth=None, color=False): - # find the max depth from the tree - if depth is None: - depth = max((t.depth+1 for t in tree), default=0) - if depth == 0: - return '' - - def branchrepr(tree, x, d, was): - for t in tree: - if t.depth == d and t.b == x: - if any(t.depth == d and t.a == x - for t in tree): - return '+-', t.color, t.color - elif any(t.depth == d - and x > min(t.a, t.b) - and x < max(t.a, t.b) - for t in tree): - return '|-', t.color, t.color - elif t.a < t.b: - return '\'-', t.color, t.color - else: - return '.-', t.color, t.color - for t in tree: - if t.depth == d and t.a == x: - return '+ ', t.color, None - for t in tree: - if (t.depth == d - and x > min(t.a, t.b) - and x < max(t.a, t.b)): - return '| ', t.color, was - if was: - return '--', was, was - return ' ', None, None - - trunk = [] - was = None - for d in range(depth): - t, c, was = branchrepr(tree, x, d, was) - - trunk.append('%s%s%s%s' % ( - '\x1b[33m' if color and c == 'y' - else '\x1b[31m' if color and c == 'r' - else '\x1b[90m' if color and c == 'b' - else '', - t, - ('>' if was else ' ') if d == depth-1 else '', - '\x1b[m' if color and c else '')) - - return '%s ' % ''.join(trunk) - # compute the difference between two paths, returning everything # in a after the paths diverge, as well as the relevant index def pathdelta(a, b): @@ -1023,115 +910,6 @@ class Rbyd: return best - # create an rbyd tree for debugging - def _tree_rtree(self, **args): - trunks = co.defaultdict(lambda: (-1, 0)) - alts = co.defaultdict(lambda: {}) - - for rid, rattr, path in self.rattrs(path=True): - # keep track of trunks/alts - trunks[rattr.toff] = (rid, rattr.tag) - - for ralt in path: - if ralt.followed: - alts[ralt.toff] |= {'f': ralt.joff, 'c': ralt.color} - else: - alts[ralt.toff] |= {'nf': ralt.off, 'c': ralt.color} - - if args.get('tree_rbyd'): - # treat unreachable alts as converging paths - for j_, alt in alts.items(): - if 'f' not in alt: - alt['f'] = alt['nf'] - elif 'nf' not in alt: - alt['nf'] = alt['f'] - - else: - # prune any alts with unreachable edges - pruned = {} - for j, alt in alts.items(): - if 'f' not in alt: - pruned[j] = alt['nf'] - elif 'nf' not in alt: - pruned[j] = alt['f'] - for j in pruned.keys(): - del alts[j] - - for j, alt in alts.items(): - while alt['f'] in pruned: - alt['f'] = pruned[alt['f']] - while alt['nf'] in pruned: - alt['nf'] = pruned[alt['nf']] - - # find the trunk and depth of each alt - def rec_trunk(j): - if j not in alts: - return trunks[j] - else: - if 'nft' not in alts[j]: - alts[j]['nft'] = rec_trunk(alts[j]['nf']) - return alts[j]['nft'] - - for j in alts.keys(): - rec_trunk(j) - for j, alt in alts.items(): - if alt['f'] in alts: - alt['ft'] = alts[alt['f']]['nft'] - else: - alt['ft'] = trunks[alt['f']] - - def rec_height(j): - if j not in alts: - return 0 - else: - if 'h' not in alts[j]: - alts[j]['h'] = max( - rec_height(alts[j]['f']), - rec_height(alts[j]['nf'])) + 1 - return alts[j]['h'] - - for j in alts.keys(): - rec_height(j) - - t_depth = max((alt['h']+1 for alt in alts.values()), default=0) - - # convert to more general tree representation - tree = set() - for j, alt in alts.items(): - # note all non-trunk edges should be colored black - tree.add(TreeBranch( - alt['nft'], - alt['nft'], - t_depth-1 - alt['h'], - alt['c'])) - if alt['ft'] != alt['nft']: - tree.add(TreeBranch( - alt['nft'], - alt['ft'], - t_depth-1 - alt['h'], - 'b')) - - return tree - - # create a btree tree for debugging - def _tree_btree(self, **args): - # for rbyds this is just a pointer to ever rid - tree = set() - root = None - for rid, name in self.rids(): - b = (rid, name.tag) - if root is None: - root = b - tree.add(TreeBranch(root, b)) - return tree - - # create tree representation for debugging - def tree(self, **args): - if args.get('tree_btree'): - return self._tree_btree(**args) - else: - return self._tree_rtree(**args) - # our rbyd btree type class Btree: @@ -1516,148 +1294,6 @@ class Btree: else: return bid, name - # create an rbyd tree for debugging - def _tree_rtree(self, *, - depth=None, - inner=False, - **args): - # precompute rbyd trees so we know the max depth at each layer - # to nicely align trees - rtrees = {} - rdepths = {} - for bid, rbyd, path in self.traverse(path=True, depth=depth): - if not rbyd: - continue - - rtrees[rbyd] = rbyd.tree(**args) - rdepths[len(path)] = max( - rdepths.get(len(path), 0), - max((t.depth+1 for t in rtrees[rbyd]), default=0)) - - # map rbyd branches into our btree space - tree = set() - for bid, rbyd, path in self.traverse(path=True, depth=depth): - if not rbyd: - continue - - # yes we can find new rbyds if disk is being mutated, just - # ignore these - if rbyd not in rtrees: - continue - - rtree = rtrees[rbyd] - rdepth = max((t.depth+1 for t in rtree), default=0) - d = sum(rdepths[d]+1 for d in range(len(path))) - - # map into our btree space - for t in rtree: - # note we adjust our bid to be left-leaning, this allows - # a global order and makes tree rendering quite a bit easier - a_rid, a_tag = t.a - b_rid, b_tag = t.b - _, (_, a_w, _) = rbyd.lookupnext(a_rid) - _, (_, b_w, _) = rbyd.lookupnext(b_rid) - tree.add(TreeBranch( - (bid-(rbyd.weight-1)+a_rid-(a_w-1), len(path), a_tag), - (bid-(rbyd.weight-1)+b_rid-(b_w-1), len(path), b_tag), - d + rdepths[len(path)]-rdepth + t.depth, - t.color)) - - # connect rbyd branches to rbyd roots - if path: - l_bid, l_rbyd, l_rid, l_name = path[-1] - l_branch = l_rbyd.lookup(l_rid, TAG_BRANCH, 0x3) - - if rtree: - r_rid, r_tag = min(rtree, key=lambda t: t.depth).a - _, (_, r_w, _) = rbyd.lookupnext(r_rid) - else: - r_rid, (r_tag, r_w, _) = rbyd.lookupnext(-1) - - tree.add(TreeBranch( - (l_bid-(l_name.weight-1), len(path)-1, l_branch.tag), - (bid-(rbyd.weight-1)+r_rid-(r_w-1), len(path), r_tag), - d-1)) - - # remap branches to leaves if we aren't showing inner branches - if not inner: - # step through each btree layer backwards - b_depth = max((t.a[1]+1 for t in tree), default=0) - - for d in reversed(range(b_depth-1)): - # find bid ranges at this level - bids = set() - for t in tree: - if t.b[1] == d: - bids.add(t.b[0]) - bids = sorted(bids) - - # find the best root for each bid range - roots = {} - for i in range(len(bids)): - for t in tree: - if (t.a[1] > d - and t.a[0] >= bids[i] - and (i == len(bids)-1 or t.a[0] < bids[i+1]) - and (bids[i] not in roots - or t < roots[bids[i]])): - roots[bids[i]] = t - - # remap branches to leaf-roots - tree = {t.map( - lambda x: x[1] == d and x[0] in roots, - lambda x: roots[x[0]].a) - for t in tree} - - return tree - - # create a btree tree for debugging - def _tree_btree(self, *, - depth=None, - inner=False, - **args): - # find all branches - tree = set() - root = None - branches = {} - for bid, name, path in self.bids( - path=True, - depth=depth): - # create branch for each jump in path - # - # note we adjust our bid to be left-leaning, this allows - # a global order and makes tree rendering quite a bit easier - a = root - for d, (bid_, rbyd_, rid_, name_) in enumerate(path): - # map into our btree space - bid__ = bid_-(name_.weight-1) - b = (bid__, d, name_.tag) - - # remap branches to leaves if we aren't showing inner - # branches - if not inner: - if b not in branches: - bid_, rbyd_, rid_, name_ = path[-1] - bid__ = bid_-(name_.weight-1) - branches[b] = (bid__, len(path)-1, name_.tag) - b = branches[b] - - # render the root path on first rid, this is arbitrary - if root is None: - root, a = b, b - - tree.add(TreeBranch(a, b, d)) - a = b - - return tree - - # create tree representation for debugging - def tree(self, **args): - if args.get('tree_btree'): - return self._tree_btree(**args) - else: - return self._tree_rtree(**args) - # a metadata id, this includes mbits for convenience class Mid: @@ -1898,24 +1534,6 @@ class Mdir: return Mid(self.mid, rid), name_ - # create tree representation for debugging - def tree(self, **args): - tree = self.rbyd.tree(**args) - - # map to mid - tree_ = set() - for t in tree: - a_rid, a_tag = t.a - b_rid, b_tag = t.b - tree_.add(TreeBranch( - (Mid(self.mid, a_rid), a_tag), - (Mid(self.mid, b_rid), b_tag), - t.depth, - t.color)) - tree = tree_ - - return tree - # the mtree, the skeletal structure of littlefs class Mtree: def __init__(self, bd, mrootchain, mtree, *, @@ -2577,9 +2195,249 @@ class Mtree: return mid_, mdir, name_, path_+[(mid_, mdir, name_)] else: return mid_, mdir, name_ - - # create an rbyd tree for debugging - def _tree_rtree(self, *, + + + +# tree renderer +class TreeArt: + # tree branches are an abstract thing for tree rendering + class Branch(co.namedtuple('Branch', ['a', 'b', 'z', 'color'])): + __slots__ = () + def __new__(cls, a, b, z=0, color='b'): + # a and b are context specific + return super().__new__(cls, a, b, z, color) + + def __repr__(self): + return '%s(%s, %s, %s, %s)' % ( + self.__class__.__name__, + self.a, + self.b, + self.z, + self.color) + + # don't include color in branch comparisons, or else our tree + # renderings can end up with inconsistent colors between runs + def __eq__(self, other): + return (self.a, self.b, self.z) == (other.a, other.b, other.z) + + def __ne__(self, other): + return (self.a, self.b, self.z) != (other.a, other.b, other.z) + + def __hash__(self): + return hash((self.a, self.b, self.z)) + + # also order by z first, which can be useful for reproducibly + # prioritizing branches when simplifying trees + def __lt__(self, other): + return (self.z, self.a, self.b) < (other.z, other.a, other.b) + + def __le__(self, other): + return (self.z, self.a, self.b) <= (other.z, other.a, other.b) + + def __gt__(self, other): + return (self.z, self.a, self.b) > (other.z, other.a, other.b) + + def __ge__(self, other): + return (self.z, self.a, self.b) >= (other.z, other.a, other.b) + + # apply a function to a/b while trying to avoid copies + def map(self, filter_, map_=None): + if map_ is None: + filter_, map_ = None, filter_ + + a = self.a + if filter_ is None or filter_(a): + a = map_(a) + + b = self.b + if filter_ is None or filter_(b): + b = map_(b) + + if a != self.a or b != self.b: + return self.__class__( + a if a != self.a else self.a, + b if b != self.b else self.b, + self.z, + self.color) + else: + return self + + def __init__(self, tree): + self.tree = tree + self.depth = max((t.z+1 for t in tree), default=0) + if self.depth > 0: + self.width = 2*self.depth + 2 + else: + self.width = 0 + + def __iter__(self): + return iter(self.tree) + + # render an rbyd rbyd tree for debugging + @classmethod + def _fromrbydrtree(cls, rbyd, **args): + trunks = co.defaultdict(lambda: (-1, 0)) + alts = co.defaultdict(lambda: {}) + + for rid, rattr, path in rbyd.rattrs(path=True): + # keep track of trunks/alts + trunks[rattr.toff] = (rid, rattr.tag) + + for ralt in path: + if ralt.followed: + alts[ralt.toff] |= {'f': ralt.joff, 'c': ralt.color} + else: + alts[ralt.toff] |= {'nf': ralt.off, 'c': ralt.color} + + if args.get('tree_rbyd'): + # treat unreachable alts as converging paths + for j_, alt in alts.items(): + if 'f' not in alt: + alt['f'] = alt['nf'] + elif 'nf' not in alt: + alt['nf'] = alt['f'] + + else: + # prune any alts with unreachable edges + pruned = {} + for j, alt in alts.items(): + if 'f' not in alt: + pruned[j] = alt['nf'] + elif 'nf' not in alt: + pruned[j] = alt['f'] + for j in pruned.keys(): + del alts[j] + + for j, alt in alts.items(): + while alt['f'] in pruned: + alt['f'] = pruned[alt['f']] + while alt['nf'] in pruned: + alt['nf'] = pruned[alt['nf']] + + # find the trunk and depth of each alt + def rec_trunk(j): + if j not in alts: + return trunks[j] + else: + if 'nft' not in alts[j]: + alts[j]['nft'] = rec_trunk(alts[j]['nf']) + return alts[j]['nft'] + + for j in alts.keys(): + rec_trunk(j) + for j, alt in alts.items(): + if alt['f'] in alts: + alt['ft'] = alts[alt['f']]['nft'] + else: + alt['ft'] = trunks[alt['f']] + + def rec_height(j): + if j not in alts: + return 0 + else: + if 'h' not in alts[j]: + alts[j]['h'] = max( + rec_height(alts[j]['f']), + rec_height(alts[j]['nf'])) + 1 + return alts[j]['h'] + + for j in alts.keys(): + rec_height(j) + + t_depth = max((alt['h']+1 for alt in alts.values()), default=0) + + # convert to more general tree representation + tree = set() + for j, alt in alts.items(): + # note all non-trunk edges should be colored black + tree.add(cls.Branch( + alt['nft'], + alt['nft'], + t_depth-1 - alt['h'], + alt['c'])) + if alt['ft'] != alt['nft']: + tree.add(cls.Branch( + alt['nft'], + alt['ft'], + t_depth-1 - alt['h'], + 'b')) + + return cls(tree) + + # render an rbyd btree tree for debugging + @classmethod + def _fromrbydbtree(cls, rbyd, **args): + # for rbyds this is just a pointer to every rid + tree = set() + root = None + for rid, name in rbyd.rids(): + b = (rid, name.tag) + if root is None: + root = b + tree.add(cls.Branch(root, b)) + return cls(tree) + + # render an rbyd tree for debugging + @classmethod + def fromrbyd(cls, rbyd, **args): + if args.get('tree_btree'): + return cls._fromrbydbtree(rbyd, **args) + else: + return cls._fromrbydrtree(rbyd, **args) + + # render some nice ascii trees + def repr(self, x, color=False): + if self.depth == 0: + return '' + + def branchrepr(tree, x, d, was): + for t in tree: + if t.z == d and t.b == x: + if any(t.z == d and t.a == x + for t in tree): + return '+-', t.color, t.color + elif any(t.z == d + and x > min(t.a, t.b) + and x < max(t.a, t.b) + for t in tree): + return '|-', t.color, t.color + elif t.a < t.b: + return '\'-', t.color, t.color + else: + return '.-', t.color, t.color + for t in tree: + if t.z == d and t.a == x: + return '+ ', t.color, None + for t in tree: + if (t.z == d + and x > min(t.a, t.b) + and x < max(t.a, t.b)): + return '| ', t.color, was + if was: + return '--', was, was + return ' ', None, None + + trunk = [] + was = None + for d in range(self.depth): + t, c, was = branchrepr(self.tree, x, d, was) + + trunk.append('%s%s%s%s' % ( + '\x1b[33m' if color and c == 'y' + else '\x1b[31m' if color and c == 'r' + else '\x1b[90m' if color and c == 'b' + else '', + t, + ('>' if was else ' ') if d == self.depth-1 else '', + '\x1b[m' if color and c else '')) + + return '%s ' % ''.join(trunk) + + # some more renderers + + # render a btree rbyd tree for debugging + @classmethod + def _frombtreertree(cls, btree, *, depth=None, inner=False, **args): @@ -2587,7 +2445,151 @@ class Mtree: # to nicely align trees rtrees = {} rdepths = {} - for mdir, path in self.traverse(path=True, depth=depth): + for bid, rbyd, path in btree.traverse(path=True, depth=depth): + if not rbyd: + continue + + rtree = cls.fromrbyd(rbyd, **args) + rtrees[rbyd] = rtree + rdepths[len(path)] = max(rdepths.get(len(path), 0), rtree.depth) + + # map rbyd branches into our btree space + tree = set() + for bid, rbyd, path in btree.traverse(path=True, depth=depth): + if not rbyd: + continue + + # yes we can find new rbyds if disk is being mutated, just + # ignore these + if rbyd not in rtrees: + continue + + rtree = rtrees[rbyd] + rz = max((t.z+1 for t in rtree), default=0) + d = sum(rdepths[d]+1 for d in range(len(path))) + + # map into our btree space + for t in rtree: + # note we adjust our bid to be left-leaning, this allows + # a global order and makes tree rendering quite a bit easier + a_rid, a_tag = t.a + b_rid, b_tag = t.b + _, (_, a_w, _) = rbyd.lookupnext(a_rid) + _, (_, b_w, _) = rbyd.lookupnext(b_rid) + tree.add(cls.Branch( + (bid-(rbyd.weight-1)+a_rid-(a_w-1), len(path), a_tag), + (bid-(rbyd.weight-1)+b_rid-(b_w-1), len(path), b_tag), + d + rdepths[len(path)]-rz + t.z, + t.color)) + + # connect rbyd branches to rbyd roots + if path: + l_bid, l_rbyd, l_rid, l_name = path[-1] + l_branch = l_rbyd.lookup(l_rid, TAG_BRANCH, 0x3) + + if rtree: + r_rid, r_tag = min(rtree, key=lambda t: t.z).a + _, (_, r_w, _) = rbyd.lookupnext(r_rid) + else: + r_rid, (r_tag, r_w, _) = rbyd.lookupnext(-1) + + tree.add(cls.Branch( + (l_bid-(l_name.weight-1), len(path)-1, l_branch.tag), + (bid-(rbyd.weight-1)+r_rid-(r_w-1), len(path), r_tag), + d-1)) + + # remap branches to leaves if we aren't showing inner branches + if not inner: + # step through each btree layer backwards + b_depth = max((t.a[1]+1 for t in tree), default=0) + + for d in reversed(range(b_depth-1)): + # find bid ranges at this level + bids = set() + for t in tree: + if t.b[1] == d: + bids.add(t.b[0]) + bids = sorted(bids) + + # find the best root for each bid range + roots = {} + for i in range(len(bids)): + for t in tree: + if (t.a[1] > d + and t.a[0] >= bids[i] + and (i == len(bids)-1 or t.a[0] < bids[i+1]) + and (bids[i] not in roots + or t < roots[bids[i]])): + roots[bids[i]] = t + + # remap branches to leaf-roots + tree = {t.map( + lambda x: x[1] == d and x[0] in roots, + lambda x: roots[x[0]].a) + for t in tree} + + return cls(tree) + + # render a btree btree tree for debugging + @classmethod + def _frombtreebtree(cls, btree, *, + depth=None, + inner=False, + **args): + # find all branches + tree = set() + root = None + branches = {} + for bid, name, path in btree.bids( + path=True, + depth=depth): + # create branch for each jump in path + # + # note we adjust our bid to be left-leaning, this allows + # a global order and makes tree rendering quite a bit easier + a = root + for d, (bid_, rbyd_, rid_, name_) in enumerate(path): + # map into our btree space + bid__ = bid_-(name_.weight-1) + b = (bid__, d, name_.tag) + + # remap branches to leaves if we aren't showing inner + # branches + if not inner: + if b not in branches: + bid_, rbyd_, rid_, name_ = path[-1] + bid__ = bid_-(name_.weight-1) + branches[b] = (bid__, len(path)-1, name_.tag) + b = branches[b] + + # render the root path on first rid, this is arbitrary + if root is None: + root, a = b, b + + tree.add(cls.Branch(a, b, d)) + a = b + + return cls(tree) + + # render a btree tree for debugging + @classmethod + def frombtree(cls, btree, **args): + if args.get('tree_btree'): + return cls._frombtreebtree(btree, **args) + else: + return cls._frombtreertree(btree, **args) + + # render an mtree tree for debugging + @classmethod + def _frommtreertree(cls, mtree, *, + depth=None, + inner=False, + **args): + # precompute rbyd trees so we know the max depth at each layer + # to nicely align trees + rtrees = {} + rdepths = {} + for mdir, path in mtree.traverse(path=True, depth=depth): if isinstance(mdir, Mdir): if not mdir: continue @@ -2597,15 +2599,14 @@ class Mtree: if not rbyd: continue - rtrees[rbyd] = rbyd.tree(**args) - rdepths[len(path)] = max( - rdepths.get(len(path), 0), - max((t.depth+1 for t in rtrees[rbyd]), default=0)) + rtree = cls.fromrbyd(rbyd, **args) + rtrees[rbyd] = rtree + rdepths[len(path)] = max(rdepths.get(len(path), 0), rtree.depth) # map rbyd branches into our mtree space tree = set() branches = {} - for mdir, path in self.traverse(path=True, depth=depth): + for mdir, path in mtree.traverse(path=True, depth=depth): if isinstance(mdir, Mdir): if not mdir: continue @@ -2621,7 +2622,7 @@ class Mtree: continue rtree = rtrees[rbyd] - rdepth = max((t.depth+1 for t in rtree), default=0) + rz = max((t.z+1 for t in rtree), default=0) d = sum(rdepths[d]+1 for d, p in enumerate(path)) # map into our mtree space @@ -2636,16 +2637,16 @@ class Mtree: _, (_, a_w, _) = rbyd.lookupnext(a_rid) _, (_, b_w, _) = rbyd.lookupnext(b_rid) if isinstance(mdir, Mdir): - a_mid = self.mid(mdir.mid, a_rid) - b_mid = self.mid(mdir.mid, b_rid) + a_mid = mtree.mid(mdir.mid, a_rid) + b_mid = mtree.mid(mdir.mid, b_rid) else: - a_mid = self.mid(bid-(rbyd.weight-1)+a_rid-(a_w-1), -1) - b_mid = self.mid(bid-(rbyd.weight-1)+b_rid-(b_w-1), -1) + a_mid = mtree.mid(bid-(rbyd.weight-1)+a_rid-(a_w-1), -1) + b_mid = mtree.mid(bid-(rbyd.weight-1)+b_rid-(b_w-1), -1) - tree.add(TreeBranch( + tree.add(cls.Branch( (a_mid, len(path), a_tag), (b_mid, len(path), b_tag), - d + rdepths[len(path)]-rdepth + t.depth, + d + rdepths[len(path)]-rz + t.z, t.color)) # connect rbyd branches to rbyd roots @@ -2657,23 +2658,23 @@ class Mtree: or l_mdir.lookup(l_mid, TAG_MTREE, 0x3)) else: l_bid, l_rbyd, l_rid, l_name = path[-1] - l_mid = self.mid(l_bid-(l_name.weight-1), -1) + l_mid = mtree.mid(l_bid-(l_name.weight-1), -1) l_branch = (l_rbyd.lookup(l_rid, TAG_BRANCH, 0x3) or l_rbyd.lookup(l_rid, TAG_MDIR, 0x3)) # figure out root mid/rattr if rtree: - r_rid, r_tag = min(rtree, key=lambda t: t.depth).a + r_rid, r_tag = min(rtree, key=lambda t: t.z).a _, (_, r_w, _) = rbyd.lookupnext(r_rid) else: r_rid, (r_tag, r_w, _) = rbyd.lookupnext(-1) if isinstance(mdir, Mdir): - r_mid = self.mid(mdir.mid, r_rid) + r_mid = mtree.mid(mdir.mid, r_rid) else: - r_mid = self.mid(bid-(rbyd.weight-1)+r_rid-(r_w-1), -1) + r_mid = mtree.mid(bid-(rbyd.weight-1)+r_rid-(r_w-1), -1) - tree.add(TreeBranch( + tree.add(cls.Branch( (l_mid, len(path)-1, l_branch.tag), (r_mid, len(path), r_tag), d-1)) @@ -2683,7 +2684,7 @@ class Mtree: # step through each btree layer backwards b_depth = max((t.a[1]+1 for t in tree), default=0) - for d in reversed(range(len(self.mrootchain), b_depth-1)): + for d in reversed(range(len(mtree.mrootchain), b_depth-1)): # find mid ranges at this level mids = set() for t in tree: @@ -2708,17 +2709,18 @@ class Mtree: lambda x: roots[x[0]].a) for t in tree} - return tree + return cls(tree) - # create a btree tree for debugging - def _tree_btree(self, *, + # render an mtree tree for debugging + @classmethod + def _frommtreebtree(cls, mtree, *, depth=None, inner=False, **args): tree = set() root = None branches = {} - for mid, mdir, name, path in self.mids( + for mid, mdir, name, path in mtree.mids( mdirs_only=False, path=True, depth=depth): @@ -2736,7 +2738,7 @@ class Mtree: mid_, mdir_, name_ = p else: bid_, rbyd_, rid_, name_ = p - mid_ = self.mid(bid_-(name_.weight-1), -1) + mid_ = mtree.mid(bid_-(name_.weight-1), -1) b = (mid_, d, name_.tag) # remap branches to leaves if we aren't showing inner @@ -2747,7 +2749,7 @@ class Mtree: mid_, mdir_, name_ = path[-1] else: bid_, rbyd_, rid_, name_ = path[-1] - mid_ = self.mid(bid_-(name_.weight-1), -1) + mid_ = mtree.mid(bid_-(name_.weight-1), -1) branches[b] = (mid_, len(path)-1, name_.tag) b = branches[b] @@ -2755,17 +2757,18 @@ class Mtree: if root is None: root, a = b, b - tree.add(TreeBranch(a, b, d)) + tree.add(cls.Branch(a, b, d)) a = b - return tree + return cls(tree) - # create tree representation for debugging - def tree(self, **args): + # render an mtree tree for debugging + @classmethod + def frommtree(cls, mtree, **args): if args.get('tree_btree'): - return self._tree_btree(**args) + return cls._frommtreebtree(mtree, **args) else: - return self._tree_rtree(**args) + return cls._frommtreertree(mtree, **args) @@ -2828,12 +2831,8 @@ def main(disk, mroots=None, *, if (args.get('tree') or args.get('tree_rbyd') or args.get('tree_btree')): - tree = mtree.tree(**args) - - # find the max depth from the tree - t_depth = max((t.depth+1 for t in tree), default=0) - if t_depth > 0: - t_width = 2*t_depth + 2 + treeart = TreeArt.frommtree(mtree, **args) + t_width = treeart.width # dynamically size the id field w_width = max( @@ -2851,8 +2850,7 @@ def main(disk, mroots=None, *, '{%s}:' % ','.join('%04x' % block for block in mdir.blocks) if i == 0 else '', - treerepr(tree, (mid, d, rattr.tag), - t_depth, color) + treeart.repr((mid, d, rattr.tag), color) if args.get('tree') or args.get('tree_rbyd') or args.get('tree_btree') @@ -2902,10 +2900,10 @@ def main(disk, mroots=None, *, '%04x.%04x:' % (rbyd.block, rbyd.trunk) if prbyd is None or rbyd != prbyd else '', - treerepr(tree, + treeart.repr( (mtree.mid(bid-(name.weight-1), -1), d, rattr.tag), - t_depth, color) + color) if args.get('tree') or args.get('tree_rbyd') or args.get('tree_btree') diff --git a/scripts/dbgrbyd.py b/scripts/dbgrbyd.py index 08926f6a..5a323502 100755 --- a/scripts/dbgrbyd.py +++ b/scripts/dbgrbyd.py @@ -295,119 +295,6 @@ def tagrepr(tag, weight=None, size=None, *, ' w%d' % weight if weight is not None else '', ' %d' % size if size is not None else '') -# tree branches are an abstract thing for tree rendering -class TreeBranch(co.namedtuple('TreeBranch', ['a', 'b', 'depth', 'color'])): - __slots__ = () - def __new__(cls, a, b, depth=0, color='b'): - # a and b are context specific - return super().__new__(cls, a, b, depth, color) - - def __repr__(self): - return '%s(%s, %s, %s, %s)' % ( - self.__class__.__name__, - self.a, - self.b, - self.depth, - self.color) - - # don't include color in branch comparisons, or else our tree - # renderings can end up with inconsistent colors between runs - def __eq__(self, other): - return (self.a, self.b, self.depth) == (other.a, other.b, other.depth) - - def __ne__(self, other): - return (self.a, self.b, self.depth) != (other.a, other.b, other.depth) - - def __hash__(self): - return hash((self.a, self.b, self.depth)) - - # also order by depth first, which can be useful for reproducibly - # prioritizing branches when simplifying trees - def __lt__(self, other): - return (self.depth, self.a, self.b) < (other.depth, other.a, other.b) - - def __le__(self, other): - return (self.depth, self.a, self.b) <= (other.depth, other.a, other.b) - - def __gt__(self, other): - return (self.depth, self.a, self.b) > (other.depth, other.a, other.b) - - def __ge__(self, other): - return (self.depth, self.a, self.b) >= (other.depth, other.a, other.b) - - # apply a function to a/b while trying to avoid copies - def map(self, filter_, map_=None): - if map_ is None: - filter_, map_ = None, filter_ - - a = self.a - if filter_ is None or filter_(a): - a = map_(a) - - b = self.b - if filter_ is None or filter_(b): - b = map_(b) - - if a != self.a or b != self.b: - return self.__class__( - a if a != self.a else self.a, - b if b != self.b else self.b, - self.depth, - self.color) - else: - return self - -# render some nice ascii trees -def treerepr(tree, x, depth=None, color=False): - # find the max depth from the tree - if depth is None: - depth = max((t.depth+1 for t in tree), default=0) - if depth == 0: - return '' - - def branchrepr(tree, x, d, was): - for t in tree: - if t.depth == d and t.b == x: - if any(t.depth == d and t.a == x - for t in tree): - return '+-', t.color, t.color - elif any(t.depth == d - and x > min(t.a, t.b) - and x < max(t.a, t.b) - for t in tree): - return '|-', t.color, t.color - elif t.a < t.b: - return '\'-', t.color, t.color - else: - return '.-', t.color, t.color - for t in tree: - if t.depth == d and t.a == x: - return '+ ', t.color, None - for t in tree: - if (t.depth == d - and x > min(t.a, t.b) - and x < max(t.a, t.b)): - return '| ', t.color, was - if was: - return '--', was, was - return ' ', None, None - - trunk = [] - was = None - for d in range(depth): - t, c, was = branchrepr(tree, x, d, was) - - trunk.append('%s%s%s%s' % ( - '\x1b[33m' if color and c == 'y' - else '\x1b[31m' if color and c == 'r' - else '\x1b[90m' if color and c == 'b' - else '', - t, - ('>' if was else ' ') if d == depth-1 else '', - '\x1b[m' if color and c else '')) - - return '%s ' % ''.join(trunk) - # a simple wrapper over an open file with bd geometry class Bd: @@ -991,115 +878,6 @@ class Rbyd: return best - # create an rbyd tree for debugging - def _tree_rtree(self, **args): - trunks = co.defaultdict(lambda: (-1, 0)) - alts = co.defaultdict(lambda: {}) - - for rid, rattr, path in self.rattrs(path=True): - # keep track of trunks/alts - trunks[rattr.toff] = (rid, rattr.tag) - - for ralt in path: - if ralt.followed: - alts[ralt.toff] |= {'f': ralt.joff, 'c': ralt.color} - else: - alts[ralt.toff] |= {'nf': ralt.off, 'c': ralt.color} - - if args.get('tree_rbyd'): - # treat unreachable alts as converging paths - for j_, alt in alts.items(): - if 'f' not in alt: - alt['f'] = alt['nf'] - elif 'nf' not in alt: - alt['nf'] = alt['f'] - - else: - # prune any alts with unreachable edges - pruned = {} - for j, alt in alts.items(): - if 'f' not in alt: - pruned[j] = alt['nf'] - elif 'nf' not in alt: - pruned[j] = alt['f'] - for j in pruned.keys(): - del alts[j] - - for j, alt in alts.items(): - while alt['f'] in pruned: - alt['f'] = pruned[alt['f']] - while alt['nf'] in pruned: - alt['nf'] = pruned[alt['nf']] - - # find the trunk and depth of each alt - def rec_trunk(j): - if j not in alts: - return trunks[j] - else: - if 'nft' not in alts[j]: - alts[j]['nft'] = rec_trunk(alts[j]['nf']) - return alts[j]['nft'] - - for j in alts.keys(): - rec_trunk(j) - for j, alt in alts.items(): - if alt['f'] in alts: - alt['ft'] = alts[alt['f']]['nft'] - else: - alt['ft'] = trunks[alt['f']] - - def rec_height(j): - if j not in alts: - return 0 - else: - if 'h' not in alts[j]: - alts[j]['h'] = max( - rec_height(alts[j]['f']), - rec_height(alts[j]['nf'])) + 1 - return alts[j]['h'] - - for j in alts.keys(): - rec_height(j) - - t_depth = max((alt['h']+1 for alt in alts.values()), default=0) - - # convert to more general tree representation - tree = set() - for j, alt in alts.items(): - # note all non-trunk edges should be colored black - tree.add(TreeBranch( - alt['nft'], - alt['nft'], - t_depth-1 - alt['h'], - alt['c'])) - if alt['ft'] != alt['nft']: - tree.add(TreeBranch( - alt['nft'], - alt['ft'], - t_depth-1 - alt['h'], - 'b')) - - return tree - - # create a btree tree for debugging - def _tree_btree(self, **args): - # for rbyds this is just a pointer to ever rid - tree = set() - root = None - for rid, name in self.rids(): - b = (rid, name.tag) - if root is None: - root = b - tree.add(TreeBranch(root, b)) - return tree - - # create tree representation for debugging - def tree(self, **args): - if args.get('tree_btree'): - return self._tree_btree(**args) - else: - return self._tree_rtree(**args) - # jump renderer @@ -1133,18 +911,21 @@ class JumpArt: self.jumps = jumps self.width = 2*max((x for _, _, x, _ in jumps), default=0) - def collide(self): + @classmethod + def collide(cls, jumps): # figure out x-offsets to avoid collisions between jumps - for j in range(len(self.jumps)): - a, b, _, c = self.jumps[j] + for j in range(len(jumps)): + a, b, _, c = jumps[j] x = 0 while any( max(a, b) >= min(a_, b_) and max(a_, b_) >= min(a, b) and x == x_ - for a_, b_, x_, _ in self.jumps[:j]): + for a_, b_, x_, _ in jumps[:j]): x += 1 - self.jumps[j] = self.Jump(a, b, x, c) + jumps[j] = cls.Jump(a, b, x, c) + + return jumps @classmethod def fromrbyd(cls, rbyd, all=False): @@ -1170,9 +951,8 @@ class JumpArt: else: jumps.append(cls.Jump(j, j-size, 0, 'b')) - jumpart = cls(jumps) - jumpart.collide() - return jumpart + jumps = cls.collide(jumps) + return cls(jumps) def repr(self, j, color=False): # render jumps @@ -1438,6 +1218,240 @@ class LifetimeArt: self.width - sum(len(r) for r in reprs), '') +# tree renderer +class TreeArt: + # tree branches are an abstract thing for tree rendering + class Branch(co.namedtuple('Branch', ['a', 'b', 'z', 'color'])): + __slots__ = () + def __new__(cls, a, b, z=0, color='b'): + # a and b are context specific + return super().__new__(cls, a, b, z, color) + + def __repr__(self): + return '%s(%s, %s, %s, %s)' % ( + self.__class__.__name__, + self.a, + self.b, + self.z, + self.color) + + # don't include color in branch comparisons, or else our tree + # renderings can end up with inconsistent colors between runs + def __eq__(self, other): + return (self.a, self.b, self.z) == (other.a, other.b, other.z) + + def __ne__(self, other): + return (self.a, self.b, self.z) != (other.a, other.b, other.z) + + def __hash__(self): + return hash((self.a, self.b, self.z)) + + # also order by z first, which can be useful for reproducibly + # prioritizing branches when simplifying trees + def __lt__(self, other): + return (self.z, self.a, self.b) < (other.z, other.a, other.b) + + def __le__(self, other): + return (self.z, self.a, self.b) <= (other.z, other.a, other.b) + + def __gt__(self, other): + return (self.z, self.a, self.b) > (other.z, other.a, other.b) + + def __ge__(self, other): + return (self.z, self.a, self.b) >= (other.z, other.a, other.b) + + # apply a function to a/b while trying to avoid copies + def map(self, filter_, map_=None): + if map_ is None: + filter_, map_ = None, filter_ + + a = self.a + if filter_ is None or filter_(a): + a = map_(a) + + b = self.b + if filter_ is None or filter_(b): + b = map_(b) + + if a != self.a or b != self.b: + return self.__class__( + a if a != self.a else self.a, + b if b != self.b else self.b, + self.z, + self.color) + else: + return self + + def __init__(self, tree): + self.tree = tree + self.depth = max((t.z+1 for t in tree), default=0) + if self.depth > 0: + self.width = 2*self.depth + 2 + else: + self.width = 0 + + # render an rbyd rbyd tree for debugging + @classmethod + def _fromrbydrtree(cls, rbyd, **args): + trunks = co.defaultdict(lambda: (-1, 0)) + alts = co.defaultdict(lambda: {}) + + for rid, rattr, path in rbyd.rattrs(path=True): + # keep track of trunks/alts + trunks[rattr.toff] = (rid, rattr.tag) + + for ralt in path: + if ralt.followed: + alts[ralt.toff] |= {'f': ralt.joff, 'c': ralt.color} + else: + alts[ralt.toff] |= {'nf': ralt.off, 'c': ralt.color} + + if args.get('tree_rbyd'): + # treat unreachable alts as converging paths + for j_, alt in alts.items(): + if 'f' not in alt: + alt['f'] = alt['nf'] + elif 'nf' not in alt: + alt['nf'] = alt['f'] + + else: + # prune any alts with unreachable edges + pruned = {} + for j, alt in alts.items(): + if 'f' not in alt: + pruned[j] = alt['nf'] + elif 'nf' not in alt: + pruned[j] = alt['f'] + for j in pruned.keys(): + del alts[j] + + for j, alt in alts.items(): + while alt['f'] in pruned: + alt['f'] = pruned[alt['f']] + while alt['nf'] in pruned: + alt['nf'] = pruned[alt['nf']] + + # find the trunk and depth of each alt + def rec_trunk(j): + if j not in alts: + return trunks[j] + else: + if 'nft' not in alts[j]: + alts[j]['nft'] = rec_trunk(alts[j]['nf']) + return alts[j]['nft'] + + for j in alts.keys(): + rec_trunk(j) + for j, alt in alts.items(): + if alt['f'] in alts: + alt['ft'] = alts[alt['f']]['nft'] + else: + alt['ft'] = trunks[alt['f']] + + def rec_height(j): + if j not in alts: + return 0 + else: + if 'h' not in alts[j]: + alts[j]['h'] = max( + rec_height(alts[j]['f']), + rec_height(alts[j]['nf'])) + 1 + return alts[j]['h'] + + for j in alts.keys(): + rec_height(j) + + t_depth = max((alt['h']+1 for alt in alts.values()), default=0) + + # convert to more general tree representation + tree = set() + for j, alt in alts.items(): + # note all non-trunk edges should be colored black + tree.add(cls.Branch( + alt['nft'], + alt['nft'], + t_depth-1 - alt['h'], + alt['c'])) + if alt['ft'] != alt['nft']: + tree.add(cls.Branch( + alt['nft'], + alt['ft'], + t_depth-1 - alt['h'], + 'b')) + + return cls(tree) + + # render an rbyd btree tree for debugging + @classmethod + def _fromrbydbtree(cls, rbyd, **args): + # for rbyds this is just a pointer to every rid + tree = set() + root = None + for rid, name in rbyd.rids(): + b = (rid, name.tag) + if root is None: + root = b + tree.add(cls.Branch(root, b)) + return cls(tree) + + # render an rbyd tree for debugging + @classmethod + def fromrbyd(cls, rbyd, **args): + if args.get('tree_btree'): + return cls._fromrbydbtree(rbyd, **args) + else: + return cls._fromrbydrtree(rbyd, **args) + + # render some nice ascii trees + def repr(self, x, color=False): + if self.depth == 0: + return '' + + def branchrepr(tree, x, d, was): + for t in tree: + if t.z == d and t.b == x: + if any(t.z == d and t.a == x + for t in tree): + return '+-', t.color, t.color + elif any(t.z == d + and x > min(t.a, t.b) + and x < max(t.a, t.b) + for t in tree): + return '|-', t.color, t.color + elif t.a < t.b: + return '\'-', t.color, t.color + else: + return '.-', t.color, t.color + for t in tree: + if t.z == d and t.a == x: + return '+ ', t.color, None + for t in tree: + if (t.z == d + and x > min(t.a, t.b) + and x < max(t.a, t.b)): + return '| ', t.color, was + if was: + return '--', was, was + return ' ', None, None + + trunk = [] + was = None + for d in range(self.depth): + t, c, was = branchrepr(self.tree, x, d, was) + + trunk.append('%s%s%s%s' % ( + '\x1b[33m' if color and c == 'y' + else '\x1b[31m' if color and c == 'r' + else '\x1b[90m' if color and c == 'b' + else '', + t, + ('>' if was else ' ') if d == self.depth-1 else '', + '\x1b[m' if color and c else '')) + + return '%s ' % ''.join(trunk) + + + # show the rbyd log def dbg_log(rbyd, *, color=False, @@ -1608,12 +1622,8 @@ def dbg_tree(rbyd, *, if (args.get('tree') or args.get('tree_rbyd') or args.get('tree_btree')): - tree = rbyd.tree(**args) - - # find the max depth from the tree - t_depth = max((t.depth+1 for t in tree), default=0) - if t_depth > 0: - t_width = 2*t_depth + 2 + tree = TreeArt.fromrbyd(rbyd, **args) + t_width = tree.width # dynamically size the id field w_width = mt.ceil(mt.log10(max(1, rbyd.weight)+1)) @@ -1622,7 +1632,7 @@ def dbg_tree(rbyd, *, # show human-readable tag representation print('%08x: %s%*s %-*s %s' % ( rattr.toff, - treerepr(tree, (rid, rattr.tag), t_depth, color) + tree.repr((rid, rattr.tag), color) if (args.get('tree') or args.get('tree_rbyd') or args.get('tree_btree'))