From 582f92d073ce6fa530bc982f5edd6df52f511c33 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 24 Mar 2025 01:36:58 -0500 Subject: [PATCH] scripts: Reworked dbgmtree.py, adopted Mtree, Mdir, etc This is where the high-level structure of littlefs starts to reveal itself. This is also where a lot of really annoying Mtree vs Btree API questions come to a head, like should Mtree.lookup return an Mdir or an Rattr? What about Btree.lookup? What gets included in the returned path in all of these? Well, at least this is an interesting exercise in rethinking littlefs's internal APIs... New classes: - Mid - A representation of littlefs's metadata ids. I've just gone ahead and included the block_size-dependent mbits as a field in every Mid instance to try to make Mid operations easier. It's not like we care about one extra word of storage in Python. - Mdir - Again, we intentionally _don't_ inherit Rbyd to try to reduce type errors, though Mdirs really are just Rbyds in this design. - Mtree - The skeleton of littlefs. Tricky bits include traversing the mroot chain and handling mroot-inlined mdirs. Note mroots are included in the mdir/mid iteration methods. Getting the tree renderers all working again was a real pain in the ass. --- scripts/dbgbtree.py | 212 ++-- scripts/dbgmtree.py | 2963 +++++++++++++++++++++++++++---------------- scripts/dbgrbyd.py | 114 +- 3 files changed, 2047 insertions(+), 1242 deletions(-) diff --git a/scripts/dbgbtree.py b/scripts/dbgbtree.py index 84d0e77d..553cdd9c 100755 --- a/scripts/dbgbtree.py +++ b/scripts/dbgbtree.py @@ -263,13 +263,11 @@ def tagrepr(tag, weight=None, size=None, off=None): ' %d' % size if size is not None else '') # tree branches are an abstract thing for tree rendering -class TreeBranch: - def __init__(self, a, b, depth=0, color='b'): +class TreeBranch(co.namedtuple('TreeBranch', ['a', 'b', 'depth', 'color'])): + __slots__ = () + def __new__(cls, a, b, depth=0, color='b'): # a and b are context specific - self.a = a - self.b = b - self.depth = depth - self.color = color + return super().__new__(cls, a, b, depth, color) def __repr__(self): return '%s(%s, %s, %s, %s)' % ( @@ -279,16 +277,6 @@ class TreeBranch: self.depth, self.color) - def __eq__(self, other): - return ((self.a, self.b, self.depth, self.color) - == (other.a, other.b, other.depth, other.color)) - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash((self.a, self.b, self.depth)) - def treerepr(tree, x, depth=None, color=False): # find the max depth from the tree if depth is None: @@ -346,7 +334,7 @@ def pathdelta(a, b): a = list(a) i = 0 for a_, b_ in zip(a, b): - if a_ == b_: + if type(a_) == type(b_) and a_ == b_: i += 1 else: break @@ -393,9 +381,9 @@ class Rattr: return len(self.data) def __repr__(self): - return '<%s %s>' % (self.__class__.__name__, self.tagrepr()) + return '<%s %s>' % (self.__class__.__name__, self) - def tagrepr(self): + def __str__(self): return tagrepr(self.tag, self.weight, self.size) def __iter__(self): @@ -432,9 +420,9 @@ class Ralt: return self.toff - self.jump def __repr__(self): - return '<%s %s>' % (self.__class__.__name__, self.tagrepr()) + return '<%s %s>' % (self.__class__.__name__, self) - def tagrepr(self): + def __str__(self): return tagrepr(self.tag, self.weight, self.jump, self.toff) def __iter__(self): @@ -643,7 +631,7 @@ class Rbyd: gcksumdelta=gcksumdelta, corrupt=not trunk_) - def lookupnext(self, rid=-1, tag=None, *, + def lookupnext(self, rid, tag=None, *, path=False): if not self: return None, None, *(([],) if path else ()) @@ -721,21 +709,26 @@ class Rbyd: self.data[j+d:j+d+jump]), *((path_,) if path else ())) - def lookup(self, rid, tag=None, mask=None, *, + def lookup_(self, rid, tag=None, mask=None, *, path=False): if tag is None: tag, mask = 0, 0xffff + if mask is None: + mask = 0 - rid_, rattr_, *path_ = self.lookupnext(rid, tag & ~(mask or 0), + rid_, rattr_, *path_ = self.lookupnext(rid, tag & ~mask, path=path) if (rid_ is None or rid_ != rid - or (rattr_.tag & ~(mask or 0)) != (tag & ~(mask or 0))): - if path: - return None, *path_ - else: - return None + or (rattr_.tag & ~mask) != (tag & ~mask)): + return None, *path_ + return rattr_, *path_ + + def lookup(self, rid, tag=None, mask=None, *, + path=False): + rattr_, *path_ = self.lookup_(rid, tag, mask, + path=path) if path: return rattr_, *path_ else: @@ -751,26 +744,22 @@ class Rbyd: if not isinstance(key, tuple): key = (key,) - v = self.lookup(*key) - if isinstance(v, tuple): - return v[0] is not None - else: - return v is not None + return self.lookup_(*key)[0] is not None def rids(self, *, path=False): rid = -1 while True: - rid, rattr, *path_ = self.lookupnext(rid, + rid, name, *path_ = self.lookupnext(rid, path=path) # found end of tree? if rid is None: break - yield rid, rattr, *path_ + yield rid, name, *path_ rid += 1 - def rattrs(self, rid=None, *, + def rattrs_(self, rid=None, *, path=False): if rid is None: rid, tag = -1, 0 @@ -792,11 +781,21 @@ class Rbyd: if rid_ is None or rid_ != rid: break + yield rattr, *path_ + tag = rattr.tag + + def rattrs(self, rid=None, *, + path=False): + if rid is None: + yield from self.rattrs_(rid, + path=path) + else: + for rattr, *path_ in self.rattrs_(rid, + path=path): if path: yield rattr, *path_ else: yield rattr - tag = rattr.tag def __iter__(self): return self.rattrs() @@ -834,8 +833,8 @@ class Rbyd: return best - # create tree representation for debugging - def tree(self, **args): + # create an rbyd tree for debugging + def _tree_rtree(self, **args): trunks = co.defaultdict(lambda: (-1, 0)) alts = co.defaultdict(lambda: {}) @@ -924,14 +923,31 @@ class Rbyd: return tree + # create a btree tree for debugging + def _tree_btree(self, **args): + # for rbyds this is just a pointer to ever rid + tree = set() + root = None + for rid, name in self.rids(): + b = (rid, name.tag) + if root is None: + root = b + tree.add(TreeBranch(root, b)) + return tree + + # create tree representation for debugging + def tree(self, **args): + if args.get('tree_btree'): + return self._tree_btree(**args) + else: + return self._tree_rtree(**args) + # our rbyd btree type class Btree: - def __init__(self, bd, rbyd, *, - corrupt=False): + def __init__(self, bd, rbyd): self.bd = bd self.rbyd = rbyd - self.corrupt = corrupt or rbyd.corrupt @property def block(self): @@ -970,9 +986,6 @@ class Btree: self.addr(), self.weight) - def __bool__(self): - return not self.corrupt - def __eq__(self, other): return self.rbyd == other.rbyd @@ -999,13 +1012,14 @@ class Btree: path_ = [] while True: + # corrupt branch? + if not rbyd: + return (bid, rbyd, rid, None, + *((path_,) if path else ())) + # first tag indicates the branch's weight rid_, name_ = rbyd.lookupnext(rid) - # if we hit this the rbyd is probably corrupt if rid_ is None: - # still keep track of best guess in path - if path: - path_.append((bid, rbyd, rid, None)) return (None, None, None, None, *((path_,) if path else ())) @@ -1044,7 +1058,9 @@ class Btree: return None, None, None, None, None, *path_ # lookup tag in rbyd - rattr_ = rbyd_.lookup(rid_, tag, mask or 0) + rattr_ = rbyd_.lookup(rid_, tag, mask) + if rattr_ is None: + return None, None, None, None, None, *path_ return bid_, rbyd_, rid_, name_, rattr_, *path_ @@ -1066,7 +1082,7 @@ class Btree: path=False, depth=None): # include our root rbyd even if the weight is zero - if self.weight == 0 and (depth is None or depth > 0): + if self.weight == 0: yield -1, self.rbyd, *(([],) if path else()) bid = 0 @@ -1079,22 +1095,24 @@ class Btree: yield (bid-rid + (rbyd.weight-1), rbyd, *((path_[0][:-1],) if path else ())) - bid += (rbyd.weight - name.weight) + 1 + bid += rbyd.weight - rid + 1 def traverse(self, *, path=False, depth=None): - ppath_ = [] + ptrunk_ = [] for bid, rbyd, path_ in self.leaves( path=True, depth=depth): - for d, (bid_, rbyd_, rid_, name_) in pathdelta( - path_, ppath_): - yield (bid_-rid_ + (rbyd_.weight-1), rbyd_, - *((path_[:d],) if path else ())) - ppath_ = path_ - - yield bid, rbyd, *((path_,) if path else ()) + # we only care about the rbyds here + trunk_ = ([(bid_-rid_ + (rbyd_.weight-1), rbyd_) + for bid_, rbyd_, rid_, name_ in path_] + + [(bid, rbyd)]) + for d, (bid_, rbyd_) in pathdelta( + trunk_, ptrunk_): + # but include branch rids in the path if requested + yield bid_, rbyd_, *((path_[:d],) if path else ()) + ptrunk_ = trunk_ def bids(self, *, path=False, @@ -1110,7 +1128,7 @@ class Btree: rbyd, rid, name)],) if path else ())) - def rattrs(self, bid=None, *, + def rattrs_(self, bid=None, *, path=False, depth=None): if bid is None: @@ -1130,6 +1148,19 @@ class Btree: path=path, depth=depth) for rattr in rbyd.rattrs(rid): + yield rattr, *path_ + + def rattrs(self, bid=None, *, + path=False, + depth=None): + if bid is None: + yield from self.rattrs_(bid, + path=path, + depth=depth) + else: + for rattr, *path_ in self.rattrs_(bid, + path=path, + depth=depth): if path: yield rattr, *path_ else: @@ -1170,7 +1201,7 @@ class Btree: else: return found_, bid + rid_, rbyd, rid_, name_ - # btree rbyd-tree generation for debugging + # create an rbyd tree for debugging def _tree_rtree(self, *, depth=None, inner=False, @@ -1203,9 +1234,10 @@ class Btree: rdepth = max((t.depth+1 for t in rtree), default=0) d = sum(rdepths[d]+1 for d in range(len(path))) + # map into our btree space for t in rtree: - # note we adjust our bid to be left-leaning, this allows a - # global order and make tree rendering quite a bit easier + # note we adjust our bid to be left-leaning, this allows + # a global order and makes tree rendering quite a bit easier a_rid, a_tag = t.a b_rid, b_tag = t.b _, (_, a_w, _) = rbyd.lookupnext(a_rid) @@ -1225,7 +1257,7 @@ class Btree: r_rid, r_tag = min(rtree, key=lambda t: t.depth).a _, (_, r_w, _) = rbyd.lookupnext(r_rid) else: - r_rid, (_, r_w, _) = rbyd.lookupnext() + r_rid, (r_tag, r_w, _) = rbyd.lookupnext(-1) tree.add(TreeBranch( (l_bid-(l_name.weight-1), len(path)-1, l_branch.tag), @@ -1276,7 +1308,7 @@ class Btree: return tree - # btree btree generation for debugging + # create a btree tree for debugging def _tree_btree(self, *, depth=None, inner=False, @@ -1290,20 +1322,21 @@ class Btree: depth=depth): # create branch for each jump in path # - # note we adjust our bid to be left-leaning, this allows a - # global order and make tree rendering quite a bit easier - # + # note we adjust our bid to be left-leaning, this allows + # a global order and makes tree rendering quite a bit easier a = root for d, (bid_, rbyd_, rid_, name_) in enumerate(path): - b = (bid_-(name_.weight-1), d, name_.tag) + # map into our btree space + bid__ = bid_-(name_.weight-1) + b = (bid__, d, name_.tag) # remap branches to leaves if we aren't showing inner # branches if not inner: if b not in branches: bid_, rbyd_, rid_, name_ = path[-1] - branches[b] = ( - bid_-(name_.weight-1), len(path)-1, name_.tag) + bid__ = bid_-(name_.weight-1) + branches[b] = (bid__, len(path)-1, name_.tag) b = branches[b] # render the root path on first rid, this is arbitrary @@ -1359,13 +1392,15 @@ def main(disk, roots=None, *, # fetch the btree bd = Bd(f, block_size, block_count) btree = Btree.fetch(bd, roots, trunk) + + # print some information about the btree print('btree %s w%d, rev %08x, cksum %08x' % ( btree.addr(), btree.weight, btree.rev, btree.cksum)) - # precompute trees if requested + # precompute tree renderings t_width = 0 if (args.get('tree') or args.get('tree_rbyd') @@ -1402,7 +1437,7 @@ def main(disk, roots=None, *, if rattr.weight > 1 else bid if rattr.weight > 0 else '', - 21+w_width, rattr.tagrepr(), + 21+w_width, rattr, next(xxd(rattr.data, 8), '') if not args.get('raw') and not args.get('no_truncate') @@ -1427,12 +1462,18 @@ def main(disk, roots=None, *, line)) # traverse and print entries - prbyd = None ppath = [] corrupted = False for bid, rbyd, path in btree.leaves( path=True, depth=args.get('depth')): + # print inner branches if requested + if args.get('inner'): + for d, (bid_, rbyd_, rid_, name_) in pathdelta( + path, ppath): + dbg_branch(d, bid_, rbyd_, rid_, name_) + ppath = path + # corrupted? try to keep printing the tree if not rbyd: print('%04x.%04x: %*s%s%s%s' % ( @@ -1445,13 +1486,6 @@ def main(disk, roots=None, *, corrupted = True continue - # print inner branches if requested - if args.get('inner'): - for d, (bid_, rbyd_, rid_, name_) in pathdelta( - path, ppath): - dbg_branch(d, bid_, rbyd_, rid_, name_) - ppath = path - for rid, name in rbyd.rids(): bid_ = bid-(rbyd.weight-1) + rid # show the leaf entry/branch @@ -1503,15 +1537,15 @@ if __name__ == "__main__": parser.add_argument( '-t', '--tree', action='store_true', - help="Show the underlying rbyd trees.") - parser.add_argument( - '-B', '--tree-btree', - action='store_true', - help="Show the B-tree.") + help="Show the rbyd tree.") parser.add_argument( '-R', '--tree-rbyd', action='store_true', - help="Show the full underlying rbyd trees.") + help="Show the full rbyd tree.") + parser.add_argument( + '-B', '--tree-btree', + action='store_true', + help="Show a simplified btree tree.") parser.add_argument( '-i', '--inner', action='store_true', diff --git a/scripts/dbgmtree.py b/scripts/dbgmtree.py index 3be7532a..9dd06b36 100755 --- a/scripts/dbgmtree.py +++ b/scripts/dbgmtree.py @@ -155,7 +155,7 @@ def frommdir(data): block, d_ = fromleb128(data[d:]) blocks.append(block) d += d_ - return tuple(blocks) + return blocks def frombranch(data): d = 0 @@ -277,25 +277,200 @@ def tagrepr(tag, weight=None, size=None, off=None): ' w%d' % weight if weight is not None else '', ' %d' % size if size is not None else '') +# tree branches are an abstract thing for tree rendering +class TreeBranch(co.namedtuple('TreeBranch', ['a', 'b', 'depth', 'color'])): + __slots__ = () + def __new__(cls, a, b, depth=0, color='b'): + # a and b are context specific + return super().__new__(cls, a, b, depth, color) + + def __repr__(self): + return '%s(%s, %s, %s, %s)' % ( + self.__class__.__name__, + self.a, + self.b, + self.depth, + self.color) + +def treerepr(tree, x, depth=None, color=False): + # find the max depth from the tree + if depth is None: + depth = max((t.depth+1 for t in tree), default=0) + if depth == 0: + return '' + + def branchrepr(tree, x, d, was): + for t in tree: + if t.depth == d and t.b == x: + if any(t.depth == d and t.a == x + for t in tree): + return '+-', t.color, t.color + elif any(t.depth == d + and x > min(t.a, t.b) + and x < max(t.a, t.b) + for t in tree): + return '|-', t.color, t.color + elif t.a < t.b: + return '\'-', t.color, t.color + else: + return '.-', t.color, t.color + for t in tree: + if t.depth == d and t.a == x: + return '+ ', t.color, None + for t in tree: + if (t.depth == d + and x > min(t.a, t.b) + and x < max(t.a, t.b)): + return '| ', t.color, was + if was: + return '--', was, was + return ' ', None, None + + trunk = [] + was = None + for d in range(depth): + t, c, was = branchrepr(tree, x, d, was) + + trunk.append('%s%s%s%s' % ( + '\x1b[33m' if color and c == 'y' + else '\x1b[31m' if color and c == 'r' + else '\x1b[90m' if color and c == 'b' + else '', + t, + ('>' if was else ' ') if d == depth-1 else '', + '\x1b[m' if color and c else '')) + + return '%s ' % ''.join(trunk) + +# compute the difference between two paths, returning everything +# in a after the paths diverge, as well as the relevant index +def pathdelta(a, b): + if not isinstance(a, list): + a = list(a) + i = 0 + for a_, b_ in zip(a, b): + if type(a_) == type(b_) and a_ == b_: + i += 1 + else: + break + + return [(i+j, a_) for j, a_ in enumerate(a[i:])] + + +# a simple wrapper over an open file with bd geometry +class Bd: + def __init__(self, f, block_size=None, block_count=None): + self.f = f + self.block_size = block_size + self.block_count = block_count + + def __repr__(self): + return '<%s %sx%s>' % ( + self.__class__.__name__, + self.block_size, + self.block_count) + + def read(self, size=-1): + return self.f.read(size) + + def seek(self, block, off, whence=0): + pos = self.f.seek(block*self.block_size + off, whence) + return pos // block_size, pos % block_size + + def readblock(self, block): + self.f.seek(block*self.block_size) + return self.f.read(self.block_size) + +# tagged data in an rbyd +class Rattr: + def __init__(self, tag, weight, block, toff, off, data): + self.tag = tag + self.weight = weight + self.block = block + self.toff = toff + self.off = off + self.data = data + + @property + def size(self): + return len(self.data) + + def __repr__(self): + return '<%s %s>' % (self.__class__.__name__, self) + + def __str__(self): + return tagrepr(self.tag, self.weight, self.size) + + def __iter__(self): + return iter((self.tag, self.weight, self.data)) + + def __eq__(self, other): + return ((self.tag, self.weight, self.data) + == (other.tag, other.weight, other.data)) + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.tag, self.weight, self.data)) + +class Ralt: + def __init__(self, tag, weight, block, toff, off, jump, + color=None, followed=None): + self.tag = tag + self.weight = weight + self.block = block + self.toff = toff + self.off = off + self.jump = jump + + if color is not None: + self.color = color + else: + self.color = 'r' if tag & TAG_R else 'b' + self.followed = followed + + @property + def joff(self): + return self.toff - self.jump + + def __repr__(self): + return '<%s %s>' % (self.__class__.__name__, self) + + def __str__(self): + return tagrepr(self.tag, self.weight, self.jump, self.toff) + + def __iter__(self): + return iter((self.tag, self.weight, self.jump)) + + def __eq__(self, other): + return ((self.tag, self.weight, self.jump) + == (other.tag, other.weight, other.jump)) + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.tag, self.weight, self.jump)) -# this type is used for tree representations -TBranch = co.namedtuple('TBranch', 'a, b, d, c') # our core rbyd type class Rbyd: - def __init__(self, blocks, data, rev, eoff, trunk, weight, cksum, - gcksumdelta): + def __init__(self, data, blocks, trunk, weight, rev, eoff, cksum, *, + gcksumdelta=None, + corrupt=False): if isinstance(blocks, int): - blocks = (blocks,) + blocks = [blocks] - self.blocks = tuple(blocks) self.data = data - self.rev = rev - self.eoff = eoff + self.blocks = list(blocks) self.trunk = trunk self.weight = weight + self.rev = rev + self.eoff = eoff self.cksum = cksum self.gcksumdelta = gcksumdelta + self.corrupt = corrupt @property def block(self): @@ -309,15 +484,31 @@ class Rbyd: ','.join('%x' % block for block in self.blocks), self.trunk) + def __repr__(self): + return '<%s %s w%s>' % ( + self.__class__.__name__, + self.addr(), + self.weight) + + def __bool__(self): + return not self.corrupt + + def __eq__(self, other): + return ((frozenset(self.blocks), self.trunk) + == (frozenset(other.blocks), other.trunk)) + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((frozenset(self.blocks), self.trunk)) + @classmethod - def fetch(cls, f, block_size, block, trunk=None, cksum=None): - # multiple blocks? - if (not isinstance(block, int) - and not isinstance(block, Rbyd) - and len(block) > 1): + def fetch(cls, bd, blocks, trunk=None, cksum=None): + # multiple blocks? unfortunately this must be a list + if isinstance(blocks, list): # fetch all blocks - rbyds = [cls.fetch(f, block_size, block, trunk, cksum) - for block in block] + rbyds = [cls.fetch(bd, block, trunk, cksum) for block in blocks] # determine most recent revision i = 0 for i_, rbyd in enumerate(rbyds): @@ -335,26 +526,25 @@ class Rbyd: for j in range(len(rbyds)-1)) return rbyd - # block may be an rbyd, in which case we inherit the - # already-read data - # - # this helps avoid race conditions with cksums and shrubs - if isinstance(block, Rbyd): - # inherit the trunk too I guess? - if trunk is None: - trunk = block.trunk - block, data = block.block, block.data - else: - # block may encode a trunk - block = block[0] if not isinstance(block, int) else block - if isinstance(block, tuple): - if trunk is None: - trunk = block[1] - block = block[0] + block = blocks - # seek to the block - f.seek(block * block_size) - data = f.read(block_size) + # blocks may also encode trunks + block, trunk = ( + block[0] if isinstance(block, tuple) + else block, + trunk if trunk is not None + else block[1] if isinstance(block, tuple) + else None) + + # bd can be either a bd reference or preread data + # + # preread data can be useful for avoiding race conditions + # with cksums and shrubs + if isinstance(bd, Bd): + # seek/read the block + data = bd.readblock(block) + else: + data = bd # fetch the rbyd rev = fromle32(data[0:4]) @@ -391,7 +581,8 @@ class Rbyd: # found a gcksumdelta? if (tag & 0xff00) == TAG_GCKSUMDELTA: - gcksumdelta_ = (tag, w, j_-d, d, data[j_:j_+size]) + gcksumdelta_ = Rattr(tag, w, + block, j_-d, d, data[j_:j_+size]) # found a cksum? else: @@ -448,60 +639,76 @@ class Rbyd: # cksum mismatch? if cksum is not None and cksum_ != cksum: - return cls(block, data, rev, 0, 0, 0, cksum_, gcksumdelta) + return cls(data, block, 0, 0, rev, 0, cksum_, + corrupt=True) - return cls(block, data, rev, eoff, trunk_, weight, cksum_, gcksumdelta) + return cls(data, block, trunk_, weight, rev, eoff, cksum_, + gcksumdelta=gcksumdelta, + corrupt=not trunk_) - def lookup(self, rid, tag): + def lookupnext(self, rid, tag=None, *, + path=False): if not self: - return True, 0, -1, 0, 0, 0, b'', [] + return None, None, *(([],) if path else ()) - tag = max(tag, 0x1) + tag = max(tag or 0, 0x1) lower = 0 upper = self.weight - path = [] + path_ = [] # descend down tree j = self.trunk while True: - _, alt, weight_, jump, d = fromtag(self.data[j:]) + _, alt, w, jump, d = fromtag(self.data[j:]) # found an alt? if alt & TAG_ALT: # follow? - if ((rid, tag & 0xfff) > (upper-weight_-1, alt & 0xfff) + if ((rid, tag & 0xfff) > (upper-w-1, alt & 0xfff) if alt & TAG_GT else ((rid, tag & 0xfff) - <= (lower+weight_-1, alt & 0xfff))): - lower += upper-lower-weight_ if alt & TAG_GT else 0 - upper -= upper-lower-weight_ if not alt & TAG_GT else 0 + <= (lower+w-1, alt & 0xfff))): + lower += upper-lower-w if alt & TAG_GT else 0 + upper -= upper-lower-w if not alt & TAG_GT else 0 j = j - jump - # figure out which color - if alt & TAG_R: - _, nalt, _, _, _ = fromtag(self.data[j+jump+d:]) - if nalt & TAG_R: - path.append((j+jump, j, True, 'y')) + if path: + # figure out which color + if alt & TAG_R: + _, nalt, _, _, _ = fromtag(self.data[j+jump+d:]) + if nalt & TAG_R: + color = 'y' + else: + color = 'r' else: - path.append((j+jump, j, True, 'r')) - else: - path.append((j+jump, j, True, 'b')) + color = 'b' + + path_.append(Ralt( + alt, w, self.block, j+jump, j+jump+d, jump, + color=color, + followed=True)) # stay on path else: - lower += weight_ if not alt & TAG_GT else 0 - upper -= weight_ if alt & TAG_GT else 0 + lower += w if not alt & TAG_GT else 0 + upper -= w if alt & TAG_GT else 0 j = j + d - # figure out which color - if alt & TAG_R: - _, nalt, _, _, _ = fromtag(self.data[j:]) - if nalt & TAG_R: - path.append((j-d, j, False, 'y')) + if path: + # figure out which color + if alt & TAG_R: + _, nalt, _, _, _ = fromtag(self.data[j:]) + if nalt & TAG_R: + color = 'y' + else: + color = 'r' else: - path.append((j-d, j, False, 'r')) - else: - path.append((j-d, j, False, 'b')) + color = 'b' + + path_.append(Ralt( + alt, w, self.block, j-d, j, jump, + color=color, + followed=False)) # found tag else: @@ -509,55 +716,154 @@ class Rbyd: tag_ = alt w_ = upper-lower - done = not tag_ or (rid_, tag_) < (rid, tag) + if not tag_ or (rid_, tag_) < (rid, tag): + return None, None, *(([],) if path else ()) - return (done, rid_, tag_, w_, j, d, - self.data[j+d:j+d+jump], - path) + return (rid_, + Rattr(tag_, w_, self.block, j, j+d, + self.data[j+d:j+d+jump]), + *((path_,) if path else ())) - def __bool__(self): - return bool(self.trunk) + def lookup_(self, rid, tag=None, mask=None, *, + path=False): + if tag is None: + tag, mask = 0, 0xffff + if mask is None: + mask = 0 - def __eq__(self, other): - return self.block == other.block and self.trunk == other.trunk + rid_, rattr_, *path_ = self.lookupnext(rid, tag & ~mask, + path=path) + if (rid_ is None + or rid_ != rid + or (rattr_.tag & ~mask) != (tag & ~mask)): + return None, *path_ - def __ne__(self, other): - return not self.__eq__(other) + return rattr_, *path_ - def __iter__(self): - tag = 0 + def lookup(self, rid, tag=None, mask=None, *, + path=False): + rattr_, *path_ = self.lookup_(rid, tag, mask, + path=path) + if path: + return rattr_, *path_ + else: + return rattr_ + + def __getitem__(self, key): + if not isinstance(key, tuple): + key = (key,) + + return self.lookup(*key) + + def __contains__(self, key): + if not isinstance(key, tuple): + key = (key,) + + return self.lookup_(*key)[0] is not None + + def rids(self, *, + path=False): rid = -1 - while True: - done, rid, tag, w, j, d, data, _ = self.lookup(rid, tag+0x1) - if done: + rid, name, *path_ = self.lookupnext(rid, + path=path) + # found end of tree? + if rid is None: break - yield rid, tag, w, j, d, data + yield rid, name, *path_ + rid += 1 - # create tree representation for debugging - def tree(self, *, - rbyd=False): + def rattrs_(self, rid=None, *, + path=False): + if rid is None: + rid, tag = -1, 0 + while True: + rid, rattr, *path_ = self.lookupnext(rid, tag+0x1, + path=path) + # found end of tree? + if rid is None: + break + + yield rid, rattr, *path_ + tag = rattr.tag + else: + tag = 0 + while True: + rid_, rattr, *path_ = self.lookupnext(rid, tag+0x1, + path=path) + # found end of tree? + if rid_ is None or rid_ != rid: + break + + yield rattr, *path_ + tag = rattr.tag + + def rattrs(self, rid=None, *, + path=False): + if rid is None: + yield from self.rattrs_(rid, + path=path) + else: + for rattr, *path_ in self.rattrs_(rid, + path=path): + if path: + yield rattr, *path_ + else: + yield rattr + + def __iter__(self): + return self.rattrs() + + # lookup by name + def namelookup(self, did, name): + # binary search + best = (False, None, None, None, None) + lower = 0 + upper = self.weight + while lower < upper: + rid, rattr = self.lookupnext(lower + (upper-1-lower)//2) + if rid is None: + break + + # treat vestigial names as a catch-all + if ((rattr.tag == TAG_NAME and rid-(rattr.weight-1) == 0) + or (rattr.tag & 0xff00) != TAG_NAME): + did_ = 0 + name_ = b'' + else: + did_, d = fromleb128(rattr.data) + name_ = rattr.data[d:] + + # bisect search space + if (did_, name_) > (did, name): + upper = rid-(w-1) + elif (did_, name_) < (did, name): + lower = rid + 1 + # keep track of best match + best = (False, rid, rattr) + else: + # found a match + return True, rid, rattr + + return best + + # create an rbyd tree for debugging + def _tree_rtree(self, **args): trunks = co.defaultdict(lambda: (-1, 0)) alts = co.defaultdict(lambda: {}) - rid, tag = -1, 0 - while True: - done, rid, tag, w, j, d, data, path = self.lookup(rid, tag+0x1) - # found end of tree? - if done: - break - + for rid, rattr, path in self.rattrs(path=True): # keep track of trunks/alts - trunks[j] = (rid, tag) + trunks[rattr.toff] = (rid, rattr.tag) - for j_, j__, followed, c in path: - if followed: - alts[j_] |= {'f': j__, 'c': c} + for ralt in path: + if ralt.followed: + alts[ralt.toff] |= {'f': ralt.joff, 'c': ralt.color} else: - alts[j_] |= {'nf': j__, 'c': c} + alts[ralt.toff] |= {'nf': ralt.off, 'c': ralt.color} - if rbyd: + if args.get('tree_rbyd'): # treat unreachable alts as converging paths for j_, alt in alts.items(): if 'f' not in alt: @@ -568,324 +874,1396 @@ class Rbyd: else: # prune any alts with unreachable edges pruned = {} - for j_, alt in alts.items(): + for j, alt in alts.items(): if 'f' not in alt: - pruned[j_] = alt['nf'] + pruned[j] = alt['nf'] elif 'nf' not in alt: - pruned[j_] = alt['f'] - for j_ in pruned.keys(): - del alts[j_] + pruned[j] = alt['f'] + for j in pruned.keys(): + del alts[j] - for j_, alt in alts.items(): + for j, alt in alts.items(): while alt['f'] in pruned: alt['f'] = pruned[alt['f']] while alt['nf'] in pruned: alt['nf'] = pruned[alt['nf']] # find the trunk and depth of each alt - def rec_trunk(j_): - if j_ not in alts: - return trunks[j_] + def rec_trunk(j): + if j not in alts: + return trunks[j] else: - if 'nft' not in alts[j_]: - alts[j_]['nft'] = rec_trunk(alts[j_]['nf']) - return alts[j_]['nft'] + if 'nft' not in alts[j]: + alts[j]['nft'] = rec_trunk(alts[j]['nf']) + return alts[j]['nft'] - for j_ in alts.keys(): - rec_trunk(j_) - for j_, alt in alts.items(): + for j in alts.keys(): + rec_trunk(j) + for j, alt in alts.items(): if alt['f'] in alts: alt['ft'] = alts[alt['f']]['nft'] else: alt['ft'] = trunks[alt['f']] - def rec_height(j_): - if j_ not in alts: + def rec_height(j): + if j not in alts: return 0 else: - if 'h' not in alts[j_]: - alts[j_]['h'] = max( - rec_height(alts[j_]['f']), - rec_height(alts[j_]['nf'])) + 1 - return alts[j_]['h'] + if 'h' not in alts[j]: + alts[j]['h'] = max( + rec_height(alts[j]['f']), + rec_height(alts[j]['nf'])) + 1 + return alts[j]['h'] - for j_ in alts.keys(): - rec_height(j_) + for j in alts.keys(): + rec_height(j) t_depth = max((alt['h']+1 for alt in alts.values()), default=0) # convert to more general tree representation tree = set() for j, alt in alts.items(): - # note all non-trunk edges should be black - tree.add(TBranch( - a=alt['nft'], - b=alt['nft'], - d=t_depth-1 - alt['h'], - c=alt['c'], - )) + # note all non-trunk edges should be colored black + tree.add(TreeBranch( + alt['nft'], + alt['nft'], + t_depth-1 - alt['h'], + alt['c'])) if alt['ft'] != alt['nft']: - tree.add(TBranch( - a=alt['nft'], - b=alt['ft'], - d=t_depth-1 - alt['h'], - c='b', - )) + tree.add(TreeBranch( + alt['nft'], + alt['ft'], + t_depth-1 - alt['h'], + 'b')) - return tree, t_depth + return tree - # btree lookup with this rbyd as the root - def btree_lookup(self, f, block_size, bid, *, - depth=None): - rbyd = self - rid = bid - depth_ = 1 - path = [] - - # corrupted? return a corrupted block once - if not rbyd: - return bid > 0, bid, 0, rbyd, -1, [], path - - while True: - # collect all tags, normally you don't need to do this - # but we are debugging here - name = None - tags = [] - branch = None - rid_ = rid - tag = 0 - w = 0 - for i in it.count(): - done, rid__, tag, w_, j, d, data, _ = rbyd.lookup( - rid_, tag+0x1) - if done or (i != 0 and rid__ != rid_): - break - - # first tag indicates the branch's weight - if i == 0: - rid_, w = rid__, w_ - - # catch any branches - if tag & 0xfff == TAG_BRANCH: - branch = (tag, j, d, data) - - tags.append((tag, j, d, data)) - - # keep track of path - path.append((bid + (rid_-rid), w, rbyd, rid_, tags)) - - # descend down branch? - if branch is not None and ( - not depth or depth_ < depth): - tag, j, d, data = branch - block, trunk, cksum = frombranch(data) - rbyd = Rbyd.fetch(f, block_size, block, trunk, cksum) - - # corrupted? bail here so we can keep traversing the tree - if not rbyd: - return False, bid + (rid_-rid), w, rbyd, -1, [], path - - rid -= (rid_-(w-1)) - depth_ += 1 - else: - return not tags, bid + (rid_-rid), w, rbyd, rid_, tags, path - - # btree rbyd-tree generation for debugging - def btree_tree(self, f, block_size, *, - depth=None, - inner=False, - rbyd=False): - # find the max depth of each layer to nicely align trees - bdepths = {} - bid = -1 - while True: - done, bid, w, rbyd_, rid, tags, path = self.btree_lookup( - f, block_size, bid+1, depth=depth) - if done: - break - - for d, (bid, w, rbyd_, rid, tags) in enumerate(path): - _, rdepth = rbyd_.tree(rbyd=rbyd) - bdepths[d] = max(bdepths.get(d, 0), rdepth) - - # find all branches + # create a btree tree for debugging + def _tree_btree(self, **args): + # for rbyds this is just a pointer to ever rid tree = set() root = None - branches = {} - bid = -1 + for rid, name in self.rids(): + b = (rid, name.tag) + if root is None: + root = b + tree.add(TreeBranch(root, b)) + return tree + + # create tree representation for debugging + def tree(self, **args): + if args.get('tree_btree'): + return self._tree_btree(**args) + else: + return self._tree_rtree(**args) + + +# our rbyd btree type +class Btree: + def __init__(self, bd, rbyd): + self.bd = bd + self.rbyd = rbyd + + @property + def block(self): + return self.rbyd.block + + @property + def blocks(self): + return self.rbyd.blocks + + @property + def trunk(self): + return self.rbyd.trunk + + @property + def weight(self): + return self.rbyd.weight + + @property + def rev(self): + return self.rbyd.rev + + @property + def eoff(self): + return self.rbyd.eoff + + @property + def cksum(self): + return self.rbyd.cksum + + def addr(self): + return self.rbyd.addr() + + def __repr__(self): + return '<%s %s w%s>' % ( + self.__class__.__name__, + self.addr(), + self.weight) + + def __eq__(self, other): + return self.rbyd == other.rbyd + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.rbyd) + + @classmethod + def fetch(cls, bd, blocks, trunk=None, cksum=None): + # we need a real bd reference here + assert isinstance(bd, Bd) + + rbyd = Rbyd.fetch(bd, blocks, trunk, cksum) + return cls(bd, rbyd) + + def lookupleaf(self, bid, *, + path=None, + depth=None): + rbyd = self.rbyd + rid = bid + depth_ = 1 + path_ = [] + while True: - done, bid, w, rbyd_, rid, tags, path = self.btree_lookup( - f, block_size, bid+1, depth=depth) - if done: + # corrupt branch? + if not rbyd: + return (bid, rbyd, rid, None, + *((path_,) if path else ())) + + # first tag indicates the branch's weight + rid_, name_ = rbyd.lookupnext(rid) + if rid_ is None: + return (None, None, None, None, + *((path_,) if path else ())) + + # keep track of path + if path: + path_.append((bid + (rid_-rid), rbyd, rid_, name_)) + + # find branch tag if there is one + branch_ = rbyd.lookup(rid_, TAG_BRANCH, 0x3) + + # descend down branch? + if branch_ is not None and ( + not depth or depth_ < depth): + block, trunk, cksum = frombranch(branch_.data) + rbyd = Rbyd.fetch(self.bd, block, trunk, cksum) + # keep track of expected trunk/weight if corrupted + if not rbyd: + rbyd.trunk = trunk + rbyd.weight = name_.weight + + rid -= (rid_-(name_.weight-1)) + depth_ += 1 + + else: + return (bid + (rid_-rid), rbyd, rid_, name_, + *((path_,) if path else ())) + + def lookup(self, bid, tag=None, mask=None, *, + path=False, + depth=None): + # lookup rbyd in btree + bid_, rbyd_, rid_, name_, *path_ = self.lookupleaf(bid, + path=path, + depth=depth) + if bid_ is None: + return None, None, None, None, None, *path_ + + # lookup tag in rbyd + rattr_ = rbyd_.lookup(rid_, tag, mask) + if rattr_ is None: + return None, None, None, None, None, *path_ + + return bid_, rbyd_, rid_, name_, rattr_, *path_ + + def __getitem__(self, key): + if not isinstance(key, tuple): + key = (key,) + + return self.lookup(*key) + + def __contains__(self, key): + if not isinstance(key, tuple): + key = (key,) + + return self.lookup(*key)[0] is not None + + # note leaves only iterates over leaf rbyds, whereas traverse + # traverses all rbyds + def leaves(self, *, + path=False, + depth=None): + # include our root rbyd even if the weight is zero + if self.weight == 0: + yield -1, self.rbyd, *(([],) if path else()) + + bid = 0 + while True: + bid, rbyd, rid, name, *path_ = self.lookupleaf(bid, + path=path, + depth=depth) + if bid is None: break - d_ = 0 - leaf = None - for d, (bid, w, rbyd_, rid, tags) in enumerate(path): - if not tags: - continue + yield (bid-rid + (rbyd.weight-1), rbyd, + *((path_[0][:-1],) if path else ())) + bid += rbyd.weight - rid + 1 - # map rbyd tree into B-tree space - rtree, rdepth = rbyd_.tree(rbyd=rbyd) + def traverse(self, *, + path=False, + depth=None): + ptrunk_ = [] + for bid, rbyd, path_ in self.leaves( + path=True, + depth=depth): + # we only care about the rbyds here + trunk_ = ([(bid_-rid_ + (rbyd_.weight-1), rbyd_) + for bid_, rbyd_, rid_, name_ in path_] + + [(bid, rbyd)]) + for d, (bid_, rbyd_) in pathdelta( + trunk_, ptrunk_): + # but include branch rids in the path if requested + yield bid_, rbyd_, *((path_[:d],) if path else ()) + ptrunk_ = trunk_ - # note we adjust our bid/rids to be left-leaning, - # this allows a global order and make tree rendering quite - # a bit easier - rtree_ = set() - for branch in rtree: - a_rid, a_tag = branch.a - b_rid, b_tag = branch.b - _, _, _, a_w, _, _, _, _ = rbyd_.lookup(a_rid, 0) - _, _, _, b_w, _, _, _, _ = rbyd_.lookup(b_rid, 0) - rtree_.add(TBranch( - a=(a_rid-(a_w-1), a_tag), - b=(b_rid-(b_w-1), b_tag), - d=branch.d, - c=branch.c, - )) - rtree = rtree_ + def bids(self, *, + path=False, + depth=None): + for bid, rbyd, *path_ in self.leaves( + path=path, + depth=depth): + for rid, name in rbyd.rids(): + yield (bid-(rbyd.weight-1) + rid, + rbyd, rid, name, + *((path_[0]+[ + (bid-(rbyd.weight-1) + rid, + rbyd, rid, name)],) + if path else ())) - # connect our branch to the rbyd's root - if leaf is not None: - root = min(rtree, - key=lambda branch: branch.d, - default=None) + def rattrs_(self, bid=None, *, + path=False, + depth=None): + if bid is None: + for bid, rbyd, *path_ in self.leaves( + path=path, + depth=depth): + for rid, name in rbyd.rids(): + for rattr in rbyd.rattrs(rid): + yield (bid-(rbyd.weight-1) + rid, + rbyd, rid, name, rattr, + *((path_[0]+[ + (bid-(rbyd.weight-1) + rid, + rbyd, rid, name)],) + if path else ())) + else: + bid, rbyd, rid, name, *path_ = self.lookupleaf(bid, + path=path, + depth=depth) + for rattr in rbyd.rattrs(rid): + yield rattr, *path_ - if root is not None: - r_rid, r_tag = root.a - else: - r_rid, r_tag = rid-(w-1), tags[0][0] - tree.add(TBranch( - a=leaf, - b=(bid-rid+r_rid, d, r_rid, r_tag), - d=d_-1, - c='b', - )) + def rattrs(self, bid=None, *, + path=False, + depth=None): + if bid is None: + yield from self.rattrs_(bid, + path=path, + depth=depth) + else: + for rattr, *path_ in self.rattrs_(bid, + path=path, + depth=depth): + if path: + yield rattr, *path_ + else: + yield rattr - for branch in rtree: - # map rbyd branches into our btree space - a_rid, a_tag = branch.a - b_rid, b_tag = branch.b - tree.add(TBranch( - a=(bid-rid+a_rid, d, a_rid, a_tag), - b=(bid-rid+b_rid, d, b_rid, b_tag), - d=branch.d + d_ + bdepths.get(d, 0)-rdepth, - c=branch.c, - )) + def __iter__(self): + return self.rattrs() - d_ += max(bdepths.get(d, 0), 1) - leaf = (bid-(w-1), d, rid-(w-1), - next( - (tag for tag, _, _, _ in tags - if tag & 0xfff == TAG_BRANCH), - TAG_BRANCH)) + # lookup by name + def namelookup(self, did, name, *, + depth=None): + rbyd = self.rbyd + bid = 0 + depth_ = 1 + + while True: + found_, rid_, name_ = rbyd.namelookup(did, name) + + # find branch tag if there is one + branch_ = rbyd.lookup(rid_, TAG_BRANCH, 0x3) + + # found another branch + if branch_ is not None and ( + not depth or depth_ < depth): + # update our bid + bid += rid_ - (name_.weight-1) + + block, trunk, cksum = frombranch(branch_.data) + rbyd = Rbyd.fetch(self.bd, block, trunk, cksum) + # keep track of expected trunk/weight if corrupted + if not rbyd: + rbyd.trunk = trunk + rbyd.weight = name_.weight + + depth_ += 1 + + # found best match + else: + return found_, bid + rid_, rbyd, rid_, name_ + + # create an rbyd tree for debugging + def _tree_rtree(self, *, + depth=None, + inner=False, + **args): + # precompute rbyd trees so we know the max depth at each layer + # to nicely align trees + rtrees = {} + rdepths = {} + for bid, rbyd, path in self.traverse(path=True, depth=depth): + if not rbyd: + continue + + rtrees[rbyd] = rbyd.tree(**args) + rdepths[len(path)] = max( + rdepths.get(len(path), 0), + max((t.depth+1 for t in rtrees[rbyd]), default=0)) + + # map rbyd branches into our btree space + tree = set() + for bid, rbyd, path in self.traverse(path=True, depth=depth): + if not rbyd: + continue + + # yes we can find new rbyds if disk is being mutated, just + # ignore these + if rbyd not in rtrees: + continue + + rtree = rtrees[rbyd] + rdepth = max((t.depth+1 for t in rtree), default=0) + d = sum(rdepths[d]+1 for d in range(len(path))) + + # map into our btree space + for t in rtree: + # note we adjust our bid to be left-leaning, this allows + # a global order and makes tree rendering quite a bit easier + a_rid, a_tag = t.a + b_rid, b_tag = t.b + _, (_, a_w, _) = rbyd.lookupnext(a_rid) + _, (_, b_w, _) = rbyd.lookupnext(b_rid) + tree.add(TreeBranch( + (bid-(rbyd.weight-1)+a_rid-(a_w-1), len(path), a_tag), + (bid-(rbyd.weight-1)+b_rid-(b_w-1), len(path), b_tag), + d + rdepths[len(path)]-rdepth + t.depth, + t.color)) + + # connect rbyd branches to rbyd roots + if path: + l_bid, l_rbyd, l_rid, l_name = path[-1] + l_branch = l_rbyd.lookup(l_rid, TAG_BRANCH, 0x3) + + if rtree: + r_rid, r_tag = min(rtree, key=lambda t: t.depth).a + _, (_, r_w, _) = rbyd.lookupnext(r_rid) + else: + r_rid, (r_tag, r_w, _) = rbyd.lookupnext(-1) + + tree.add(TreeBranch( + (l_bid-(l_name.weight-1), len(path)-1, l_branch.tag), + (bid-(rbyd.weight-1)+r_rid-(r_w-1), len(path), r_tag), + d-1)) # remap branches to leaves if we aren't showing inner branches if not inner: - # step through each layer backwards - b_depth = max((branch.a[1]+1 for branch in tree), default=0) + # step through each btree layer backwards + b_depth = max((t.a[1]+1 for t in tree), default=0) - # keep track of the original bids, unfortunately because we - # store the bids in the branches we overwrite these - tree = {(branch.b[0] - branch.b[2], branch) for branch in tree} + for d in reversed(range(b_depth-1)): + # find bid ranges at this level + bids = set() + for t in tree: + if t.a[1] == d: + bids.add(t.a[0]) + bids = sorted(bids) - for bd in reversed(range(b_depth-1)): - # find leaf-roots at this level + # find the best root for each bid range roots = {} - for bid, branch in tree: - # choose the highest node as the root - if (branch.b[1] == b_depth-1 - and (bid not in roots - or branch.d < roots[bid].d)): - roots[bid] = branch + for i in range(len(bids)): + for t in tree: + if (t.b[1] > d + and t.b[0] >= bids[i] + and (i == len(bids)-1 or t.b[0] < bids[i+1]) + and (bids[i] not in roots + or t.depth < roots[bids[i]].depth)): + roots[bids[i]] = t # remap branches to leaf-roots tree_ = set() - for bid, branch in tree: - if branch.a[1] == bd and branch.a[0] in roots: - branch = TBranch( - a=roots[branch.a[0]].b, - b=branch.b, - d=branch.d, - c=branch.c, - ) - if branch.b[1] == bd and branch.b[0] in roots: - branch = TBranch( - a=branch.a, - b=roots[branch.b[0]].b, - d=branch.d, - c=branch.c, - ) - tree_.add((bid, branch)) + for t in tree: + if t.a[1] == d and t.a[0] in roots: + t = TreeBranch( + roots[t.a[0]].b, + t.b, + t.depth, + t.color) + if t.b[1] == d and t.b[0] in roots: + t = TreeBranch( + t.a, + roots[t.b[0]].b, + t.depth, + t.color) + tree_.add(t) tree = tree_ - # strip out bids - tree = {branch for _, branch in tree} + return tree - return tree, max((branch.d+1 for branch in tree), default=0) - - # btree B-tree generation for debugging - def btree_btree(self, f, block_size, *, + # create a btree tree for debugging + def _tree_btree(self, *, depth=None, - inner=False): + inner=False, + **args): # find all branches tree = set() root = None branches = {} - bid = -1 - while True: - done, bid, w, rbyd, rid, tags, path = self.btree_lookup( - f, block_size, bid+1, depth=depth) - if done: - break - - # if we're not showing inner nodes, prefer names higher in - # the tree since this avoids showing vestigial names - name = None - if not inner: - name = None - for bid_, w_, rbyd_, rid_, tags_ in reversed(path): - for tag_, j_, d_, data_ in tags_: - if tag_ & 0x7f00 == TAG_NAME: - name = (tag_, j_, d_, data_) - - if rid_-(w_-1) != 0: - break - + for bid, rbyd, rid, name, path in self.bids( + path=True, + depth=depth): + # create branch for each jump in path + # + # note we adjust our bid to be left-leaning, this allows + # a global order and makes tree rendering quite a bit easier a = root - for d, (bid, w, rbyd, rid, tags) in enumerate(path): - if not tags: - continue + for d, (bid_, rbyd_, rid_, name_) in enumerate(path): + # map into our btree space + bid__ = bid_-(name_.weight-1) + b = (bid__, d, name_.tag) - b = (bid-(w-1), d, rid-(w-1), - (name if name else tags[0])[0]) - - # remap branches to leaves if we aren't showing - # inner branches + # remap branches to leaves if we aren't showing inner + # branches if not inner: if b not in branches: - bid, w, rbyd, rid, tags = path[-1] - if not tags: - continue - branches[b] = ( - bid-(w-1), len(path)-1, rid-(w-1), - (name if name else tags[0])[0]) + bid_, rbyd_, rid_, name_ = path[-1] + bid__ = bid_-(name_.weight-1) + branches[b] = (bid__, len(path)-1, name_.tag) b = branches[b] - # found entry point? + # render the root path on first rid, this is arbitrary if root is None: - root = b - a = root + root, a = b, b - tree.add(TBranch( - a=a, - b=b, - d=d, - c='b', - )) + tree.add(TreeBranch(a, b, d)) a = b - return tree, max((branch.d+1 for branch in tree), default=0) + return tree + + # create tree representation for debugging + def tree(self, **args): + if args.get('tree_btree'): + return self._tree_btree(**args) + else: + return self._tree_rtree(**args) + + +# a metadata id, this includes mbits for convenience +class Mid: + def __init__(self, mbid, mrid=None, *, + mbits=None): + # we need one of these to figure out mbits + if mbits is not None: + self.mbits = mbits + elif isinstance(mbid, Mid): + self.mbits = mbid.mbits + else: + assert mbits is not None, "mbits?" + + # accept other mids which can be useful for changing mrids + if isinstance(mbid, Mid): + mbid = mbid.mbid + + # accept either merged mid or separate mbid+mrid + if mrid is None: + mid = mbid + mbid = mid | ((1 << self.mbits) - 1) + mrid = mid & ((1 << self.mbits) - 1) + + # map mrid=-1 + if mrid == ((1 << self.mbits) - 1): + mrid = -1 + + self.mbid = mbid + self.mrid = mrid + + @property + def mid(self): + return ((self.mbid & ~((1 << self.mbits) - 1)) + | (self.mrid & ((1 << self.mbits) - 1))) + + def mbid_(self): + return self.mid >> self.mbits + + def mrid_(self): + return self.mrid + + def __repr__(self): + return '<%s %s>' % (self.__class__.__name__, self) + + def __str__(self): + return '%s.%s' % (self.mbid_(), self.mrid_()) + + def __iter__(self): + return iter((self.mbid, self.mrid)) + + # note this is slightly different from mid order when mrid=-1 + def __eq__(self, other): + if isinstance(other, Mid): + return (self.mbid, self.mrid) == (other.mbid, other.mrid) + else: + return self.mid == other + + def __ne__(self, other): + if isinstance(other, Mid): + return (self.mbid, self.mrid) != (other.mbid, other.mrid) + else: + return self.mid != other + + def __hash__(self): + return hash((self.mbid, self.mrid)) + + def __lt__(self, other): + return (self.mbid, self.mrid) < (other.mbid, other.mrid) + + def __le__(self, other): + return (self.mbid, self.mrid) <= (other.mbid, other.mrid) + + def __gt__(self, other): + return (self.mbid, self.mrid) > (other.mbid, other.mrid) + + def __ge__(self, other): + return (self.mbid, self.mrid) >= (other.mbid, other.mrid) + +# mdirs, the gooey atomic center of littlefs +# +# really the only difference between this and our rbyd class is the +# implicit mbid associated with the mdir +class Mdir: + def __init__(self, mid, rbyd, *, + mbits=None, + corrupt=False): + # we need one of these to figure out mbits + if mbits is not None: + self.mbits = mbits + elif isinstance(mid, Mid): + self.mbits = mid.mbits + elif isinstance(rbyd, Mdir): + self.mbits = rbyd.mbits + else: + assert mbits is not None, "mbits?" + + # strip mrid, bugs will happen if caller relies on mrid here + self.mid = Mid(mid, -1, mbits=self.mbits) + + # accept either another mdir or rbyd + if isinstance(rbyd, Mdir): + self.rbyd = rbyd.rbyd + self.corrupt = corrupt or rbyd.corrupt + else: + self.rbyd = rbyd + self.corrupt = corrupt or rbyd.corrupt + + @property + def data(self): + return self.rbyd.data + + @property + def block(self): + return self.rbyd.block + + @property + def blocks(self): + return self.rbyd.blocks + + @property + def trunk(self): + return self.rbyd.trunk + + @property + def weight(self): + return self.rbyd.weight + + @property + def rev(self): + return self.rbyd.rev + + @property + def eoff(self): + return self.rbyd.eoff + + @property + def cksum(self): + return self.rbyd.cksum + + @property + def gcksumdelta(self): + return self.rbyd.gcksumdelta + + def addr(self): + return self.rbyd.addr() + + def __repr__(self): + return '<%s %s %s>' % ( + self.__class__.__name__, + self.mid.mbid_(), + self.addr()) + + def __bool__(self): + return not self.corrupt + + # we _don't_ care about mid for equality, or trunk even + def __eq__(self, other): + return frozenset(self.blocks) == frozenset(other.blocks) + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(frozenset(self.blocks)) + + @classmethod + def fetch(cls, bd, mid, blocks): + rbyd = Rbyd.fetch(bd, blocks) + # this affects mbits + if isinstance(bd, Bd): + return cls(mid, rbyd, mbits=Mtree.mbits_(bd)) + else: + return cls(mid, rbyd) + + def lookup_(self, mid, tag=None, mask=None, *, + path=False): + if not isinstance(mid, Mid): + mid = Mid(mid, mbits=self.mbits) + return self.rbyd.lookup_(mid.mrid, tag, + path=path) + + def lookup(self, mid, tag=None, mask=None, *, + path=False): + rattr_, *path_ = self.lookup_(mid, tag, mask, + path=path) + if path: + return rattr_, *path_ + else: + return rattr_ + + def __getitem__(self, key): + if not isinstance(key, tuple): + key = (key,) + + return self.lookup(*key) + + def __contains__(self, key): + if not isinstance(key, tuple): + key = (key,) + + return self.lookup_(*key)[0] is not None + + def mids(self, *, + path=False): + for rid, name, *path_ in self.rbyd.rids( + path=path): + mid = Mid(self.mid, rid) + yield mid, name, *path_ + + def rattrs_(self, mid=None, *, + path=False): + if mid is None: + for rid, rattr, *path_ in self.rbyd.rattrs_( + path=path): + mid = Mid(self.mid, rid) + yield mid, rattr, *path_ + else: + yield from self.rbyd.rattrs_(mid.mrid, + path=path) + + def rattrs(self, mid=None, *, + path=False): + if mid is None: + yield from self.rattrs_(mid, + path=path) + else: + for rattr, *path_ in self.rattrs_(mid, + path=path): + if path: + yield rattr, *path_ + else: + yield rattr + + def __iter__(self): + return self.rattrs() + + # lookup by name + def namelookup(self, did, name): + found, rid, rattr = self.rbyd.namelookup(did, name) + if rid is None: + return found, None, None + else: + return found, Mid(self.mid, rid), rattr + + # create tree representation for debugging + def tree(self, **args): + tree = self.rbyd.tree(**args) + + # map to mid + tree_ = set() + for t in tree: + a_rid, a_tag = t.a + b_rid, b_tag = t.b + tree_.add(TreeBranch( + (Mid(self.mid, a_rid), a_tag), + (Mid(self.mid, b_rid), b_tag), + t.depth, + t.color)) + tree = tree_ + + return tree + +# the mtree, the skeletal structure of littlefs +class Mtree: + def __init__(self, bd, mrootchain, mtree, *, + mrootpath=None, + mtreepath=None, + mbits=None): + if isinstance(mrootchain, Mdir): + mrootchain = [Mdir] + # we at least need the mrootanchor, even if it is corrupt + assert len(mrootchain) >= 1 + + self.bd = bd + if mbits is not None: + self.mbits = mbits + else: + self.mbits = Mtree.mbits_(self.bd) + + self.mrootchain = mrootchain + self.mrootanchor = mrootchain[0] + self.mroot = mrootchain[-1] + self.mtree = mtree + + # mbits is a static value derived from the block_size + @staticmethod + def mbits_(block_size): + if isinstance(block_size, Bd): + block_size = block_size.block_size + return mt.ceil(mt.log2(block_size // 8)) + + # convenience function for creating mbits-dependent mids + def mid(self, mbid, mrid=None): + return Mid(mbid, mrid, mbits=self.mbits) + + @property + def block(self): + return self.mroot.block + + @property + def blocks(self): + return self.mroot.blocks + + @property + def trunk(self): + return self.mroot.trunk + + @property + def weight(self): + if self.mtree is None: + return 0 + else: + return self.mtree.weight + + def mbweight(self): + return self.weight + + def mrweight(self): + return 1 << self.mbits + + def mbweight_(self): + return self.weight >> self.mbits + + def mrweight_(self): + return 1 << self.mbits + + @property + def rev(self): + return self.mroot.rev + + @property + def eoff(self): + return self.mroot.eoff + + @property + def cksum(self): + return self.mroot.cksum + + def addr(self): + return self.mroot.addr() + + def __repr__(self): + return '<%s %s w%s.%s>' % ( + self.__class__.__name__, + self.addr(), + self.mbweight_(), self.mrweight_()) + + def __eq__(self, other): + return self.mrootanchor == other.mrootanchor + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash(self.mrootanchor) + + @classmethod + def fetch(cls, bd, blocks=None, *, + depth=None): + # we need a real bd reference here + assert isinstance(bd, Bd) + + # default to blocks 0x{0,1} + if blocks is None: + blocks = [0, 1] + + # figure out mbits + mbits = Mtree.mbits_(bd) + + # fetch the mrootanchor + mrootanchor = Mdir.fetch(bd, -1, blocks) + + # follow the mroot chain to try to find the active mroot + mroot = mrootanchor + mrootchain = [mrootanchor] + mrootseen = set() + while True: + # corrupted? + if not mroot: + break + # cycle detected? + if mroot in mrootseen: + break + mrootseen.add(mroot) + + # stop here? + if depth and len(mrootchain) >= depth: + break + + # fetch the next mroot + rattr_ = mroot.lookup(-1, TAG_MROOT, 0x3) + if rattr_ is None: + break + blocks_ = frommdir(rattr_.data) + mroot = Mdir.fetch(bd, -1, blocks_) + mrootchain.append(mroot) + + # fetch the actual mtree, if there is one + mtree = None + if not depth or len(mrootchain) < depth: + rattr_ = mroot.lookup(-1, TAG_MTREE, 0x3) + if rattr_ is not None: + w_, block_, trunk_, cksum_ = frombtree(rattr_.data) + mtree = Btree.fetch(bd, block_, trunk_, cksum_) + + return cls(bd, mrootchain, mtree, + mbits=mbits) + + def lookupleaf_(self, mid, *, + path=None, + depth=None): + if not isinstance(mid, Mid): + mid = self.mid(mid) + + if path or depth: + # iterate over mrootchain + path_ = [] + for mroot in self.mrootchain: + path_.append((mroot.mid, mroot, + mroot.lookup(-1, TAG_MAGIC))) + # stop here? + if depth and len(path_) >= depth: + return mroot, path_ + + # no mtree? must be inlined in mroot + if self.mtree is None: + if mid.mbid >= (1 << self.mbits): + return None, *((path_,) if path else ()) + + mdir = Mdir(mid, self.mroot) + # corrupt mdir? + if not mdir: + return mdir, *((path_,) if path else ()) + # not in mdir? + if mid.mrid >= mdir.weight: + return None, *((path_,) if path else ()) + + if path: + path_.append((mid, mdir, mdir.lookup(mid))) + return mdir, *((path_,) if path else ()) + + # mtree? lookup in mtree + else: + # need to do two steps here in case lookupleaf stops early + bid_, rbyd_, rid_, name_, *path__ = ( + self.mtree.lookupleaf(mid.mid, + path=path or depth, + depth=depth-len(path_) if depth else None)) + if path or depth: + path_.extend(( + self.mid(bid__), + (bid__, rbyd__, rid__, name__), + name__) + for bid__, rbyd__, rid__, name__ in path__[0]) + if bid_ is None: + return None, *((path_,) if path else ()) + + # stop here? it's not an mdir, but we only return this if + # depth is explicitly requested + if depth and len(path_) >= depth: + return ((bid_, rbyd_, rid_, name_), + *((path_,) if path else ())) + + # fetch the mdir + rattr_ = rbyd_.lookup(rid_, TAG_MDIR, 0x3) + # mdir tag missing? weird + if rattr_ is None: + return None, *((path_,) if path else ()) + blocks_ = frommdir(rattr_.data) + mdir = Mdir.fetch(self.bd, mid, blocks_) + # corrupt mdir? + if not mdir: + return mdir, *((path_,) if path else ()) + # not in mdir? + if mid.mrid >= mdir.weight: + return None, *((path_,) if path else ()) + + if path: + path_.append((mid, mdir, mdir.lookup(mid))) + return mdir, *((path_,) if path else ()) + + def lookupleaf(self, mid, *, + path=None, + depth=None): + mdir, *path_ = self.lookupleaf_(mid, + path=path, + depth=depth) + if path: + return mdir, *path_ + else: + return mdir + + def lookup(self, mid, tag=None, mask=None, *, + path=False, + depth=None): + mdir_, *path_ = self.lookupleaf_(mid, + path=path, + depth=depth) + if (mdir_ is None + # these can happen if depth reached + or not isinstance(mdir_, Mdir) + or mdir_.mid == -1): + return None, None, *path_ + + # lookup tag in mdir + rattr_ = mdir_.lookup(mid, tag, mask) + if rattr_ is None: + return None, None, *path_ + + return mdir_, rattr_, *path_ + + def __getitem__(self, key): + if not isinstance(key, tuple): + key = (key,) + + return self.lookup(*key) + + def __contains__(self, key): + if not isinstance(key, tuple): + key = (key,) + + return self.lookup(*key)[0] is not None + + # iterate over all mdirs, this includes the mrootchain + def leaves_(self, *, + path=False, + depth=None): + # iterate over mrootchain + if path or depth: + path_ = [] + for mroot in self.mrootchain: + yield mroot, *((path_,) if path else ()) + + if path or depth: + path_.append((mroot.mid, mroot, + mroot.lookup(-1, TAG_MAGIC))) + # stop here? + if depth and len(path_) >= depth: + return + + # do we even have an mtree? + if self.mtree is not None: + # include the mtree root even if the weight is zero + if self.mtree.weight == 0: + yield (-1, self.mtree.rbyd), *((path_,) if path else ()) + + mid = self.mid(0) + while True: + mdir, *path__ = self.lookupleaf_(mid, + path=path, + depth=depth) + if mdir is None: + break + + # mdir? + if isinstance(mdir, Mdir): + yield mdir, *((path__[0][:-1],) if path else ()) + mid = self.mid(mid.mbid+1) + # btree node? + else: + bid, rbyd, rid, name = mdir + yield ((bid-rid + (rbyd.weight-1), rbyd), + *((path__[0][:-1],) if path else ())) + mid = self.mid(bid-rid + (rbyd.weight-1) + 1) + + def leaves(self, *, + path=False, + depth=None): + for mdir, *path_ in self.leaves_( + path=path, + depth=depth): + if path: + yield mdir, *path_ + else: + yield mdir + + # traverse over all mdirs and btree nodes + # - mdir => Mdir + # - btree node => (bid, rbyd) + def traverse_(self, *, + path=False, + depth=None): + ptrunk_ = [] + for mdir, path_ in self.leaves( + path=True, + depth=depth): + # we only care about the mdirs/rbyds here + trunk_ = ([mdir if isinstance(mdir, Mdir) + else (lambda bid_, rbyd_, rid_, name_: + (bid_-rid_ + (rbyd_.weight-1), rbyd_))( + *mdir) + for mid, mdir, name in path_] + + [mdir]) + for d, mdir in pathdelta( + trunk_, ptrunk_): + # but include branch mids/rids in the path if requested + yield mdir, *((path_[:d],) if path else ()) + ptrunk_ = trunk_ + + def traverse(self, *, + path=False, + depth=None): + for mdir, *path_ in self.traverse_( + path=path, + depth=depth): + if path: + yield mdir, *path_ + else: + yield mdir + + # these are just aliases + mdirs_ = leaves_ + mdirs = leaves + + def mids(self, *, + path=False, + depth=None): + for mdir, *path_ in self.leaves_( + path=path, + depth=depth): + if isinstance(mdir, Mdir): + for mid, name in mdir.mids(): + yield (mid, mdir, name, + *((path_[0]+[(mid, mdir, mdir.lookup(mid))],) + if path else ())) + else: + bid, rbyd = mdir + for rid, name in rbyd.rids(): + bid_ = bid-(rbyd.weight-1) + rid + mid_ = self.mid(bid_) + mdir_ = (bid_, rbyd, rid, name) + yield (mid_, mdir_, name, + *((path_[0]+[(mid_, mdir_, name)],) + if path else ())) + + def rattrs_(self, mid=None, *, + path=False, + depth=None): + if mid is None: + for mdir, *path_ in self.leaves_( + path=path, + depth=depth): + if isinstance(mdir, Mdir): + for mid, rattr in mdir.rattrs(): + yield (mid, mdir, rattr, + *((path_[0]+[(mid, mdir, mdir.lookup(mid))],) + if path else ())) + else: + bid, rbyd = mdir + for rid, name in rbyd.rids(): + bid_ = bid-(rbyd.weight-1) + rid + mid_ = self.mid(bid_) + mdir_ = (bid_, rbyd, rid, name) + for rattr in rbyd.rattrs(rid): + yield (mid_, mdir_, rattr, + *((path_[0]+[(mid_, mdir_, name)],) + if path else ())) + else: + if not isinstance(mid, Mid): + mid = self.mid(mid) + + mdir, *path_ = self.lookupleaf_(mid, + path=path, + depth=depth) + if isinstance(mdir, Mdir): + for rattr in mdir.rattrs(mid): + yield rattr, *path_ + else: + bid, rbyd, rid, name = mdir + for rattr in rbyd.rattrs(rid): + yield rattr, *path_ + + def rattrs(self, mid=None, *, + path=False, + depth=None): + if mid is None: + yield from self.rattrs_(mid, + path=path, + depth=depth) + else: + for rattr, *path_ in self.rattrs_(mid, + path=path, + depth=depth): + if path: + yield rattr, *path_ + else: + yield rattr + + def __iter__(self): + return self.rattrs() + + # lookup by name + def namelookup(self, did, name, *, + depth=None): + # TODO + pass + + # create an rbyd tree for debugging + def _tree_rtree(self, *, + depth=None, + inner=False, + **args): + # precompute rbyd trees so we know the max depth at each layer + # to nicely align trees + rtrees = {} + rdepths = {} + for mdir, path in self.traverse(path=True, depth=depth): + if isinstance(mdir, Mdir): + if not mdir: + continue + rbyd = mdir.rbyd + else: + bid, rbyd = mdir + if not rbyd: + continue + + rtrees[rbyd] = rbyd.tree(**args) + rdepths[len(path)] = max( + rdepths.get(len(path), 0), + max((t.depth+1 for t in rtrees[rbyd]), default=0)) + + # map rbyd branches into our mtree space + tree = set() + branches = {} + for mdir, path in self.traverse(path=True, depth=depth): + if isinstance(mdir, Mdir): + if not mdir: + continue + rbyd = mdir.rbyd + else: + bid, rbyd = mdir + if not rbyd: + continue + + # yes we can find new rbyds if disk is being mutated, just + # ignore these + if rbyd not in rtrees: + continue + + rtree = rtrees[rbyd] + rdepth = max((t.depth+1 for t in rtree), default=0) + d = sum(rdepths[d]+1 for d in range(len(path))) + + # map into our mtree space + for t in rtree: + # note we adjust our mid/bid to be left-leaning, this allows + # a global order and makes tree rendering quite a bit easier + # + # we also need to give btree nodes mrid=-1 so they come + # before and mrid=-1 mdir attrs + a_rid, a_tag = t.a + b_rid, b_tag = t.b + _, (_, a_w, _) = rbyd.lookupnext(a_rid) + _, (_, b_w, _) = rbyd.lookupnext(b_rid) + if isinstance(mdir, Mdir): + a_mid = self.mid(mdir.mid, a_rid) + b_mid = self.mid(mdir.mid, b_rid) + else: + a_mid = self.mid(bid-(rbyd.weight-1)+a_rid-(a_w-1), -1) + b_mid = self.mid(bid-(rbyd.weight-1)+b_rid-(b_w-1), -1) + + tree.add(TreeBranch( + (a_mid, len(path), a_tag), + (b_mid, len(path), b_tag), + d + rdepths[len(path)]-rdepth + t.depth, + t.color)) + + # connect rbyd branches to rbyd roots + if path: + # figure out branch mid/attr + l_mid, l_mdir, l_name = path[-1] + if isinstance(l_mdir, Mdir): + l_branch = (l_mdir.lookup(l_mid, TAG_MROOT, 0x3) + or l_mdir.lookup(l_mid, TAG_MTREE, 0x3)) + else: + l_bid, l_rbyd, l_rid, l_name = l_mdir + l_mid = self.mid(l_bid-(l_name.weight-1), -1) + l_branch = (l_rbyd.lookup(l_rid, TAG_BRANCH, 0x3) + or l_rbyd.lookup(l_rid, TAG_MDIR, 0x3)) + + # figure out root mid/rattr + if rtree: + r_rid, r_tag = min(rtree, key=lambda t: t.depth).a + _, (_, r_w, _) = rbyd.lookupnext(r_rid) + else: + r_rid, (r_tag, r_w, _) = rbyd.lookupnext(-1) + + if isinstance(mdir, Mdir): + r_mid = self.mid(mdir.mid, r_rid) + else: + r_mid = self.mid(bid-(rbyd.weight-1)+r_rid-(r_w-1), -1) + + tree.add(TreeBranch( + (l_mid, len(path)-1, l_branch.tag), + (r_mid, len(path), r_tag), + d-1)) + + # remap branches to leaves if we aren't showing inner branches + if not inner: + # step through each btree layer backwards + b_depth = max((t.a[1]+1 for t in tree), default=0) + + for d in reversed(range(len(self.mrootchain), b_depth-1)): + # find mid ranges at this level + mids = set() + for t in tree: + if t.a[1] == d: + mids.add(t.a[0]) + mids = sorted(mids) + + # find the best root for each mid range + roots = {} + for i in range(len(mids)): + for t in tree: + if (t.b[1] > d + and t.b[0] >= mids[i] + and (i == len(mids)-1 or t.b[0] < mids[i+1]) + and (mids[i] not in roots + or t.depth < roots[mids[i]].depth)): + roots[mids[i]] = t + + # remap branches to leaf-roots + tree_ = set() + for t in tree: + if t.a[1] == d and t.a[0] in roots: + t = TreeBranch( + roots[t.a[0]].b, + t.b, + t.depth, + t.color) + if t.b[1] == d and t.b[0] in roots: + t = TreeBranch( + t.a, + roots[t.b[0]].b, + t.depth, + t.color) + tree_.add(t) + tree = tree_ + + return tree + + # create a btree tree for debugging + def _tree_btree(self, *, + depth=None, + inner=False, + **args): + tree = set() + root = None + branches = {} + for mid, mdir, name, path in self.mids( + path=True, + depth=depth): + # create branch for each jump in path + # + # note we adjust our mid/bid to be left-leaning, this allows + # a global order and makes tree rendering quite a bit easier + # + # we also need to give btree nodes mrid=-1 so they come + # before and mrid=-1 mdir attrs + a = root + for d, (mid_, mdir_, name_) in enumerate(path): + # map into our mtree space + if not isinstance(mdir_, Mdir): + bid_, rbyd_, rid_, name_ = mdir_ + mid_ = self.mid(bid_-(name_.weight-1), -1) + b = (mid_, d, name_.tag) + + # remap branches to leaves if we aren't showing inner + # branches + if not inner: + if b not in branches: + mid_, mdir_, name_ = path[-1] + if not isinstance(mdir_, Mdir): + bid_, rbyd_, rid_, name_ = mdir_ + mid_ = self.mid(bid_-(name_.weight-1), -1) + branches[b] = (mid_, len(path)-1, name_.tag) + b = branches[b] + + # render the root path on first rid, this is arbitrary + if root is None: + root, a = b, b + + tree.add(TreeBranch(a, b, d)) + a = b + + return tree + + # create tree representation for debugging + def tree(self, **args): + if args.get('tree_btree'): + return self._tree_btree(**args) + else: + return self._tree_rtree(**args) + def main(disk, mroots=None, *, @@ -910,7 +2288,7 @@ def main(disk, mroots=None, *, # flatten mroots, default to 0x{0,1} if not mroots: mroots = [(0,1)] - mroots = tuple(block for mroots_ in mroots for block in mroots_) + mroots = [block for mroots_ in mroots for block in mroots_] # we seek around a bunch, so just keep the disk open with open(disk, 'rb') as f: @@ -919,633 +2297,110 @@ def main(disk, mroots=None, *, f.seek(0, os.SEEK_END) block_size = f.tell() - # determine the mleaf_weight from the block_size, this is just for - # printing purposes - mleaf_weight = 1 << mt.ceil(mt.log2(block_size // 8)) + # fetch the mtree + bd = Bd(f, block_size, block_count) + mtree = Mtree.fetch(bd, mroots, + depth=args.get('depth')) - # before we print, we need to do a pass for a few things: - # - find the actual mroot - # - find the total weight - bweight = 0 - rweight = 0 + # print some information about the mtree + print('mtree %s w%s.%s, rev %08x, cksum %08x' % ( + mtree.addr(), + mtree.mbweight_(), mtree.mrweight_(), + mtree.rev, + mtree.cksum)) - mroot = Rbyd.fetch(f, block_size, mroots) - mdepth = 1 - mseen = set() - while True: - # corrupted? - if not mroot: - break - # cycle detected? - elif mroot.blocks in mseen: - break - - mseen.add(mroot.blocks) - - rweight = max(rweight, mroot.weight) - - # stop here? - if args.get('depth') and mdepth >= args.get('depth'): - break - - # fetch the next mroot - done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MROOT) - if not (not done and rid == -1 and tag == TAG_MROOT): - break - - blocks = frommdir(data) - mroot = Rbyd.fetch(f, block_size, blocks) - mdepth += 1 - - # fetch the mdir, if there is one - mdir = None - if not args.get('depth') or mdepth < args.get('depth'): - done, rid, tag, w, j, _, data, _ = mroot.lookup(-1, TAG_MDIR) - if not done and rid == -1 and tag == TAG_MDIR: - blocks = frommdir(data) - mdir = Rbyd.fetch(f, block_size, blocks) - - # corrupted? - if mdir: - rweight = max(rweight, mdir.weight) - - # fetch the actual mtree, if there is one - mtree = None - if not args.get('depth') or mdepth < args.get('depth'): - done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MTREE) - if not done and rid == -1 and tag == TAG_MTREE: - w, block, trunk, cksum = frombtree(data) - mtree = Rbyd.fetch(f, block_size, block, trunk, cksum) - - bweight = w - - # traverse entries - mbid = -1 - while True: - done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup( - f, block_size, mbid+1, - depth=args.get('depth', mdepth)-mdepth) - if done: - break - - # corrupted? - if not rbyd: - continue - - mdir__ = None - if (not args.get('depth') - or mdepth+len(path) < args.get('depth')): - mdir__ = next( - ((tag, j, d, data) - for tag, j, d, data in tags - if tag == TAG_MDIR), - None) - - if mdir__: - # fetch the mdir - _, _, _, data = mdir__ - blocks = frommdir(data) - mdir_ = Rbyd.fetch(f, block_size, blocks) - - # corrupted? - if mdir_: - rweight = max(rweight, mdir_.weight) - - # precompute rbyd-tree if requested + # precompute tree renderings t_width = 0 - if args.get('tree') or args.get('rbyd'): - # compute mroot chain "tree", prefix our actual mtree with this - tree = set() - d_ = 0 - mroot_ = Rbyd.fetch(f, block_size, mroots) - mdepth_ = 1 - mseen_ = set() - for d in it.count(): - # corrupted? - if not mroot_: - break - # cycle detected? - elif mroot_.blocks in mseen_: - break + if (args.get('tree') + or args.get('tree_rbyd') + or args.get('tree_btree')): + tree = mtree.tree(**args) - mseen_.add(mroot_.blocks) - - # compute the mroots rbyd-tree - rtree, rdepth = mroot_.tree(rbyd=args.get('rbyd')) - - # connect branch to our root - if d > 0: - root = min(rtree, - key=lambda branch: branch.d, - default=None) - - if root: - r_rid, r_tag = root.a - else: - _, r_rid, r_tag, _, _, _, _, _ = mroot_.lookup(-1, 0x1) - tree.add(TBranch( - a=(-1, d-1, 0, -1, TAG_MROOT), - b=(-1, d, 0, r_rid, r_tag), - d=d_-1, - c='b', - )) - - # map the tree into our metadata space - for branch in rtree: - a_rid, a_tag = branch.a - b_rid, b_tag = branch.b - tree.add(TBranch( - a=(-1, d, 0, a_rid, a_tag), - b=(-1, d, 0, b_rid, b_tag), - d=d_ + branch.d, - c=branch.c, - )) - d_ += rdepth - - # stop here? - if args.get('depth') and mdepth_ >= args.get('depth'): - break - - # fetch the next mroot - done, rid, tag, w, j, _, data, _ = mroot_.lookup(-1, TAG_MROOT) - if not (not done and rid == -1 and tag == TAG_MROOT): - break - - blocks = frommdir(data) - mroot_ = Rbyd.fetch(f, block_size, blocks) - mdepth_ += 1 - - # compute mdir's rbyd-tree if there is one - if mdir: - rtree, rdepth = mdir.tree(rbyd=args.get('rbyd')) - - # connect branch to our root - root = min(rtree, - key=lambda branch: branch.d, - default=None) - - if root: - r_rid, r_tag = root.a - else: - _, r_rid, r_tag, _, _, _, _, _ = mdir.lookup(-1, 0x1) - tree.add(TBranch( - a=(-1, d, 0, -1, TAG_MDIR), - b=(0, 0, 0, r_rid, r_tag), - d=d_-1, - c='b', - )) - - # map the tree into our metadata space - for branch in rtree: - a_rid, a_tag = branch.a - b_rid, b_tag = branch.b - tree.add(TBranch( - a=(0, 0, 0, a_rid, a_tag), - b=(0, 0, 0, b_rid, b_tag), - d=d_ + branch.d, - c=branch.c, - )) - - # compute the mtree's rbyd-tree if there is one - if mtree: - tree_, tdepth = mtree.btree_tree( - f, block_size, - depth=args.get('depth', mdepth)-mdepth, - inner=args.get('inner'), - rbyd=args.get('rbyd')) - - # connect a branch to the root of the tree - root = min(tree_, key=lambda branch: branch.d, default=None) - if root: - r_bid, r_bd, r_rid, r_tag = root.a - tree.add(TBranch( - a=(-1, d, 0, -1, TAG_MTREE), - b=(r_bid, r_bd, r_rid, 0, r_tag), - d=d_-1, - c='b', - )) - - # map the tree into our metadata space - for branch in tree_: - a_bid, a_bd, a_rid, a_tag = branch.a - b_bid, b_bd, b_rid, b_tag = branch.b - tree.add(TBranch( - a=(a_bid, a_bd, a_rid, 0, a_tag), - b=(b_bid, b_bd, b_rid, 0, b_tag), - d=d_ + branch.d, - c=branch.c, - )) - - # find the max depth of each mdir to nicely align trees - mdepth_ = 0 - mbid = -1 - while True: - done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup( - f, block_size, mbid+1, - depth=args.get('depth', mdepth)-mdepth) - if done: - break - - # corrupted? - if not rbyd: - continue - - mdir__ = None - if (not args.get('depth') - or mdepth+len(path) < args.get('depth')): - mdir__ = next( - ((tag, j, d, data) - for tag, j, d, data in tags - if tag == TAG_MDIR), - None) - - if mdir__: - # fetch the mdir - _, _, _, data = mdir__ - blocks = frommdir(data) - mdir_ = Rbyd.fetch(f, block_size, blocks) - - rtree, rdepth = mdir_.tree(rbyd=args.get('rbyd')) - mdepth_ = max(mdepth_, rdepth) - - # compute the rbyd-tree for each mdir - mbid = -1 - while True: - done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup( - f, block_size, mbid+1, - depth=args.get('depth', mdepth)-mdepth) - if done: - break - - # corrupted? - if not rbyd: - continue - - mdir__ = None - if (not args.get('depth') - or mdepth+len(path) < args.get('depth')): - mdir__ = next( - ((tag, j, d, data) - for tag, j, d, data in tags - if tag == TAG_MDIR), - None) - - if mdir__: - # fetch the mdir - _, _, _, data = mdir__ - blocks = frommdir(data) - mdir_ = Rbyd.fetch(f, block_size, blocks) - - rtree, rdepth = mdir_.tree(rbyd=args.get('rbyd')) - - # connect the root to the mtree - branch = max( - (branch for branch in tree - if branch.b[0] == mbid-(mw-1)), - key=lambda branch: branch.d, - default=None) - if branch: - root = min(rtree, - key=lambda branch: branch.d, - default=None) - if root: - r_rid, r_tag = root.a - else: - _, r_rid, r_tag, _, _, _, _, _ = ( - mdir_.lookup(-1, 0x1)) - tree.add(TBranch( - a=branch.b, - b=(mbid-(mw-1), len(path), 0, r_rid, r_tag), - d=d_ + tdepth, - c='b', - )) - - # map the tree into our metadata space - for branch in rtree: - a_rid, a_tag = branch.a - b_rid, b_tag = branch.b - tree.add(TBranch( - a=(mbid-(mw-1), len(path), 0, a_rid, a_tag), - b=(mbid-(mw-1), len(path), 0, b_rid, b_tag), - d=(d_ + tdepth + 1 - + branch.d + mdepth_-rdepth), - c=branch.c, - )) - - # remap branches to leaves if we aren't showing inner branches - if not args.get('inner'): - # step through each layer backwards - b_depth = max((branch.b[1]+1 for branch in tree), default=0) - - # keep track of the original bids, unfortunately because we - # store the bids in the branches we overwrite these - tree = {(branch.b[0] - branch.b[2], branch) - for branch in tree} - - for bd in reversed(range(b_depth-1)): - # find leaf-roots at this level - roots = {} - for bid, branch in tree: - # choose the highest node as the root - if (branch.b[1] == b_depth-1 - and (bid not in roots - or branch.d < roots[bid].d)): - roots[bid] = branch - - # remap branches to leaf-roots - tree_ = set() - for bid, branch in tree: - # note we ignore mroot branches, we don't collapse - # normally these - if (branch.a[0] != -1 - and branch.a[1] == bd - and branch.a[0] in roots): - branch = TBranch( - a=roots[branch.a[0]].b, - b=branch.b, - d=branch.d, - c=branch.c, - ) - if (branch.b[0] != -1 - and branch.b[1] == bd - and branch.b[0] in roots): - branch = TBranch( - a=branch.a, - b=roots[branch.b[0]].b, - d=branch.d, - c=branch.c, - ) - tree_.add((bid, branch)) - tree = tree_ - - # strip out bids - tree = {branch for _, branch in tree} - - # precompute B-tree if requested - elif args.get('btree'): - # compute mroot chain "tree", prefix our actual mtree with this - tree = set() - mroot_ = Rbyd.fetch(f, block_size, mroots) - mdepth_ = 1 - mseen_ = set() - for d in it.count(): - # corrupted? - if not mroot_: - break - # cycle detected? - elif mroot_.blocks in mseen_: - break - - mseen_.add(mroot_.blocks) - - # connect branch to our first tag - if d > 0: - done, rid, tag, w, j, _, data, _ = mroot_.lookup(-1, 0x1) - if not done: - tree.add(TBranch( - a=(-1, d-1, 0, -1, TAG_MROOT), - b=(-1, d, 0, rid, tag), - d=0, - c='b', - )) - - # stop here? - if args.get('depth') and mdepth_ >= args.get('depth'): - break - - # fetch the next mroot - done, rid, tag, w, j, _, data, _ = mroot_.lookup(-1, TAG_MROOT) - if not (not done and rid == -1 and tag == TAG_MROOT): - break - - blocks = frommdir(data) - mroot_ = Rbyd.fetch(f, block_size, blocks) - mdepth_ += 1 - - # create a branch to our mdir if there is one - if mdir: - # connect branch to our first tag - done, rid, tag, w, j, _, data, _ = mdir.lookup(-1, 0x1) - if not done: - tree.add(TBranch( - a=(-1, d, 0, -1, TAG_MDIR), - b=(0, 0, 0, rid, tag), - d=0, - c='b', - )) - - # compute the mtree's B-tree if there is one - if mtree: - tree_, tdepth = mtree.btree_btree( - f, block_size, - depth=args.get('depth', mdepth)-mdepth, - inner=args.get('inner')) - - # connect a branch to the root of the tree - root = min(tree_, key=lambda branch: branch.d, default=None) - if root: - r_bid, r_bd, r_rid, r_tag = root.a - tree.add(TBranch( - a=(-1, d, 0, -1, TAG_MTREE), - b=(r_bid, r_bd, r_rid, 0, r_tag), - d=0, - c='b', - )) - - # map the tree into our metadata space - for branch in tree_: - a_bid, a_bd, a_rid, a_tag = branch.a - b_bid, b_bd, b_rid, b_tag = branch.b - tree.add(TBranch( - a=(a_bid, a_bd, a_rid, 0, a_tag), - b=(b_bid, b_bd, b_rid, 0, b_tag), - d=1 + branch.d, - c=branch.c, - )) - - # remap branches to leaves if we aren't showing inner branches - if not args.get('inner'): - mbid = -1 - while True: - done, mbid, mw, rbyd, rid, tags, path = ( - mtree.btree_lookup( - f, block_size, mbid+1, - depth=args.get('depth', mdepth)-mdepth)) - if done: - break - - # corrupted? - if not rbyd: - continue - - mdir__ = None - if (not args.get('depth') - or mdepth+len(path) < args.get('depth')): - mdir__ = next( - ((tag, j, d, data) - for tag, j, d, data in tags - if tag == TAG_MDIR), - None) - - if mdir__: - # fetch the mdir - _, _, _, data = mdir__ - blocks = frommdir(data) - mdir_ = Rbyd.fetch(f, block_size, blocks) - - # find the first entry in the mdir, map branches - # to this entry - done, rid, tag, _, j, d, data, _ = ( - mdir_.lookup(-1, 0x1)) - - tree_ = set() - for branch in tree: - if branch.a[0] == mbid-(mw-1): - a_bid, a_bd, _, _, _ = branch.a - branch = TBranch( - a=(a_bid, a_bd+1, 0, rid, tag), - b=branch.b, - d=branch.d, - c=branch.c, - ) - if branch.b[0] == mbid-(mw-1): - b_bid, b_bd, _, _, _ = branch.b - branch = TBranch( - a=branch.a, - b=(b_bid, b_bd+1, 0, rid, tag), - d=branch.d, - c=branch.c, - ) - tree_.add(branch) - tree = tree_ - - # common tree renderer - if args.get('tree') or args.get('rbyd') or args.get('btree'): # find the max depth from the tree - t_depth = max((branch.d+1 for branch in tree), default=0) + t_depth = max((t.depth+1 for t in tree), default=0) if t_depth > 0: t_width = 2*t_depth + 2 - def treerepr(mbid, mw, md, mrid, rid, tag): - if t_depth == 0: - return '' + # dynamically size the id field + w_width = max( + mt.ceil(mt.log10(max(1, mtree.mbweight_())+1)), + mt.ceil(mt.log10(max(1, mtree.mrweight_())+1)), + # in case of -1.-1 + 2) - def branchrepr(x, d, was): - for branch in tree: - if branch.d == d and branch.b == x: - if any(branch.d == d and branch.a == x - for branch in tree): - return '+-', branch.c, branch.c - elif any(branch.d == d - and x > min(branch.a, branch.b) - and x < max(branch.a, branch.b) - for branch in tree): - return '|-', branch.c, branch.c - elif branch.a < branch.b: - return '\'-', branch.c, branch.c - else: - return '.-', branch.c, branch.c - for branch in tree: - if branch.d == d and branch.a == x: - return '+ ', branch.c, None - for branch in tree: - if (branch.d == d - and x > min(branch.a, branch.b) - and x < max(branch.a, branch.b)): - return '| ', branch.c, was - if was: - return '--', was, was - return ' ', None, None - - trunk = [] - was = None - for d in range(t_depth): - t, c, was = branchrepr( - (mbid-max(mw-1, 0), md, - mrid-max(mw-1, 0), rid, tag), - d, was) - - trunk.append('%s%s%s%s' % ( - '\x1b[33m' if color and c == 'y' - else '\x1b[31m' if color and c == 'r' - else '\x1b[90m' if color and c == 'b' - else '', - t, - ('>' if was else ' ') if d == t_depth-1 else '', - '\x1b[m' if color and c else '')) - - return '%s ' % ''.join(trunk) - - - def dbg_mdir(mdir, mbid, mw, md): - for i, (rid, tag, w, j, d, data) in enumerate(mdir): - # show human-readable tag representation + def dbg_mdir(d, mdir): + # show human-readable tag representation + for i, (mid, rattr) in enumerate(mdir.rattrs()): print('%12s %s%s' % ( '{%s}:' % ','.join('%04x' % block for block in mdir.blocks) if i == 0 else '', - treerepr(mbid-max(mw-1, 0), 0, md, 0, rid, tag) + treerepr(tree, (mid, d, rattr.tag), + t_depth, color) if args.get('tree') - or args.get('rbyd') - or args.get('btree') + or args.get('tree_rbyd') + or args.get('tree_btree') else '', '%*s %-*s%s' % ( 2*w_width+1, '%d.%d-%d' % ( - mbid//mleaf_weight, rid-(w-1), rid) - if w > 1 - else '%d.%d' % (mbid//mleaf_weight, rid) - if w > 0 or i == 0 + mid.mbid_(), + mid.mrid_()-(rattr.weight-1), + mid.mrid_()) + if rattr.weight > 1 + else '%d.%d' % (mid.mbid_(), mid.mrid_()) + if rattr.weight > 0 or i == 0 else '', - 21+w_width, tagrepr(tag, w, len(data), j), - ' %s' % next(xxd(data, 8), '') + 21+w_width, rattr, + ' %s' % next(xxd(rattr.data, 8), '') if not args.get('raw') and not args.get('no_truncate') else ''))) # show on-disk encoding of tags if args.get('raw'): - for o, line in enumerate(xxd(mdir.data[j:j+d])): + for o, line in enumerate(xxd( + mdir.data[rattr.toff:rattr.off])): print('%11s: %*s%*s %s' % ( - '%04x' % (j + o*16), + '%04x' % (rattr.toff + o*16), t_width, '', 2*w_width+1, '', line)) if args.get('raw') or args.get('no_truncate'): - if not tag & TAG_ALT: - for o, line in enumerate(xxd(data)): - print('%11s: %*s%*s %s' % ( - '%04x' % (j+d + o*16), - t_width, '', - 2*w_width+1, '', - line)) + for o, line in enumerate(xxd(rattr.data)): + print('%11s: %*s%*s %s' % ( + '%04x' % (rattr.off + o*16), + t_width, '', + 2*w_width+1, '', + line)) # prbyd here means the last rendered rbyd, we update # in dbg_branch to always print interleaved addresses prbyd = None - def dbg_branch(bid, w, rbyd, rid, tags, bd): + def dbg_branch(d, bid, rbyd, rid, name): nonlocal prbyd # show human-readable representation - for i, (tag, j, d, data) in enumerate(tags): + for rattr in rbyd.rattrs(rid): print('%12s %s%*s %-*s %s' % ( '%04x.%04x:' % (rbyd.block, rbyd.trunk) if prbyd is None or rbyd != prbyd else '', - treerepr(bid, w, bd, rid, 0, tag) + treerepr(tree, + (mtree.mid(bid-(name.weight-1), -1), + d, rattr.tag), + t_depth, color) if args.get('tree') - or args.get('rbyd') - or args.get('btree') + or args.get('tree_rbyd') + or args.get('tree_btree') else '', - 2*w_width+1, '' if i != 0 - else '%d-%d' % ( - (bid-(w-1))//mleaf_weight, - bid//mleaf_weight) - if (w//mleaf_weight) > 1 - else bid//mleaf_weight if w > 0 + 2*w_width+1, '%d-%d' % ( + (bid-(rattr.weight-1)) >> mtree.mbits, + bid >> mtree.mbits) + if (rattr.weight >> mtree.mbits) > 1 + else bid >> mtree.mbits if rattr.weight > 0 else '', - 21+w_width, tagrepr( - tag, w if i == 0 else 0, len(data), None), - next(xxd(data, 8), '') + 21+w_width, rattr, + next(xxd(rattr.data, 8), '') if not args.get('raw') and not args.get('no_truncate') else '')) @@ -1553,204 +2408,94 @@ def main(disk, mroots=None, *, # show on-disk encoding of tags/data if args.get('raw'): - for o, line in enumerate(xxd(rbyd.data[j:j+d])): + for o, line in enumerate(xxd( + rbyd.data[rattr.toff:rattr.off])): print('%11s: %*s%*s %s' % ( - '%04x' % (j + o*16), + '%04x' % (rattr.toff + o*16), t_width, '', 2*w_width+1, '', line)) if args.get('raw') or args.get('no_truncate'): - for o, line in enumerate(xxd(data)): + for o, line in enumerate(xxd(rattr.data)): print('%11s: %*s%*s %s' % ( - '%04x' % (j+d + o*16), + '%04x' % (rattr.off + o*16), t_width, '', 2*w_width+1, '', line)) - - #### actual debugging begins here - - # print some information about the mtree - print('mtree %s w%d.%d, rev %08x, cksum %08x' % ( - mroot.addr(), - bweight//mleaf_weight, 1*mleaf_weight, - mroot.rev, - mroot.cksum)) - - # dynamically size the id field - w_width = max( - mt.ceil(mt.log10(max(1, bweight//mleaf_weight)+1)), - mt.ceil(mt.log10(max(1, rweight)+1)), - # in case of -1.-1 - 2) - - # show each mroot + # traverse and print entries prbyd = None ppath = [] + mrootseen = set() corrupted = False - mroot = Rbyd.fetch(f, block_size, mroots) - mdepth = 1 - mseen = set() - for d in it.count(): - # corrupted? - if not mroot: - print('{%s}: %s%s%s' % ( - ','.join('%04x' % block - for block in mroot.blocks), - '\x1b[31m' if color else '', - '(corrupted mroot %s)' % mroot.addr(), - '\x1b[m' if color else '')) - corrupted = True - break - # cycle detected? - elif mroot.blocks in mseen: - print('{%s}: %s%s%s' % ( - ','.join('%04x' % block - for block in mroot.blocks), - '\x1b[31m' if color else '', - '(mroot cycle detected %s)' % mroot.addr(), - '\x1b[m' if color else '')) - corrupted = True - break - else: - # show the mdir - dbg_mdir(mroot, -1, 0, d) - - mseen.add(mroot.blocks) - - # stop here? - if args.get('depth') and mdepth >= args.get('depth'): - break - - # fetch the next mroot - done, rid, tag, w, j, _, data, _ = mroot.lookup(-1, TAG_MROOT) - if not (not done and rid == -1 and tag == TAG_MROOT): - break - - blocks = frommdir(data) - mroot = Rbyd.fetch(f, block_size, blocks) - mdepth += 1 - - # show the mdir, if there is one - if not args.get('depth') or mdepth < args.get('depth'): - done, rid, tag, w, j, _, data, _ = mroot.lookup(-1, TAG_MDIR) - if not done and rid == -1 and tag == TAG_MDIR: - blocks = frommdir(data) - mdir = Rbyd.fetch(f, block_size, blocks) + for mdir, path in mtree.leaves( + path=True, + depth=args.get('depth')): + # print inner branches if requested + if args.get('inner'): + for d, (mid_, (bid_, rbyd_, rid_, name_), name_) in pathdelta( + # skip the mrootchain + path[len(mtree.mrootchain):], + ppath[len(mtree.mrootchain):]): + dbg_branch(len(mtree.mrootchain)+d, + bid_, rbyd_, rid_, name_) + ppath = path + # mdir? + if isinstance(mdir, Mdir): # corrupted? if not mdir: print('{%s}: %s%s%s' % ( ','.join('%04x' % block for block in mdir.blocks), '\x1b[31m' if color else '', - '(corrupted mdir %s)' % mdir.addr(), + '(corrupted %s %s)' % ( + 'mroot' if mdir.mid == -1 else 'mdir', + mdir.addr()), '\x1b[m' if color else '')) corrupted = True - else: - # show the mdir - dbg_mdir(mdir, 0, 0, 0) + prbyd = None + continue - # fetch the actual mtree, if there is one - if not args.get('depth') or mdepth < args.get('depth'): - done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MTREE) - if not done and rid == -1 and tag == TAG_MTREE: - w, block, trunk, cksum = frombtree(data) - mtree = Rbyd.fetch(f, block_size, block, trunk, cksum) - - # traverse entries - mbid = -1 - while True: - done, mbid, mw, rbyd, rid, tags, path = mtree.btree_lookup( - f, block_size, mbid+1, - depth=args.get('depth', mdepth)-mdepth) - if done: - break - - # print inner btree entries if requested - if args.get('inner'): - changed = False - for (x, px) in it.zip_longest( - enumerate(path[:-1]), - enumerate(ppath[:-1])): - if x is None: - break - if not (changed or px is None or x != px): - continue - changed = True - - # show the inner entry - d, (mid_, w_, rbyd_, rid_, tags_) = x - dbg_branch(mid_, w_, rbyd_, rid_, tags_, d) - ppath = path - - # corrupted? try to keep printing the tree - if not rbyd: - print('%11s: %*s%s%s%s' % ( - '%04x.%04x' % (rbyd.block, rbyd.trunk), - t_width, '', - '\x1b[31m' if color else '', - '(corrupted rbyd %s)' % rbyd.addr(), - '\x1b[m' if color else '')) - prbyd = rbyd - corrupted = True - continue - - # if we're not showing inner nodes, prefer names higher in - # the tree since this avoids showing vestigial names - if not args.get('inner'): - name = None - for mid_, w_, rbyd_, rid_, tags_ in reversed(path): - for tag_, j_, d_, data_ in tags_: - if tag_ & 0x7f00 == TAG_NAME: - name = (tag_, j_, d_, data_) - - if rid_-(w_-1) != 0: - break - - if name is not None: - tags = [name] + [(tag, j, d, data) - for tag, j, d, data in tags - if tag & 0x7f00 != TAG_NAME] - - # found an mdir in the tags? - mdir__ = None - if (not args.get('depth') - or mdepth+len(path) < args.get('depth')): - mdir__ = next( - ((tag, j, d, data) - for tag, j, d, data in tags - if tag == TAG_MDIR), - None) - - # show other btree entries in certain cases - if args.get('inner') or not mdir__: - dbg_branch(mbid, mw, rbyd, rid, tags, len(path)-1) - - if not mdir__: - continue - - # fetch the mdir - _, _, _, data = mdir__ - blocks = frommdir(data) - mdir_ = Rbyd.fetch(f, block_size, blocks) - - # corrupted? - if not mdir_: - print('{%s}: %*s%s%s%s' % ( + # cycle detected? + if mdir.mid == -1: + if mdir in mrootseen: + print('{%s}: %s%s%s' % ( ','.join('%04x' % block - for block in mdir_.blocks), - t_width, '', + for block in mdir.blocks), '\x1b[31m' if color else '', - '(corrupted mdir %s)' % mdir_.addr(), + '(mroot cycle detected %s)' % mdir.addr(), '\x1b[m' if color else '')) corrupted = True - else: - # show the mdir - dbg_mdir(mdir_, mbid, mw, len(path)) - - # force next btree entry to be shown prbyd = None + continue + mrootseen.add(mdir) + + # show the mdir + dbg_mdir(len(path), mdir) + + # force next btree entry to be shown + prbyd = None + + # btree node? + else: + bid, rbyd = mdir + # corrupted? try to keep printing the tree + if not rbyd: + print('%11s: %*s%s%s%s' % ( + '%04x.%04x' % (rbyd.block, rbyd.trunk), + t_width, '', + '\x1b[31m' if color else '', + '(corrupted rbyd %s)' % rbyd.addr(), + '\x1b[m' if color else '')) + corrupted = True + prbyd = rbyd + continue + + for rid, name in rbyd.rids(): + bid_ = bid-(rbyd.weight-1) + rid + # show the leaf entry/branch + dbg_branch(len(path), bid_, rbyd, rid, name) if args.get('error_on_corrupt') and corrupted: sys.exit(2) @@ -1794,15 +2539,15 @@ if __name__ == "__main__": parser.add_argument( '-t', '--tree', action='store_true', - help="Show the underlying rbyd trees.") + help="Show the rbyd tree.") parser.add_argument( - '-B', '--btree', + '-R', '--tree-rbyd', action='store_true', - help="Show the underlying B-trees.") + help="Show the full rbyd tree.") parser.add_argument( - '-R', '--rbyd', + '-B', '--tree-btree', action='store_true', - help="Show the full underlying rbyd trees.") + help="Show a simplified btree tree.") parser.add_argument( '-i', '--inner', action='store_true', diff --git a/scripts/dbgrbyd.py b/scripts/dbgrbyd.py index 83123000..77c4d2a4 100755 --- a/scripts/dbgrbyd.py +++ b/scripts/dbgrbyd.py @@ -266,13 +266,11 @@ def tagrepr(tag, weight=None, size=None, off=None): ' %d' % size if size is not None else '') # tree branches are an abstract thing for tree rendering -class TreeBranch: - def __init__(self, a, b, depth=0, color='b'): +class TreeBranch(co.namedtuple('TreeBranch', ['a', 'b', 'depth', 'color'])): + __slots__ = () + def __new__(cls, a, b, depth=0, color='b'): # a and b are context specific - self.a = a - self.b = b - self.depth = depth - self.color = color + return super().__new__(cls, a, b, depth, color) def __repr__(self): return '%s(%s, %s, %s, %s)' % ( @@ -282,16 +280,6 @@ class TreeBranch: self.depth, self.color) - def __eq__(self, other): - return ((self.a, self.b, self.depth, self.color) - == (other.a, other.b, other.depth, other.color)) - - def __ne__(self, other): - return not self.__eq__(other) - - def __hash__(self): - return hash((self.a, self.b, self.depth)) - def treerepr(tree, x, depth=None, color=False): # find the max depth from the tree if depth is None: @@ -382,9 +370,9 @@ class Rattr: return len(self.data) def __repr__(self): - return '<%s %s>' % (self.__class__.__name__, self.tagrepr()) + return '<%s %s>' % (self.__class__.__name__, self) - def tagrepr(self): + def __str__(self): return tagrepr(self.tag, self.weight, self.size) def __iter__(self): @@ -421,9 +409,9 @@ class Ralt: return self.toff - self.jump def __repr__(self): - return '<%s %s>' % (self.__class__.__name__, self.tagrepr()) + return '<%s %s>' % (self.__class__.__name__, self) - def tagrepr(self): + def __str__(self): return tagrepr(self.tag, self.weight, self.jump, self.toff) def __iter__(self): @@ -632,7 +620,7 @@ class Rbyd: gcksumdelta=gcksumdelta, corrupt=not trunk_) - def lookupnext(self, rid=-1, tag=None, *, + def lookupnext(self, rid, tag=None, *, path=False): if not self: return None, None, *(([],) if path else ()) @@ -710,21 +698,26 @@ class Rbyd: self.data[j+d:j+d+jump]), *((path_,) if path else ())) - def lookup(self, rid, tag=None, mask=None, *, + def lookup_(self, rid, tag=None, mask=None, *, path=False): if tag is None: tag, mask = 0, 0xffff + if mask is None: + mask = 0 - rid_, rattr_, *path_ = self.lookupnext(rid, tag & ~(mask or 0), + rid_, rattr_, *path_ = self.lookupnext(rid, tag & ~mask, path=path) if (rid_ is None or rid_ != rid - or (rattr_.tag_ & ~(mask or 0)) != (tag & ~(mask or 0))): - if path: - return None, *path_ - else: - return None + or (rattr_.tag & ~mask) != (tag & ~mask)): + return None, *path_ + return rattr_, *path_ + + def lookup(self, rid, tag=None, mask=None, *, + path=False): + rattr_, *path_ = self.lookup_(rid, tag, mask, + path=path) if path: return rattr_, *path_ else: @@ -740,26 +733,22 @@ class Rbyd: if not isinstance(key, tuple): key = (key,) - v = self.lookup(*key) - if isinstance(v, tuple): - return v[0] is not None - else: - return v is not None + return self.lookup_(*key)[0] is not None def rids(self, *, path=False): rid = -1 while True: - rid, rattr, *path_ = self.lookupnext(rid, + rid, name, *path_ = self.lookupnext(rid, path=path) # found end of tree? if rid is None: break - yield rid, rattr, *path_ + yield rid, name, *path_ rid += 1 - def rattrs(self, rid=None, *, + def rattrs_(self, rid=None, *, path=False): if rid is None: rid, tag = -1, 0 @@ -781,11 +770,21 @@ class Rbyd: if rid_ is None or rid_ != rid: break + yield rattr, *path_ + tag = rattr.tag + + def rattrs(self, rid=None, *, + path=False): + if rid is None: + yield from self.rattrs_(rid, + path=path) + else: + for rattr, *path_ in self.rattrs_(rid, + path=path): if path: yield rattr, *path_ else: yield rattr - tag = rattr.tag def __iter__(self): return self.rattrs() @@ -823,8 +822,8 @@ class Rbyd: return best - # create tree representation for debugging - def tree(self, **args): + # create an rbyd tree for debugging + def _tree_rtree(self, **args): trunks = co.defaultdict(lambda: (-1, 0)) alts = co.defaultdict(lambda: {}) @@ -913,6 +912,25 @@ class Rbyd: return tree + # create a btree tree for debugging + def _tree_btree(self, **args): + # for rbyds this is just a pointer to ever rid + tree = set() + root = None + for rid, name in self.rids(): + b = (rid, name.tag) + if root is None: + root = b + tree.add(TreeBranch(root, b)) + return tree + + # create tree representation for debugging + def tree(self, **args): + if args.get('tree_btree'): + return self._tree_btree(**args) + else: + return self._tree_rtree(**args) + def dbg_log(rbyd, *, block_size, @@ -1291,9 +1309,11 @@ def dbg_tree(rbyd, *, data = rbyd.data - # precompute tree + # precompute tree renderings t_width = 0 - if args.get('tree') or args.get('tree_rbyd'): + if (args.get('tree') + or args.get('tree_rbyd') + or args.get('tree_btree')): tree = rbyd.tree(**args) # find the max depth from the tree @@ -1309,13 +1329,15 @@ def dbg_tree(rbyd, *, print('%08x: %s%*s %-*s %s' % ( rattr.toff, treerepr(tree, (rid, rattr.tag), t_depth, color) - if args.get('tree') or args.get('tree_rbyd') + if (args.get('tree') + or args.get('tree_rbyd') + or args.get('tree_btree')) else '', 2*w_width+1, '%d-%d' % (rid-(rattr.weight-1), rid) if rattr.weight > 1 else rid if rattr.weight > 0 or i == 0 else '', - 21+w_width, rattr.tagrepr(), + 21+w_width, rattr, next(xxd(rattr.data[:8], 8), '') if not args.get('raw') and not args.get('no_truncate') @@ -1375,6 +1397,7 @@ def main(disk, blocks=None, *, bd = Bd(f, block_size, block_count) rbyd = Rbyd.fetch(bd, blocks) + # print some information about the rbyd print('rbyd %s w%d, rev %08x, size %d, cksum %08x' % ( rbyd.addr(), rbyd.weight, @@ -1448,11 +1471,14 @@ if __name__ == "__main__": '-t', '--tree', action='store_true', help="Show the rbyd tree.") - # TODO adopt this rename in all scripts parser.add_argument( '-R', '--tree-rbyd', action='store_true', help="Show the full rbyd tree.") + parser.add_argument( + '-B', '--tree-btree', + action='store_true', + help="Show a simplified btree tree.") parser.add_argument( '-j', '--jumps', action='store_true',