diff --git a/scripts/dbgrbyd.py b/scripts/dbgrbyd.py index 7b14cef4..6c1886e9 100755 --- a/scripts/dbgrbyd.py +++ b/scripts/dbgrbyd.py @@ -88,11 +88,12 @@ def bdgeom(s): else: return int(s, b) +# TODO sync across scripts # parse some rbyd addr encodings -# 0xa -> (0xa,) -# 0xa.c -> ((0xa, 0xc),) -# 0x{a,b} -> (0xa, 0xb) -# 0x{a,b}.c -> ((0xa, 0xc), (0xb, 0xc)) +# 0xa -> [0xa] +# 0xa.c -> [(0xa, 0xc)] +# 0x{a,b} -> [0xa, 0xb] +# 0x{a,b}.c -> [(0xa, 0xc), (0xb, 0xc)] def rbydaddr(s): s = s.strip() b = 10 @@ -123,7 +124,7 @@ def rbydaddr(s): else: addr.append(int(s, b)) - return tuple(addr) + return addr def crc32c(data, crc=0): crc ^= 0xffffffff @@ -168,11 +169,12 @@ def xxd(data, width=16): b if b >= ' ' and b <= '~' else '.' for b in map(chr, data[i:i+width]))) -def tagrepr(tag, w=None, size=None, off=None): +# TODO sync across scripts +def tagrepr(tag, weight=None, size=None, off=None): if (tag & 0x6fff) == TAG_NULL: return '%snull%s%s' % ( 'shrub' if tag & TAG_SHRUB else '', - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %d' % size if size else '') elif (tag & 0x6f00) == TAG_CONFIG: return '%s%s%s%s' % ( @@ -186,14 +188,14 @@ def tagrepr(tag, w=None, size=None, off=None): else 'namelimit' if (tag & 0xfff) == TAG_NAMELIMIT else 'filelimit' if (tag & 0xfff) == TAG_FILELIMIT else 'config 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_GDELTA: return '%s%s%s%s' % ( 'shrub' if tag & TAG_SHRUB else '', 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA else 'gdelta 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_NAME: return '%s%s%s%s' % ( @@ -204,7 +206,7 @@ def tagrepr(tag, w=None, size=None, off=None): else 'bookmark' if (tag & 0xfff) == TAG_BOOKMARK else 'stickynote' if (tag & 0xfff) == TAG_STICKYNOTE else 'name 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') elif (tag & 0x6f00) == TAG_STRUCT: return '%s%s%s%s' % ( @@ -219,21 +221,21 @@ def tagrepr(tag, w=None, size=None, off=None): else 'did' if (tag & 0xfff) == TAG_DID else 'branch' if (tag & 0xfff) == TAG_BRANCH else 'struct 0x%02x' % (tag & 0xff), - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') elif (tag & 0x6e00) == TAG_ATTR: return '%s%sattr 0x%02x%s%s' % ( 'shrub' if tag & TAG_SHRUB else '', 's' if tag & 0x100 else 'u', ((tag & 0x100) >> 1) ^ (tag & 0xff), - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') elif tag & TAG_ALT: return 'alt%s%s 0x%03x%s%s' % ( 'r' if tag & TAG_R else 'b', 'gt' if tag & TAG_GT else 'le', tag & 0x0fff, - ' w%d' % w if w is not None else '', + ' w%d' % weight if weight is not None else '', ' 0x%x' % (0xffffffff & (off-size)) if size and off is not None else ' -%d' % size if size @@ -242,38 +244,559 @@ def tagrepr(tag, w=None, size=None, off=None): return 'cksum%s%s%s%s' % ( 'p' if not tag & 0xfe and tag & TAG_P else '', ' 0x%02x' % (tag & 0xff) if tag & 0xfe else '', - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') elif (tag & 0x7f00) == TAG_NOTE: return 'note%s%s%s' % ( ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') elif (tag & 0x7f00) == TAG_ECKSUM: return 'ecksum%s%s%s' % ( ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') elif (tag & 0x7f00) == TAG_GCKSUMDELTA: return 'gcksumdelta%s%s%s' % ( ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', + ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') else: return '0x%04x%s%s' % ( tag, - ' w%d' % w if w is not None else '', + ' w%d' % weight if weight is not None else '', ' %d' % size if size is not None else '') +# tagged data in an rbyd +class Rattr: + def __init__(self, tag, weight, block, toff, off, data): + self.tag = tag + self.weight = weight + self.block = block + self.toff = toff + self.off = off + self.data = data -def dbg_log(data, block_size, rev, eoff, weight, *, + @property + def size(self): + return len(self.data) + + def __repr__(self): + return '<%s %s>' % (self.__class__.__name__, self.tagrepr()) + + def tagrepr(self): + return tagrepr(self.tag, self.weight, self.size) + + def __bool__(self): + return bool(self.data) + + def __len__(self): + return len(self.data) + + def __getitem__(self, key): + return self.data[key] + + def __iter__(self): + return iter(self.data) + +class Ralt: + def __init__(self, tag, weight, block, toff, off, jump, color=None): + self.tag = tag + self.weight = weight + self.block = block + self.toff = toff + self.off = off + self.jump = jump + + if color is not None: + self.color = color + else: + self.color = 'r' if tag & TAG_R else 'b' + + @property + def joff(self): + return self.toff - self.jump + + def __repr__(self): + return '<%s %s>' % (self.__class__.__name__, self.tagrepr()) + + def tagrepr(self): + return tagrepr(self.tag, self.weight, self.jump, self.toff) + +# tree branches are an abstract thing for tree rendering +class TreeBranch: + def __init__(self, a, b, depth, color): + # note a and b are context specific + self.a = a + self.b = b + self.depth = depth + self.color = color + + def __repr__(self): + return '%s(%s, %s, %s, %s)' % ( + self.__class__.__name__, + self.a, + self.b, + self.depth, + self.color) + +# our core rbyd type +class Rbyd: + def __init__(self, data, blocks, trunk, weight, rev, eoff, cksum, *, + gcksumdelta=None, + corrupt=False): + if isinstance(blocks, int): + blocks = [blocks] + + self.data = data + self.blocks = list(blocks) + self.trunk = trunk + self.weight = weight + self.rev = rev + self.eoff = eoff + self.cksum = cksum + self.gcksumdelta = gcksumdelta + self.corrupt = corrupt + + @property + def block(self): + return self.blocks[0] + + def addr(self): + if len(self.blocks) == 1: + return '0x%x.%x' % (self.block, self.trunk) + else: + return '0x{%s}.%x' % ( + ','.join('%x' % block for block in self.blocks), + self.trunk) + + def __repr__(self): + return '<%s %s>' % (self.__class__.__name__, self.addr()) + + def __bool__(self): + return not self.corrupt + + def __eq__(self, other): + return (self.blocks, self.trunk) == (other.blocks, other.trunk) + + def __ne__(self, other): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.blocks, self.trunk)) + + @classmethod + def fetch(cls, data, block, trunk=None, cksum=None): + # multiple blocks? + if isinstance(block, list): + # fetch all blocks + rbyds = [cls.fetch(data, block, trunk, cksum) for block in block] + # determine most recent revision + i = 0 + for i_, rbyd in enumerate(rbyds): + # compare with sequence arithmetic + if rbyd and ( + not rbyds[i] + or not ((rbyd.rev - rbyds[i].rev) & 0x80000000) + or (rbyd.rev == rbyds[i].rev + and rbyd.trunk > rbyds[i].trunk)): + i = i_ + # keep track of the other blocks + rbyd = rbyds[i] + rbyd.blocks += tuple( + rbyds[(i+1+j) % len(rbyds)].block + for j in range(len(rbyds)-1)) + return rbyd + + # block may encode a trunk + block, trunk = ( + block[0] if isinstance(block, tuple) + else block, + trunk if trunk is not None + else block[1] if isinstance(block, tuple) + else None) + + # data can be either disk + block_size tuple or data + # + # note preread data can be useful for avoiding race conditions + # with cksums and shrubs + if isinstance(data, tuple): + f, block_size, *_ = data + # seek to the block + f.seek(block * block_size) + data = f.read(block_size) + + # fetch the rbyd + rev = fromle32(data[0:4]) + cksum_ = 0 + cksum__ = crc32c(data[0:4]) + cksum___ = cksum__ + perturb = False + eoff = 0 + eoff_ = None + j_ = 4 + trunk_ = 0 + trunk__ = 0 + trunk___ = 0 + weight = 0 + weight_ = 0 + weight__ = 0 + gcksumdelta = None + gcksumdelta_ = None + while j_ < len(data) and (not trunk or eoff <= trunk): + # read next tag + v, tag, w, size, d = fromtag(data[j_:]) + if v != parity(cksum___): + break + cksum___ ^= 0x00000080 if v else 0 + cksum___ = crc32c(data[j_:j_+d], cksum___) + j_ += d + if not tag & TAG_ALT and j_ + size > len(data): + break + + # take care of cksums + if not tag & TAG_ALT: + if (tag & 0xff00) != TAG_CKSUM: + cksum___ = crc32c(data[j_:j_+size], cksum___) + + # found a gcksumdelta? + if (tag & 0xff00) == TAG_GCKSUMDELTA: + gcksumdelta_ = Rattr(tag, w, + block, j_-d, d, data[j_:j_+size]) + + # found a cksum? + else: + # check cksum + cksum____ = fromle32(data[j_:j_+4]) + if cksum___ != cksum____: + break + # commit what we have + eoff = eoff_ if eoff_ else j_ + size + cksum_ = cksum__ + trunk_ = trunk__ + weight = weight_ + gcksumdelta = gcksumdelta_ + gcksumdelta_ = None + # update perturb bit + perturb = tag & TAG_P + # revert to data cksum and perturb + cksum___ = cksum__ ^ (0xfca42daf if perturb else 0) + + # evaluate trunks + if (tag & 0xf000) != TAG_CKSUM: + if not (trunk and j_-d > trunk and not trunk___): + # new trunk? + if not trunk___: + trunk___ = j_-d + weight__ = 0 + + # keep track of weight + weight__ += w + + # end of trunk? + if not tag & TAG_ALT: + # update trunk/weight unless we found a shrub or an + # explicit trunk (which may be a shrub) is requested + if not tag & TAG_SHRUB or trunk___ == trunk: + trunk__ = trunk___ + weight_ = weight__ + # keep track of eoff for best matching trunk + if trunk and j_ + size > trunk: + eoff_ = j_ + size + eoff = eoff_ + cksum_ = cksum___ ^ ( + 0xfca42daf if perturb else 0) + trunk_ = trunk__ + weight = weight_ + gcksumdelta = gcksumdelta_ + trunk___ = 0 + + # update canonical checksum, xoring out any perturb state + cksum__ = cksum___ ^ (0xfca42daf if perturb else 0) + + if not tag & TAG_ALT: + j_ += size + + # cksum mismatch? + if cksum is not None and cksum_ != cksum: + return cls(data, block, trunk or 0, 0, rev, 0, cksum_, + corrupt=True) + + return cls(data, block, trunk_, weight, rev, eoff, cksum_, + gcksumdelta=gcksumdelta, + corrupt=not trunk_) + + def lookupnext(self, rid, tag=None, *, + path=False): + if not self: + return None, None, None, *(([],) if path else ()) + + tag = max(tag or 0, 0x1) + lower = 0 + upper = self.weight + path_ = [] + + # descend down tree + j = self.trunk + while True: + _, alt, w, jump, d = fromtag(self.data[j:]) + + # found an alt? + if alt & TAG_ALT: + # follow? + if ((rid, tag & 0xfff) > (upper-w-1, alt & 0xfff) + if alt & TAG_GT + else ((rid, tag & 0xfff) + <= (lower+w-1, alt & 0xfff))): + lower += upper-lower-w if alt & TAG_GT else 0 + upper -= upper-lower-w if not alt & TAG_GT else 0 + j = j - jump + + if path: + # figure out which color + if alt & TAG_R: + _, nalt, _, _, _ = fromtag(self.data[j+jump+d:]) + if nalt & TAG_R: + color = 'y' + else: + color = 'r' + else: + color = 'b' + + path_.append(( + Ralt(alt, w, self.block, j+jump, j+jump+d, + jump, color), + True)) + + # stay on path + else: + lower += w if not alt & TAG_GT else 0 + upper -= w if alt & TAG_GT else 0 + j = j + d + + if path: + # figure out which color + if alt & TAG_R: + _, nalt, _, _, _ = fromtag(self.data[j:]) + if nalt & TAG_R: + color = 'y' + else: + color = 'r' + else: + color = 'b' + + path_.append(( + Ralt(alt, w, self.block, j-d, j, + jump, color), + False)) + + # found tag + else: + rid_ = upper-1 + tag_ = alt + w_ = upper-lower + + if not tag_ or (rid_, tag_) < (rid, tag): + return None, None, None, *(([],) if path else ()) + + return (rid_, tag_, + Rattr(tag_, w_, self.block, j, j+d, + self.data[j+d:j+d+jump]), + *((path_,) if path else ())) + + def lookup(self, rid, tag=None, mask=None, *, + path=False): + if tag is None: + tag, mask = 0, 0xffff + + rid_, tag_, rattr_, *path_ = self.lookupnext(rid, tag & ~(mask or 0), + path=path) + if (rid_ is None + or rid_ != rid + or (tag_ & ~(mask or 0)) != (tag & ~(mask or 0))): + if mask is not None: + return None, None, *path_ + elif path: + return None, *path_ + else: + return None + + if mask is not None: + return tag_, rattr_, *path_ + elif path: + return rattr_, *path_ + else: + return rattr_ + + def __getitem__(self, key): + if not isinstance(key, tuple): + key = (key,) + + return self.lookup(*key) + + def __contains__(self, key): + if not isinstance(key, tuple): + key = (key,) + + v = self.lookup(*key) + if isinstance(v, tuple): + return v[0] is not None + else: + return v is not None + + def __iter__(self): + rid, tag = -1, 0 + while True: + rid, tag, rattr = self.lookupnext(rid, tag+0x1) + # found end of tree? + if rid is None: + break + + yield rid, tag, rattr + + # lookup by name + def namelookup(self, did, name): + # binary search + best = (False, None, None, None) + lower = 0 + upper = self.weight + while lower < upper: + rid, tag, rattr = self.lookupnext(lower + (upper-1-lower)//2) + if rid is None: + break + + # treat vestigial names as a catch-all + if ((tag == TAG_NAME and rid-(rattr.weight-1) == 0) + or (tag & 0xff00) != TAG_NAME): + did_ = 0 + name_ = b'' + else: + did_, d = fromleb128(rattr[:]) + name_ = rattr[d:] + + # bisect search space + if (did_, name_) > (did, name): + upper = rid-(rattr.weight-1) + elif (did_, name_) < (did, name): + lower = rid + 1 + # keep track of best match + best = (False, rid, tag, rattr) + else: + # found a match + return True, rid, tag, rattr + + return best + + # create tree representation for debugging + def tree(self, *, + rbyd=False): + trunks = co.defaultdict(lambda: (-1, 0)) + alts = co.defaultdict(lambda: {}) + + rid, tag = -1, 0 + while True: + rid, tag, rattr, path = self.lookupnext(rid, tag+0x1, + path=True) + # found end of tree? + if rid is None: + break + + # keep track of trunks/alts + trunks[rattr.toff] = (rid, tag) + + for ralt, followed in path: + if followed: + alts[ralt.toff] |= {'f': ralt.joff, 'c': ralt.color} + else: + alts[ralt.toff] |= {'nf': ralt.off, 'c': ralt.color} + + if rbyd: + # treat unreachable alts as converging paths + for j_, alt in alts.items(): + if 'f' not in alt: + alt['f'] = alt['nf'] + elif 'nf' not in alt: + alt['nf'] = alt['f'] + + else: + # prune any alts with unreachable edges + pruned = {} + for j, alt in alts.items(): + if 'f' not in alt: + pruned[j] = alt['nf'] + elif 'nf' not in alt: + pruned[j] = alt['f'] + for j in pruned.keys(): + del alts[j] + + for j, alt in alts.items(): + while alt['f'] in pruned: + alt['f'] = pruned[alt['f']] + while alt['nf'] in pruned: + alt['nf'] = pruned[alt['nf']] + + # find the trunk and depth of each alt + def rec_trunk(j): + if j not in alts: + return trunks[j] + else: + if 'nft' not in alts[j]: + alts[j]['nft'] = rec_trunk(alts[j]['nf']) + return alts[j]['nft'] + + for j in alts.keys(): + rec_trunk(j) + for j, alt in alts.items(): + if alt['f'] in alts: + alt['ft'] = alts[alt['f']]['nft'] + else: + alt['ft'] = trunks[alt['f']] + + def rec_height(j): + if j not in alts: + return 0 + else: + if 'h' not in alts[j]: + alts[j]['h'] = max( + rec_height(alts[j]['f']), + rec_height(alts[j]['nf'])) + 1 + return alts[j]['h'] + + for j in alts.keys(): + rec_height(j) + + t_depth = max((alt['h']+1 for alt in alts.values()), default=0) + + # convert to more general tree representation + tree = set() + for j, alt in alts.items(): + # note all non-trunk edges should be colored black + tree.add(TreeBranch( + alt['nft'], + alt['nft'], + depth=t_depth-1 - alt['h'], + color=alt['c'])) + if alt['ft'] != alt['nft']: + tree.add(TreeBranch( + alt['nft'], + alt['ft'], + depth=t_depth-1 - alt['h'], + color='b')) + + return tree + + +def dbg_log(rbyd, *, + block_size, color=False, **args): + data = rbyd.data + # preprocess jumps if args.get('jumps'): jumps = [] j_ = 4 - while j_ < (block_size if args.get('all') else eoff): + while j_ < (block_size if args.get('all') else rbyd.eoff): j = j_ v, tag, w, size, d = fromtag(data[j_:]) j_ += d @@ -370,7 +893,7 @@ def dbg_log(data, block_size, rev, eoff, weight, *, weight_ = 0 trunk_ = 0 j_ = 4 - while j_ < (block_size if args.get('all') else eoff): + while j_ < (block_size if args.get('all') else rbyd.eoff): j = j_ v, tag, w, size, d = fromtag(data[j_:]) j_ += d @@ -498,7 +1021,7 @@ def dbg_log(data, block_size, rev, eoff, weight, *, weight__ = 0 trunk_ = 0 j_ = 4 - while j_ < (block_size if args.get('all') else eoff): + while j_ < (block_size if args.get('all') else rbyd.eoff): j = j_ v, tag, w, size, d = fromtag(data[j_:]) j_ += d @@ -536,7 +1059,7 @@ def dbg_log(data, block_size, rev, eoff, weight, *, lower_, upper_ = 0, 0 trunk_ = 0 j_ = 4 - while j_ < (block_size if args.get('all') else eoff): + while j_ < (block_size if args.get('all') else rbyd.eoff): notes = [] # read next tag @@ -586,13 +1109,13 @@ def dbg_log(data, block_size, rev, eoff, weight, *, # show human-readable tag representation print('%s%08x:%s %*s%s%*s %-*s%s%s%s' % ( - '\x1b[90m' if color and j >= eoff else '', + '\x1b[90m' if color and j >= rbyd.eoff else '', j, - '\x1b[m' if color and j >= eoff else '', + '\x1b[m' if color and j >= rbyd.eoff else '', lifetime_width, lifetimerepr(j) if args.get('lifetimes') else '', - '\x1b[90m' if color and j >= eoff else '', + '\x1b[90m' if color and j >= rbyd.eoff else '', 2*w_width+1, '' if (tag & 0xe000) != 0x0000 else '%d-%d' % (rid-(w-1), rid) if w > 1 else rid, @@ -604,7 +1127,7 @@ def dbg_log(data, block_size, rev, eoff, weight, *, and not tag & TAG_ALT else ''), ' (%s)' % ', '.join(notes) if notes else '', - '\x1b[m' if color and j >= eoff else '', + '\x1b[m' if color and j >= rbyd.eoff else '', ' %s' % jumprepr(j) if args.get('jumps') and not notes else '')) @@ -613,189 +1136,40 @@ def dbg_log(data, block_size, rev, eoff, weight, *, if args.get('raw'): for o, line in enumerate(xxd(data[j:j+d])): print('%s%8s: %*s%*s %s%s' % ( - '\x1b[90m' if color and j >= eoff else '', + '\x1b[90m' if color and j >= rbyd.eoff else '', '%04x' % (j + o*16), lifetime_width, '', 2*w_width+1, '', line, - '\x1b[m' if color and j >= eoff else '')) + '\x1b[m' if color and j >= rbyd.eoff else '')) if args.get('raw') or args.get('no_truncate'): if not tag & TAG_ALT: for o, line in enumerate(xxd(data[j+d:j+d+size])): print('%s%8s: %*s%*s %s%s' % ( - '\x1b[90m' if color and j >= eoff else '', + '\x1b[90m' if color and j >= rbyd.eoff else '', '%04x' % (j+d + o*16), lifetime_width, '', 2*w_width+1, '', line, - '\x1b[m' if color and j >= eoff else '')) + '\x1b[m' if color and j >= rbyd.eoff else '')) -def dbg_tree(data, block_size, rev, trunk, weight, *, +def dbg_tree(rbyd, *, + block_size, color=False, **args): - if not trunk: + if not rbyd: return - # lookup a tag, returning also the search path for decoration - # purposes - def lookup(rid, tag): - tag = max(tag, 0x1) - lower = 0 - upper = weight - path = [] - - # descend down tree - j = trunk - while True: - _, alt, w, jump, d = fromtag(data[j:]) - - # found an alt? - if alt & TAG_ALT: - # follow? - if ((rid, tag & 0xfff) > (upper-w-1, alt & 0xfff) - if alt & TAG_GT - else ((rid, tag & 0xfff) <= (lower+w-1, alt & 0xfff))): - lower += upper-lower-w if alt & TAG_GT else 0 - upper -= upper-lower-w if not alt & TAG_GT else 0 - j = j - jump - - # figure out which color - if alt & TAG_R: - _, nalt, _, _, _ = fromtag(data[j+jump+d:]) - if nalt & TAG_R: - path.append((j+jump, j, True, 'y')) - else: - path.append((j+jump, j, True, 'r')) - else: - path.append((j+jump, j, True, 'b')) - - # stay on path - else: - lower += w if not alt & TAG_GT else 0 - upper -= w if alt & TAG_GT else 0 - j = j + d - - # figure out which color - if alt & TAG_R: - _, nalt, _, _, _ = fromtag(data[j:]) - if nalt & TAG_R: - path.append((j-d, j, False, 'y')) - else: - path.append((j-d, j, False, 'r')) - else: - path.append((j-d, j, False, 'b')) - - # found tag - else: - rid_ = upper-1 - tag_ = alt - w_ = upper-lower - - done = not tag_ or (rid_, tag_) < (rid, tag) - - return done, rid_, tag_, w_, j, d, jump, path + data = rbyd.data # precompute tree t_width = 0 - if args.get('tree') or args.get('rbyd'): - trunks = co.defaultdict(lambda: (-1, 0)) - alts = co.defaultdict(lambda: {}) - - rid, tag = -1, 0 - while True: - done, rid, tag, w, j, d, size, path = lookup(rid, tag+0x1) - # found end of tree? - if done: - break - - # keep track of trunks/alts - trunks[j] = (rid, tag) - - for j_, j__, followed, c in path: - if followed: - alts[j_] |= {'f': j__, 'c': c} - else: - alts[j_] |= {'nf': j__, 'c': c} - - if args.get('rbyd'): - # treat unreachable alts as converging paths - for j_, alt in alts.items(): - if 'f' not in alt: - alt['f'] = alt['nf'] - elif 'nf' not in alt: - alt['nf'] = alt['f'] - - else: - # prune any alts with unreachable edges - pruned = {} - for j_, alt in alts.items(): - if 'f' not in alt: - pruned[j_] = alt['nf'] - elif 'nf' not in alt: - pruned[j_] = alt['f'] - for j_ in pruned.keys(): - del alts[j_] - - for j_, alt in alts.items(): - while alt['f'] in pruned: - alt['f'] = pruned[alt['f']] - while alt['nf'] in pruned: - alt['nf'] = pruned[alt['nf']] - - # find the trunk and depth of each alt - def rec_trunk(j_): - if j_ not in alts: - return trunks[j_] - else: - if 'nft' not in alts[j_]: - alts[j_]['nft'] = rec_trunk(alts[j_]['nf']) - return alts[j_]['nft'] - - for j_ in alts.keys(): - rec_trunk(j_) - for j_, alt in alts.items(): - if alt['f'] in alts: - alt['ft'] = alts[alt['f']]['nft'] - else: - alt['ft'] = trunks[alt['f']] - - def rec_height(j_): - if j_ not in alts: - return 0 - else: - if 'h' not in alts[j_]: - alts[j_]['h'] = max( - rec_height(alts[j_]['f']), - rec_height(alts[j_]['nf'])) + 1 - return alts[j_]['h'] - - for j_ in alts.keys(): - rec_height(j_) - - t_depth = max((alt['h']+1 for alt in alts.values()), default=0) - - # convert to more general tree representation - TBranch = co.namedtuple('TBranch', 'a, b, d, c') - tree = set() - for j, alt in alts.items(): - # note all non-trunk edges should be black - tree.add(TBranch( - a=alt['nft'], - b=alt['nft'], - d=t_depth-1 - alt['h'], - c=alt['c'], - )) - if alt['ft'] != alt['nft']: - tree.add(TBranch( - a=alt['nft'], - b=alt['ft'], - d=t_depth-1 - alt['h'], - c='b', - )) + if args.get('tree') or args.get('tree_rbyd'): + tree = rbyd.tree(rbyd=args.get('tree_rbyd')) # find the max depth from the tree - t_depth = max((branch.d+1 for branch in tree), default=0) + t_depth = max((b.depth+1 for b in tree), default=0) if t_depth > 0: t_width = 2*t_depth + 2 @@ -804,28 +1178,28 @@ def dbg_tree(data, block_size, rev, trunk, weight, *, return '' def branchrepr(x, d, was): - for branch in tree: - if branch.d == d and branch.b == x: - if any(branch.d == d and branch.a == x - for branch in tree): - return '+-', branch.c, branch.c - elif any(branch.d == d - and x > min(branch.a, branch.b) - and x < max(branch.a, branch.b) - for branch in tree): - return '|-', branch.c, branch.c - elif branch.a < branch.b: - return '\'-', branch.c, branch.c + for b in tree: + if b.depth == d and b.b == x: + if any(b.depth == d and b.a == x + for b in tree): + return '+-', b.color, b.color + elif any(b.depth == d + and x > min(b.a, b.b) + and x < max(b.a, b.b) + for b in tree): + return '|-', b.color, b.color + elif b.a < b.b: + return '\'-', b.color, b.color else: - return '.-', branch.c, branch.c - for branch in tree: - if branch.d == d and branch.a == x: - return '+ ', branch.c, None - for branch in tree: - if (branch.d == d - and x > min(branch.a, branch.b) - and x < max(branch.a, branch.b)): - return '| ', branch.c, was + return '.-', b.color, b.color + for b in tree: + if b.depth == d and b.a == x: + return '+ ', b.color, None + for b in tree: + if (b.depth == d + and x > min(b.a, b.b) + and x < max(b.a, b.b)): + return '| ', b.color, was if was: return '--', was, was return ' ', None, None @@ -848,26 +1222,21 @@ def dbg_tree(data, block_size, rev, trunk, weight, *, # dynamically size the id field - w_width = mt.ceil(mt.log10(max(1, weight)+1)) - - rid, tag = -1, 0 - for i in it.count(): - done, rid, tag, w, j, d, size, path = lookup(rid, tag+0x1) - # found end of tree? - if done: - break + w_width = mt.ceil(mt.log10(max(1, rbyd.weight)+1)) + for i, (rid, tag, rattr) in enumerate(rbyd): # show human-readable tag representation print('%08x: %s%*s %-*s %s' % ( - j, + rattr.toff, treerepr(rid, tag) - if args.get('tree') or args.get('rbyd') + if args.get('tree') or args.get('tree_rbyd') else '', - 2*w_width+1, '%d-%d' % (rid-(w-1), rid) if w > 1 - else rid if w > 0 or i == 0 + 2*w_width+1, '%d-%d' % (rid-(rattr.weight-1), rid) + if rattr.weight > 1 + else rid if rattr.weight > 0 or i == 0 else '', - 21+w_width, tagrepr(tag, w, size, j), - next(xxd(data[j+d:j+d+min(size, 8)], 8), '') + 21+w_width, rattr.tagrepr(), + next(xxd(rattr[:8], 8), '') if not args.get('raw') and not args.get('no_truncate') and not tag & TAG_ALT @@ -875,17 +1244,17 @@ def dbg_tree(data, block_size, rev, trunk, weight, *, # show on-disk encoding of tags if args.get('raw'): - for o, line in enumerate(xxd(data[j:j+d])): + for o, line in enumerate(xxd(data[rattr.toff:rattr.off])): print('%8s: %*s%*s %s' % ( - '%04x' % (j + o*16), + '%04x' % (rattr.toff + o*16), t_width, '', 2*w_width+1, '', line)) if args.get('raw') or args.get('no_truncate'): if not tag & TAG_ALT: - for o, line in enumerate(xxd(data[j+d:j+d+size])): + for o, line in enumerate(xxd(rattr[:])): print('%8s: %*s%*s %s' % ( - '%04x' % (j+d + o*16), + '%04x' % (rattr.off + o*16), t_width, '', 2*w_width+1, '', line)) @@ -922,154 +1291,28 @@ def main(disk, blocks=None, *, f.seek(0, os.SEEK_END) block_size = f.tell() - # blocks may also encode trunks - blocks, trunks = ( - [block[0] if isinstance(block, tuple) else block - for block in blocks], - [trunk if trunk is not None - else block[1] if isinstance(block, tuple) - else None - for block in blocks]) - - # read each block - datas = [] - for block in blocks: - f.seek(block * block_size) - datas.append(f.read(block_size)) - - # first figure out which block as the most recent revision - def fetch(data, trunk): - rev = fromle32(data[0:4]) - cksum = 0 - cksum_ = crc32c(data[0:4]) - cksum__ = cksum_ - perturb = False - eoff = 0 - eoff_ = None - j_ = 4 - trunk_ = 0 - trunk__ = 0 - trunk___ = 0 - weight = 0 - weight_ = 0 - weight__ = 0 - while j_ < len(data) and (not trunk or eoff <= trunk): - # read next tag - v, tag, w, size, d = fromtag(data[j_:]) - if v != parity(cksum__): - break - cksum__ ^= 0x00000080 if v else 0 - cksum__ = crc32c(data[j_:j_+d], cksum__) - j_ += d - if not tag & TAG_ALT and j_ + size > len(data): - break - - # take care of cksums - if not tag & TAG_ALT: - if (tag & 0xff00) != TAG_CKSUM: - cksum__ = crc32c(data[j_:j_+size], cksum__) - # found a cksum? - else: - # check cksum - cksum___ = fromle32(data[j_:j_+4]) - if cksum__ != cksum___: - break - # commit what we have - eoff = eoff_ if eoff_ else j_ + size - cksum = cksum_ - trunk_ = trunk__ - weight = weight_ - # update perturb bit - perturb = tag & TAG_P - # revert to data cksum and perturb - cksum__ = cksum_ ^ (0xfca42daf if perturb else 0) - - # evaluate trunks - if (tag & 0xf000) != TAG_CKSUM: - if not (trunk and j_-d > trunk and not trunk___): - # new trunk? - if not trunk___: - trunk___ = j_-d - weight__ = 0 - - # keep track of weight - weight__ += w - - # end of trunk? - if not tag & TAG_ALT: - # update trunk/weight unless we found a shrub or an - # explicit trunk (which may be a shrub) is requested - if not tag & TAG_SHRUB or trunk___ == trunk: - trunk__ = trunk___ - weight_ = weight__ - # keep track of eoff for best matching trunk - if trunk and j_ + size > trunk: - eoff_ = j_ + size - eoff = eoff_ - cksum = cksum__ ^ ( - 0xfca42daf if perturb else 0) - trunk_ = trunk__ - weight = weight_ - trunk___ = 0 - - # update canonical checksum, xoring out any perturb state - cksum_ = cksum__ ^ (0xfca42daf if perturb else 0) - - if not tag & TAG_ALT: - j_ += size - - return rev, eoff, trunk_, weight, cksum - - revs, eoffs, trunks_, weights, cksums = [], [], [], [], [] - i = 0 - for i_, (data, trunk_) in enumerate(zip(datas, trunks)): - rev, eoff, trunk_, weight, cksum = fetch(data, trunk_) - revs.append(rev) - eoffs.append(eoff) - trunks_.append(trunk_) - weights.append(weight) - cksums.append(cksum) - - # compare with sequence arithmetic - if trunk_ and ( - not trunks_[i] - or not ((rev - revs[i]) & 0x80000000) - or (rev == revs[i] and trunk_ > trunks_[i])): - i = i_ - - # print contents of the winning metadata block - block, data, rev, eoff, trunk_, weight, cksum = ( - blocks[i], - datas[i], - revs[i], - eoffs[i], - trunks_[i], - weights[i], - cksums[i]) + # fetch the rbyd + rbyd = Rbyd.fetch((f, block_size), blocks) print('rbyd %s w%d, rev %08x, size %d, cksum %08x' % ( - '0x%x.%x' % (block, trunk_) - if len(blocks) == 1 - else '0x{%x,%s}.%x' % ( - block, - ','.join('%x' % blocks[(i+1+j) % len(blocks)] - for j in range(len(blocks)-1)), - trunk_), - weight, - rev, - eoff, - cksum)) + rbyd.addr(), + rbyd.weight, + rbyd.rev, + rbyd.eoff, + rbyd.cksum)) if args.get('log'): - dbg_log(data, block_size, rev, eoff, weight, + dbg_log(rbyd, + block_size=block_size, color=color, **args) else: - dbg_tree(data, block_size, rev, trunk_, weight, + dbg_tree(rbyd, + block_size=block_size, color=color, **args) - if args.get('error_on_corrupt') and eoff == 0: + if args.get('error_on_corrupt') and not rbyd: sys.exit(2) @@ -1124,8 +1367,9 @@ if __name__ == "__main__": '-t', '--tree', action='store_true', help="Show the rbyd tree.") + # TODO adopt this rename in all scripts parser.add_argument( - '-R', '--rbyd', + '-R', '--tree-rbyd', action='store_true', help="Show the full rbyd tree.") parser.add_argument(