Christopher Haster
5511c100ed
This started with adding -d/--diff support to dbgflags.py, which is very
useful for comparing flags during test failure.
The flag asserts in our tests generally look like this:
tests/test_mount.toml:171:assert: assert failed with 33570064,
expected eq 33570576
assert(fsinfo.flags == (
Which can now be quickly compared with dbgflags.py:
$ ./scripts/dbgflags.py +i 33570064 -d 33570576
LFS3_I_GBMAP 0x02000000 Global on-disk block-map in use
LFS3_I_REVPERTURB 0x00000010 Mounted with LFS3_M_REVPERTURB
LFS3_I_MKCONSISTENT 0x00000100 Filesystem needs mkconsistent to write
-LFS3_I_LOOKAHEAD 0x00000200 Lookahead buffer is not full
LFS3_I_PREERASE 0x00000400 Blocks can be pre-erased
LFS3_I_COMPACT 0x00000800 Filesystem may have uncompacted metadata
LFS3_I_CKMETA 0x00001000 Metadata checksums not checked recently
LFS3_I_CKDATA 0x00002000 Data checksums not checked recently
The assert print is a bit more annoying than it needs to be, as it only
prints in decimal. But, since our prettyasserts.py only works at the
syntax layer, it's not possible to make it any smarter.
---
To make this diffing work required a couple more features in our
self-parsing Flag class:
- Keep track of lineno, mainly for ordering things
- Moved find logic into a staticmethod on all classes
- Added _sentinel based defaults to find functions
- Allowed self to be non-class in line functions to deduplicate "Unknown
flag" messages
I went ahead and extended these to the other self-parsing classes (Err
and Tag) in case they're useful in the future.
1964 lines
61 KiB
Python
Executable File
1964 lines
61 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# prevent local imports
|
|
if __name__ == "__main__":
|
|
__import__('sys').path.pop(0)
|
|
|
|
import bisect
|
|
import collections as co
|
|
import functools as ft
|
|
import itertools as it
|
|
import math as mt
|
|
import os
|
|
import struct
|
|
|
|
try:
|
|
import crc32c as crc32c_lib
|
|
except ModuleNotFoundError:
|
|
crc32c_lib = None
|
|
|
|
|
|
TAG_NULL = 0x0000 ## v--- ---- +--- ----
|
|
TAG_INTERNAL = 0x0000 ## v--- ---- +ttt tttt
|
|
TAG_CONFIG = 0x0100 ## v--- ---1 +ttt tttt
|
|
TAG_MAGIC = 0x0131 # v--- ---1 +-11 --rr
|
|
TAG_VERSION = 0x0134 # v--- ---1 +-11 -1--
|
|
TAG_RCOMPAT = 0x0135 # v--- ---1 +-11 -1-1
|
|
TAG_WCOMPAT = 0x0136 # v--- ---1 +-11 -11-
|
|
TAG_OCOMPAT = 0x0137 # v--- ---1 +-11 -111
|
|
TAG_GEOMETRY = 0x0138 # v--- ---1 +-11 1---
|
|
TAG_NAMELIMIT = 0x0139 # v--- ---1 +-11 1--1
|
|
TAG_FILELIMIT = 0x013a # v--- ---1 +-11 1-1-
|
|
TAG_GDELTA = 0x0200 ## v--- --1- +ttt tttt
|
|
TAG_GRMDELTA = 0x0230 # v--- --1- +-11 --++
|
|
TAG_GBMAPDELTA = 0x0234 # v--- --1- +-11 -1rr
|
|
TAG_NAME = 0x0300 ## v--- --11 +ttt tttt
|
|
TAG_BNAME = 0x0300 # v--- --11 +--- ----
|
|
TAG_REG = 0x0301 # v--- --11 +--- ---1
|
|
TAG_DIR = 0x0302 # v--- --11 +--- --1-
|
|
TAG_STICKYNOTE = 0x0303 # v--- --11 +--- --11
|
|
TAG_BOOKMARK = 0x0304 # v--- --11 +--- -1--
|
|
TAG_MNAME = 0x0330 # v--- --11 +-11 ----
|
|
TAG_STRUCT = 0x0400 ## v--- -1-- +ttt tttt
|
|
TAG_BRANCH = 0x0400 # v--- -1-- +--- --rr
|
|
TAG_DATA = 0x0404 # v--- -1-- +--- -1rr
|
|
TAG_BLOCK = 0x0408 # v--- -1-- +--- 1err
|
|
TAG_DID = 0x0420 # v--- -1-- +-1- ----
|
|
TAG_BSHRUB = 0x0428 # v--- -1-- +-1- 1-rr
|
|
TAG_BTREE = 0x042c # v--- -1-- +-1- 11rr
|
|
TAG_MROOT = 0x0431 # v--- -1-- +-11 --rr
|
|
TAG_MDIR = 0x0435 # v--- -1-- +-11 -1rr
|
|
TAG_MTREE = 0x043c # v--- -1-- +-11 11rr
|
|
TAG_BMRANGE = 0x0440 # v--- -1-- +1-- ++uu
|
|
TAG_BMFREE = 0x0440 # v--- -1-- +1-- ----
|
|
TAG_BMINUSE = 0x0441 # v--- -1-- +1-- ---1
|
|
TAG_BMERASED = 0x0442 # v--- -1-- +1-- --1-
|
|
TAG_BMBAD = 0x0443 # v--- -1-- +1-- --11
|
|
TAG_ATTR = 0x0600 ## v--- -11a +aaa aaaa
|
|
TAG_UATTR = 0x0600 # v--- -11- +aaa aaaa
|
|
TAG_SATTR = 0x0700 # v--- -111 +aaa aaaa
|
|
TAG_SHRUB = 0x1000 ## v--1 kkkk +kkk kkkk
|
|
TAG_ALT = 0x4000 ## v1cd kkkk +kkk kkkk
|
|
TAG_B = 0x0000
|
|
TAG_R = 0x2000
|
|
TAG_LE = 0x0000
|
|
TAG_GT = 0x1000
|
|
TAG_CKSUM = 0x3000 ## v-11 ---- ++++ +pqq
|
|
TAG_PHASE = 0x0003
|
|
TAG_PERTURB = 0x0004
|
|
TAG_NOTE = 0x3100 ## v-11 ---1 ++++ ++++
|
|
TAG_ECKSUM = 0x3200 ## v-11 --1- ++++ ++++
|
|
TAG_GCKSUMDELTA = 0x3300 ## v-11 --11 ++++ ++++
|
|
|
|
|
|
# self-parsing tag repr
|
|
class Tag:
|
|
def __init__(self, name, tag, encoding, help, *,
|
|
lineno=0):
|
|
self.name = name
|
|
self.tag = tag
|
|
self.encoding = encoding
|
|
self.help = help
|
|
self.lineno = lineno
|
|
# derive mask from encoding
|
|
self.mask = sum(
|
|
(1 if x in 'v-01' else 0) << len(self.encoding)-1-i
|
|
for i, x in enumerate(self.encoding))
|
|
|
|
def __repr__(self):
|
|
return 'Tag(%r, %r, %r)' % (
|
|
self.name,
|
|
self.tag,
|
|
self.encoding)
|
|
|
|
def __eq__(self, other):
|
|
return self.name == other.name
|
|
|
|
def __ne__(self, other):
|
|
return self.name != other.name
|
|
|
|
def __hash__(self):
|
|
return hash(self.name)
|
|
|
|
def line(self):
|
|
# substitute mask chars when zero
|
|
tag = '0x%s' % ''.join(
|
|
n if n != '0' else next(
|
|
(x for x in self.encoding[i*4:i*4+4]
|
|
if x not in 'v-01+'),
|
|
'0')
|
|
for i, n in enumerate('%04x' % self.tag))
|
|
# group into nibbles
|
|
encoding = ' '.join(self.encoding[i*4:i*4+4]
|
|
for i in range(len(self.encoding)//4))
|
|
return ('LFS3_%s' % self.name, tag, encoding)
|
|
|
|
def specificity(self):
|
|
return sum(1 for x in self.encoding if x in 'v-01')
|
|
|
|
def matches(self, tag):
|
|
return (tag & self.mask) == (self.tag & self.mask)
|
|
|
|
def get(self, chars, tag):
|
|
return sum(
|
|
tag & ((1 if x in chars else 0) << len(self.encoding)-1-i)
|
|
for i, x in enumerate(self.encoding))
|
|
|
|
def max(self, chars):
|
|
return max(len(self.encoding)-1-i
|
|
for i, x in enumerate(self.encoding) if x in chars)
|
|
|
|
def min(self, chars):
|
|
return min(len(self.encoding)-1-i
|
|
for i, x in enumerate(self.encoding) if x in chars)
|
|
|
|
def width(self, chars):
|
|
return self.max(chars) - self.min(chars)
|
|
|
|
def __contains__(self, chars):
|
|
return any(x in self.encoding for x in chars)
|
|
|
|
@staticmethod
|
|
@ft.cache
|
|
def tags():
|
|
# parse our script's source to figure out tags
|
|
import inspect
|
|
import re
|
|
tags = []
|
|
tag_pattern = re.compile(
|
|
'^(?P<name>TAG_[^ ]*) *= *(?P<tag>[^#]*?) *'
|
|
'#+ *(?P<encoding>(?:[^ ] *?){16}) *(?P<help>.*)$')
|
|
for i, line in enumerate(
|
|
inspect.getsource(inspect.getmodule(inspect.currentframe()))
|
|
.replace('\\\n', '')
|
|
.splitlines()):
|
|
m = tag_pattern.match(line)
|
|
if m:
|
|
tags.append(Tag(
|
|
m.group('name'),
|
|
globals()[m.group('name')],
|
|
m.group('encoding').replace(' ', ''),
|
|
m.group('help'),
|
|
lineno=1+i))
|
|
return tags
|
|
|
|
# find best matching tag
|
|
_sentinel = object()
|
|
@staticmethod
|
|
def find(tag, *, default=_sentinel):
|
|
# find tags, note this is cached
|
|
tags__ = Tag.tags()
|
|
|
|
# find the most specific matching tag, ignoring valid bits
|
|
t = max((t for t in tags__ if t.matches(tag & 0x7fff)),
|
|
key=lambda t: t.specificity(),
|
|
default=None)
|
|
if t is not None:
|
|
return t
|
|
elif default is Tag._sentinel:
|
|
raise KeyError(tag)
|
|
else:
|
|
return default
|
|
|
|
# human readable tag repr
|
|
@staticmethod
|
|
def repr(tag, weight=None, size=None, *,
|
|
global_=False,
|
|
toff=None):
|
|
# find the most specific matching tag, ignoring the shrub bit
|
|
t = Tag.find(
|
|
tag & ~(TAG_SHRUB if tag & 0x7000 == TAG_SHRUB else 0),
|
|
default=None)
|
|
|
|
# build repr
|
|
r = []
|
|
# normal tag?
|
|
if not tag & TAG_ALT:
|
|
if t is not None:
|
|
# prefix shrub tags with shrub
|
|
if tag & 0x7000 == TAG_SHRUB:
|
|
r.append('shrub')
|
|
# lowercase name
|
|
r.append(t.name.split('_', 1)[1].lower())
|
|
# gstate tag?
|
|
if global_:
|
|
if r[-1] == 'gdelta':
|
|
r[-1] = 'gstate'
|
|
elif r[-1].endswith('delta'):
|
|
r[-1] = r[-1][:-len('delta')]
|
|
# include perturb/phase bits
|
|
if 'q' in t:
|
|
r.append('q%d' % t.get('q', tag))
|
|
if 'p' in t and tag & TAG_PERTURB:
|
|
r.append('p')
|
|
|
|
# include unmatched fields, but not just redund, and
|
|
# only reserved bits if non-zero
|
|
if 'tua' in t or ('+' in t and t.get('+', tag) != 0):
|
|
r.append(' 0x%0*x' % (
|
|
(t.width('tuar+')+4-1)//4,
|
|
t.get('tuar+', tag)))
|
|
# unknown tag?
|
|
else:
|
|
r.append('0x%04x' % tag)
|
|
|
|
# weight?
|
|
if weight:
|
|
r.append(' w%d' % weight)
|
|
# size? don't include if null
|
|
if size is not None and (size or tag & 0x7fff):
|
|
r.append(' %d' % size)
|
|
|
|
# alt pointer?
|
|
else:
|
|
r.append('alt')
|
|
r.append('r' if tag & TAG_R else 'b')
|
|
r.append('gt' if tag & TAG_GT else 'le')
|
|
r.append(' 0x%0*x' % (
|
|
(t.width('k')+4-1)//4,
|
|
t.get('k', tag)))
|
|
|
|
# weight?
|
|
if weight is not None:
|
|
r.append(' w%d' % weight)
|
|
# jump?
|
|
if size and toff is not None:
|
|
r.append(' 0x%x' % (0xffffffff & (toff-size)))
|
|
elif size:
|
|
r.append(' -%d' % size)
|
|
|
|
return ''.join(r)
|
|
|
|
|
|
# some ways of block geometry representations
|
|
# 512 -> 512
|
|
# 512x16 -> (512, 16)
|
|
# 0x200x10 -> (512, 16)
|
|
def bdgeom(s):
|
|
s = s.strip()
|
|
b = 10
|
|
if s.startswith('0x') or s.startswith('0X'):
|
|
s = s[2:]
|
|
b = 16
|
|
elif s.startswith('0o') or s.startswith('0O'):
|
|
s = s[2:]
|
|
b = 8
|
|
elif s.startswith('0b') or s.startswith('0B'):
|
|
s = s[2:]
|
|
b = 2
|
|
|
|
if 'x' in s:
|
|
s, s_ = s.split('x', 1)
|
|
return (int(s, b), int(s_, b))
|
|
else:
|
|
return int(s, b)
|
|
|
|
# parse some rbyd addr encodings
|
|
# 0xa -> (0xa,)
|
|
# 0xa.c -> ((0xa, 0xc),)
|
|
# 0x{a,b} -> (0xa, 0xb)
|
|
# 0x{a,b}.c -> ((0xa, 0xc), (0xb, 0xc))
|
|
def rbydaddr(s):
|
|
s = s.strip()
|
|
b = 10
|
|
if s.startswith('0x') or s.startswith('0X'):
|
|
s = s[2:]
|
|
b = 16
|
|
elif s.startswith('0o') or s.startswith('0O'):
|
|
s = s[2:]
|
|
b = 8
|
|
elif s.startswith('0b') or s.startswith('0B'):
|
|
s = s[2:]
|
|
b = 2
|
|
|
|
trunk = None
|
|
if '.' in s:
|
|
s, s_ = s.split('.', 1)
|
|
trunk = int(s_, b)
|
|
|
|
if s.startswith('{') and '}' in s:
|
|
ss = s[1:s.find('}')].split(',')
|
|
else:
|
|
ss = [s]
|
|
|
|
addr = []
|
|
for s in ss:
|
|
if trunk is not None:
|
|
addr.append((int(s, b), trunk))
|
|
else:
|
|
addr.append(int(s, b))
|
|
|
|
return tuple(addr)
|
|
|
|
def crc32c(data, crc=0):
|
|
if crc32c_lib is not None:
|
|
return crc32c_lib.crc32c(data, crc)
|
|
else:
|
|
crc ^= 0xffffffff
|
|
for b in data:
|
|
crc ^= b
|
|
for j in range(8):
|
|
crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78)
|
|
return 0xffffffff ^ crc
|
|
|
|
def popc(x):
|
|
return bin(x).count('1')
|
|
|
|
def parity(x):
|
|
return popc(x) & 1
|
|
|
|
def fromle32(data, j=0):
|
|
return struct.unpack('<I', data[j:j+4].ljust(4, b'\0'))[0]
|
|
|
|
def fromleb128(data, j=0):
|
|
word = 0
|
|
d = 0
|
|
while j+d < len(data):
|
|
b = data[j+d]
|
|
word |= (b & 0x7f) << 7*d
|
|
word &= 0xffffffff
|
|
if not b & 0x80:
|
|
return word, d+1
|
|
d += 1
|
|
return word, d
|
|
|
|
def fromtag(data, j=0):
|
|
d = 0
|
|
tag = struct.unpack('>H', data[j:j+2].ljust(2, b'\0'))[0]; d += 2
|
|
weight, d_ = fromleb128(data, j+d); d += d_
|
|
size, d_ = fromleb128(data, j+d); d += d_
|
|
return tag>>15, tag&0x7fff, weight, size, d
|
|
|
|
def frombranch(data, j=0):
|
|
d = 0
|
|
block, d_ = fromleb128(data, j+d); d += d_
|
|
trunk, d_ = fromleb128(data, j+d); d += d_
|
|
cksum = fromle32(data, j+d); d += 4
|
|
return block, trunk, cksum, d
|
|
|
|
def xxd(data, width=16):
|
|
for i in range(0, len(data), width):
|
|
yield '%-*s %-*s' % (
|
|
3*width,
|
|
' '.join('%02x' % b for b in data[i:i+width]),
|
|
width,
|
|
''.join(
|
|
b if b >= ' ' and b <= '~' else '.'
|
|
for b in map(chr, data[i:i+width])))
|
|
|
|
# compute the difference between two paths, returning everything
|
|
# in a after the paths diverge, as well as the relevant index
|
|
def pathdelta(a, b):
|
|
if not isinstance(a, list):
|
|
a = list(a)
|
|
i = 0
|
|
for a_, b_ in zip(a, b):
|
|
try:
|
|
if type(a_) == type(b_) and a_ == b_:
|
|
i += 1
|
|
else:
|
|
break
|
|
# treat exceptions here as failure to match, most likely
|
|
# the compared types are incompatible, it's the caller's
|
|
# problem
|
|
except Exception:
|
|
break
|
|
|
|
return [(i+j, a_) for j, a_ in enumerate(a[i:])]
|
|
|
|
|
|
# a simple wrapper over an open file with bd geometry
|
|
class Bd:
|
|
def __init__(self, f, block_size=None, block_count=None):
|
|
self.f = f
|
|
self.block_size = block_size
|
|
self.block_count = block_count
|
|
|
|
def __repr__(self):
|
|
return '<%s %s>' % (self.__class__.__name__, self.repr())
|
|
|
|
def repr(self):
|
|
return 'bd %sx%s' % (self.block_size, self.block_count)
|
|
|
|
def read(self, block, off, size):
|
|
self.f.seek(block*self.block_size + off)
|
|
return self.f.read(size)
|
|
|
|
def readblock(self, block):
|
|
self.f.seek(block*self.block_size)
|
|
return self.f.read(self.block_size)
|
|
|
|
# tagged data in an rbyd
|
|
class Rattr:
|
|
def __init__(self, tag, weight, blocks, toff, tdata, data):
|
|
self.tag = tag
|
|
self.weight = weight
|
|
if isinstance(blocks, int):
|
|
self.blocks = (blocks,)
|
|
else:
|
|
self.blocks = blocks
|
|
self.toff = toff
|
|
self.tdata = tdata
|
|
self.data = data
|
|
|
|
@property
|
|
def block(self):
|
|
return self.blocks[0]
|
|
|
|
@property
|
|
def tsize(self):
|
|
return len(self.tdata)
|
|
|
|
@property
|
|
def off(self):
|
|
return self.toff + len(self.tdata)
|
|
|
|
@property
|
|
def size(self):
|
|
return len(self.data)
|
|
|
|
def __bytes__(self):
|
|
return self.data
|
|
|
|
def __repr__(self):
|
|
return '<%s %s>' % (self.__class__.__name__, self.repr())
|
|
|
|
def repr(self):
|
|
return Tag.repr(self.tag, self.weight, self.size)
|
|
|
|
def __iter__(self):
|
|
return iter((self.tag, self.weight, self.data))
|
|
|
|
def __eq__(self, other):
|
|
return ((self.tag, self.weight, self.data)
|
|
== (other.tag, other.weight, other.data))
|
|
|
|
def __ne__(self, other):
|
|
return not self.__eq__(other)
|
|
|
|
def __hash__(self):
|
|
return hash((self.tag, self.weight, self.data))
|
|
|
|
# convenience for did/name access
|
|
def _parse_name(self):
|
|
# note we return a null name for non-name tags, this is so
|
|
# vestigial names in btree nodes act as a catch-all
|
|
if (self.tag & 0xff00) != TAG_NAME:
|
|
did = 0
|
|
name = b''
|
|
else:
|
|
did, d = fromleb128(self.data)
|
|
name = self.data[d:]
|
|
|
|
# cache both
|
|
self.did = did
|
|
self.name = name
|
|
|
|
@ft.cached_property
|
|
def did(self):
|
|
self._parse_name()
|
|
return self.did
|
|
|
|
@ft.cached_property
|
|
def name(self):
|
|
self._parse_name()
|
|
return self.name
|
|
|
|
class Ralt:
|
|
def __init__(self, tag, weight, blocks, toff, tdata, jump,
|
|
color=None, followed=None):
|
|
self.tag = tag
|
|
self.weight = weight
|
|
if isinstance(blocks, int):
|
|
self.blocks = (blocks,)
|
|
else:
|
|
self.blocks = blocks
|
|
self.toff = toff
|
|
self.tdata = tdata
|
|
self.jump = jump
|
|
|
|
if color is not None:
|
|
self.color = color
|
|
else:
|
|
self.color = 'r' if tag & TAG_R else 'b'
|
|
self.followed = followed
|
|
|
|
@property
|
|
def block(self):
|
|
return self.blocks[0]
|
|
|
|
@property
|
|
def tsize(self):
|
|
return len(self.tdata)
|
|
|
|
@property
|
|
def off(self):
|
|
return self.toff + len(self.tdata)
|
|
|
|
@property
|
|
def joff(self):
|
|
return self.toff - self.jump
|
|
|
|
def __repr__(self):
|
|
return '<%s %s>' % (self.__class__.__name__, self.repr())
|
|
|
|
def repr(self):
|
|
return Tag.repr(self.tag, self.weight, self.jump, toff=self.toff)
|
|
|
|
def __iter__(self):
|
|
return iter((self.tag, self.weight, self.jump))
|
|
|
|
def __eq__(self, other):
|
|
return ((self.tag, self.weight, self.jump)
|
|
== (other.tag, other.weight, other.jump))
|
|
|
|
def __ne__(self, other):
|
|
return not self.__eq__(other)
|
|
|
|
def __hash__(self):
|
|
return hash((self.tag, self.weight, self.jump))
|
|
|
|
|
|
# our core rbyd type
|
|
class Rbyd:
|
|
def __init__(self, blocks, trunk, weight, rev, eoff, cksum, data, *,
|
|
shrub=False,
|
|
gcksumdelta=None,
|
|
redund=0):
|
|
if isinstance(blocks, int):
|
|
self.blocks = (blocks,)
|
|
else:
|
|
self.blocks = blocks
|
|
self.trunk = trunk
|
|
self.weight = weight
|
|
self.rev = rev
|
|
self.eoff = eoff
|
|
self.cksum = cksum
|
|
self.data = data
|
|
|
|
self.shrub = shrub
|
|
self.gcksumdelta = gcksumdelta
|
|
self.redund = redund
|
|
|
|
@property
|
|
def block(self):
|
|
return self.blocks[0]
|
|
|
|
@property
|
|
def corrupt(self):
|
|
# use redund=-1 to indicate corrupt rbyds
|
|
return self.redund >= 0
|
|
|
|
def addr(self):
|
|
if len(self.blocks) == 1:
|
|
return '0x%x.%x' % (self.block, self.trunk)
|
|
else:
|
|
return '0x{%s}.%x' % (
|
|
','.join('%x' % block for block in self.blocks),
|
|
self.trunk)
|
|
|
|
def __repr__(self):
|
|
return '<%s %s>' % (self.__class__.__name__, self.repr())
|
|
|
|
def repr(self):
|
|
return 'rbyd %s w%s' % (self.addr(), self.weight)
|
|
|
|
def __bool__(self):
|
|
# use redund=-1 to indicate corrupt rbyds
|
|
return self.redund >= 0
|
|
|
|
def __eq__(self, other):
|
|
return ((frozenset(self.blocks), self.trunk)
|
|
== (frozenset(other.blocks), other.trunk))
|
|
|
|
def __ne__(self, other):
|
|
return not self.__eq__(other)
|
|
|
|
def __hash__(self):
|
|
return hash((frozenset(self.blocks), self.trunk))
|
|
|
|
@classmethod
|
|
def _fetch(cls, data, block, trunk=None):
|
|
# fetch the rbyd
|
|
rev = fromle32(data, 0)
|
|
cksum = 0
|
|
cksum_ = crc32c(data[0:4])
|
|
cksum__ = cksum_
|
|
perturb = False
|
|
eoff = 0
|
|
eoff_ = None
|
|
j_ = 4
|
|
trunk_ = 0
|
|
trunk__ = 0
|
|
trunk___ = 0
|
|
weight = 0
|
|
weight_ = 0
|
|
weight__ = 0
|
|
gcksumdelta = None
|
|
gcksumdelta_ = None
|
|
while j_ < len(data) and (not trunk or eoff <= trunk):
|
|
# read next tag
|
|
v, tag, w, size, d = fromtag(data, j_)
|
|
if v != parity(cksum__):
|
|
break
|
|
cksum__ ^= 0x00000080 if v else 0
|
|
cksum__ = crc32c(data[j_:j_+d], cksum__)
|
|
j_ += d
|
|
if not tag & TAG_ALT and j_ + size > len(data):
|
|
break
|
|
|
|
# take care of cksums
|
|
if not tag & TAG_ALT:
|
|
if (tag & 0xff00) != TAG_CKSUM:
|
|
cksum__ = crc32c(data[j_:j_+size], cksum__)
|
|
|
|
# found a gcksumdelta?
|
|
if (tag & 0xff00) == TAG_GCKSUMDELTA:
|
|
gcksumdelta_ = Rattr(tag, w, block, j_-d,
|
|
data[j_-d:j_],
|
|
data[j_:j_+size])
|
|
|
|
# found a cksum?
|
|
else:
|
|
# check cksum
|
|
cksum___ = fromle32(data, j_)
|
|
if cksum__ != cksum___:
|
|
break
|
|
# commit what we have
|
|
eoff = eoff_ if eoff_ else j_ + size
|
|
cksum = cksum_
|
|
trunk_ = trunk__
|
|
weight = weight_
|
|
gcksumdelta = gcksumdelta_
|
|
gcksumdelta_ = None
|
|
# update perturb bit
|
|
perturb = bool(tag & TAG_PERTURB)
|
|
# revert to data cksum and perturb
|
|
cksum__ = cksum_ ^ (0xfca42daf if perturb else 0)
|
|
|
|
# evaluate trunks
|
|
if (tag & 0xf000) != TAG_CKSUM:
|
|
if not (trunk and j_-d > trunk and not trunk___):
|
|
# new trunk?
|
|
if not trunk___:
|
|
trunk___ = j_-d
|
|
weight__ = 0
|
|
|
|
# keep track of weight
|
|
weight__ += w
|
|
|
|
# end of trunk?
|
|
if not tag & TAG_ALT:
|
|
# update trunk/weight unless we found a shrub or an
|
|
# explicit trunk (which may be a shrub) is requested
|
|
if not tag & TAG_SHRUB or trunk___ == trunk:
|
|
trunk__ = trunk___
|
|
weight_ = weight__
|
|
# keep track of eoff for best matching trunk
|
|
if trunk and j_ + size > trunk:
|
|
eoff_ = j_ + size
|
|
eoff = eoff_
|
|
cksum = cksum__ ^ (
|
|
0xfca42daf if perturb else 0)
|
|
trunk_ = trunk__
|
|
weight = weight_
|
|
gcksumdelta = gcksumdelta_
|
|
trunk___ = 0
|
|
|
|
# update canonical checksum, xoring out any perturb state
|
|
cksum_ = cksum__ ^ (0xfca42daf if perturb else 0)
|
|
|
|
if not tag & TAG_ALT:
|
|
j_ += size
|
|
|
|
return cls(block, trunk_, weight, rev, eoff, cksum, data,
|
|
gcksumdelta=gcksumdelta,
|
|
redund=0 if trunk_ else -1)
|
|
|
|
@classmethod
|
|
def fetch(cls, bd, blocks, trunk=None):
|
|
# multiple blocks?
|
|
if not isinstance(blocks, int):
|
|
# fetch all blocks
|
|
rbyds = [cls.fetch(bd, block, trunk) for block in blocks]
|
|
|
|
# determine most recent revision/trunk
|
|
rev, trunk = None, None
|
|
for rbyd in rbyds:
|
|
# compare with sequence arithmetic
|
|
if rbyd and (
|
|
rev is None
|
|
or not ((rbyd.rev - rev) & 0x80000000)
|
|
or (rbyd.rev == rev and rbyd.trunk > trunk)):
|
|
rev, trunk = rbyd.rev, rbyd.trunk
|
|
# sort for reproducibility
|
|
rbyds.sort(key=lambda rbyd: (
|
|
# prioritize valid redund blocks
|
|
0 if rbyd and rbyd.rev == rev and rbyd.trunk == trunk
|
|
else 1,
|
|
# default to sorting by block
|
|
rbyd.block))
|
|
|
|
# choose an active rbyd
|
|
rbyd = rbyds[0]
|
|
# keep track of the other blocks
|
|
rbyd.blocks = tuple(rbyd.block for rbyd in rbyds)
|
|
# keep track of how many redund blocks are valid
|
|
rbyd.redund = -1 + sum(1 for rbyd in rbyds
|
|
if rbyd and rbyd.rev == rev and rbyd.trunk == trunk)
|
|
# and patch the gcksumdelta if we have one
|
|
if rbyd.gcksumdelta is not None:
|
|
rbyd.gcksumdelta.blocks = rbyd.blocks
|
|
return rbyd
|
|
|
|
# seek/read the block
|
|
block = blocks
|
|
data = bd.readblock(block)
|
|
|
|
# fetch the rbyd
|
|
return cls._fetch(data, block, trunk)
|
|
|
|
@classmethod
|
|
def fetchck(cls, bd, blocks, trunk, weight, cksum):
|
|
# try to fetch the rbyd normally
|
|
rbyd = cls.fetch(bd, blocks, trunk)
|
|
|
|
# cksum mismatch? trunk/weight mismatch?
|
|
if (rbyd.cksum != cksum
|
|
or rbyd.trunk != trunk
|
|
or rbyd.weight != weight):
|
|
# mark as corrupt and keep track of expected trunk/weight
|
|
rbyd.redund = -1
|
|
rbyd.trunk = trunk
|
|
rbyd.weight = weight
|
|
|
|
return rbyd
|
|
|
|
@classmethod
|
|
def fetchshrub(cls, rbyd, trunk):
|
|
# steal the original rbyd's data
|
|
#
|
|
# this helps avoid race conditions with cksums and stuff
|
|
shrub = cls._fetch(rbyd.data, rbyd.block, trunk)
|
|
shrub.blocks = rbyd.blocks
|
|
shrub.shrub = True
|
|
return shrub
|
|
|
|
def lookupnext(self, rid, tag=None, *,
|
|
path=False):
|
|
if not self or rid >= self.weight:
|
|
if path:
|
|
return None, None, []
|
|
else:
|
|
return None, None
|
|
|
|
tag = max(tag or 0, 0x1)
|
|
lower = 0
|
|
upper = self.weight
|
|
path_ = []
|
|
|
|
# descend down tree
|
|
j = self.trunk
|
|
while True:
|
|
_, alt, w, jump, d = fromtag(self.data, j)
|
|
|
|
# found an alt?
|
|
if alt & TAG_ALT:
|
|
# follow?
|
|
if ((rid, tag & 0xfff) > (upper-w-1, alt & 0xfff)
|
|
if alt & TAG_GT
|
|
else ((rid, tag & 0xfff)
|
|
<= (lower+w-1, alt & 0xfff))):
|
|
lower += upper-lower-w if alt & TAG_GT else 0
|
|
upper -= upper-lower-w if not alt & TAG_GT else 0
|
|
j = j - jump
|
|
|
|
if path:
|
|
# figure out which color
|
|
if alt & TAG_R:
|
|
_, nalt, _, _, _ = fromtag(self.data, j+jump+d)
|
|
if nalt & TAG_R:
|
|
color = 'y'
|
|
else:
|
|
color = 'r'
|
|
else:
|
|
color = 'b'
|
|
|
|
path_.append(Ralt(
|
|
alt, w, self.blocks, j+jump,
|
|
self.data[j+jump:j+jump+d], jump,
|
|
color=color,
|
|
followed=True))
|
|
|
|
# stay on path
|
|
else:
|
|
lower += w if not alt & TAG_GT else 0
|
|
upper -= w if alt & TAG_GT else 0
|
|
j = j + d
|
|
|
|
if path:
|
|
# figure out which color
|
|
if alt & TAG_R:
|
|
_, nalt, _, _, _ = fromtag(self.data, j)
|
|
if nalt & TAG_R:
|
|
color = 'y'
|
|
else:
|
|
color = 'r'
|
|
else:
|
|
color = 'b'
|
|
|
|
path_.append(Ralt(
|
|
alt, w, self.blocks, j-d,
|
|
self.data[j-d:j], jump,
|
|
color=color,
|
|
followed=False))
|
|
|
|
# found tag
|
|
else:
|
|
rid_ = upper-1
|
|
tag_ = alt
|
|
w_ = upper-lower
|
|
|
|
if not tag_ or (rid_, tag_) < (rid, tag):
|
|
if path:
|
|
return None, None, path_
|
|
else:
|
|
return None, None
|
|
|
|
rattr_ = Rattr(tag_, w_, self.blocks, j,
|
|
self.data[j:j+d],
|
|
self.data[j+d:j+d+jump])
|
|
if path:
|
|
return rid_, rattr_, path_
|
|
else:
|
|
return rid_, rattr_
|
|
|
|
def lookup(self, rid, tag=None, mask=None, *,
|
|
path=False):
|
|
if tag is None:
|
|
tag, mask = 0, 0xffff
|
|
if mask is None:
|
|
mask = 0
|
|
|
|
r = self.lookupnext(rid, tag & ~mask,
|
|
path=path)
|
|
if path:
|
|
rid_, rattr_, path_ = r
|
|
else:
|
|
rid_, rattr_ = r
|
|
if (rid_ is None
|
|
or rid_ != rid
|
|
or (rattr_.tag & ~mask & 0xfff)
|
|
!= (tag & ~mask & 0xfff)):
|
|
if path:
|
|
return None, path_
|
|
else:
|
|
return None
|
|
|
|
if path:
|
|
return rattr_, path_
|
|
else:
|
|
return rattr_
|
|
|
|
def rids(self, *,
|
|
path=False):
|
|
rid = -1
|
|
while True:
|
|
r = self.lookupnext(rid,
|
|
path=path)
|
|
if path:
|
|
rid, name, path_ = r
|
|
else:
|
|
rid, name = r
|
|
# found end of tree?
|
|
if rid is None:
|
|
break
|
|
|
|
if path:
|
|
yield rid, name, path_
|
|
else:
|
|
yield rid, name
|
|
rid += 1
|
|
|
|
def rattrs(self, rid=None, tag=None, mask=None, *,
|
|
path=False):
|
|
if rid is None:
|
|
rid, tag = -1, 0
|
|
while True:
|
|
r = self.lookupnext(rid, tag+0x1,
|
|
path=path)
|
|
if path:
|
|
rid, rattr, path_ = r
|
|
else:
|
|
rid, rattr = r
|
|
# found end of tree?
|
|
if rid is None:
|
|
break
|
|
|
|
if path:
|
|
yield rid, rattr, path_
|
|
else:
|
|
yield rid, rattr
|
|
tag = rattr.tag
|
|
else:
|
|
if tag is None:
|
|
tag, mask = 0, 0xffff
|
|
if mask is None:
|
|
mask = 0
|
|
|
|
tag_ = max((tag & ~mask) - 1, 0)
|
|
while True:
|
|
r = self.lookupnext(rid, tag_+0x1,
|
|
path=path)
|
|
if path:
|
|
rid_, rattr_, path_ = r
|
|
else:
|
|
rid_, rattr_ = r
|
|
# found end of tree?
|
|
if (rid_ is None
|
|
or rid_ != rid
|
|
or (rattr_.tag & ~mask & 0xfff)
|
|
!= (tag & ~mask & 0xfff)):
|
|
break
|
|
|
|
if path:
|
|
yield rattr_, path_
|
|
else:
|
|
yield rattr_
|
|
tag_ = rattr_.tag
|
|
|
|
# lookup by name
|
|
def namelookup(self, did, name):
|
|
# binary search
|
|
best = None, None
|
|
lower = 0
|
|
upper = self.weight
|
|
while lower < upper:
|
|
rid, name_ = self.lookupnext(
|
|
lower + (upper-1-lower)//2)
|
|
if rid is None:
|
|
break
|
|
|
|
# bisect search space
|
|
if (name_.did, name_.name) > (did, name):
|
|
upper = rid-(name_.weight-1)
|
|
elif (name_.did, name_.name) < (did, name):
|
|
lower = rid + 1
|
|
# keep track of best match
|
|
best = rid, name_
|
|
else:
|
|
# found a match
|
|
return rid, name_
|
|
|
|
return best
|
|
|
|
|
|
# our rbyd btree type
|
|
class Btree:
|
|
def __init__(self, bd, rbyd):
|
|
self.bd = bd
|
|
self.rbyd = rbyd
|
|
|
|
@property
|
|
def block(self):
|
|
return self.rbyd.block
|
|
|
|
@property
|
|
def blocks(self):
|
|
return self.rbyd.blocks
|
|
|
|
@property
|
|
def trunk(self):
|
|
return self.rbyd.trunk
|
|
|
|
@property
|
|
def weight(self):
|
|
return self.rbyd.weight
|
|
|
|
@property
|
|
def rev(self):
|
|
return self.rbyd.rev
|
|
|
|
@property
|
|
def cksum(self):
|
|
return self.rbyd.cksum
|
|
|
|
@property
|
|
def shrub(self):
|
|
return self.rbyd.shrub
|
|
|
|
def addr(self):
|
|
return self.rbyd.addr()
|
|
|
|
def __repr__(self):
|
|
return '<%s %s>' % (self.__class__.__name__, self.repr())
|
|
|
|
def repr(self):
|
|
return 'btree %s w%s' % (self.addr(), self.weight)
|
|
|
|
def __eq__(self, other):
|
|
return self.rbyd == other.rbyd
|
|
|
|
def __ne__(self, other):
|
|
return not self.__eq__(other)
|
|
|
|
def __hash__(self):
|
|
return hash(self.rbyd)
|
|
|
|
@classmethod
|
|
def fetch(cls, bd, blocks, trunk=None):
|
|
# rbyd fetch does most of the work here
|
|
rbyd = Rbyd.fetch(bd, blocks, trunk)
|
|
return cls(bd, rbyd)
|
|
|
|
@classmethod
|
|
def fetchck(cls, bd, blocks, trunk, weight, cksum):
|
|
# rbyd fetchck does most of the work here
|
|
rbyd = Rbyd.fetchck(bd, blocks, trunk, weight, cksum)
|
|
return cls(bd, rbyd)
|
|
|
|
@classmethod
|
|
def fetchshrub(cls, bd, rbyd, trunk):
|
|
shrub = Rbyd.fetchshrub(rbyd, trunk)
|
|
return cls(bd, shrub)
|
|
|
|
def lookupnext_(self, bid, *,
|
|
path=False,
|
|
depth=None):
|
|
if not self or bid >= self.weight:
|
|
if path:
|
|
return None, None, None, None, []
|
|
else:
|
|
return None, None, None, None
|
|
|
|
rbyd = self.rbyd
|
|
rid = bid
|
|
depth_ = 1
|
|
path_ = []
|
|
|
|
while True:
|
|
# corrupt branch?
|
|
if not rbyd:
|
|
if path:
|
|
return bid, rbyd, rid, None, path_
|
|
else:
|
|
return bid, rbyd, rid, None
|
|
|
|
# first tag indicates the branch's weight
|
|
rid_, name_ = rbyd.lookupnext(rid)
|
|
if rid_ is None:
|
|
if path:
|
|
return None, None, None, None, path_
|
|
else:
|
|
return None, None, None, None
|
|
|
|
# keep track of path
|
|
if path:
|
|
path_.append((bid + (rid_-rid), rbyd, rid_, name_))
|
|
|
|
# find branch tag if there is one
|
|
branch_ = rbyd.lookup(rid_, TAG_BRANCH, 0x3)
|
|
|
|
# descend down branch?
|
|
if branch_ is not None and (
|
|
not depth or depth_ < depth):
|
|
block, trunk, cksum, _ = frombranch(branch_.data)
|
|
rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight,
|
|
cksum)
|
|
|
|
rid -= (rid_-(name_.weight-1))
|
|
depth_ += 1
|
|
|
|
else:
|
|
if path:
|
|
return bid + (rid_-rid), rbyd, rid_, name_, path_
|
|
else:
|
|
return bid + (rid_-rid), rbyd, rid_, name_
|
|
|
|
# the non-leaf variants discard the rbyd info, these can be a bit
|
|
# more convenient, but at a performance cost
|
|
def lookupnext(self, bid, *,
|
|
path=False,
|
|
depth=None):
|
|
# just discard the rbyd info
|
|
r = self.lookupnext_(bid,
|
|
path=path,
|
|
depth=depth)
|
|
if path:
|
|
bid, rbyd, rid, name, path_ = r
|
|
else:
|
|
bid, rbyd, rid, name = r
|
|
|
|
if path:
|
|
return bid, name, path_
|
|
else:
|
|
return bid, name
|
|
|
|
def lookup(self, bid, tag=None, mask=None, *,
|
|
path=False,
|
|
depth=None):
|
|
# lookup rbyd in btree
|
|
#
|
|
# note this function expects bid to be known, use lookupnext
|
|
# first if you don't care about the exact bid (or better yet,
|
|
# lookupnext_ and call lookup on the returned rbyd)
|
|
#
|
|
# this matches rbyd's lookup behavior, which needs a known rid
|
|
# to avoid a double lookup
|
|
r = self.lookupnext_(bid,
|
|
path=path,
|
|
depth=depth)
|
|
if path:
|
|
bid_, rbyd_, rid_, name_, path_ = r
|
|
else:
|
|
bid_, rbyd_, rid_, name_ = r
|
|
if bid_ is None or bid_ != bid:
|
|
if path:
|
|
return None, path_
|
|
else:
|
|
return None
|
|
|
|
# lookup tag in rbyd
|
|
rattr_ = rbyd_.lookup(rid_, tag, mask)
|
|
if rattr_ is None:
|
|
if path:
|
|
return None, path_
|
|
else:
|
|
return None
|
|
|
|
if path:
|
|
return rattr_, path_
|
|
else:
|
|
return rattr_
|
|
|
|
# note leaves only iterates over leaf rbyds, whereas traverse
|
|
# traverses all rbyds
|
|
def leaves(self, *,
|
|
path=False,
|
|
depth=None):
|
|
# include our root rbyd even if the weight is zero
|
|
if self.weight == 0:
|
|
if path:
|
|
yield -1, self.rbyd, []
|
|
else:
|
|
yield -1, self.rbyd
|
|
return
|
|
|
|
bid = 0
|
|
while True:
|
|
r = self.lookupnext_(bid,
|
|
path=path,
|
|
depth=depth)
|
|
if r:
|
|
bid, rbyd, rid, name, path_ = r
|
|
else:
|
|
bid, rbyd, rid, name = r
|
|
if bid is None:
|
|
break
|
|
|
|
if path:
|
|
yield (bid-rid + (rbyd.weight-1), rbyd,
|
|
# path tail is usually redundant unless corrupt
|
|
path_[:-1]
|
|
if path_ and path_[-1][1] == rbyd
|
|
else path_)
|
|
else:
|
|
yield bid-rid + (rbyd.weight-1), rbyd
|
|
bid += rbyd.weight - rid + 1
|
|
|
|
def traverse(self, *,
|
|
path=False,
|
|
depth=None):
|
|
ptrunk_ = []
|
|
for bid, rbyd, path_ in self.leaves(
|
|
path=True,
|
|
depth=depth):
|
|
# we only care about the rbyds here
|
|
trunk_ = ([(bid_-rid_ + (rbyd_.weight-1), rbyd_)
|
|
for bid_, rbyd_, rid_, name_ in path_]
|
|
+ [(bid, rbyd)])
|
|
for d, (bid_, rbyd_) in pathdelta(
|
|
trunk_, ptrunk_):
|
|
# but include branch rids in the path if requested
|
|
if path:
|
|
yield bid_, rbyd_, path_[:d]
|
|
else:
|
|
yield bid_, rbyd_
|
|
ptrunk_ = trunk_
|
|
|
|
# note bids/rattrs do _not_ include corrupt btree nodes!
|
|
def bids(self, *,
|
|
leaves=False,
|
|
path=False,
|
|
depth=None):
|
|
for r in self.leaves(
|
|
path=path,
|
|
depth=depth):
|
|
if path:
|
|
bid, rbyd, path_ = r
|
|
else:
|
|
bid, rbyd = r
|
|
for rid, name in rbyd.rids():
|
|
bid_ = bid-(rbyd.weight-1) + rid
|
|
if leaves:
|
|
if path:
|
|
yield (bid_, rbyd, rid, name,
|
|
path_+[(bid_, rbyd, rid, name)])
|
|
else:
|
|
yield bid_, rbyd, rid, name
|
|
else:
|
|
if path:
|
|
yield (bid_, name,
|
|
path_+[(bid_, rbyd, rid, name)])
|
|
else:
|
|
yield bid_, name
|
|
|
|
def rattrs(self, bid=None, tag=None, mask=None, *,
|
|
leaves=False,
|
|
path=False,
|
|
depth=None):
|
|
if bid is None:
|
|
for r in self.leaves(
|
|
path=path,
|
|
depth=depth):
|
|
if path:
|
|
bid, rbyd, path_ = r
|
|
else:
|
|
bid, rbyd = r
|
|
for rid, name in rbyd.rids():
|
|
bid_ = bid-(rbyd.weight-1) + rid
|
|
for rattr in rbyd.rattrs(rid):
|
|
if leaves:
|
|
if path:
|
|
yield (bid_, rbyd, rid, rattr,
|
|
path_+[(bid_, rbyd, rid, name)])
|
|
else:
|
|
yield bid_, rbyd, rid, rattr
|
|
else:
|
|
if path:
|
|
yield (bid_, rattr,
|
|
path_+[(bid_, rbyd, rid, name)])
|
|
else:
|
|
yield bid_, rattr
|
|
else:
|
|
r = self.lookupnext_(bid,
|
|
path=path,
|
|
depth=depth)
|
|
if path:
|
|
bid, rbyd, rid, name, path_ = r
|
|
else:
|
|
bid, rbyd, rid, name = r
|
|
if bid is None:
|
|
return
|
|
|
|
for rattr in rbyd.rattrs(rid, tag, mask):
|
|
if leaves:
|
|
if path:
|
|
yield rbyd, rid, rattr, path_
|
|
else:
|
|
yield rbyd, rid, rattr
|
|
else:
|
|
if path:
|
|
yield rattr, path_
|
|
else:
|
|
yield rattr
|
|
|
|
# lookup by name
|
|
def namelookup_(self, did, name, *,
|
|
path=False,
|
|
depth=None):
|
|
rbyd = self.rbyd
|
|
bid = 0
|
|
depth_ = 1
|
|
path_ = []
|
|
|
|
while True:
|
|
# corrupt branch?
|
|
if not rbyd:
|
|
bid_ = bid+(rbyd.weight-1)
|
|
if path:
|
|
return bid_, rbyd, rbyd.weight-1, None, path_
|
|
else:
|
|
return bid_, rbyd, rbyd.weight-1, None
|
|
|
|
rid_, name_ = rbyd.namelookup(did, name)
|
|
|
|
# keep track of path
|
|
if path:
|
|
path_.append((bid + rid_, rbyd, rid_, name_))
|
|
|
|
# find branch tag if there is one
|
|
branch_ = rbyd.lookup(rid_, TAG_BRANCH, 0x3)
|
|
|
|
# found another branch
|
|
if branch_ is not None and (
|
|
not depth or depth_ < depth):
|
|
block, trunk, cksum, _ = frombranch(branch_.data)
|
|
rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight,
|
|
cksum)
|
|
|
|
# update our bid
|
|
bid += rid_ - (name_.weight-1)
|
|
depth_ += 1
|
|
|
|
# found best match
|
|
else:
|
|
if path:
|
|
return bid + rid_, rbyd, rid_, name_, path_
|
|
else:
|
|
return bid + rid_, rbyd, rid_, name_
|
|
|
|
def namelookup(self, bid, *,
|
|
path=False,
|
|
depth=None):
|
|
# just discard the rbyd info
|
|
r = self.namelookup_(did, name,
|
|
path=path,
|
|
depth=depth)
|
|
if path:
|
|
bid, rbyd, rid, name, path_ = r
|
|
else:
|
|
bid, rbyd, rid, name = r
|
|
|
|
if path:
|
|
return bid, name, path_
|
|
else:
|
|
return bid, name
|
|
|
|
|
|
|
|
# tree renderer
|
|
class TreeArt:
|
|
# tree branches are an abstract thing for tree rendering
|
|
class Branch(co.namedtuple('Branch', ['a', 'b', 'z', 'color'])):
|
|
__slots__ = ()
|
|
def __new__(cls, a, b, z=0, color='b'):
|
|
# a and b are context specific
|
|
return super().__new__(cls, a, b, z, color)
|
|
|
|
def __repr__(self):
|
|
return '%s(%s, %s, %s, %s)' % (
|
|
self.__class__.__name__,
|
|
self.a,
|
|
self.b,
|
|
self.z,
|
|
self.color)
|
|
|
|
# don't include color in branch comparisons, or else our tree
|
|
# renderings can end up with inconsistent colors between runs
|
|
def __eq__(self, other):
|
|
return (self.a, self.b, self.z) == (other.a, other.b, other.z)
|
|
|
|
def __ne__(self, other):
|
|
return (self.a, self.b, self.z) != (other.a, other.b, other.z)
|
|
|
|
def __hash__(self):
|
|
return hash((self.a, self.b, self.z))
|
|
|
|
# also order by z first, which can be useful for reproducibly
|
|
# prioritizing branches when simplifying trees
|
|
def __lt__(self, other):
|
|
return (self.z, self.a, self.b) < (other.z, other.a, other.b)
|
|
|
|
def __le__(self, other):
|
|
return (self.z, self.a, self.b) <= (other.z, other.a, other.b)
|
|
|
|
def __gt__(self, other):
|
|
return (self.z, self.a, self.b) > (other.z, other.a, other.b)
|
|
|
|
def __ge__(self, other):
|
|
return (self.z, self.a, self.b) >= (other.z, other.a, other.b)
|
|
|
|
# apply a function to a/b while trying to avoid copies
|
|
def map(self, filter_, map_=None):
|
|
if map_ is None:
|
|
filter_, map_ = None, filter_
|
|
|
|
a = self.a
|
|
if filter_ is None or filter_(a):
|
|
a = map_(a)
|
|
|
|
b = self.b
|
|
if filter_ is None or filter_(b):
|
|
b = map_(b)
|
|
|
|
if a != self.a or b != self.b:
|
|
return self.__class__(
|
|
a if a != self.a else self.a,
|
|
b if b != self.b else self.b,
|
|
self.z,
|
|
self.color)
|
|
else:
|
|
return self
|
|
|
|
def __init__(self, tree):
|
|
self.tree = tree
|
|
self.depth = max((t.z+1 for t in tree), default=0)
|
|
if self.depth > 0:
|
|
self.width = 2*self.depth + 2
|
|
else:
|
|
self.width = 0
|
|
|
|
def __iter__(self):
|
|
return iter(self.tree)
|
|
|
|
def __bool__(self):
|
|
return bool(self.tree)
|
|
|
|
def __len__(self):
|
|
return len(self.tree)
|
|
|
|
# render an rbyd rbyd tree for debugging
|
|
@classmethod
|
|
def _fromrbydrtree(cls, rbyd, **args):
|
|
trunks = co.defaultdict(lambda: (-1, 0))
|
|
alts = co.defaultdict(lambda: {})
|
|
|
|
for rid, rattr, path in rbyd.rattrs(path=True):
|
|
# keep track of trunks/alts
|
|
trunks[rattr.toff] = (rid, rattr.tag)
|
|
|
|
for ralt in path:
|
|
if ralt.followed:
|
|
alts[ralt.toff] |= {'f': ralt.joff, 'c': ralt.color}
|
|
else:
|
|
alts[ralt.toff] |= {'nf': ralt.off, 'c': ralt.color}
|
|
|
|
if args.get('tree_rbyd_all'):
|
|
# treat unreachable alts as converging paths
|
|
for j_, alt in alts.items():
|
|
if 'f' not in alt:
|
|
alt['f'] = alt['nf']
|
|
elif 'nf' not in alt:
|
|
alt['nf'] = alt['f']
|
|
|
|
else:
|
|
# prune any alts with unreachable edges
|
|
pruned = {}
|
|
for j, alt in alts.items():
|
|
if 'f' not in alt:
|
|
pruned[j] = alt['nf']
|
|
elif 'nf' not in alt:
|
|
pruned[j] = alt['f']
|
|
for j in pruned.keys():
|
|
del alts[j]
|
|
|
|
for j, alt in alts.items():
|
|
while alt['f'] in pruned:
|
|
alt['f'] = pruned[alt['f']]
|
|
while alt['nf'] in pruned:
|
|
alt['nf'] = pruned[alt['nf']]
|
|
|
|
# find the trunk and depth of each alt
|
|
def rec_trunk(j):
|
|
if j not in alts:
|
|
return trunks[j]
|
|
else:
|
|
if 'nft' not in alts[j]:
|
|
alts[j]['nft'] = rec_trunk(alts[j]['nf'])
|
|
return alts[j]['nft']
|
|
|
|
for j in alts.keys():
|
|
rec_trunk(j)
|
|
for j, alt in alts.items():
|
|
if alt['f'] in alts:
|
|
alt['ft'] = alts[alt['f']]['nft']
|
|
else:
|
|
alt['ft'] = trunks[alt['f']]
|
|
|
|
def rec_height(j):
|
|
if j not in alts:
|
|
return 0
|
|
else:
|
|
if 'h' not in alts[j]:
|
|
alts[j]['h'] = max(
|
|
rec_height(alts[j]['f']),
|
|
rec_height(alts[j]['nf'])) + 1
|
|
return alts[j]['h']
|
|
|
|
for j in alts.keys():
|
|
rec_height(j)
|
|
|
|
t_depth = max((alt['h']+1 for alt in alts.values()), default=0)
|
|
|
|
# convert to more general tree representation
|
|
tree = set()
|
|
for j, alt in alts.items():
|
|
# note all non-trunk edges should be colored black
|
|
tree.add(cls.Branch(
|
|
alt['nft'],
|
|
alt['nft'],
|
|
t_depth-1 - alt['h'],
|
|
alt['c']))
|
|
if alt['ft'] != alt['nft']:
|
|
tree.add(cls.Branch(
|
|
alt['nft'],
|
|
alt['ft'],
|
|
t_depth-1 - alt['h'],
|
|
'b'))
|
|
|
|
return cls(tree)
|
|
|
|
# render an rbyd btree tree for debugging
|
|
@classmethod
|
|
def _fromrbydbtree(cls, rbyd, **args):
|
|
# for rbyds this is just a pointer to every rid
|
|
tree = set()
|
|
root = None
|
|
for rid, name in rbyd.rids():
|
|
b = (rid, name.tag)
|
|
if root is None:
|
|
root = b
|
|
tree.add(cls.Branch(root, b))
|
|
return cls(tree)
|
|
|
|
# render an rbyd tree for debugging
|
|
@classmethod
|
|
def fromrbyd(cls, rbyd, **args):
|
|
if args.get('tree_btree'):
|
|
return cls._fromrbydbtree(rbyd, **args)
|
|
else:
|
|
return cls._fromrbydrtree(rbyd, **args)
|
|
|
|
# render some nice ascii trees
|
|
def repr(self, x, color=False):
|
|
if self.depth == 0:
|
|
return ''
|
|
|
|
def branchrepr(tree, x, d, was):
|
|
for t in tree:
|
|
if t.z == d and t.b == x:
|
|
if any(t.z == d and t.a == x
|
|
for t in tree):
|
|
return '+-', t.color, t.color
|
|
elif any(t.z == d
|
|
and x > min(t.a, t.b)
|
|
and x < max(t.a, t.b)
|
|
for t in tree):
|
|
return '|-', t.color, t.color
|
|
elif t.a < t.b:
|
|
return '\'-', t.color, t.color
|
|
else:
|
|
return '.-', t.color, t.color
|
|
for t in tree:
|
|
if t.z == d and t.a == x:
|
|
return '+ ', t.color, None
|
|
for t in tree:
|
|
if (t.z == d
|
|
and x > min(t.a, t.b)
|
|
and x < max(t.a, t.b)):
|
|
return '| ', t.color, was
|
|
if was:
|
|
return '--', was, was
|
|
return ' ', None, None
|
|
|
|
trunk = []
|
|
was = None
|
|
for d in range(self.depth):
|
|
t, c, was = branchrepr(self.tree, x, d, was)
|
|
|
|
trunk.append('%s%s%s%s' % (
|
|
'\x1b[33m' if color and c == 'y'
|
|
else '\x1b[31m' if color and c == 'r'
|
|
else '\x1b[1;30m' if color and c == 'b'
|
|
else '',
|
|
t,
|
|
('>' if was else ' ') if d == self.depth-1 else '',
|
|
'\x1b[m' if color and c else ''))
|
|
|
|
return '%s ' % ''.join(trunk)
|
|
|
|
# some more renderers
|
|
|
|
# render a btree rbyd tree for debugging
|
|
@classmethod
|
|
def _treeartfrombtreertree(cls, btree, *,
|
|
depth=None,
|
|
inner=False,
|
|
**args):
|
|
# precompute rbyd trees so we know the max depth at each layer
|
|
# to nicely align trees
|
|
rtrees = {}
|
|
rdepths = {}
|
|
for bid, rbyd, path in btree.traverse(path=True, depth=depth):
|
|
if not rbyd:
|
|
continue
|
|
|
|
rtree = cls.fromrbyd(rbyd, **args)
|
|
rtrees[rbyd] = rtree
|
|
rdepths[len(path)] = max(rdepths.get(len(path), 0), rtree.depth)
|
|
|
|
# map rbyd branches into our btree space
|
|
tree = set()
|
|
for bid, rbyd, path in btree.traverse(path=True, depth=depth):
|
|
if not rbyd:
|
|
continue
|
|
|
|
# yes we can find new rbyds if disk is being mutated, just
|
|
# ignore these
|
|
if rbyd not in rtrees:
|
|
continue
|
|
|
|
rtree = rtrees[rbyd]
|
|
rz = max((t.z+1 for t in rtree), default=0)
|
|
d = sum(rdepths[d]+1 for d in range(len(path)))
|
|
|
|
# map into our btree space
|
|
for t in rtree:
|
|
# note we adjust our bid to be left-leaning, this allows
|
|
# a global order and makes tree rendering quite a bit easier
|
|
a_rid, a_tag = t.a
|
|
b_rid, b_tag = t.b
|
|
_, (_, a_w, _) = rbyd.lookupnext(a_rid)
|
|
_, (_, b_w, _) = rbyd.lookupnext(b_rid)
|
|
tree.add(cls.Branch(
|
|
(bid-(rbyd.weight-1)+a_rid-(a_w-1), len(path), a_tag),
|
|
(bid-(rbyd.weight-1)+b_rid-(b_w-1), len(path), b_tag),
|
|
d + rdepths[len(path)]-rz + t.z,
|
|
t.color))
|
|
|
|
# connect rbyd branches to rbyd roots
|
|
if path:
|
|
l_bid, l_rbyd, l_rid, l_name = path[-1]
|
|
l_branch = l_rbyd.lookup(l_rid, TAG_BRANCH, 0x3)
|
|
|
|
if rtree:
|
|
r_rid, r_tag = min(rtree, key=lambda t: t.z).a
|
|
_, (_, r_w, _) = rbyd.lookupnext(r_rid)
|
|
else:
|
|
r_rid, (r_tag, r_w, _) = rbyd.lookupnext(-1)
|
|
|
|
tree.add(cls.Branch(
|
|
(l_bid-(l_name.weight-1), len(path)-1, l_branch.tag),
|
|
(bid-(rbyd.weight-1)+r_rid-(r_w-1), len(path), r_tag),
|
|
d-1))
|
|
|
|
# remap branches to leaves if we aren't showing inner branches
|
|
if not inner:
|
|
# step through each btree layer backwards
|
|
b_depth = max((t.a[1]+1 for t in tree), default=0)
|
|
|
|
for d in reversed(range(b_depth-1)):
|
|
# find bid ranges at this level
|
|
bids = set()
|
|
for t in tree:
|
|
if t.b[1] == d:
|
|
bids.add(t.b[0])
|
|
bids = sorted(bids)
|
|
|
|
# find the best root for each bid range
|
|
roots = {}
|
|
for i in range(len(bids)):
|
|
for t in tree:
|
|
if (t.a[1] > d
|
|
and t.a[0] >= bids[i]
|
|
and (i == len(bids)-1 or t.a[0] < bids[i+1])
|
|
and (bids[i] not in roots
|
|
or t < roots[bids[i]])):
|
|
roots[bids[i]] = t
|
|
|
|
# remap branches to leaf-roots
|
|
tree = {t.map(
|
|
lambda x: x[1] == d and x[0] in roots,
|
|
lambda x: roots[x[0]].a)
|
|
for t in tree}
|
|
|
|
return cls(tree)
|
|
|
|
# render a btree btree tree for debugging
|
|
@classmethod
|
|
def _treeartfrombtreebtree(cls, btree, *,
|
|
depth=None,
|
|
inner=False,
|
|
**args):
|
|
# find all branches
|
|
tree = set()
|
|
root = None
|
|
branches = {}
|
|
for bid, name, path in btree.bids(
|
|
path=True,
|
|
depth=depth):
|
|
# create branch for each jump in path
|
|
#
|
|
# note we adjust our bid to be left-leaning, this allows
|
|
# a global order and makes tree rendering quite a bit easier
|
|
a = root
|
|
for d, (bid_, rbyd_, rid_, name_) in enumerate(path):
|
|
# map into our btree space
|
|
bid__ = bid_-(name_.weight-1)
|
|
b = (bid__, d, name_.tag)
|
|
|
|
# remap branches to leaves if we aren't showing inner
|
|
# branches
|
|
if not inner:
|
|
if b not in branches:
|
|
bid_, rbyd_, rid_, name_ = path[-1]
|
|
bid__ = bid_-(name_.weight-1)
|
|
branches[b] = (bid__, len(path)-1, name_.tag)
|
|
b = branches[b]
|
|
|
|
# render the root path on first rid, this is arbitrary
|
|
if root is None:
|
|
root, a = b, b
|
|
|
|
tree.add(cls.Branch(a, b, d))
|
|
a = b
|
|
|
|
return cls(tree)
|
|
|
|
# render a btree tree for debugging
|
|
@classmethod
|
|
def treeartfrombtree(cls, btree, **args):
|
|
if args.get('tree_btree'):
|
|
return cls._frombtreebtree(btree, **args)
|
|
else:
|
|
return cls._frombtreertree(btree, **args)
|
|
|
|
TreeArt._frombtreertree = _treeartfrombtreertree
|
|
TreeArt._frombtreebtree = _treeartfrombtreebtree
|
|
TreeArt.frombtree = treeartfrombtree
|
|
|
|
|
|
|
|
def main(disk, roots=None, *,
|
|
trunk=None,
|
|
block_size=None,
|
|
block_count=None,
|
|
quiet=False,
|
|
color='auto',
|
|
**args):
|
|
# figure out what color should be
|
|
if color == 'auto':
|
|
color = sys.stdout.isatty()
|
|
elif color == 'always':
|
|
color = True
|
|
else:
|
|
color = False
|
|
|
|
# is bd geometry specified?
|
|
if isinstance(block_size, tuple):
|
|
block_size, block_count_ = block_size
|
|
if block_count is None:
|
|
block_count = block_count_
|
|
|
|
# flatten roots, default to block 0
|
|
roots = list(it.chain.from_iterable(roots)) if roots else [0]
|
|
|
|
# roots may also encode trunks
|
|
roots, trunk = (
|
|
[block[0] if isinstance(block, tuple)
|
|
else block
|
|
for block in roots],
|
|
trunk if trunk is not None
|
|
else ft.reduce(
|
|
lambda x, y: y,
|
|
(block[1] for block in roots
|
|
if isinstance(block, tuple)),
|
|
None))
|
|
|
|
# we seek around a bunch, so just keep the disk open
|
|
with open(disk, 'rb') as f:
|
|
# if block_size is omitted, assume the block device is one big block
|
|
if block_size is None:
|
|
f.seek(0, os.SEEK_END)
|
|
block_size = f.tell()
|
|
|
|
# fetch the btree
|
|
bd = Bd(f, block_size, block_count)
|
|
btree = Btree.fetch(bd, roots, trunk)
|
|
|
|
# print some information about the btree
|
|
if not quiet:
|
|
print('btree %s w%d, rev %08x, cksum %08x' % (
|
|
btree.addr(),
|
|
btree.weight,
|
|
btree.rev,
|
|
btree.cksum))
|
|
|
|
# precompute tree renderings
|
|
t_width = 0
|
|
if (args.get('tree_rbyd')
|
|
or args.get('tree_rbyd_all')
|
|
or args.get('tree_btree')):
|
|
treeart = TreeArt.frombtree(btree, **args)
|
|
t_width = treeart.width
|
|
|
|
# dynamically size the id field
|
|
w_width = mt.ceil(mt.log10(max(1, btree.weight)+1))
|
|
|
|
# prbyd keeps track of the last rendered rbyd, we update
|
|
# in dbg_branch to always print interleaved addresses
|
|
prbyd = None
|
|
def dbg_branch(d, bid, rbyd, rid, name):
|
|
nonlocal prbyd
|
|
|
|
# show human-readable representation
|
|
for rattr in rbyd.rattrs(rid):
|
|
print('%10s %s%*s %-*s %s' % (
|
|
'%04x.%04x:' % (rbyd.block, rbyd.trunk)
|
|
if prbyd is None or rbyd != prbyd
|
|
else '',
|
|
treeart.repr((bid-(name.weight-1), d, rattr.tag), color)
|
|
if args.get('tree_rbyd')
|
|
or args.get('tree_rbyd_all')
|
|
or args.get('tree_btree')
|
|
else '',
|
|
2*w_width+1, '%d-%d' % (bid-(rattr.weight-1), bid)
|
|
if rattr.weight > 1
|
|
else bid if rattr.weight > 0
|
|
else '',
|
|
21+w_width, rattr.repr(),
|
|
next(xxd(rattr.data, 8), '')
|
|
if not args.get('raw')
|
|
and not args.get('no_truncate')
|
|
else ''))
|
|
prbyd = rbyd
|
|
|
|
# show on-disk encoding of tags/data
|
|
if args.get('raw'):
|
|
for o, line in enumerate(xxd(rattr.tdata)):
|
|
print('%9s: %*s%*s %s' % (
|
|
'%04x' % (rattr.toff + o*16),
|
|
t_width, '',
|
|
2*w_width+1, '',
|
|
line))
|
|
if args.get('raw') or args.get('no_truncate'):
|
|
for o, line in enumerate(xxd(rattr.data)):
|
|
print('%9s: %*s%*s %s' % (
|
|
'%04x' % (rattr.off + o*16),
|
|
t_width, '',
|
|
2*w_width+1, '',
|
|
line))
|
|
|
|
# traverse and print entries
|
|
ppath = []
|
|
corrupted = False
|
|
for bid, rbyd, path in btree.leaves(
|
|
path=True,
|
|
depth=args.get('depth')):
|
|
# print inner branches if requested
|
|
if args.get('inner') and not quiet:
|
|
for d, (bid_, rbyd_, rid_, name_) in pathdelta(
|
|
path, ppath):
|
|
dbg_branch(d, bid_, rbyd_, rid_, name_)
|
|
ppath = path
|
|
|
|
# corrupted? try to keep printing the tree
|
|
if not rbyd:
|
|
if not quiet:
|
|
print('%s%04x.%04x: %*s%s%s' % (
|
|
'\x1b[31m' if color else '',
|
|
rbyd.block, rbyd.trunk,
|
|
t_width, '',
|
|
'(corrupted rbyd %s)' % rbyd.addr(),
|
|
'\x1b[m' if color else ''))
|
|
prbyd = None
|
|
corrupted = True
|
|
continue
|
|
|
|
if not quiet:
|
|
for rid, name in rbyd.rids():
|
|
bid_ = bid-(rbyd.weight-1) + rid
|
|
# show the leaf entry/branch
|
|
dbg_branch(len(path), bid_, rbyd, rid, name)
|
|
|
|
if args.get('error_on_corrupt') and corrupted:
|
|
sys.exit(2)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import argparse
|
|
import sys
|
|
parser = argparse.ArgumentParser(
|
|
description="Debug rbyd B-trees.",
|
|
allow_abbrev=False)
|
|
parser.add_argument(
|
|
'disk',
|
|
help="File containing the block device.")
|
|
parser.add_argument(
|
|
'roots',
|
|
nargs='*',
|
|
type=rbydaddr,
|
|
help="Block address of the roots of the tree.")
|
|
parser.add_argument(
|
|
'--trunk',
|
|
type=lambda x: int(x, 0),
|
|
help="Use this offset as the trunk of the tree.")
|
|
parser.add_argument(
|
|
'-b', '--block-size',
|
|
type=bdgeom,
|
|
help="Block size/geometry in bytes. Accepts <size>x<count>.")
|
|
parser.add_argument(
|
|
'--block-count',
|
|
type=lambda x: int(x, 0),
|
|
help="Block count in blocks.")
|
|
parser.add_argument(
|
|
'-q', '--quiet',
|
|
action='store_true',
|
|
help="Don't show anything, useful when checking for errors.")
|
|
parser.add_argument(
|
|
'--color',
|
|
choices=['never', 'always', 'auto'],
|
|
default='auto',
|
|
help="When to use terminal colors. Defaults to 'auto'.")
|
|
parser.add_argument(
|
|
'-x', '--raw',
|
|
action='store_true',
|
|
help="Show the raw data including tag encodings.")
|
|
parser.add_argument(
|
|
'-T', '--no-truncate',
|
|
action='store_true',
|
|
help="Don't truncate, show the full contents.")
|
|
parser.add_argument(
|
|
'-R', '--tree', '--rbyd', '--tree-rbyd',
|
|
dest='tree_rbyd',
|
|
action='store_true',
|
|
help="Show the rbyd tree.")
|
|
parser.add_argument(
|
|
'-Y', '--rbyd-all', '--tree-rbyd-all',
|
|
dest='tree_rbyd_all',
|
|
action='store_true',
|
|
help="Show the full rbyd tree.")
|
|
parser.add_argument(
|
|
'-B', '--btree', '--tree-btree',
|
|
dest='tree_btree',
|
|
action='store_true',
|
|
help="Show a simplified btree tree.")
|
|
parser.add_argument(
|
|
'-i', '--inner',
|
|
action='store_true',
|
|
help="Show inner branches.")
|
|
parser.add_argument(
|
|
'-z', '--depth',
|
|
nargs='?',
|
|
type=lambda x: int(x, 0),
|
|
const=0,
|
|
help="Depth of the btree to show.")
|
|
parser.add_argument(
|
|
'-e', '--error-on-corrupt',
|
|
action='store_true',
|
|
help="Error if B-tree is corrupt.")
|
|
sys.exit(main(**{k: v
|
|
for k, v in vars(parser.parse_intermixed_args()).items()
|
|
if v is not None}))
|