diff --git a/scripts/dbgbmap.py b/scripts/dbgbmap.py index d538647e..6bc07319 100755 --- a/scripts/dbgbmap.py +++ b/scripts/dbgbmap.py @@ -200,61 +200,64 @@ def popc(x): def parity(x): return popc(x) & 1 -def fromle32(data): - return struct.unpack('H', data[:2])[0] - weight, d = fromleb128(data[2:]) - size, d_ = fromleb128(data[2+d:]) - return tag>>15, tag&0x7fff, weight, size, 2+d+d_ +def fromtag(data, j=0): + d = 0 + tag = struct.unpack('>H', data[j:j+2].ljust(2, b'\0'))[0]; d += 2 + weight, d_ = fromleb128(data, j+d); d += d_ + size, d_ = fromleb128(data, j+d); d += d_ + return tag>>15, tag&0x7fff, weight, size, d -def frommdir(data): +def frombranch(data, j=0): + d = 0 + block, d_ = fromleb128(data, j+d); d += d_ + trunk, d_ = fromleb128(data, j+d); d += d_ + cksum = fromle32(data, j+d); d += 4 + return block, trunk, cksum, d + +def frombtree(data, j=0): + d = 0 + w, d_ = fromleb128(data, j+d); d += d_ + block, trunk, cksum, d_ = frombranch(data, j+d); d += d_ + return w, block, trunk, cksum, d + +def frommdir(data, j=0): blocks = [] d = 0 - while d < len(data): - block, d_ = fromleb128(data[d:]) + while j+d < len(data): + block, d_ = fromleb128(data, j+d) blocks.append(block) d += d_ - return blocks + return tuple(blocks), d -def fromshrub(data): +def fromshrub(data, j=0): d = 0 - weight, d_ = fromleb128(data[d:]); d += d_ - trunk, d_ = fromleb128(data[d:]); d += d_ - return weight, trunk + weight, d_ = fromleb128(data, j+d); d += d_ + trunk, d_ = fromleb128(data, j+d); d += d_ + return weight, trunk, d -def frombranch(data): +def frombptr(data, j=0): d = 0 - block, d_ = fromleb128(data[d:]); d += d_ - trunk, d_ = fromleb128(data[d:]); d += d_ - cksum = fromle32(data[d:]); d += 4 - return block, trunk, cksum - -def frombtree(data): - d = 0 - w, d_ = fromleb128(data[d:]); d += d_ - block, trunk, cksum = frombranch(data[d:]) - return w, block, trunk, cksum - -def frombptr(data): - d = 0 - size, d_ = fromleb128(data[d:]); d += d_ - block, d_ = fromleb128(data[d:]); d += d_ - off, d_ = fromleb128(data[d:]); d += d_ - cksize, d_ = fromleb128(data[d:]); d += d_ - cksum = fromle32(data[d:]); d += 4 - return size, block, off, cksize, cksum + size, d_ = fromleb128(data, j+d); d += d_ + block, d_ = fromleb128(data, j+d); d += d_ + off, d_ = fromleb128(data, j+d); d += d_ + cksize, d_ = fromleb128(data, j+d); d += d_ + cksum = fromle32(data, j+d); d += 4 + return size, block, off, cksize, cksum, d def xxd(data, width=16): for i in range(0, len(data), width): @@ -626,7 +629,7 @@ class Rbyd: @classmethod def _fetch(cls, data, block, trunk=None): # fetch the rbyd - rev = fromle32(data[0:4]) + rev = fromle32(data, 0) cksum = 0 cksum_ = crc32c(data[0:4]) cksum__ = cksum_ @@ -644,7 +647,7 @@ class Rbyd: gcksumdelta_ = None while j_ < len(data) and (not trunk or eoff <= trunk): # read next tag - v, tag, w, size, d = fromtag(data[j_:]) + v, tag, w, size, d = fromtag(data, j_) if v != parity(cksum__): break cksum__ ^= 0x00000080 if v else 0 @@ -667,7 +670,7 @@ class Rbyd: # found a cksum? else: # check cksum - cksum___ = fromle32(data[j_:j_+4]) + cksum___ = fromle32(data, j_) if cksum__ != cksum___: break # commit what we have @@ -806,7 +809,7 @@ class Rbyd: # descend down tree j = self.trunk while True: - _, alt, w, jump, d = fromtag(self.data[j:]) + _, alt, w, jump, d = fromtag(self.data, j) # found an alt? if alt & TAG_ALT: @@ -822,7 +825,7 @@ class Rbyd: if path: # figure out which color if alt & TAG_R: - _, nalt, _, _, _ = fromtag(self.data[j+jump+d:]) + _, nalt, _, _, _ = fromtag(self.data, j+jump+d) if nalt & TAG_R: color = 'y' else: @@ -845,7 +848,7 @@ class Rbyd: if path: # figure out which color if alt & TAG_R: - _, nalt, _, _, _ = fromtag(self.data[j:]) + _, nalt, _, _, _ = fromtag(self.data, j) if nalt & TAG_R: color = 'y' else: @@ -1106,7 +1109,7 @@ class Btree: # descend down branch? if branch_ is not None and ( not depth or depth_ < depth): - block, trunk, cksum = frombranch(branch_.data) + block, trunk, cksum, _ = frombranch(branch_.data) rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, cksum) @@ -1337,7 +1340,7 @@ class Btree: # found another branch if branch_ is not None and ( not depth or depth_ < depth): - block, trunk, cksum = frombranch(branch_.data) + block, trunk, cksum, _ = frombranch(branch_.data) rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, cksum) @@ -1731,7 +1734,7 @@ class Mtree: rattr_ = mroot.lookup(-1, TAG_MROOT, 0x3) if rattr_ is None: break - blocks_ = frommdir(rattr_.data) + blocks_, _ = frommdir(rattr_.data) mroot = Mdir.fetch(bd, -1, blocks_) mrootchain.append(mroot) @@ -1740,7 +1743,7 @@ class Mtree: if not depth or len(mrootchain) < depth: rattr_ = mroot.lookup(-1, TAG_MTREE, 0x3) if rattr_ is not None: - w_, block_, trunk_, cksum_ = frombtree(rattr_.data) + w_, block_, trunk_, cksum_, _ = frombtree(rattr_.data) mtree = Btree.fetchck(bd, block_, trunk_, w_, cksum_) return cls(bd, mrootchain, mtree, @@ -1820,7 +1823,7 @@ class Mtree: return (bid_, rbyd_, rid_), path_ else: return (bid_, rbyd_, rid_) - blocks_ = frommdir(rattr_.data) + blocks_, _ = frommdir(rattr_.data) mdir = Mdir.fetch(self.bd, mid, blocks_) if path: return mdir, path_ @@ -2194,7 +2197,7 @@ class Mtree: return (bid_, rbyd_, rid_), path_ else: return (bid_, rbyd_, rid_) - blocks_ = frommdir(rattr_.data) + blocks_, _ = frommdir(rattr_.data) mdir = Mdir.fetch(self.bd, self.mid(bid_), blocks_) if path: return mdir, path_ @@ -2458,8 +2461,8 @@ class Config: def __init__(self, mroot, tag, rattr): super().__init__(mroot, tag, rattr) d = 0 - self.major, d_ = fromleb128(self.data[d:]); d += d_ - self.minor, d_ = fromleb128(self.data[d:]); d += d_ + self.major, d_ = fromleb128(self.data, d); d += d_ + self.minor, d_ = fromleb128(self.data, d); d += d_ @property def tuple(self): @@ -2498,8 +2501,8 @@ class Config: def __init__(self, mroot, tag, rattr): super().__init__(mroot, tag, rattr) d = 0 - block_size, d_ = fromleb128(self.data[d:]); d += d_ - block_count, d_ = fromleb128(self.data[d:]); d += d_ + block_size, d_ = fromleb128(self.data, d); d += d_ + block_count, d_ = fromleb128(self.data, d); d += d_ # these are offset by 1 to avoid overflow issues self.block_size = block_size + 1 self.block_count = block_count + 1 @@ -2703,11 +2706,11 @@ class Gstate: def __init__(self, mtree, tag, gdeltas): super().__init__(mtree, tag, gdeltas) d = 0 - count, d_ = fromleb128(self.data[d:]); d += d_ + count, d_ = fromleb128(self.data, d); d += d_ rms = [] if count <= 2: for _ in range(count): - mid, d_ = fromleb128(self.data[d:]); d += d_ + mid, d_ = fromleb128(self.data, d); d += d_ rms.append(mtree.mid(mid)) self.count = count self.rms = rms @@ -3242,11 +3245,11 @@ class Lfs: self.bshrub = None if (self.struct is not None and (self.struct.tag & ~0x3) == TAG_BSHRUB): - weight, trunk = fromshrub(self.struct.data) + weight, trunk, _ = fromshrub(self.struct.data) self.bshrub = Btree.fetchshrub(lfs.bd, mdir.rbyd, trunk) elif (self.struct is not None and (self.struct.tag & ~0x3) == TAG_BTREE): - weight, block, trunk, cksum = frombtree(self.struct.data) + weight, block, trunk, cksum, _ = frombtree(self.struct.data) self.bshrub = Btree.fetchck( lfs.bd, block, trunk, weight, cksum) @@ -3366,7 +3369,7 @@ class Lfs: return bid-(rattr.weight-1), rattr # block pointer? elif (rattr.tag & ~0x1003) == TAG_BLOCK: - size, block, off, cksize, cksum = frombptr(rattr.data) + size, block, off, cksize, cksum, _ = frombptr(rattr.data) bptr = Bptr.fetchck(self.lfs.bd, rattr, block, off, size, cksize, cksum) if path: diff --git a/scripts/dbgbmapd3.py b/scripts/dbgbmapd3.py index 15944b6a..e505d994 100755 --- a/scripts/dbgbmapd3.py +++ b/scripts/dbgbmapd3.py @@ -230,61 +230,64 @@ def popc(x): def parity(x): return popc(x) & 1 -def fromle32(data): - return struct.unpack('H', data[:2])[0] - weight, d = fromleb128(data[2:]) - size, d_ = fromleb128(data[2+d:]) - return tag>>15, tag&0x7fff, weight, size, 2+d+d_ +def fromtag(data, j=0): + d = 0 + tag = struct.unpack('>H', data[j:j+2].ljust(2, b'\0'))[0]; d += 2 + weight, d_ = fromleb128(data, j+d); d += d_ + size, d_ = fromleb128(data, j+d); d += d_ + return tag>>15, tag&0x7fff, weight, size, d -def frommdir(data): +def frombranch(data, j=0): + d = 0 + block, d_ = fromleb128(data, j+d); d += d_ + trunk, d_ = fromleb128(data, j+d); d += d_ + cksum = fromle32(data, j+d); d += 4 + return block, trunk, cksum, d + +def frombtree(data, j=0): + d = 0 + w, d_ = fromleb128(data, j+d); d += d_ + block, trunk, cksum, d_ = frombranch(data, j+d); d += d_ + return w, block, trunk, cksum, d + +def frommdir(data, j=0): blocks = [] d = 0 - while d < len(data): - block, d_ = fromleb128(data[d:]) + while j+d < len(data): + block, d_ = fromleb128(data, j+d) blocks.append(block) d += d_ - return blocks + return tuple(blocks), d -def fromshrub(data): +def fromshrub(data, j=0): d = 0 - weight, d_ = fromleb128(data[d:]); d += d_ - trunk, d_ = fromleb128(data[d:]); d += d_ - return weight, trunk + weight, d_ = fromleb128(data, j+d); d += d_ + trunk, d_ = fromleb128(data, j+d); d += d_ + return weight, trunk, d -def frombranch(data): +def frombptr(data, j=0): d = 0 - block, d_ = fromleb128(data[d:]); d += d_ - trunk, d_ = fromleb128(data[d:]); d += d_ - cksum = fromle32(data[d:]); d += 4 - return block, trunk, cksum - -def frombtree(data): - d = 0 - w, d_ = fromleb128(data[d:]); d += d_ - block, trunk, cksum = frombranch(data[d:]) - return w, block, trunk, cksum - -def frombptr(data): - d = 0 - size, d_ = fromleb128(data[d:]); d += d_ - block, d_ = fromleb128(data[d:]); d += d_ - off, d_ = fromleb128(data[d:]); d += d_ - cksize, d_ = fromleb128(data[d:]); d += d_ - cksum = fromle32(data[d:]); d += 4 - return size, block, off, cksize, cksum + size, d_ = fromleb128(data, j+d); d += d_ + block, d_ = fromleb128(data, j+d); d += d_ + off, d_ = fromleb128(data, j+d); d += d_ + cksize, d_ = fromleb128(data, j+d); d += d_ + cksum = fromle32(data, j+d); d += 4 + return size, block, off, cksize, cksum, d def xxd(data, width=16): for i in range(0, len(data), width): @@ -656,7 +659,7 @@ class Rbyd: @classmethod def _fetch(cls, data, block, trunk=None): # fetch the rbyd - rev = fromle32(data[0:4]) + rev = fromle32(data, 0) cksum = 0 cksum_ = crc32c(data[0:4]) cksum__ = cksum_ @@ -674,7 +677,7 @@ class Rbyd: gcksumdelta_ = None while j_ < len(data) and (not trunk or eoff <= trunk): # read next tag - v, tag, w, size, d = fromtag(data[j_:]) + v, tag, w, size, d = fromtag(data, j_) if v != parity(cksum__): break cksum__ ^= 0x00000080 if v else 0 @@ -697,7 +700,7 @@ class Rbyd: # found a cksum? else: # check cksum - cksum___ = fromle32(data[j_:j_+4]) + cksum___ = fromle32(data, j_) if cksum__ != cksum___: break # commit what we have @@ -836,7 +839,7 @@ class Rbyd: # descend down tree j = self.trunk while True: - _, alt, w, jump, d = fromtag(self.data[j:]) + _, alt, w, jump, d = fromtag(self.data, j) # found an alt? if alt & TAG_ALT: @@ -852,7 +855,7 @@ class Rbyd: if path: # figure out which color if alt & TAG_R: - _, nalt, _, _, _ = fromtag(self.data[j+jump+d:]) + _, nalt, _, _, _ = fromtag(self.data, j+jump+d) if nalt & TAG_R: color = 'y' else: @@ -875,7 +878,7 @@ class Rbyd: if path: # figure out which color if alt & TAG_R: - _, nalt, _, _, _ = fromtag(self.data[j:]) + _, nalt, _, _, _ = fromtag(self.data, j) if nalt & TAG_R: color = 'y' else: @@ -1136,7 +1139,7 @@ class Btree: # descend down branch? if branch_ is not None and ( not depth or depth_ < depth): - block, trunk, cksum = frombranch(branch_.data) + block, trunk, cksum, _ = frombranch(branch_.data) rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, cksum) @@ -1367,7 +1370,7 @@ class Btree: # found another branch if branch_ is not None and ( not depth or depth_ < depth): - block, trunk, cksum = frombranch(branch_.data) + block, trunk, cksum, _ = frombranch(branch_.data) rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, cksum) @@ -1761,7 +1764,7 @@ class Mtree: rattr_ = mroot.lookup(-1, TAG_MROOT, 0x3) if rattr_ is None: break - blocks_ = frommdir(rattr_.data) + blocks_, _ = frommdir(rattr_.data) mroot = Mdir.fetch(bd, -1, blocks_) mrootchain.append(mroot) @@ -1770,7 +1773,7 @@ class Mtree: if not depth or len(mrootchain) < depth: rattr_ = mroot.lookup(-1, TAG_MTREE, 0x3) if rattr_ is not None: - w_, block_, trunk_, cksum_ = frombtree(rattr_.data) + w_, block_, trunk_, cksum_, _ = frombtree(rattr_.data) mtree = Btree.fetchck(bd, block_, trunk_, w_, cksum_) return cls(bd, mrootchain, mtree, @@ -1850,7 +1853,7 @@ class Mtree: return (bid_, rbyd_, rid_), path_ else: return (bid_, rbyd_, rid_) - blocks_ = frommdir(rattr_.data) + blocks_, _ = frommdir(rattr_.data) mdir = Mdir.fetch(self.bd, mid, blocks_) if path: return mdir, path_ @@ -2224,7 +2227,7 @@ class Mtree: return (bid_, rbyd_, rid_), path_ else: return (bid_, rbyd_, rid_) - blocks_ = frommdir(rattr_.data) + blocks_, _ = frommdir(rattr_.data) mdir = Mdir.fetch(self.bd, self.mid(bid_), blocks_) if path: return mdir, path_ @@ -2488,8 +2491,8 @@ class Config: def __init__(self, mroot, tag, rattr): super().__init__(mroot, tag, rattr) d = 0 - self.major, d_ = fromleb128(self.data[d:]); d += d_ - self.minor, d_ = fromleb128(self.data[d:]); d += d_ + self.major, d_ = fromleb128(self.data, d); d += d_ + self.minor, d_ = fromleb128(self.data, d); d += d_ @property def tuple(self): @@ -2528,8 +2531,8 @@ class Config: def __init__(self, mroot, tag, rattr): super().__init__(mroot, tag, rattr) d = 0 - block_size, d_ = fromleb128(self.data[d:]); d += d_ - block_count, d_ = fromleb128(self.data[d:]); d += d_ + block_size, d_ = fromleb128(self.data, d); d += d_ + block_count, d_ = fromleb128(self.data, d); d += d_ # these are offset by 1 to avoid overflow issues self.block_size = block_size + 1 self.block_count = block_count + 1 @@ -2733,11 +2736,11 @@ class Gstate: def __init__(self, mtree, tag, gdeltas): super().__init__(mtree, tag, gdeltas) d = 0 - count, d_ = fromleb128(self.data[d:]); d += d_ + count, d_ = fromleb128(self.data, d); d += d_ rms = [] if count <= 2: for _ in range(count): - mid, d_ = fromleb128(self.data[d:]); d += d_ + mid, d_ = fromleb128(self.data, d); d += d_ rms.append(mtree.mid(mid)) self.count = count self.rms = rms @@ -3272,11 +3275,11 @@ class Lfs: self.bshrub = None if (self.struct is not None and (self.struct.tag & ~0x3) == TAG_BSHRUB): - weight, trunk = fromshrub(self.struct.data) + weight, trunk, _ = fromshrub(self.struct.data) self.bshrub = Btree.fetchshrub(lfs.bd, mdir.rbyd, trunk) elif (self.struct is not None and (self.struct.tag & ~0x3) == TAG_BTREE): - weight, block, trunk, cksum = frombtree(self.struct.data) + weight, block, trunk, cksum, _ = frombtree(self.struct.data) self.bshrub = Btree.fetchck( lfs.bd, block, trunk, weight, cksum) @@ -3396,7 +3399,7 @@ class Lfs: return bid-(rattr.weight-1), rattr # block pointer? elif (rattr.tag & ~0x1003) == TAG_BLOCK: - size, block, off, cksize, cksum = frombptr(rattr.data) + size, block, off, cksize, cksum, _ = frombptr(rattr.data) bptr = Bptr.fetchck(self.lfs.bd, rattr, block, off, size, cksize, cksum) if path: diff --git a/scripts/dbgbtree.py b/scripts/dbgbtree.py index 6606f58e..3854be01 100755 --- a/scripts/dbgbtree.py +++ b/scripts/dbgbtree.py @@ -138,31 +138,34 @@ def popc(x): def parity(x): return popc(x) & 1 -def fromle32(data): - return struct.unpack('H', data[:2])[0] - weight, d = fromleb128(data[2:]) - size, d_ = fromleb128(data[2+d:]) - return tag>>15, tag&0x7fff, weight, size, 2+d+d_ - -def frombranch(data): +def fromtag(data, j=0): d = 0 - block, d_ = fromleb128(data[d:]); d += d_ - trunk, d_ = fromleb128(data[d:]); d += d_ - cksum = fromle32(data[d:]); d += 4 - return block, trunk, cksum + tag = struct.unpack('>H', data[j:j+2].ljust(2, b'\0'))[0]; d += 2 + weight, d_ = fromleb128(data, j+d); d += d_ + size, d_ = fromleb128(data, j+d); d += d_ + return tag>>15, tag&0x7fff, weight, size, d + +def frombranch(data, j=0): + d = 0 + block, d_ = fromleb128(data, j+d); d += d_ + trunk, d_ = fromleb128(data, j+d); d += d_ + cksum = fromle32(data, j+d); d += 4 + return block, trunk, cksum, d def xxd(data, width=16): for i in range(0, len(data), width): @@ -534,7 +537,7 @@ class Rbyd: @classmethod def _fetch(cls, data, block, trunk=None): # fetch the rbyd - rev = fromle32(data[0:4]) + rev = fromle32(data, 0) cksum = 0 cksum_ = crc32c(data[0:4]) cksum__ = cksum_ @@ -552,7 +555,7 @@ class Rbyd: gcksumdelta_ = None while j_ < len(data) and (not trunk or eoff <= trunk): # read next tag - v, tag, w, size, d = fromtag(data[j_:]) + v, tag, w, size, d = fromtag(data, j_) if v != parity(cksum__): break cksum__ ^= 0x00000080 if v else 0 @@ -575,7 +578,7 @@ class Rbyd: # found a cksum? else: # check cksum - cksum___ = fromle32(data[j_:j_+4]) + cksum___ = fromle32(data, j_) if cksum__ != cksum___: break # commit what we have @@ -714,7 +717,7 @@ class Rbyd: # descend down tree j = self.trunk while True: - _, alt, w, jump, d = fromtag(self.data[j:]) + _, alt, w, jump, d = fromtag(self.data, j) # found an alt? if alt & TAG_ALT: @@ -730,7 +733,7 @@ class Rbyd: if path: # figure out which color if alt & TAG_R: - _, nalt, _, _, _ = fromtag(self.data[j+jump+d:]) + _, nalt, _, _, _ = fromtag(self.data, j+jump+d) if nalt & TAG_R: color = 'y' else: @@ -753,7 +756,7 @@ class Rbyd: if path: # figure out which color if alt & TAG_R: - _, nalt, _, _, _ = fromtag(self.data[j:]) + _, nalt, _, _, _ = fromtag(self.data, j) if nalt & TAG_R: color = 'y' else: @@ -1014,7 +1017,7 @@ class Btree: # descend down branch? if branch_ is not None and ( not depth or depth_ < depth): - block, trunk, cksum = frombranch(branch_.data) + block, trunk, cksum, _ = frombranch(branch_.data) rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, cksum) @@ -1245,7 +1248,7 @@ class Btree: # found another branch if branch_ is not None and ( not depth or depth_ < depth): - block, trunk, cksum = frombranch(branch_.data) + block, trunk, cksum, _ = frombranch(branch_.data) rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, cksum) diff --git a/scripts/dbgle32.py b/scripts/dbgle32.py index 16e16c04..6c2a5d6c 100755 --- a/scripts/dbgle32.py +++ b/scripts/dbgle32.py @@ -21,14 +21,14 @@ def openio(path, mode='r', buffering=-1): else: return open(path, mode, buffering) -def fromle32(data): - return struct.unpack('H', data[:2])[0] - weight, d = fromleb128(data[2:]) - size, d_ = fromleb128(data[2+d:]) - return tag>>15, tag&0x7fff, weight, size, 2+d+d_ +def fromtag(data, j=0): + d = 0 + tag = struct.unpack('>H', data[j:j+2].ljust(2, b'\0'))[0]; d += 2 + weight, d_ = fromleb128(data, j+d); d += d_ + size, d_ = fromleb128(data, j+d); d += d_ + return tag>>15, tag&0x7fff, weight, size, d -def frommdir(data): +def frombranch(data, j=0): + d = 0 + block, d_ = fromleb128(data, j+d); d += d_ + trunk, d_ = fromleb128(data, j+d); d += d_ + cksum = fromle32(data, j+d); d += 4 + return block, trunk, cksum, d + +def frombtree(data, j=0): + d = 0 + w, d_ = fromleb128(data, j+d); d += d_ + block, trunk, cksum, d_ = frombranch(data, j+d); d += d_ + return w, block, trunk, cksum, d + +def frommdir(data, j=0): blocks = [] d = 0 - while d < len(data): - block, d_ = fromleb128(data[d:]) + while j+d < len(data): + block, d_ = fromleb128(data, j+d) blocks.append(block) d += d_ - return blocks + return tuple(blocks), d -def fromshrub(data): +def fromshrub(data, j=0): d = 0 - weight, d_ = fromleb128(data[d:]); d += d_ - trunk, d_ = fromleb128(data[d:]); d += d_ - return weight, trunk + weight, d_ = fromleb128(data, j+d); d += d_ + trunk, d_ = fromleb128(data, j+d); d += d_ + return weight, trunk, d -def frombranch(data): +def frombptr(data, j=0): d = 0 - block, d_ = fromleb128(data[d:]); d += d_ - trunk, d_ = fromleb128(data[d:]); d += d_ - cksum = fromle32(data[d:]); d += 4 - return block, trunk, cksum - -def frombtree(data): - d = 0 - w, d_ = fromleb128(data[d:]); d += d_ - block, trunk, cksum = frombranch(data[d:]) - return w, block, trunk, cksum - -def frombptr(data): - d = 0 - size, d_ = fromleb128(data[d:]); d += d_ - block, d_ = fromleb128(data[d:]); d += d_ - off, d_ = fromleb128(data[d:]); d += d_ - cksize, d_ = fromleb128(data[d:]); d += d_ - cksum = fromle32(data[d:]); d += 4 - return size, block, off, cksize, cksum + size, d_ = fromleb128(data, j+d); d += d_ + block, d_ = fromleb128(data, j+d); d += d_ + off, d_ = fromleb128(data, j+d); d += d_ + cksize, d_ = fromleb128(data, j+d); d += d_ + cksum = fromle32(data, j+d); d += 4 + return size, block, off, cksize, cksum, d def xxd(data, width=16): for i in range(0, len(data), width): @@ -583,7 +586,7 @@ class Rbyd: @classmethod def _fetch(cls, data, block, trunk=None): # fetch the rbyd - rev = fromle32(data[0:4]) + rev = fromle32(data, 0) cksum = 0 cksum_ = crc32c(data[0:4]) cksum__ = cksum_ @@ -601,7 +604,7 @@ class Rbyd: gcksumdelta_ = None while j_ < len(data) and (not trunk or eoff <= trunk): # read next tag - v, tag, w, size, d = fromtag(data[j_:]) + v, tag, w, size, d = fromtag(data, j_) if v != parity(cksum__): break cksum__ ^= 0x00000080 if v else 0 @@ -624,7 +627,7 @@ class Rbyd: # found a cksum? else: # check cksum - cksum___ = fromle32(data[j_:j_+4]) + cksum___ = fromle32(data, j_) if cksum__ != cksum___: break # commit what we have @@ -763,7 +766,7 @@ class Rbyd: # descend down tree j = self.trunk while True: - _, alt, w, jump, d = fromtag(self.data[j:]) + _, alt, w, jump, d = fromtag(self.data, j) # found an alt? if alt & TAG_ALT: @@ -779,7 +782,7 @@ class Rbyd: if path: # figure out which color if alt & TAG_R: - _, nalt, _, _, _ = fromtag(self.data[j+jump+d:]) + _, nalt, _, _, _ = fromtag(self.data, j+jump+d) if nalt & TAG_R: color = 'y' else: @@ -802,7 +805,7 @@ class Rbyd: if path: # figure out which color if alt & TAG_R: - _, nalt, _, _, _ = fromtag(self.data[j:]) + _, nalt, _, _, _ = fromtag(self.data, j) if nalt & TAG_R: color = 'y' else: @@ -1063,7 +1066,7 @@ class Btree: # descend down branch? if branch_ is not None and ( not depth or depth_ < depth): - block, trunk, cksum = frombranch(branch_.data) + block, trunk, cksum, _ = frombranch(branch_.data) rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, cksum) @@ -1294,7 +1297,7 @@ class Btree: # found another branch if branch_ is not None and ( not depth or depth_ < depth): - block, trunk, cksum = frombranch(branch_.data) + block, trunk, cksum, _ = frombranch(branch_.data) rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, cksum) @@ -1688,7 +1691,7 @@ class Mtree: rattr_ = mroot.lookup(-1, TAG_MROOT, 0x3) if rattr_ is None: break - blocks_ = frommdir(rattr_.data) + blocks_, _ = frommdir(rattr_.data) mroot = Mdir.fetch(bd, -1, blocks_) mrootchain.append(mroot) @@ -1697,7 +1700,7 @@ class Mtree: if not depth or len(mrootchain) < depth: rattr_ = mroot.lookup(-1, TAG_MTREE, 0x3) if rattr_ is not None: - w_, block_, trunk_, cksum_ = frombtree(rattr_.data) + w_, block_, trunk_, cksum_, _ = frombtree(rattr_.data) mtree = Btree.fetchck(bd, block_, trunk_, w_, cksum_) return cls(bd, mrootchain, mtree, @@ -1777,7 +1780,7 @@ class Mtree: return (bid_, rbyd_, rid_), path_ else: return (bid_, rbyd_, rid_) - blocks_ = frommdir(rattr_.data) + blocks_, _ = frommdir(rattr_.data) mdir = Mdir.fetch(self.bd, mid, blocks_) if path: return mdir, path_ @@ -2151,7 +2154,7 @@ class Mtree: return (bid_, rbyd_, rid_), path_ else: return (bid_, rbyd_, rid_) - blocks_ = frommdir(rattr_.data) + blocks_, _ = frommdir(rattr_.data) mdir = Mdir.fetch(self.bd, self.mid(bid_), blocks_) if path: return mdir, path_ @@ -2415,8 +2418,8 @@ class Config: def __init__(self, mroot, tag, rattr): super().__init__(mroot, tag, rattr) d = 0 - self.major, d_ = fromleb128(self.data[d:]); d += d_ - self.minor, d_ = fromleb128(self.data[d:]); d += d_ + self.major, d_ = fromleb128(self.data, d); d += d_ + self.minor, d_ = fromleb128(self.data, d); d += d_ @property def tuple(self): @@ -2455,8 +2458,8 @@ class Config: def __init__(self, mroot, tag, rattr): super().__init__(mroot, tag, rattr) d = 0 - block_size, d_ = fromleb128(self.data[d:]); d += d_ - block_count, d_ = fromleb128(self.data[d:]); d += d_ + block_size, d_ = fromleb128(self.data, d); d += d_ + block_count, d_ = fromleb128(self.data, d); d += d_ # these are offset by 1 to avoid overflow issues self.block_size = block_size + 1 self.block_count = block_count + 1 @@ -2660,11 +2663,11 @@ class Gstate: def __init__(self, mtree, tag, gdeltas): super().__init__(mtree, tag, gdeltas) d = 0 - count, d_ = fromleb128(self.data[d:]); d += d_ + count, d_ = fromleb128(self.data, d); d += d_ rms = [] if count <= 2: for _ in range(count): - mid, d_ = fromleb128(self.data[d:]); d += d_ + mid, d_ = fromleb128(self.data, d); d += d_ rms.append(mtree.mid(mid)) self.count = count self.rms = rms @@ -3199,11 +3202,11 @@ class Lfs: self.bshrub = None if (self.struct is not None and (self.struct.tag & ~0x3) == TAG_BSHRUB): - weight, trunk = fromshrub(self.struct.data) + weight, trunk, _ = fromshrub(self.struct.data) self.bshrub = Btree.fetchshrub(lfs.bd, mdir.rbyd, trunk) elif (self.struct is not None and (self.struct.tag & ~0x3) == TAG_BTREE): - weight, block, trunk, cksum = frombtree(self.struct.data) + weight, block, trunk, cksum, _ = frombtree(self.struct.data) self.bshrub = Btree.fetchck( lfs.bd, block, trunk, weight, cksum) @@ -3323,7 +3326,7 @@ class Lfs: return bid-(rattr.weight-1), rattr # block pointer? elif (rattr.tag & ~0x1003) == TAG_BLOCK: - size, block, off, cksize, cksum = frombptr(rattr.data) + size, block, off, cksize, cksum, _ = frombptr(rattr.data) bptr = Bptr.fetchck(self.lfs.bd, rattr, block, off, size, cksize, cksum) if path: diff --git a/scripts/dbgmtree.py b/scripts/dbgmtree.py index 9f759938..1e5bc9cc 100755 --- a/scripts/dbgmtree.py +++ b/scripts/dbgmtree.py @@ -138,46 +138,49 @@ def popc(x): def parity(x): return popc(x) & 1 -def fromle32(data): - return struct.unpack('H', data[:2])[0] - weight, d = fromleb128(data[2:]) - size, d_ = fromleb128(data[2+d:]) - return tag>>15, tag&0x7fff, weight, size, 2+d+d_ +def fromtag(data, j=0): + d = 0 + tag = struct.unpack('>H', data[j:j+2].ljust(2, b'\0'))[0]; d += 2 + weight, d_ = fromleb128(data, j+d); d += d_ + size, d_ = fromleb128(data, j+d); d += d_ + return tag>>15, tag&0x7fff, weight, size, d -def frommdir(data): +def frombranch(data, j=0): + d = 0 + block, d_ = fromleb128(data, j+d); d += d_ + trunk, d_ = fromleb128(data, j+d); d += d_ + cksum = fromle32(data, j+d); d += 4 + return block, trunk, cksum, d + +def frombtree(data, j=0): + d = 0 + w, d_ = fromleb128(data, j+d); d += d_ + block, trunk, cksum, d_ = frombranch(data, j+d); d += d_ + return w, block, trunk, cksum, d + +def frommdir(data, j=0): blocks = [] d = 0 - while d < len(data): - block, d_ = fromleb128(data[d:]) + while j+d < len(data): + block, d_ = fromleb128(data, j+d) blocks.append(block) d += d_ - return blocks - -def frombranch(data): - d = 0 - block, d_ = fromleb128(data[d:]); d += d_ - trunk, d_ = fromleb128(data[d:]); d += d_ - cksum = fromle32(data[d:]); d += 4 - return block, trunk, cksum - -def frombtree(data): - d = 0 - w, d_ = fromleb128(data[d:]); d += d_ - block, trunk, cksum = frombranch(data[d:]) - return w, block, trunk, cksum + return tuple(blocks), d def xxd(data, width=16): for i in range(0, len(data), width): @@ -549,7 +552,7 @@ class Rbyd: @classmethod def _fetch(cls, data, block, trunk=None): # fetch the rbyd - rev = fromle32(data[0:4]) + rev = fromle32(data, 0) cksum = 0 cksum_ = crc32c(data[0:4]) cksum__ = cksum_ @@ -567,7 +570,7 @@ class Rbyd: gcksumdelta_ = None while j_ < len(data) and (not trunk or eoff <= trunk): # read next tag - v, tag, w, size, d = fromtag(data[j_:]) + v, tag, w, size, d = fromtag(data, j_) if v != parity(cksum__): break cksum__ ^= 0x00000080 if v else 0 @@ -590,7 +593,7 @@ class Rbyd: # found a cksum? else: # check cksum - cksum___ = fromle32(data[j_:j_+4]) + cksum___ = fromle32(data, j_) if cksum__ != cksum___: break # commit what we have @@ -729,7 +732,7 @@ class Rbyd: # descend down tree j = self.trunk while True: - _, alt, w, jump, d = fromtag(self.data[j:]) + _, alt, w, jump, d = fromtag(self.data, j) # found an alt? if alt & TAG_ALT: @@ -745,7 +748,7 @@ class Rbyd: if path: # figure out which color if alt & TAG_R: - _, nalt, _, _, _ = fromtag(self.data[j+jump+d:]) + _, nalt, _, _, _ = fromtag(self.data, j+jump+d) if nalt & TAG_R: color = 'y' else: @@ -768,7 +771,7 @@ class Rbyd: if path: # figure out which color if alt & TAG_R: - _, nalt, _, _, _ = fromtag(self.data[j:]) + _, nalt, _, _, _ = fromtag(self.data, j) if nalt & TAG_R: color = 'y' else: @@ -1029,7 +1032,7 @@ class Btree: # descend down branch? if branch_ is not None and ( not depth or depth_ < depth): - block, trunk, cksum = frombranch(branch_.data) + block, trunk, cksum, _ = frombranch(branch_.data) rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, cksum) @@ -1260,7 +1263,7 @@ class Btree: # found another branch if branch_ is not None and ( not depth or depth_ < depth): - block, trunk, cksum = frombranch(branch_.data) + block, trunk, cksum, _ = frombranch(branch_.data) rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, cksum) @@ -1654,7 +1657,7 @@ class Mtree: rattr_ = mroot.lookup(-1, TAG_MROOT, 0x3) if rattr_ is None: break - blocks_ = frommdir(rattr_.data) + blocks_, _ = frommdir(rattr_.data) mroot = Mdir.fetch(bd, -1, blocks_) mrootchain.append(mroot) @@ -1663,7 +1666,7 @@ class Mtree: if not depth or len(mrootchain) < depth: rattr_ = mroot.lookup(-1, TAG_MTREE, 0x3) if rattr_ is not None: - w_, block_, trunk_, cksum_ = frombtree(rattr_.data) + w_, block_, trunk_, cksum_, _ = frombtree(rattr_.data) mtree = Btree.fetchck(bd, block_, trunk_, w_, cksum_) return cls(bd, mrootchain, mtree, @@ -1743,7 +1746,7 @@ class Mtree: return (bid_, rbyd_, rid_), path_ else: return (bid_, rbyd_, rid_) - blocks_ = frommdir(rattr_.data) + blocks_, _ = frommdir(rattr_.data) mdir = Mdir.fetch(self.bd, mid, blocks_) if path: return mdir, path_ @@ -2117,7 +2120,7 @@ class Mtree: return (bid_, rbyd_, rid_), path_ else: return (bid_, rbyd_, rid_) - blocks_ = frommdir(rattr_.data) + blocks_, _ = frommdir(rattr_.data) mdir = Mdir.fetch(self.bd, self.mid(bid_), blocks_) if path: return mdir, path_ diff --git a/scripts/dbgrbyd.py b/scripts/dbgrbyd.py index 4e075997..a919cd02 100755 --- a/scripts/dbgrbyd.py +++ b/scripts/dbgrbyd.py @@ -148,24 +148,27 @@ def popc(x): def parity(x): return popc(x) & 1 -def fromle32(data): - return struct.unpack('H', data[:2])[0] - weight, d = fromleb128(data[2:]) - size, d_ = fromleb128(data[2+d:]) - return tag>>15, tag&0x7fff, weight, size, 2+d+d_ +def fromtag(data, j=0): + d = 0 + tag = struct.unpack('>H', data[j:j+2].ljust(2, b'\0'))[0]; d += 2 + weight, d_ = fromleb128(data, j+d); d += d_ + size, d_ = fromleb128(data, j+d); d += d_ + return tag>>15, tag&0x7fff, weight, size, d def xxd(data, width=16): for i in range(0, len(data), width): @@ -517,7 +520,7 @@ class Rbyd: @classmethod def _fetch(cls, data, block, trunk=None): # fetch the rbyd - rev = fromle32(data[0:4]) + rev = fromle32(data, 0) cksum = 0 cksum_ = crc32c(data[0:4]) cksum__ = cksum_ @@ -535,7 +538,7 @@ class Rbyd: gcksumdelta_ = None while j_ < len(data) and (not trunk or eoff <= trunk): # read next tag - v, tag, w, size, d = fromtag(data[j_:]) + v, tag, w, size, d = fromtag(data, j_) if v != parity(cksum__): break cksum__ ^= 0x00000080 if v else 0 @@ -558,7 +561,7 @@ class Rbyd: # found a cksum? else: # check cksum - cksum___ = fromle32(data[j_:j_+4]) + cksum___ = fromle32(data, j_) if cksum__ != cksum___: break # commit what we have @@ -697,7 +700,7 @@ class Rbyd: # descend down tree j = self.trunk while True: - _, alt, w, jump, d = fromtag(self.data[j:]) + _, alt, w, jump, d = fromtag(self.data, j) # found an alt? if alt & TAG_ALT: @@ -713,7 +716,7 @@ class Rbyd: if path: # figure out which color if alt & TAG_R: - _, nalt, _, _, _ = fromtag(self.data[j+jump+d:]) + _, nalt, _, _, _ = fromtag(self.data, j+jump+d) if nalt & TAG_R: color = 'y' else: @@ -736,7 +739,7 @@ class Rbyd: if path: # figure out which color if alt & TAG_R: - _, nalt, _, _, _ = fromtag(self.data[j:]) + _, nalt, _, _, _ = fromtag(self.data, j) if nalt & TAG_R: color = 'y' else: @@ -944,7 +947,7 @@ class JumpArt: j_ = 4 while j_ < (len(rbyd.data) if all_ else rbyd.eoff): j = j_ - v, tag, w, size, d = fromtag(rbyd.data[j_:]) + v, tag, w, size, d = fromtag(rbyd.data, j_) j_ += d if not tag & TAG_ALT: j_ += size @@ -952,7 +955,7 @@ class JumpArt: if tag & TAG_ALT and size: # figure out which alt color if tag & TAG_R: - _, ntag, _, _, _ = fromtag(rbyd.data[j_:]) + _, ntag, _, _, _ = fromtag(rbyd.data, j_) if ntag & TAG_R: jumps.append(cls.Jump(j, j-size, 0, 'y')) else: @@ -1104,7 +1107,7 @@ class LifetimeArt: j_ = 4 while j_ < (len(rbyd.data) if all_ else rbyd.eoff): j = j_ - v, tag, w, size, d = fromtag(rbyd.data[j_:]) + v, tag, w, size, d = fromtag(rbyd.data, j_) j_ += d if not tag & TAG_ALT: j_ += size @@ -1498,7 +1501,7 @@ def dbg_log(rbyd, *, j_ = 4 while j_ < (len(data) if args.get('all') else rbyd.eoff): j = j_ - v, tag, w, size, d = fromtag(data[j_:]) + v, tag, w, size, d = fromtag(data, j_) j_ += d if not tag & TAG_ALT: @@ -1539,7 +1542,7 @@ def dbg_log(rbyd, *, # read next tag j = j_ - v, tag, w, size, d = fromtag(data[j_:]) + v, tag, w, size, d = fromtag(data, j_) if v != parity(cksum_): notes.append('v!=%x' % parity(cksum_)) cksum_ ^= 0x00000080 if v else 0 @@ -1553,7 +1556,7 @@ def dbg_log(rbyd, *, # found a cksum? else: # check cksum - cksum__ = fromle32(data[j_:j_+4]) + cksum__ = fromle32(data, j_) if cksum_ != cksum__: notes.append('cksum!=%08x' % cksum__) # update perturb bit diff --git a/scripts/dbgtag.py b/scripts/dbgtag.py index a3bef20c..d2b1bbe7 100755 --- a/scripts/dbgtag.py +++ b/scripts/dbgtag.py @@ -64,21 +64,24 @@ def openio(path, mode='r', buffering=-1): else: return open(path, mode, buffering) -def fromleb128(data): +def fromleb128(data, j=0): word = 0 - for i, b in enumerate(data): - word |= ((b & 0x7f) << 7*i) + d = 0 + while j+d < len(data): + b = data[j+d] + word |= (b & 0x7f) << 7*d word &= 0xffffffff if not b & 0x80: - return word, i+1 + return word, d+1 + d += 1 return word, len(data) -def fromtag(data): - data = data.ljust(4, b'\0') - tag = struct.unpack('>H', data[:2])[0] - weight, d = fromleb128(data[2:]) - size, d_ = fromleb128(data[2+d:]) - return tag>>15, tag&0x7fff, weight, size, 2+d+d_ +def fromtag(data, j=0): + d = 0 + tag = struct.unpack('>H', data[j:j+2].ljust(2, b'\0'))[0]; d += 2 + weight, d_ = fromleb128(data, j+d); d += d_ + size, d_ = fromleb128(data, j+d); d += d_ + return tag>>15, tag&0x7fff, weight, size, d # human readable tag repr def tagrepr(tag, weight=None, size=None, *, @@ -244,7 +247,7 @@ def dbg_tags(data): else: j = 0 while j < len(data): - v, tag, w, size, d = fromtag(data[j:]) + v, tag, w, size, d = fromtag(data, j) lines.append(( ' '.join('%02x' % b for b in data[j:j+d]), tagrepr(tag, w, size)))