Github/littlefs
1
Fork 0
Code Actions Activity

scripts: Fixed O(n^2) slicing in Rbyd.fetch

Browse Source 
Do you see the O(n^2) behavior in this loop?

  j = 0
  while j < len(data):
      word, d = fromleb(data[j:])
      j += d

The slice, data[j:], creates a O(n) copy every iteration of the loop.

A bit tricky. Or at least I found it tricky to notice. Maybe because
array indexing being cheap is baked into my brain...

Long story short, this repeated slicing resulted in O(n^2) behavior in
Rbyd.fetch and probably some other functions. Even though we don't care
_too_ much about performance in these scripts, having Rbyd.fetch run in
O(n^2) isn't great.

Tweaking all from* functions to take an optional index solves this, at
least on paper.

---

In practice I didn't actually find any measurable performance gain. I
guess array slicing in Python is optimized enough that the constant
factor takes over?

(Maybe it's being helped by us limiting Rbyd.fetch to block_size in most
scripts? I haven't tested NAND block sizes yet...)

Still, it's good to at least know this isn't a bottleneck.
This commit is contained in:
Christopher Haster
2025-04-14 14:27:44 -05:00
parent 8b11cea3f2
commit 0cea8b96fb
9 changed files with 317 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/dbgbmap.py
+64 -61
View File
@@ -200,61 +200,64 @@ def popc(x):
def parity(x): def parity(x):
return popc(x) & 1 return popc(x) & 1
def fromle32(data): def fromle32(data, j=0):
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0] return struct.unpack('<I', data[j:j+4].ljust(4, b'\0'))[0]
def fromleb128(data): def fromleb128(data, j=0):
word = 0 word = 0
for i, b in enumerate(data): d = 0
word |= ((b & 0x7f) << 7*i) while j+d < len(data):
b = data[j+d]
word |= (b & 0x7f) << 7*d
word &= 0xffffffff word &= 0xffffffff
if not b & 0x80: if not b & 0x80:
return word, i+1 return word, d+1
d += 1
return word, len(data) return word, len(data)
def fromtag(data): def fromtag(data, j=0):
data = data.ljust(4, b'\0') d = 0
tag = struct.unpack('>H', data[:2])[0] tag = struct.unpack('>H', data[j:j+2].ljust(2, b'\0'))[0]; d += 2
weight, d = fromleb128(data[2:]) weight, d_ = fromleb128(data, j+d); d += d_
size, d_ = fromleb128(data[2+d:]) size, d_ = fromleb128(data, j+d); d += d_
return tag>>15, tag&0x7fff, weight, size, 2+d+d_ return tag>>15, tag&0x7fff, weight, size, d
def frommdir(data): def frombranch(data, j=0):
d = 0
block, d_ = fromleb128(data, j+d); d += d_
trunk, d_ = fromleb128(data, j+d); d += d_
cksum = fromle32(data, j+d); d += 4
return block, trunk, cksum, d
def frombtree(data, j=0):
d = 0
w, d_ = fromleb128(data, j+d); d += d_
block, trunk, cksum, d_ = frombranch(data, j+d); d += d_
return w, block, trunk, cksum, d
def frommdir(data, j=0):
blocks = [] blocks = []
d = 0 d = 0
while d < len(data): while j+d < len(data):
block, d_ = fromleb128(data[d:]) block, d_ = fromleb128(data, j+d)
blocks.append(block) blocks.append(block)
d += d_ d += d_
return blocks return tuple(blocks), d
def fromshrub(data): def fromshrub(data, j=0):
d = 0 d = 0
weight, d_ = fromleb128(data[d:]); d += d_ weight, d_ = fromleb128(data, j+d); d += d_
trunk, d_ = fromleb128(data[d:]); d += d_ trunk, d_ = fromleb128(data, j+d); d += d_
return weight, trunk return weight, trunk, d
def frombranch(data): def frombptr(data, j=0):
d = 0 d = 0
block, d_ = fromleb128(data[d:]); d += d_ size, d_ = fromleb128(data, j+d); d += d_
trunk, d_ = fromleb128(data[d:]); d += d_ block, d_ = fromleb128(data, j+d); d += d_
cksum = fromle32(data[d:]); d += 4 off, d_ = fromleb128(data, j+d); d += d_
return block, trunk, cksum cksize, d_ = fromleb128(data, j+d); d += d_
cksum = fromle32(data, j+d); d += 4
def frombtree(data): return size, block, off, cksize, cksum, d
d = 0
w, d_ = fromleb128(data[d:]); d += d_
block, trunk, cksum = frombranch(data[d:])
return w, block, trunk, cksum
def frombptr(data):
d = 0
size, d_ = fromleb128(data[d:]); d += d_
block, d_ = fromleb128(data[d:]); d += d_
off, d_ = fromleb128(data[d:]); d += d_
cksize, d_ = fromleb128(data[d:]); d += d_
cksum = fromle32(data[d:]); d += 4
return size, block, off, cksize, cksum
def xxd(data, width=16): def xxd(data, width=16):
for i in range(0, len(data), width): for i in range(0, len(data), width):
@@ -626,7 +629,7 @@ class Rbyd:
@classmethod @classmethod
def _fetch(cls, data, block, trunk=None): def _fetch(cls, data, block, trunk=None):
# fetch the rbyd # fetch the rbyd
rev = fromle32(data[0:4]) rev = fromle32(data, 0)
cksum = 0 cksum = 0
cksum_ = crc32c(data[0:4]) cksum_ = crc32c(data[0:4])
cksum__ = cksum_ cksum__ = cksum_
@@ -644,7 +647,7 @@ class Rbyd:
gcksumdelta_ = None gcksumdelta_ = None
while j_ < len(data) and (not trunk or eoff <= trunk): while j_ < len(data) and (not trunk or eoff <= trunk):
# read next tag # read next tag
v, tag, w, size, d = fromtag(data[j_:]) v, tag, w, size, d = fromtag(data, j_)
if v != parity(cksum__): if v != parity(cksum__):
break break
cksum__ ^= 0x00000080 if v else 0 cksum__ ^= 0x00000080 if v else 0
@@ -667,7 +670,7 @@ class Rbyd:
# found a cksum? # found a cksum?
else: else:
# check cksum # check cksum
cksum___ = fromle32(data[j_:j_+4]) cksum___ = fromle32(data, j_)
if cksum__ != cksum___: if cksum__ != cksum___:
break break
# commit what we have # commit what we have
@@ -806,7 +809,7 @@ class Rbyd:
# descend down tree # descend down tree
j = self.trunk j = self.trunk
while True: while True:
_, alt, w, jump, d = fromtag(self.data[j:]) _, alt, w, jump, d = fromtag(self.data, j)
# found an alt? # found an alt?
if alt & TAG_ALT: if alt & TAG_ALT:
@@ -822,7 +825,7 @@ class Rbyd:
if path: if path:
# figure out which color # figure out which color
if alt & TAG_R: if alt & TAG_R:
_, nalt, _, _, _ = fromtag(self.data[j+jump+d:]) _, nalt, _, _, _ = fromtag(self.data, j+jump+d)
if nalt & TAG_R: if nalt & TAG_R:
color = 'y' color = 'y'
else: else:
@@ -845,7 +848,7 @@ class Rbyd:
if path: if path:
# figure out which color # figure out which color
if alt & TAG_R: if alt & TAG_R:
_, nalt, _, _, _ = fromtag(self.data[j:]) _, nalt, _, _, _ = fromtag(self.data, j)
if nalt & TAG_R: if nalt & TAG_R:
color = 'y' color = 'y'
else: else:
@@ -1106,7 +1109,7 @@ class Btree:
# descend down branch? # descend down branch?
if branch_ is not None and ( if branch_ is not None and (
not depth or depth_ < depth): not depth or depth_ < depth):
block, trunk, cksum = frombranch(branch_.data) block, trunk, cksum, _ = frombranch(branch_.data)
rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight,
cksum) cksum)
@@ -1337,7 +1340,7 @@ class Btree:
# found another branch # found another branch
if branch_ is not None and ( if branch_ is not None and (
not depth or depth_ < depth): not depth or depth_ < depth):
block, trunk, cksum = frombranch(branch_.data) block, trunk, cksum, _ = frombranch(branch_.data)
rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight,
cksum) cksum)
@@ -1731,7 +1734,7 @@ class Mtree:
rattr_ = mroot.lookup(-1, TAG_MROOT, 0x3) rattr_ = mroot.lookup(-1, TAG_MROOT, 0x3)
if rattr_ is None: if rattr_ is None:
break break
blocks_ = frommdir(rattr_.data) blocks_, _ = frommdir(rattr_.data)
mroot = Mdir.fetch(bd, -1, blocks_) mroot = Mdir.fetch(bd, -1, blocks_)
mrootchain.append(mroot) mrootchain.append(mroot)
@@ -1740,7 +1743,7 @@ class Mtree:
if not depth or len(mrootchain) < depth: if not depth or len(mrootchain) < depth:
rattr_ = mroot.lookup(-1, TAG_MTREE, 0x3) rattr_ = mroot.lookup(-1, TAG_MTREE, 0x3)
if rattr_ is not None: if rattr_ is not None:
w_, block_, trunk_, cksum_ = frombtree(rattr_.data) w_, block_, trunk_, cksum_, _ = frombtree(rattr_.data)
mtree = Btree.fetchck(bd, block_, trunk_, w_, cksum_) mtree = Btree.fetchck(bd, block_, trunk_, w_, cksum_)
return cls(bd, mrootchain, mtree, return cls(bd, mrootchain, mtree,
@@ -1820,7 +1823,7 @@ class Mtree:
return (bid_, rbyd_, rid_), path_ return (bid_, rbyd_, rid_), path_
else: else:
return (bid_, rbyd_, rid_) return (bid_, rbyd_, rid_)
blocks_ = frommdir(rattr_.data) blocks_, _ = frommdir(rattr_.data)
mdir = Mdir.fetch(self.bd, mid, blocks_) mdir = Mdir.fetch(self.bd, mid, blocks_)
if path: if path:
return mdir, path_ return mdir, path_
@@ -2194,7 +2197,7 @@ class Mtree:
return (bid_, rbyd_, rid_), path_ return (bid_, rbyd_, rid_), path_
else: else:
return (bid_, rbyd_, rid_) return (bid_, rbyd_, rid_)
blocks_ = frommdir(rattr_.data) blocks_, _ = frommdir(rattr_.data)
mdir = Mdir.fetch(self.bd, self.mid(bid_), blocks_) mdir = Mdir.fetch(self.bd, self.mid(bid_), blocks_)
if path: if path:
return mdir, path_ return mdir, path_
@@ -2458,8 +2461,8 @@ class Config:
def __init__(self, mroot, tag, rattr): def __init__(self, mroot, tag, rattr):
super().__init__(mroot, tag, rattr) super().__init__(mroot, tag, rattr)
d = 0 d = 0
self.major, d_ = fromleb128(self.data[d:]); d += d_ self.major, d_ = fromleb128(self.data, d); d += d_
self.minor, d_ = fromleb128(self.data[d:]); d += d_ self.minor, d_ = fromleb128(self.data, d); d += d_
@property @property
def tuple(self): def tuple(self):
@@ -2498,8 +2501,8 @@ class Config:
def __init__(self, mroot, tag, rattr): def __init__(self, mroot, tag, rattr):
super().__init__(mroot, tag, rattr) super().__init__(mroot, tag, rattr)
d = 0 d = 0
block_size, d_ = fromleb128(self.data[d:]); d += d_ block_size, d_ = fromleb128(self.data, d); d += d_
block_count, d_ = fromleb128(self.data[d:]); d += d_ block_count, d_ = fromleb128(self.data, d); d += d_
# these are offset by 1 to avoid overflow issues # these are offset by 1 to avoid overflow issues
self.block_size = block_size + 1 self.block_size = block_size + 1
self.block_count = block_count + 1 self.block_count = block_count + 1
@@ -2703,11 +2706,11 @@ class Gstate:
def __init__(self, mtree, tag, gdeltas): def __init__(self, mtree, tag, gdeltas):
super().__init__(mtree, tag, gdeltas) super().__init__(mtree, tag, gdeltas)
d = 0 d = 0
count, d_ = fromleb128(self.data[d:]); d += d_ count, d_ = fromleb128(self.data, d); d += d_
rms = [] rms = []
if count <= 2: if count <= 2:
for _ in range(count): for _ in range(count):
mid, d_ = fromleb128(self.data[d:]); d += d_ mid, d_ = fromleb128(self.data, d); d += d_
rms.append(mtree.mid(mid)) rms.append(mtree.mid(mid))
self.count = count self.count = count
self.rms = rms self.rms = rms
@@ -3242,11 +3245,11 @@ class Lfs:
self.bshrub = None self.bshrub = None
if (self.struct is not None if (self.struct is not None
and (self.struct.tag & ~0x3) == TAG_BSHRUB): and (self.struct.tag & ~0x3) == TAG_BSHRUB):
weight, trunk = fromshrub(self.struct.data) weight, trunk, _ = fromshrub(self.struct.data)
self.bshrub = Btree.fetchshrub(lfs.bd, mdir.rbyd, trunk) self.bshrub = Btree.fetchshrub(lfs.bd, mdir.rbyd, trunk)
elif (self.struct is not None elif (self.struct is not None
and (self.struct.tag & ~0x3) == TAG_BTREE): and (self.struct.tag & ~0x3) == TAG_BTREE):
weight, block, trunk, cksum = frombtree(self.struct.data) weight, block, trunk, cksum, _ = frombtree(self.struct.data)
self.bshrub = Btree.fetchck( self.bshrub = Btree.fetchck(
lfs.bd, block, trunk, weight, cksum) lfs.bd, block, trunk, weight, cksum)
@@ -3366,7 +3369,7 @@ class Lfs:
return bid-(rattr.weight-1), rattr return bid-(rattr.weight-1), rattr
# block pointer? # block pointer?
elif (rattr.tag & ~0x1003) == TAG_BLOCK: elif (rattr.tag & ~0x1003) == TAG_BLOCK:
size, block, off, cksize, cksum = frombptr(rattr.data) size, block, off, cksize, cksum, _ = frombptr(rattr.data)
bptr = Bptr.fetchck(self.lfs.bd, rattr, bptr = Bptr.fetchck(self.lfs.bd, rattr,
block, off, size, cksize, cksum) block, off, size, cksize, cksum)
if path: if path:
scripts/dbgbmapd3.py
+64 -61
View File
@@ -230,61 +230,64 @@ def popc(x):
def parity(x): def parity(x):
return popc(x) & 1 return popc(x) & 1
def fromle32(data): def fromle32(data, j=0):
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0] return struct.unpack('<I', data[j:j+4].ljust(4, b'\0'))[0]
def fromleb128(data): def fromleb128(data, j=0):
word = 0 word = 0
for i, b in enumerate(data): d = 0
word |= ((b & 0x7f) << 7*i) while j+d < len(data):
b = data[j+d]
word |= (b & 0x7f) << 7*d
word &= 0xffffffff word &= 0xffffffff
if not b & 0x80: if not b & 0x80:
return word, i+1 return word, d+1
d += 1
return word, len(data) return word, len(data)
def fromtag(data): def fromtag(data, j=0):
data = data.ljust(4, b'\0') d = 0
tag = struct.unpack('>H', data[:2])[0] tag = struct.unpack('>H', data[j:j+2].ljust(2, b'\0'))[0]; d += 2
weight, d = fromleb128(data[2:]) weight, d_ = fromleb128(data, j+d); d += d_
size, d_ = fromleb128(data[2+d:]) size, d_ = fromleb128(data, j+d); d += d_
return tag>>15, tag&0x7fff, weight, size, 2+d+d_ return tag>>15, tag&0x7fff, weight, size, d
def frommdir(data): def frombranch(data, j=0):
d = 0
block, d_ = fromleb128(data, j+d); d += d_
trunk, d_ = fromleb128(data, j+d); d += d_
cksum = fromle32(data, j+d); d += 4
return block, trunk, cksum, d
def frombtree(data, j=0):
d = 0
w, d_ = fromleb128(data, j+d); d += d_
block, trunk, cksum, d_ = frombranch(data, j+d); d += d_
return w, block, trunk, cksum, d
def frommdir(data, j=0):
blocks = [] blocks = []
d = 0 d = 0
while d < len(data): while j+d < len(data):
block, d_ = fromleb128(data[d:]) block, d_ = fromleb128(data, j+d)
blocks.append(block) blocks.append(block)
d += d_ d += d_
return blocks return tuple(blocks), d
def fromshrub(data): def fromshrub(data, j=0):
d = 0 d = 0
weight, d_ = fromleb128(data[d:]); d += d_ weight, d_ = fromleb128(data, j+d); d += d_
trunk, d_ = fromleb128(data[d:]); d += d_ trunk, d_ = fromleb128(data, j+d); d += d_
return weight, trunk return weight, trunk, d
def frombranch(data): def frombptr(data, j=0):
d = 0 d = 0
block, d_ = fromleb128(data[d:]); d += d_ size, d_ = fromleb128(data, j+d); d += d_
trunk, d_ = fromleb128(data[d:]); d += d_ block, d_ = fromleb128(data, j+d); d += d_
cksum = fromle32(data[d:]); d += 4 off, d_ = fromleb128(data, j+d); d += d_
return block, trunk, cksum cksize, d_ = fromleb128(data, j+d); d += d_
cksum = fromle32(data, j+d); d += 4
def frombtree(data): return size, block, off, cksize, cksum, d
d = 0
w, d_ = fromleb128(data[d:]); d += d_
block, trunk, cksum = frombranch(data[d:])
return w, block, trunk, cksum
def frombptr(data):
d = 0
size, d_ = fromleb128(data[d:]); d += d_
block, d_ = fromleb128(data[d:]); d += d_
off, d_ = fromleb128(data[d:]); d += d_
cksize, d_ = fromleb128(data[d:]); d += d_
cksum = fromle32(data[d:]); d += 4
return size, block, off, cksize, cksum
def xxd(data, width=16): def xxd(data, width=16):
for i in range(0, len(data), width): for i in range(0, len(data), width):
@@ -656,7 +659,7 @@ class Rbyd:
@classmethod @classmethod
def _fetch(cls, data, block, trunk=None): def _fetch(cls, data, block, trunk=None):
# fetch the rbyd # fetch the rbyd
rev = fromle32(data[0:4]) rev = fromle32(data, 0)
cksum = 0 cksum = 0
cksum_ = crc32c(data[0:4]) cksum_ = crc32c(data[0:4])
cksum__ = cksum_ cksum__ = cksum_
@@ -674,7 +677,7 @@ class Rbyd:
gcksumdelta_ = None gcksumdelta_ = None
while j_ < len(data) and (not trunk or eoff <= trunk): while j_ < len(data) and (not trunk or eoff <= trunk):
# read next tag # read next tag
v, tag, w, size, d = fromtag(data[j_:]) v, tag, w, size, d = fromtag(data, j_)
if v != parity(cksum__): if v != parity(cksum__):
break break
cksum__ ^= 0x00000080 if v else 0 cksum__ ^= 0x00000080 if v else 0
@@ -697,7 +700,7 @@ class Rbyd:
# found a cksum? # found a cksum?
else: else:
# check cksum # check cksum
cksum___ = fromle32(data[j_:j_+4]) cksum___ = fromle32(data, j_)
if cksum__ != cksum___: if cksum__ != cksum___:
break break
# commit what we have # commit what we have
@@ -836,7 +839,7 @@ class Rbyd:
# descend down tree # descend down tree
j = self.trunk j = self.trunk
while True: while True:
_, alt, w, jump, d = fromtag(self.data[j:]) _, alt, w, jump, d = fromtag(self.data, j)
# found an alt? # found an alt?
if alt & TAG_ALT: if alt & TAG_ALT:
@@ -852,7 +855,7 @@ class Rbyd:
if path: if path:
# figure out which color # figure out which color
if alt & TAG_R: if alt & TAG_R:
_, nalt, _, _, _ = fromtag(self.data[j+jump+d:]) _, nalt, _, _, _ = fromtag(self.data, j+jump+d)
if nalt & TAG_R: if nalt & TAG_R:
color = 'y' color = 'y'
else: else:
@@ -875,7 +878,7 @@ class Rbyd:
if path: if path:
# figure out which color # figure out which color
if alt & TAG_R: if alt & TAG_R:
_, nalt, _, _, _ = fromtag(self.data[j:]) _, nalt, _, _, _ = fromtag(self.data, j)
if nalt & TAG_R: if nalt & TAG_R:
color = 'y' color = 'y'
else: else:
@@ -1136,7 +1139,7 @@ class Btree:
# descend down branch? # descend down branch?
if branch_ is not None and ( if branch_ is not None and (
not depth or depth_ < depth): not depth or depth_ < depth):
block, trunk, cksum = frombranch(branch_.data) block, trunk, cksum, _ = frombranch(branch_.data)
rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight,
cksum) cksum)
@@ -1367,7 +1370,7 @@ class Btree:
# found another branch # found another branch
if branch_ is not None and ( if branch_ is not None and (
not depth or depth_ < depth): not depth or depth_ < depth):
block, trunk, cksum = frombranch(branch_.data) block, trunk, cksum, _ = frombranch(branch_.data)
rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight,
cksum) cksum)
@@ -1761,7 +1764,7 @@ class Mtree:
rattr_ = mroot.lookup(-1, TAG_MROOT, 0x3) rattr_ = mroot.lookup(-1, TAG_MROOT, 0x3)
if rattr_ is None: if rattr_ is None:
break break
blocks_ = frommdir(rattr_.data) blocks_, _ = frommdir(rattr_.data)
mroot = Mdir.fetch(bd, -1, blocks_) mroot = Mdir.fetch(bd, -1, blocks_)
mrootchain.append(mroot) mrootchain.append(mroot)
@@ -1770,7 +1773,7 @@ class Mtree:
if not depth or len(mrootchain) < depth: if not depth or len(mrootchain) < depth:
rattr_ = mroot.lookup(-1, TAG_MTREE, 0x3) rattr_ = mroot.lookup(-1, TAG_MTREE, 0x3)
if rattr_ is not None: if rattr_ is not None:
w_, block_, trunk_, cksum_ = frombtree(rattr_.data) w_, block_, trunk_, cksum_, _ = frombtree(rattr_.data)
mtree = Btree.fetchck(bd, block_, trunk_, w_, cksum_) mtree = Btree.fetchck(bd, block_, trunk_, w_, cksum_)
return cls(bd, mrootchain, mtree, return cls(bd, mrootchain, mtree,
@@ -1850,7 +1853,7 @@ class Mtree:
return (bid_, rbyd_, rid_), path_ return (bid_, rbyd_, rid_), path_
else: else:
return (bid_, rbyd_, rid_) return (bid_, rbyd_, rid_)
blocks_ = frommdir(rattr_.data) blocks_, _ = frommdir(rattr_.data)
mdir = Mdir.fetch(self.bd, mid, blocks_) mdir = Mdir.fetch(self.bd, mid, blocks_)
if path: if path:
return mdir, path_ return mdir, path_
@@ -2224,7 +2227,7 @@ class Mtree:
return (bid_, rbyd_, rid_), path_ return (bid_, rbyd_, rid_), path_
else: else:
return (bid_, rbyd_, rid_) return (bid_, rbyd_, rid_)
blocks_ = frommdir(rattr_.data) blocks_, _ = frommdir(rattr_.data)
mdir = Mdir.fetch(self.bd, self.mid(bid_), blocks_) mdir = Mdir.fetch(self.bd, self.mid(bid_), blocks_)
if path: if path:
return mdir, path_ return mdir, path_
@@ -2488,8 +2491,8 @@ class Config:
def __init__(self, mroot, tag, rattr): def __init__(self, mroot, tag, rattr):
super().__init__(mroot, tag, rattr) super().__init__(mroot, tag, rattr)
d = 0 d = 0
self.major, d_ = fromleb128(self.data[d:]); d += d_ self.major, d_ = fromleb128(self.data, d); d += d_
self.minor, d_ = fromleb128(self.data[d:]); d += d_ self.minor, d_ = fromleb128(self.data, d); d += d_
@property @property
def tuple(self): def tuple(self):
@@ -2528,8 +2531,8 @@ class Config:
def __init__(self, mroot, tag, rattr): def __init__(self, mroot, tag, rattr):
super().__init__(mroot, tag, rattr) super().__init__(mroot, tag, rattr)
d = 0 d = 0
block_size, d_ = fromleb128(self.data[d:]); d += d_ block_size, d_ = fromleb128(self.data, d); d += d_
block_count, d_ = fromleb128(self.data[d:]); d += d_ block_count, d_ = fromleb128(self.data, d); d += d_
# these are offset by 1 to avoid overflow issues # these are offset by 1 to avoid overflow issues
self.block_size = block_size + 1 self.block_size = block_size + 1
self.block_count = block_count + 1 self.block_count = block_count + 1
@@ -2733,11 +2736,11 @@ class Gstate:
def __init__(self, mtree, tag, gdeltas): def __init__(self, mtree, tag, gdeltas):
super().__init__(mtree, tag, gdeltas) super().__init__(mtree, tag, gdeltas)
d = 0 d = 0
count, d_ = fromleb128(self.data[d:]); d += d_ count, d_ = fromleb128(self.data, d); d += d_
rms = [] rms = []
if count <= 2: if count <= 2:
for _ in range(count): for _ in range(count):
mid, d_ = fromleb128(self.data[d:]); d += d_ mid, d_ = fromleb128(self.data, d); d += d_
rms.append(mtree.mid(mid)) rms.append(mtree.mid(mid))
self.count = count self.count = count
self.rms = rms self.rms = rms
@@ -3272,11 +3275,11 @@ class Lfs:
self.bshrub = None self.bshrub = None
if (self.struct is not None if (self.struct is not None
and (self.struct.tag & ~0x3) == TAG_BSHRUB): and (self.struct.tag & ~0x3) == TAG_BSHRUB):
weight, trunk = fromshrub(self.struct.data) weight, trunk, _ = fromshrub(self.struct.data)
self.bshrub = Btree.fetchshrub(lfs.bd, mdir.rbyd, trunk) self.bshrub = Btree.fetchshrub(lfs.bd, mdir.rbyd, trunk)
elif (self.struct is not None elif (self.struct is not None
and (self.struct.tag & ~0x3) == TAG_BTREE): and (self.struct.tag & ~0x3) == TAG_BTREE):
weight, block, trunk, cksum = frombtree(self.struct.data) weight, block, trunk, cksum, _ = frombtree(self.struct.data)
self.bshrub = Btree.fetchck( self.bshrub = Btree.fetchck(
lfs.bd, block, trunk, weight, cksum) lfs.bd, block, trunk, weight, cksum)
@@ -3396,7 +3399,7 @@ class Lfs:
return bid-(rattr.weight-1), rattr return bid-(rattr.weight-1), rattr
# block pointer? # block pointer?
elif (rattr.tag & ~0x1003) == TAG_BLOCK: elif (rattr.tag & ~0x1003) == TAG_BLOCK:
size, block, off, cksize, cksum = frombptr(rattr.data) size, block, off, cksize, cksum, _ = frombptr(rattr.data)
bptr = Bptr.fetchck(self.lfs.bd, rattr, bptr = Bptr.fetchck(self.lfs.bd, rattr,
block, off, size, cksize, cksum) block, off, size, cksize, cksum)
if path: if path:
scripts/dbgbtree.py
+29 -26
View File
@@ -138,31 +138,34 @@ def popc(x):
def parity(x): def parity(x):
return popc(x) & 1 return popc(x) & 1
def fromle32(data): def fromle32(data, j=0):
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0] return struct.unpack('<I', data[j:j+4].ljust(4, b'\0'))[0]
def fromleb128(data): def fromleb128(data, j=0):
word = 0 word = 0
for i, b in enumerate(data): d = 0
word |= ((b & 0x7f) << 7*i) while j+d < len(data):
b = data[j+d]
word |= (b & 0x7f) << 7*d
word &= 0xffffffff word &= 0xffffffff
if not b & 0x80: if not b & 0x80:
return word, i+1 return word, d+1
d += 1
return word, len(data) return word, len(data)
def fromtag(data): def fromtag(data, j=0):
data = data.ljust(4, b'\0')
tag = struct.unpack('>H', data[:2])[0]
weight, d = fromleb128(data[2:])
size, d_ = fromleb128(data[2+d:])
return tag>>15, tag&0x7fff, weight, size, 2+d+d_
def frombranch(data):
d = 0 d = 0
block, d_ = fromleb128(data[d:]); d += d_ tag = struct.unpack('>H', data[j:j+2].ljust(2, b'\0'))[0]; d += 2
trunk, d_ = fromleb128(data[d:]); d += d_ weight, d_ = fromleb128(data, j+d); d += d_
cksum = fromle32(data[d:]); d += 4 size, d_ = fromleb128(data, j+d); d += d_
return block, trunk, cksum return tag>>15, tag&0x7fff, weight, size, d
def frombranch(data, j=0):
d = 0
block, d_ = fromleb128(data, j+d); d += d_
trunk, d_ = fromleb128(data, j+d); d += d_
cksum = fromle32(data, j+d); d += 4
return block, trunk, cksum, d
def xxd(data, width=16): def xxd(data, width=16):
for i in range(0, len(data), width): for i in range(0, len(data), width):
@@ -534,7 +537,7 @@ class Rbyd:
@classmethod @classmethod
def _fetch(cls, data, block, trunk=None): def _fetch(cls, data, block, trunk=None):
# fetch the rbyd # fetch the rbyd
rev = fromle32(data[0:4]) rev = fromle32(data, 0)
cksum = 0 cksum = 0
cksum_ = crc32c(data[0:4]) cksum_ = crc32c(data[0:4])
cksum__ = cksum_ cksum__ = cksum_
@@ -552,7 +555,7 @@ class Rbyd:
gcksumdelta_ = None gcksumdelta_ = None
while j_ < len(data) and (not trunk or eoff <= trunk): while j_ < len(data) and (not trunk or eoff <= trunk):
# read next tag # read next tag
v, tag, w, size, d = fromtag(data[j_:]) v, tag, w, size, d = fromtag(data, j_)
if v != parity(cksum__): if v != parity(cksum__):
break break
cksum__ ^= 0x00000080 if v else 0 cksum__ ^= 0x00000080 if v else 0
@@ -575,7 +578,7 @@ class Rbyd:
# found a cksum? # found a cksum?
else: else:
# check cksum # check cksum
cksum___ = fromle32(data[j_:j_+4]) cksum___ = fromle32(data, j_)
if cksum__ != cksum___: if cksum__ != cksum___:
break break
# commit what we have # commit what we have
@@ -714,7 +717,7 @@ class Rbyd:
# descend down tree # descend down tree
j = self.trunk j = self.trunk
while True: while True:
_, alt, w, jump, d = fromtag(self.data[j:]) _, alt, w, jump, d = fromtag(self.data, j)
# found an alt? # found an alt?
if alt & TAG_ALT: if alt & TAG_ALT:
@@ -730,7 +733,7 @@ class Rbyd:
if path: if path:
# figure out which color # figure out which color
if alt & TAG_R: if alt & TAG_R:
_, nalt, _, _, _ = fromtag(self.data[j+jump+d:]) _, nalt, _, _, _ = fromtag(self.data, j+jump+d)
if nalt & TAG_R: if nalt & TAG_R:
color = 'y' color = 'y'
else: else:
@@ -753,7 +756,7 @@ class Rbyd:
if path: if path:
# figure out which color # figure out which color
if alt & TAG_R: if alt & TAG_R:
_, nalt, _, _, _ = fromtag(self.data[j:]) _, nalt, _, _, _ = fromtag(self.data, j)
if nalt & TAG_R: if nalt & TAG_R:
color = 'y' color = 'y'
else: else:
@@ -1014,7 +1017,7 @@ class Btree:
# descend down branch? # descend down branch?
if branch_ is not None and ( if branch_ is not None and (
not depth or depth_ < depth): not depth or depth_ < depth):
block, trunk, cksum = frombranch(branch_.data) block, trunk, cksum, _ = frombranch(branch_.data)
rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight,
cksum) cksum)
@@ -1245,7 +1248,7 @@ class Btree:
# found another branch # found another branch
if branch_ is not None and ( if branch_ is not None and (
not depth or depth_ < depth): not depth or depth_ < depth):
block, trunk, cksum = frombranch(branch_.data) block, trunk, cksum, _ = frombranch(branch_.data)
rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight,
cksum) cksum)
scripts/dbgle32.py
+3 -3
View File
@@ -21,14 +21,14 @@ def openio(path, mode='r', buffering=-1):
else: else:
return open(path, mode, buffering) return open(path, mode, buffering)
def fromle32(data): def fromle32(data, j=0):
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0] return struct.unpack('<I', data[j:j+4].ljust(4, b'\0'))[0]
def dbg_le32s(data): def dbg_le32s(data):
lines = [] lines = []
j = 0 j = 0
while j < len(data): while j < len(data):
word = fromle32(data[j:]) word = fromle32(data, j)
lines.append(( lines.append((
' '.join('%02x' % b for b in data[j:j+4]), ' '.join('%02x' % b for b in data[j:j+4]),
word)) word))
scripts/dbgleb128.py
+8 -5
View File
@@ -21,20 +21,23 @@ def openio(path, mode='r', buffering=-1):
else: else:
return open(path, mode, buffering) return open(path, mode, buffering)
def fromleb128(data): def fromleb128(data, j=0):
word = 0 word = 0
for i, b in enumerate(data): d = 0
word |= ((b & 0x7f) << 7*i) while j+d < len(data):
b = data[j+d]
word |= (b & 0x7f) << 7*d
word &= 0xffffffff word &= 0xffffffff
if not b & 0x80: if not b & 0x80:
return word, i+1 return word, d+1
d += 1
return word, len(data) return word, len(data)
def dbg_leb128s(data): def dbg_leb128s(data):
lines = [] lines = []
j = 0 j = 0
while j < len(data): while j < len(data):
word, d = fromleb128(data[j:]) word, d = fromleb128(data, j)
lines.append(( lines.append((
' '.join('%02x' % b for b in data[j:j+d]), ' '.join('%02x' % b for b in data[j:j+d]),
word)) word))
scripts/dbglfs.py
+64 -61
View File
@@ -157,61 +157,64 @@ def popc(x):
def parity(x): def parity(x):
return popc(x) & 1 return popc(x) & 1
def fromle32(data): def fromle32(data, j=0):
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0] return struct.unpack('<I', data[j:j+4].ljust(4, b'\0'))[0]
def fromleb128(data): def fromleb128(data, j=0):
word = 0 word = 0
for i, b in enumerate(data): d = 0
word |= ((b & 0x7f) << 7*i) while j+d < len(data):
b = data[j+d]
word |= (b & 0x7f) << 7*d
word &= 0xffffffff word &= 0xffffffff
if not b & 0x80: if not b & 0x80:
return word, i+1 return word, d+1
d += 1
return word, len(data) return word, len(data)
def fromtag(data): def fromtag(data, j=0):
data = data.ljust(4, b'\0') d = 0
tag = struct.unpack('>H', data[:2])[0] tag = struct.unpack('>H', data[j:j+2].ljust(2, b'\0'))[0]; d += 2
weight, d = fromleb128(data[2:]) weight, d_ = fromleb128(data, j+d); d += d_
size, d_ = fromleb128(data[2+d:]) size, d_ = fromleb128(data, j+d); d += d_
return tag>>15, tag&0x7fff, weight, size, 2+d+d_ return tag>>15, tag&0x7fff, weight, size, d
def frommdir(data): def frombranch(data, j=0):
d = 0
block, d_ = fromleb128(data, j+d); d += d_
trunk, d_ = fromleb128(data, j+d); d += d_
cksum = fromle32(data, j+d); d += 4
return block, trunk, cksum, d
def frombtree(data, j=0):
d = 0
w, d_ = fromleb128(data, j+d); d += d_
block, trunk, cksum, d_ = frombranch(data, j+d); d += d_
return w, block, trunk, cksum, d
def frommdir(data, j=0):
blocks = [] blocks = []
d = 0 d = 0
while d < len(data): while j+d < len(data):
block, d_ = fromleb128(data[d:]) block, d_ = fromleb128(data, j+d)
blocks.append(block) blocks.append(block)
d += d_ d += d_
return blocks return tuple(blocks), d
def fromshrub(data): def fromshrub(data, j=0):
d = 0 d = 0
weight, d_ = fromleb128(data[d:]); d += d_ weight, d_ = fromleb128(data, j+d); d += d_
trunk, d_ = fromleb128(data[d:]); d += d_ trunk, d_ = fromleb128(data, j+d); d += d_
return weight, trunk return weight, trunk, d
def frombranch(data): def frombptr(data, j=0):
d = 0 d = 0
block, d_ = fromleb128(data[d:]); d += d_ size, d_ = fromleb128(data, j+d); d += d_
trunk, d_ = fromleb128(data[d:]); d += d_ block, d_ = fromleb128(data, j+d); d += d_
cksum = fromle32(data[d:]); d += 4 off, d_ = fromleb128(data, j+d); d += d_
return block, trunk, cksum cksize, d_ = fromleb128(data, j+d); d += d_
cksum = fromle32(data, j+d); d += 4
def frombtree(data): return size, block, off, cksize, cksum, d
d = 0
w, d_ = fromleb128(data[d:]); d += d_
block, trunk, cksum = frombranch(data[d:])
return w, block, trunk, cksum
def frombptr(data):
d = 0
size, d_ = fromleb128(data[d:]); d += d_
block, d_ = fromleb128(data[d:]); d += d_
off, d_ = fromleb128(data[d:]); d += d_
cksize, d_ = fromleb128(data[d:]); d += d_
cksum = fromle32(data[d:]); d += 4
return size, block, off, cksize, cksum
def xxd(data, width=16): def xxd(data, width=16):
for i in range(0, len(data), width): for i in range(0, len(data), width):
@@ -583,7 +586,7 @@ class Rbyd:
@classmethod @classmethod
def _fetch(cls, data, block, trunk=None): def _fetch(cls, data, block, trunk=None):
# fetch the rbyd # fetch the rbyd
rev = fromle32(data[0:4]) rev = fromle32(data, 0)
cksum = 0 cksum = 0
cksum_ = crc32c(data[0:4]) cksum_ = crc32c(data[0:4])
cksum__ = cksum_ cksum__ = cksum_
@@ -601,7 +604,7 @@ class Rbyd:
gcksumdelta_ = None gcksumdelta_ = None
while j_ < len(data) and (not trunk or eoff <= trunk): while j_ < len(data) and (not trunk or eoff <= trunk):
# read next tag # read next tag
v, tag, w, size, d = fromtag(data[j_:]) v, tag, w, size, d = fromtag(data, j_)
if v != parity(cksum__): if v != parity(cksum__):
break break
cksum__ ^= 0x00000080 if v else 0 cksum__ ^= 0x00000080 if v else 0
@@ -624,7 +627,7 @@ class Rbyd:
# found a cksum? # found a cksum?
else: else:
# check cksum # check cksum
cksum___ = fromle32(data[j_:j_+4]) cksum___ = fromle32(data, j_)
if cksum__ != cksum___: if cksum__ != cksum___:
break break
# commit what we have # commit what we have
@@ -763,7 +766,7 @@ class Rbyd:
# descend down tree # descend down tree
j = self.trunk j = self.trunk
while True: while True:
_, alt, w, jump, d = fromtag(self.data[j:]) _, alt, w, jump, d = fromtag(self.data, j)
# found an alt? # found an alt?
if alt & TAG_ALT: if alt & TAG_ALT:
@@ -779,7 +782,7 @@ class Rbyd:
if path: if path:
# figure out which color # figure out which color
if alt & TAG_R: if alt & TAG_R:
_, nalt, _, _, _ = fromtag(self.data[j+jump+d:]) _, nalt, _, _, _ = fromtag(self.data, j+jump+d)
if nalt & TAG_R: if nalt & TAG_R:
color = 'y' color = 'y'
else: else:
@@ -802,7 +805,7 @@ class Rbyd:
if path: if path:
# figure out which color # figure out which color
if alt & TAG_R: if alt & TAG_R:
_, nalt, _, _, _ = fromtag(self.data[j:]) _, nalt, _, _, _ = fromtag(self.data, j)
if nalt & TAG_R: if nalt & TAG_R:
color = 'y' color = 'y'
else: else:
@@ -1063,7 +1066,7 @@ class Btree:
# descend down branch? # descend down branch?
if branch_ is not None and ( if branch_ is not None and (
not depth or depth_ < depth): not depth or depth_ < depth):
block, trunk, cksum = frombranch(branch_.data) block, trunk, cksum, _ = frombranch(branch_.data)
rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight,
cksum) cksum)
@@ -1294,7 +1297,7 @@ class Btree:
# found another branch # found another branch
if branch_ is not None and ( if branch_ is not None and (
not depth or depth_ < depth): not depth or depth_ < depth):
block, trunk, cksum = frombranch(branch_.data) block, trunk, cksum, _ = frombranch(branch_.data)
rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight,
cksum) cksum)
@@ -1688,7 +1691,7 @@ class Mtree:
rattr_ = mroot.lookup(-1, TAG_MROOT, 0x3) rattr_ = mroot.lookup(-1, TAG_MROOT, 0x3)
if rattr_ is None: if rattr_ is None:
break break
blocks_ = frommdir(rattr_.data) blocks_, _ = frommdir(rattr_.data)
mroot = Mdir.fetch(bd, -1, blocks_) mroot = Mdir.fetch(bd, -1, blocks_)
mrootchain.append(mroot) mrootchain.append(mroot)
@@ -1697,7 +1700,7 @@ class Mtree:
if not depth or len(mrootchain) < depth: if not depth or len(mrootchain) < depth:
rattr_ = mroot.lookup(-1, TAG_MTREE, 0x3) rattr_ = mroot.lookup(-1, TAG_MTREE, 0x3)
if rattr_ is not None: if rattr_ is not None:
w_, block_, trunk_, cksum_ = frombtree(rattr_.data) w_, block_, trunk_, cksum_, _ = frombtree(rattr_.data)
mtree = Btree.fetchck(bd, block_, trunk_, w_, cksum_) mtree = Btree.fetchck(bd, block_, trunk_, w_, cksum_)
return cls(bd, mrootchain, mtree, return cls(bd, mrootchain, mtree,
@@ -1777,7 +1780,7 @@ class Mtree:
return (bid_, rbyd_, rid_), path_ return (bid_, rbyd_, rid_), path_
else: else:
return (bid_, rbyd_, rid_) return (bid_, rbyd_, rid_)
blocks_ = frommdir(rattr_.data) blocks_, _ = frommdir(rattr_.data)
mdir = Mdir.fetch(self.bd, mid, blocks_) mdir = Mdir.fetch(self.bd, mid, blocks_)
if path: if path:
return mdir, path_ return mdir, path_
@@ -2151,7 +2154,7 @@ class Mtree:
return (bid_, rbyd_, rid_), path_ return (bid_, rbyd_, rid_), path_
else: else:
return (bid_, rbyd_, rid_) return (bid_, rbyd_, rid_)
blocks_ = frommdir(rattr_.data) blocks_, _ = frommdir(rattr_.data)
mdir = Mdir.fetch(self.bd, self.mid(bid_), blocks_) mdir = Mdir.fetch(self.bd, self.mid(bid_), blocks_)
if path: if path:
return mdir, path_ return mdir, path_
@@ -2415,8 +2418,8 @@ class Config:
def __init__(self, mroot, tag, rattr): def __init__(self, mroot, tag, rattr):
super().__init__(mroot, tag, rattr) super().__init__(mroot, tag, rattr)
d = 0 d = 0
self.major, d_ = fromleb128(self.data[d:]); d += d_ self.major, d_ = fromleb128(self.data, d); d += d_
self.minor, d_ = fromleb128(self.data[d:]); d += d_ self.minor, d_ = fromleb128(self.data, d); d += d_
@property @property
def tuple(self): def tuple(self):
@@ -2455,8 +2458,8 @@ class Config:
def __init__(self, mroot, tag, rattr): def __init__(self, mroot, tag, rattr):
super().__init__(mroot, tag, rattr) super().__init__(mroot, tag, rattr)
d = 0 d = 0
block_size, d_ = fromleb128(self.data[d:]); d += d_ block_size, d_ = fromleb128(self.data, d); d += d_
block_count, d_ = fromleb128(self.data[d:]); d += d_ block_count, d_ = fromleb128(self.data, d); d += d_
# these are offset by 1 to avoid overflow issues # these are offset by 1 to avoid overflow issues
self.block_size = block_size + 1 self.block_size = block_size + 1
self.block_count = block_count + 1 self.block_count = block_count + 1
@@ -2660,11 +2663,11 @@ class Gstate:
def __init__(self, mtree, tag, gdeltas): def __init__(self, mtree, tag, gdeltas):
super().__init__(mtree, tag, gdeltas) super().__init__(mtree, tag, gdeltas)
d = 0 d = 0
count, d_ = fromleb128(self.data[d:]); d += d_ count, d_ = fromleb128(self.data, d); d += d_
rms = [] rms = []
if count <= 2: if count <= 2:
for _ in range(count): for _ in range(count):
mid, d_ = fromleb128(self.data[d:]); d += d_ mid, d_ = fromleb128(self.data, d); d += d_
rms.append(mtree.mid(mid)) rms.append(mtree.mid(mid))
self.count = count self.count = count
self.rms = rms self.rms = rms
@@ -3199,11 +3202,11 @@ class Lfs:
self.bshrub = None self.bshrub = None
if (self.struct is not None if (self.struct is not None
and (self.struct.tag & ~0x3) == TAG_BSHRUB): and (self.struct.tag & ~0x3) == TAG_BSHRUB):
weight, trunk = fromshrub(self.struct.data) weight, trunk, _ = fromshrub(self.struct.data)
self.bshrub = Btree.fetchshrub(lfs.bd, mdir.rbyd, trunk) self.bshrub = Btree.fetchshrub(lfs.bd, mdir.rbyd, trunk)
elif (self.struct is not None elif (self.struct is not None
and (self.struct.tag & ~0x3) == TAG_BTREE): and (self.struct.tag & ~0x3) == TAG_BTREE):
weight, block, trunk, cksum = frombtree(self.struct.data) weight, block, trunk, cksum, _ = frombtree(self.struct.data)
self.bshrub = Btree.fetchck( self.bshrub = Btree.fetchck(
lfs.bd, block, trunk, weight, cksum) lfs.bd, block, trunk, weight, cksum)
@@ -3323,7 +3326,7 @@ class Lfs:
return bid-(rattr.weight-1), rattr return bid-(rattr.weight-1), rattr
# block pointer? # block pointer?
elif (rattr.tag & ~0x1003) == TAG_BLOCK: elif (rattr.tag & ~0x1003) == TAG_BLOCK:
size, block, off, cksize, cksum = frombptr(rattr.data) size, block, off, cksize, cksum, _ = frombptr(rattr.data)
bptr = Bptr.fetchck(self.lfs.bd, rattr, bptr = Bptr.fetchck(self.lfs.bd, rattr,
block, off, size, cksize, cksum) block, off, size, cksize, cksum)
if path: if path:
scripts/dbgmtree.py
+44 -41
View File
@@ -138,46 +138,49 @@ def popc(x):
def parity(x): def parity(x):
return popc(x) & 1 return popc(x) & 1
def fromle32(data): def fromle32(data, j=0):
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0] return struct.unpack('<I', data[j:j+4].ljust(4, b'\0'))[0]
def fromleb128(data): def fromleb128(data, j=0):
word = 0 word = 0
for i, b in enumerate(data): d = 0
word |= ((b & 0x7f) << 7*i) while j+d < len(data):
b = data[j+d]
word |= (b & 0x7f) << 7*d
word &= 0xffffffff word &= 0xffffffff
if not b & 0x80: if not b & 0x80:
return word, i+1 return word, d+1
d += 1
return word, len(data) return word, len(data)
def fromtag(data): def fromtag(data, j=0):
data = data.ljust(4, b'\0') d = 0
tag = struct.unpack('>H', data[:2])[0] tag = struct.unpack('>H', data[j:j+2].ljust(2, b'\0'))[0]; d += 2
weight, d = fromleb128(data[2:]) weight, d_ = fromleb128(data, j+d); d += d_
size, d_ = fromleb128(data[2+d:]) size, d_ = fromleb128(data, j+d); d += d_
return tag>>15, tag&0x7fff, weight, size, 2+d+d_ return tag>>15, tag&0x7fff, weight, size, d
def frommdir(data): def frombranch(data, j=0):
d = 0
block, d_ = fromleb128(data, j+d); d += d_
trunk, d_ = fromleb128(data, j+d); d += d_
cksum = fromle32(data, j+d); d += 4
return block, trunk, cksum, d
def frombtree(data, j=0):
d = 0
w, d_ = fromleb128(data, j+d); d += d_
block, trunk, cksum, d_ = frombranch(data, j+d); d += d_
return w, block, trunk, cksum, d
def frommdir(data, j=0):
blocks = [] blocks = []
d = 0 d = 0
while d < len(data): while j+d < len(data):
block, d_ = fromleb128(data[d:]) block, d_ = fromleb128(data, j+d)
blocks.append(block) blocks.append(block)
d += d_ d += d_
return blocks return tuple(blocks), d
def frombranch(data):
d = 0
block, d_ = fromleb128(data[d:]); d += d_
trunk, d_ = fromleb128(data[d:]); d += d_
cksum = fromle32(data[d:]); d += 4
return block, trunk, cksum
def frombtree(data):
d = 0
w, d_ = fromleb128(data[d:]); d += d_
block, trunk, cksum = frombranch(data[d:])
return w, block, trunk, cksum
def xxd(data, width=16): def xxd(data, width=16):
for i in range(0, len(data), width): for i in range(0, len(data), width):
@@ -549,7 +552,7 @@ class Rbyd:
@classmethod @classmethod
def _fetch(cls, data, block, trunk=None): def _fetch(cls, data, block, trunk=None):
# fetch the rbyd # fetch the rbyd
rev = fromle32(data[0:4]) rev = fromle32(data, 0)
cksum = 0 cksum = 0
cksum_ = crc32c(data[0:4]) cksum_ = crc32c(data[0:4])
cksum__ = cksum_ cksum__ = cksum_
@@ -567,7 +570,7 @@ class Rbyd:
gcksumdelta_ = None gcksumdelta_ = None
while j_ < len(data) and (not trunk or eoff <= trunk): while j_ < len(data) and (not trunk or eoff <= trunk):
# read next tag # read next tag
v, tag, w, size, d = fromtag(data[j_:]) v, tag, w, size, d = fromtag(data, j_)
if v != parity(cksum__): if v != parity(cksum__):
break break
cksum__ ^= 0x00000080 if v else 0 cksum__ ^= 0x00000080 if v else 0
@@ -590,7 +593,7 @@ class Rbyd:
# found a cksum? # found a cksum?
else: else:
# check cksum # check cksum
cksum___ = fromle32(data[j_:j_+4]) cksum___ = fromle32(data, j_)
if cksum__ != cksum___: if cksum__ != cksum___:
break break
# commit what we have # commit what we have
@@ -729,7 +732,7 @@ class Rbyd:
# descend down tree # descend down tree
j = self.trunk j = self.trunk
while True: while True:
_, alt, w, jump, d = fromtag(self.data[j:]) _, alt, w, jump, d = fromtag(self.data, j)
# found an alt? # found an alt?
if alt & TAG_ALT: if alt & TAG_ALT:
@@ -745,7 +748,7 @@ class Rbyd:
if path: if path:
# figure out which color # figure out which color
if alt & TAG_R: if alt & TAG_R:
_, nalt, _, _, _ = fromtag(self.data[j+jump+d:]) _, nalt, _, _, _ = fromtag(self.data, j+jump+d)
if nalt & TAG_R: if nalt & TAG_R:
color = 'y' color = 'y'
else: else:
@@ -768,7 +771,7 @@ class Rbyd:
if path: if path:
# figure out which color # figure out which color
if alt & TAG_R: if alt & TAG_R:
_, nalt, _, _, _ = fromtag(self.data[j:]) _, nalt, _, _, _ = fromtag(self.data, j)
if nalt & TAG_R: if nalt & TAG_R:
color = 'y' color = 'y'
else: else:
@@ -1029,7 +1032,7 @@ class Btree:
# descend down branch? # descend down branch?
if branch_ is not None and ( if branch_ is not None and (
not depth or depth_ < depth): not depth or depth_ < depth):
block, trunk, cksum = frombranch(branch_.data) block, trunk, cksum, _ = frombranch(branch_.data)
rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight,
cksum) cksum)
@@ -1260,7 +1263,7 @@ class Btree:
# found another branch # found another branch
if branch_ is not None and ( if branch_ is not None and (
not depth or depth_ < depth): not depth or depth_ < depth):
block, trunk, cksum = frombranch(branch_.data) block, trunk, cksum, _ = frombranch(branch_.data)
rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight, rbyd = Rbyd.fetchck(self.bd, block, trunk, name_.weight,
cksum) cksum)
@@ -1654,7 +1657,7 @@ class Mtree:
rattr_ = mroot.lookup(-1, TAG_MROOT, 0x3) rattr_ = mroot.lookup(-1, TAG_MROOT, 0x3)
if rattr_ is None: if rattr_ is None:
break break
blocks_ = frommdir(rattr_.data) blocks_, _ = frommdir(rattr_.data)
mroot = Mdir.fetch(bd, -1, blocks_) mroot = Mdir.fetch(bd, -1, blocks_)
mrootchain.append(mroot) mrootchain.append(mroot)
@@ -1663,7 +1666,7 @@ class Mtree:
if not depth or len(mrootchain) < depth: if not depth or len(mrootchain) < depth:
rattr_ = mroot.lookup(-1, TAG_MTREE, 0x3) rattr_ = mroot.lookup(-1, TAG_MTREE, 0x3)
if rattr_ is not None: if rattr_ is not None:
w_, block_, trunk_, cksum_ = frombtree(rattr_.data) w_, block_, trunk_, cksum_, _ = frombtree(rattr_.data)
mtree = Btree.fetchck(bd, block_, trunk_, w_, cksum_) mtree = Btree.fetchck(bd, block_, trunk_, w_, cksum_)
return cls(bd, mrootchain, mtree, return cls(bd, mrootchain, mtree,
@@ -1743,7 +1746,7 @@ class Mtree:
return (bid_, rbyd_, rid_), path_ return (bid_, rbyd_, rid_), path_
else: else:
return (bid_, rbyd_, rid_) return (bid_, rbyd_, rid_)
blocks_ = frommdir(rattr_.data) blocks_, _ = frommdir(rattr_.data)
mdir = Mdir.fetch(self.bd, mid, blocks_) mdir = Mdir.fetch(self.bd, mid, blocks_)
if path: if path:
return mdir, path_ return mdir, path_
@@ -2117,7 +2120,7 @@ class Mtree:
return (bid_, rbyd_, rid_), path_ return (bid_, rbyd_, rid_), path_
else: else:
return (bid_, rbyd_, rid_) return (bid_, rbyd_, rid_)
blocks_ = frommdir(rattr_.data) blocks_, _ = frommdir(rattr_.data)
mdir = Mdir.fetch(self.bd, self.mid(bid_), blocks_) mdir = Mdir.fetch(self.bd, self.mid(bid_), blocks_)
if path: if path:
return mdir, path_ return mdir, path_
scripts/dbgrbyd.py
+27 -24
View File
@@ -148,24 +148,27 @@ def popc(x):
def parity(x): def parity(x):
return popc(x) & 1 return popc(x) & 1
def fromle32(data): def fromle32(data, j=0):
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0] return struct.unpack('<I', data[j:j+4].ljust(4, b'\0'))[0]
def fromleb128(data): def fromleb128(data, j=0):
word = 0 word = 0
for i, b in enumerate(data): d = 0
word |= ((b & 0x7f) << 7*i) while j+d < len(data):
b = data[j+d]
word |= (b & 0x7f) << 7*d
word &= 0xffffffff word &= 0xffffffff
if not b & 0x80: if not b & 0x80:
return word, i+1 return word, d+1
d += 1
return word, len(data) return word, len(data)
def fromtag(data): def fromtag(data, j=0):
data = data.ljust(4, b'\0') d = 0
tag = struct.unpack('>H', data[:2])[0] tag = struct.unpack('>H', data[j:j+2].ljust(2, b'\0'))[0]; d += 2
weight, d = fromleb128(data[2:]) weight, d_ = fromleb128(data, j+d); d += d_
size, d_ = fromleb128(data[2+d:]) size, d_ = fromleb128(data, j+d); d += d_
return tag>>15, tag&0x7fff, weight, size, 2+d+d_ return tag>>15, tag&0x7fff, weight, size, d
def xxd(data, width=16): def xxd(data, width=16):
for i in range(0, len(data), width): for i in range(0, len(data), width):
@@ -517,7 +520,7 @@ class Rbyd:
@classmethod @classmethod
def _fetch(cls, data, block, trunk=None): def _fetch(cls, data, block, trunk=None):
# fetch the rbyd # fetch the rbyd
rev = fromle32(data[0:4]) rev = fromle32(data, 0)
cksum = 0 cksum = 0
cksum_ = crc32c(data[0:4]) cksum_ = crc32c(data[0:4])
cksum__ = cksum_ cksum__ = cksum_
@@ -535,7 +538,7 @@ class Rbyd:
gcksumdelta_ = None gcksumdelta_ = None
while j_ < len(data) and (not trunk or eoff <= trunk): while j_ < len(data) and (not trunk or eoff <= trunk):
# read next tag # read next tag
v, tag, w, size, d = fromtag(data[j_:]) v, tag, w, size, d = fromtag(data, j_)
if v != parity(cksum__): if v != parity(cksum__):
break break
cksum__ ^= 0x00000080 if v else 0 cksum__ ^= 0x00000080 if v else 0
@@ -558,7 +561,7 @@ class Rbyd:
# found a cksum? # found a cksum?
else: else:
# check cksum # check cksum
cksum___ = fromle32(data[j_:j_+4]) cksum___ = fromle32(data, j_)
if cksum__ != cksum___: if cksum__ != cksum___:
break break
# commit what we have # commit what we have
@@ -697,7 +700,7 @@ class Rbyd:
# descend down tree # descend down tree
j = self.trunk j = self.trunk
while True: while True:
_, alt, w, jump, d = fromtag(self.data[j:]) _, alt, w, jump, d = fromtag(self.data, j)
# found an alt? # found an alt?
if alt & TAG_ALT: if alt & TAG_ALT:
@@ -713,7 +716,7 @@ class Rbyd:
if path: if path:
# figure out which color # figure out which color
if alt & TAG_R: if alt & TAG_R:
_, nalt, _, _, _ = fromtag(self.data[j+jump+d:]) _, nalt, _, _, _ = fromtag(self.data, j+jump+d)
if nalt & TAG_R: if nalt & TAG_R:
color = 'y' color = 'y'
else: else:
@@ -736,7 +739,7 @@ class Rbyd:
if path: if path:
# figure out which color # figure out which color
if alt & TAG_R: if alt & TAG_R:
_, nalt, _, _, _ = fromtag(self.data[j:]) _, nalt, _, _, _ = fromtag(self.data, j)
if nalt & TAG_R: if nalt & TAG_R:
color = 'y' color = 'y'
else: else:
@@ -944,7 +947,7 @@ class JumpArt:
j_ = 4 j_ = 4
while j_ < (len(rbyd.data) if all_ else rbyd.eoff): while j_ < (len(rbyd.data) if all_ else rbyd.eoff):
j = j_ j = j_
v, tag, w, size, d = fromtag(rbyd.data[j_:]) v, tag, w, size, d = fromtag(rbyd.data, j_)
j_ += d j_ += d
if not tag & TAG_ALT: if not tag & TAG_ALT:
j_ += size j_ += size
@@ -952,7 +955,7 @@ class JumpArt:
if tag & TAG_ALT and size: if tag & TAG_ALT and size:
# figure out which alt color # figure out which alt color
if tag & TAG_R: if tag & TAG_R:
_, ntag, _, _, _ = fromtag(rbyd.data[j_:]) _, ntag, _, _, _ = fromtag(rbyd.data, j_)
if ntag & TAG_R: if ntag & TAG_R:
jumps.append(cls.Jump(j, j-size, 0, 'y')) jumps.append(cls.Jump(j, j-size, 0, 'y'))
else: else:
@@ -1104,7 +1107,7 @@ class LifetimeArt:
j_ = 4 j_ = 4
while j_ < (len(rbyd.data) if all_ else rbyd.eoff): while j_ < (len(rbyd.data) if all_ else rbyd.eoff):
j = j_ j = j_
v, tag, w, size, d = fromtag(rbyd.data[j_:]) v, tag, w, size, d = fromtag(rbyd.data, j_)
j_ += d j_ += d
if not tag & TAG_ALT: if not tag & TAG_ALT:
j_ += size j_ += size
@@ -1498,7 +1501,7 @@ def dbg_log(rbyd, *,
j_ = 4 j_ = 4
while j_ < (len(data) if args.get('all') else rbyd.eoff): while j_ < (len(data) if args.get('all') else rbyd.eoff):
j = j_ j = j_
v, tag, w, size, d = fromtag(data[j_:]) v, tag, w, size, d = fromtag(data, j_)
j_ += d j_ += d
if not tag & TAG_ALT: if not tag & TAG_ALT:
@@ -1539,7 +1542,7 @@ def dbg_log(rbyd, *,
# read next tag # read next tag
j = j_ j = j_
v, tag, w, size, d = fromtag(data[j_:]) v, tag, w, size, d = fromtag(data, j_)
if v != parity(cksum_): if v != parity(cksum_):
notes.append('v!=%x' % parity(cksum_)) notes.append('v!=%x' % parity(cksum_))
cksum_ ^= 0x00000080 if v else 0 cksum_ ^= 0x00000080 if v else 0
@@ -1553,7 +1556,7 @@ def dbg_log(rbyd, *,
# found a cksum? # found a cksum?
else: else:
# check cksum # check cksum
cksum__ = fromle32(data[j_:j_+4]) cksum__ = fromle32(data, j_)
if cksum_ != cksum__: if cksum_ != cksum__:
notes.append('cksum!=%08x' % cksum__) notes.append('cksum!=%08x' % cksum__)
# update perturb bit # update perturb bit
scripts/dbgtag.py
+14 -11
View File
@@ -64,21 +64,24 @@ def openio(path, mode='r', buffering=-1):
else: else:
return open(path, mode, buffering) return open(path, mode, buffering)
def fromleb128(data): def fromleb128(data, j=0):
word = 0 word = 0
for i, b in enumerate(data): d = 0
word |= ((b & 0x7f) << 7*i) while j+d < len(data):
b = data[j+d]
word |= (b & 0x7f) << 7*d
word &= 0xffffffff word &= 0xffffffff
if not b & 0x80: if not b & 0x80:
return word, i+1 return word, d+1
d += 1
return word, len(data) return word, len(data)
def fromtag(data): def fromtag(data, j=0):
data = data.ljust(4, b'\0') d = 0
tag = struct.unpack('>H', data[:2])[0] tag = struct.unpack('>H', data[j:j+2].ljust(2, b'\0'))[0]; d += 2
weight, d = fromleb128(data[2:]) weight, d_ = fromleb128(data, j+d); d += d_
size, d_ = fromleb128(data[2+d:]) size, d_ = fromleb128(data, j+d); d += d_
return tag>>15, tag&0x7fff, weight, size, 2+d+d_ return tag>>15, tag&0x7fff, weight, size, d
# human readable tag repr # human readable tag repr
def tagrepr(tag, weight=None, size=None, *, def tagrepr(tag, weight=None, size=None, *,
@@ -244,7 +247,7 @@ def dbg_tags(data):
else: else:
j = 0 j = 0
while j < len(data): while j < len(data):
v, tag, w, size, d = fromtag(data[j:]) v, tag, w, size, d = fromtag(data, j)
lines.append(( lines.append((
' '.join('%02x' % b for b in data[j:j+d]), ' '.join('%02x' % b for b in data[j:j+d]),
tagrepr(tag, w, size))) tagrepr(tag, w, size)))