scripts: Fixed O(n^2) slicing in Rbyd.fetch
Do you see the O(n^2) behavior in this loop?
j = 0
while j < len(data):
word, d = fromleb(data[j:])
j += d
The slice, data[j:], creates a O(n) copy every iteration of the loop.
A bit tricky. Or at least I found it tricky to notice. Maybe because
array indexing being cheap is baked into my brain...
Long story short, this repeated slicing resulted in O(n^2) behavior in
Rbyd.fetch and probably some other functions. Even though we don't care
_too_ much about performance in these scripts, having Rbyd.fetch run in
O(n^2) isn't great.
Tweaking all from* functions to take an optional index solves this, at
least on paper.
---
In practice I didn't actually find any measurable performance gain. I
guess array slicing in Python is optimized enough that the constant
factor takes over?
(Maybe it's being helped by us limiting Rbyd.fetch to block_size in most
scripts? I haven't tested NAND block sizes yet...)
Still, it's good to at least know this isn't a bottleneck.
This commit is contained in:
Christopher Haster
+27
-24
@@ -148,24 +148,27 @@ def popc(x):
|
||||
def parity(x):
|
||||
return popc(x) & 1
|
||||
|
||||
def fromle32(data):
|
||||
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0]
|
||||
def fromle32(data, j=0):
|
||||
return struct.unpack('<I', data[j:j+4].ljust(4, b'\0'))[0]
|
||||
|
||||
def fromleb128(data):
|
||||
def fromleb128(data, j=0):
|
||||
word = 0
|
||||
for i, b in enumerate(data):
|
||||
word |= ((b & 0x7f) << 7*i)
|
||||
d = 0
|
||||
while j+d < len(data):
|
||||
b = data[j+d]
|
||||
word |= (b & 0x7f) << 7*d
|
||||
word &= 0xffffffff
|
||||
if not b & 0x80:
|
||||
return word, i+1
|
||||
return word, d+1
|
||||
d += 1
|
||||
return word, len(data)
|
||||
|
||||
def fromtag(data):
|
||||
data = data.ljust(4, b'\0')
|
||||
tag = struct.unpack('>H', data[:2])[0]
|
||||
weight, d = fromleb128(data[2:])
|
||||
size, d_ = fromleb128(data[2+d:])
|
||||
return tag>>15, tag&0x7fff, weight, size, 2+d+d_
|
||||
def fromtag(data, j=0):
|
||||
d = 0
|
||||
tag = struct.unpack('>H', data[j:j+2].ljust(2, b'\0'))[0]; d += 2
|
||||
weight, d_ = fromleb128(data, j+d); d += d_
|
||||
size, d_ = fromleb128(data, j+d); d += d_
|
||||
return tag>>15, tag&0x7fff, weight, size, d
|
||||
|
||||
def xxd(data, width=16):
|
||||
for i in range(0, len(data), width):
|
||||
@@ -517,7 +520,7 @@ class Rbyd:
|
||||
@classmethod
|
||||
def _fetch(cls, data, block, trunk=None):
|
||||
# fetch the rbyd
|
||||
rev = fromle32(data[0:4])
|
||||
rev = fromle32(data, 0)
|
||||
cksum = 0
|
||||
cksum_ = crc32c(data[0:4])
|
||||
cksum__ = cksum_
|
||||
@@ -535,7 +538,7 @@ class Rbyd:
|
||||
gcksumdelta_ = None
|
||||
while j_ < len(data) and (not trunk or eoff <= trunk):
|
||||
# read next tag
|
||||
v, tag, w, size, d = fromtag(data[j_:])
|
||||
v, tag, w, size, d = fromtag(data, j_)
|
||||
if v != parity(cksum__):
|
||||
break
|
||||
cksum__ ^= 0x00000080 if v else 0
|
||||
@@ -558,7 +561,7 @@ class Rbyd:
|
||||
# found a cksum?
|
||||
else:
|
||||
# check cksum
|
||||
cksum___ = fromle32(data[j_:j_+4])
|
||||
cksum___ = fromle32(data, j_)
|
||||
if cksum__ != cksum___:
|
||||
break
|
||||
# commit what we have
|
||||
@@ -697,7 +700,7 @@ class Rbyd:
|
||||
# descend down tree
|
||||
j = self.trunk
|
||||
while True:
|
||||
_, alt, w, jump, d = fromtag(self.data[j:])
|
||||
_, alt, w, jump, d = fromtag(self.data, j)
|
||||
|
||||
# found an alt?
|
||||
if alt & TAG_ALT:
|
||||
@@ -713,7 +716,7 @@ class Rbyd:
|
||||
if path:
|
||||
# figure out which color
|
||||
if alt & TAG_R:
|
||||
_, nalt, _, _, _ = fromtag(self.data[j+jump+d:])
|
||||
_, nalt, _, _, _ = fromtag(self.data, j+jump+d)
|
||||
if nalt & TAG_R:
|
||||
color = 'y'
|
||||
else:
|
||||
@@ -736,7 +739,7 @@ class Rbyd:
|
||||
if path:
|
||||
# figure out which color
|
||||
if alt & TAG_R:
|
||||
_, nalt, _, _, _ = fromtag(self.data[j:])
|
||||
_, nalt, _, _, _ = fromtag(self.data, j)
|
||||
if nalt & TAG_R:
|
||||
color = 'y'
|
||||
else:
|
||||
@@ -944,7 +947,7 @@ class JumpArt:
|
||||
j_ = 4
|
||||
while j_ < (len(rbyd.data) if all_ else rbyd.eoff):
|
||||
j = j_
|
||||
v, tag, w, size, d = fromtag(rbyd.data[j_:])
|
||||
v, tag, w, size, d = fromtag(rbyd.data, j_)
|
||||
j_ += d
|
||||
if not tag & TAG_ALT:
|
||||
j_ += size
|
||||
@@ -952,7 +955,7 @@ class JumpArt:
|
||||
if tag & TAG_ALT and size:
|
||||
# figure out which alt color
|
||||
if tag & TAG_R:
|
||||
_, ntag, _, _, _ = fromtag(rbyd.data[j_:])
|
||||
_, ntag, _, _, _ = fromtag(rbyd.data, j_)
|
||||
if ntag & TAG_R:
|
||||
jumps.append(cls.Jump(j, j-size, 0, 'y'))
|
||||
else:
|
||||
@@ -1104,7 +1107,7 @@ class LifetimeArt:
|
||||
j_ = 4
|
||||
while j_ < (len(rbyd.data) if all_ else rbyd.eoff):
|
||||
j = j_
|
||||
v, tag, w, size, d = fromtag(rbyd.data[j_:])
|
||||
v, tag, w, size, d = fromtag(rbyd.data, j_)
|
||||
j_ += d
|
||||
if not tag & TAG_ALT:
|
||||
j_ += size
|
||||
@@ -1498,7 +1501,7 @@ def dbg_log(rbyd, *,
|
||||
j_ = 4
|
||||
while j_ < (len(data) if args.get('all') else rbyd.eoff):
|
||||
j = j_
|
||||
v, tag, w, size, d = fromtag(data[j_:])
|
||||
v, tag, w, size, d = fromtag(data, j_)
|
||||
j_ += d
|
||||
|
||||
if not tag & TAG_ALT:
|
||||
@@ -1539,7 +1542,7 @@ def dbg_log(rbyd, *,
|
||||
|
||||
# read next tag
|
||||
j = j_
|
||||
v, tag, w, size, d = fromtag(data[j_:])
|
||||
v, tag, w, size, d = fromtag(data, j_)
|
||||
if v != parity(cksum_):
|
||||
notes.append('v!=%x' % parity(cksum_))
|
||||
cksum_ ^= 0x00000080 if v else 0
|
||||
@@ -1553,7 +1556,7 @@ def dbg_log(rbyd, *,
|
||||
# found a cksum?
|
||||
else:
|
||||
# check cksum
|
||||
cksum__ = fromle32(data[j_:j_+4])
|
||||
cksum__ = fromle32(data, j_)
|
||||
if cksum_ != cksum__:
|
||||
notes.append('cksum!=%08x' % cksum__)
|
||||
# update perturb bit
|
||||
|
||||
Reference in New Issue
Block a user