Github/littlefs
1
Fork 0
Code Actions Activity

Added ability to bypass rbyd fetch during B-tree lookups

Browse Source 
This is an absurd optimization that stems from the observation that the
branch encoding for the inner-rbyds in a B-tree is enough information to
jump directly to the trunk of the rbyd without needing an lfsr_rbyd_fetch.

This results in a pretty ridiculous performance jump from O(m log_m(n/m))
to O(log(m) log_m(n/m)).

If the complexity analysis isn't impressive enough, look at some rough
benchmarking of read operations for 4KiB-block, 1K-entry B-trees:

   12KiB ^     ::  :. :: .: .: :. : .: :. : : .. : : . : .: : : :
         |    .:: .::.::.:: ::.::::::::::::.::::::::.::::::::::::.
         |    : :::':: ::'::'::':: :' :':: :'::::::::': ::::::': :
before   |  ::: ::' :' :' :: :' '' '  ' '' : : : '' ' ' '
         | :::            ''
         |:
      0B :'------------------------------------------------------>

  .17KiB ^               ............:::::::::::::::::::::::::::::
         |   .   .....:::::'''''''''  '         '          '
         |  .::::::::::::
after    |  :':''
         |.::
         .:'
      0B :------------------------------------------------------->
         0                                                      1K

In order for this to work, the branch encoding did need to be tweaked
slightly. Before it stored block+off, now it stores block+trunk where
"trunk" is the offset of the entry point into the rbyd tree. Both off
and trunk are enough info to know when to stop fetching, if necessary,
but trunk allows lookups to jump directly into the branches rbyd tree
without a fetch.

With the change to trunk, lfsr_rbyd_fetch has also be extended to allow
fetching of any internal trunks, not just the last trunk in the commit.
This is very useful for dbgrbyd.py, but doesn't currently have a use in
littlefs itself. But it's at least valuable to have the feature available
in case it does become useful.

Note that two cases still requires the slower O(m log_m(n/m)) lookup
with lfsr_rbyd_fetch:

1. Name lookups, since we currently use a linear-search O(m) to find names.

2. Validating B-tree rbyd's, which requires a linear fetch O(m) to
   validate the checksums. We will need to do this at least once
   after mount.

It's also worth mentioning this will likely have a large impact on B-tree
traversal speed. Which is huge as I am expecting B-tree traversal to be
the main bottleneck once garbage-collection (or its replacement) is
involved.
This commit is contained in:
Christopher Haster
2023-04-09 02:21:02 -05:00
parent ed8d8c0c24
commit a511696bad
7 changed files with 1304 additions and 1130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+483 -372
View File
File diff suppressed because it is too large Load Diff
lfs.h
+40 -20
View File
@@ -330,12 +330,16 @@ typedef struct lfs_cache {
uint8_t *buffer;
} lfs_cache_t;
// TODO do we get ram savings with a lfsr_rorbyd_t substruct? need to measure
typedef struct lfsr_rbyd {
// note this lines up with weight in lfsr_btree_t
lfs_size_t weight;
lfs_block_t block;
// off=0, trunk=0 => not yet committed
// off=0, trunk>0 => not yet fetched
// off=block_size => rbyd not erased/needs compaction
lfs_off_t off;
lfs_off_t trunk;
lfs_size_t weight;
uint32_t rev;
uint32_t crc;
} lfsr_rbyd_t;
@@ -358,27 +362,43 @@ typedef struct lfsr_rbyd {
// - block addresses => 1 leb128 => 5 bytes (worst case)
#define LFSR_BTREE_INLINE_SIZE 5
typedef struct lfsr_branch {
lfs_block_t block;
lfs_size_t limit;
} lfsr_branch_t;
typedef struct lfsr_btree {
lfs_size_t weight;
// TODO do we need full tag actually? this fits in a byte?
lfsr_tag_t tag;
// how can we take advantage of byte packing with union alignment?
union {
struct {
uint8_t size;
uint8_t buf[LFSR_BTREE_INLINE_SIZE];
} inlined;
// if we're not inlined, point to the trunk rbyd block of the btree
lfsr_branch_t trunk;
} u;
typedef union lfsr_btree {
// note this lines up with weight in lfsr_rbyd_t
//
// weight=0 => null btree
// weight<0 => inlined btree
// weight>0 => normal btree
lfs_ssize_t weight;
lfsr_rbyd_t root;
struct {
lfs_ssize_t weight;
lfsr_tag_t tag;
uint16_t len;
uint8_t buf[LFSR_BTREE_INLINE_SIZE];
} inlined;
} lfsr_btree_t;
//typedef struct lfsr_branch {
// lfs_block_t block;
// lfs_size_t limit;
//} lfsr_branch_t;
//
//typedef struct lfsr_btree {
// lfs_size_t weight;
// // TODO do we need full tag actually? this fits in a byte?
// lfsr_tag_t tag;
// // how can we take advantage of byte packing with union alignment?
// union {
// struct {
// uint8_t size;
// uint8_t buf[LFSR_BTREE_INLINE_SIZE];
// } inlined;
//
// // if we're not inlined, point to the trunk rbyd block of the btree
// lfsr_branch_t trunk;
// } u;
//} lfsr_btree_t;
typedef struct lfs_mdir {
lfs_block_t pair[2];
uint32_t rev;
lfs_util.h
+5
View File
@@ -146,6 +146,11 @@ static inline uint16_t lfs_min16(uint16_t a, uint16_t b) {
return (a < b) ? a : b;
}
// Absolute value of signed numbers
static inline int32_t lfs_abs32(int32_t a) {
return a < 0 ? -a : a;
}
// TODO how many of these do we actually need
// Swap two 16-bit numbers
static inline void lfs_swap16(uint16_t *a, uint16_t *b) {
scripts/dbgbtree.py
+92 -81
View File
@@ -22,7 +22,7 @@ TAG_ALT = 0x0008
TAG_CRC = 0x0004
TAG_FCRC = 0x1004
def blocklim(s):
def rbydaddr(s):
if '.' in s:
s = s.strip()
b = 10
@@ -50,9 +50,10 @@ def crc32c(data, crc=0):
return 0xffffffff ^ crc
def fromle16(data):
if len(data) < 2:
return 0
return struct.unpack('<H', data[:2])[0]
return struct.unpack('<H', data[0:2].ljust(2, b'\0'))[0]
def fromle32(data):
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0]
def fromleb128(data):
word = 0
@@ -65,9 +66,9 @@ def fromleb128(data):
def fromtag(data):
tag = fromle16(data)
weight, delta = fromleb128(data[2:])
size, delta_ = fromleb128(data[2+delta:])
return tag&1, tag&~1, weight, size, 2+delta+delta_
weight, d = fromleb128(data[2:])
size, d_ = fromleb128(data[2+d:])
return tag&1, tag&~1, weight, size, 2+d+d_
def popc(x):
return bin(x).count('1')
@@ -134,9 +135,8 @@ def tagrepr(tag, w, size, off=None):
return '0x%04x w%d %d' % (tag, w, size)
class Rbyd:
def __init__(self, block, limit, data, rev, off, trunk, weight):
def __init__(self, block, data, rev, off, trunk, weight):
self.block = block
self.limit = limit
self.data = data
self.rev = rev
self.off = off
@@ -144,60 +144,74 @@ class Rbyd:
self.weight = weight
@classmethod
def fetch(cls, f, block_size, block, limit):
def fetch(cls, f, block_size, block, trunk):
# seek to the block
f.seek(block * block_size)
data = f.read(limit)
data = f.read(block_size)
# fetch the rbyd
rev, = struct.unpack('<I', data[0:4].ljust(4, b'\0'))
crc = crc32c(data[0:4])
rev = fromle32(data[0:4])
crc = 0
crc_ = crc32c(data[0:4])
off = 0
j_ = 4
trunk = None
trunk_ = None
trunk_ = 0
trunk__ = 0
weight = 0
lower_, upper_ = 0, 0
weight_ = 0
wastrunk = False
while j_ < limit:
v, tag, w, size, delta = fromtag(data[j_:])
if v != (popc(crc) & 1):
trunkoff = None
while j_ < len(data) and (not trunk or off <= trunk):
v, tag, w, size, d = fromtag(data[j_:])
if v != (popc(crc_) & 1):
break
crc_ = crc32c(data[j_:j_+d], crc_)
j_ += d
if not tag & 0x8 and j_ + size > len(data):
break
crc = crc32c(data[j_:j_+delta], crc)
j_ += delta
# find trunk
if not wastrunk and (tag & 0xc) != 0x4:
trunk_ = j_ - delta
lower_, upper_ = 0, 0
wastrunk = not not tag & 0x8
# keep track of weight
if tag & 0x8:
if tag & 0x4:
upper_ += w
else:
lower_ += w
elif (tag & 0xc) == 0x0:
weight_ = lower_+upper_+w
# take care of crcs
if not tag & 0x8:
if (tag & 0xf00f) != TAG_CRC:
crc = crc32c(data[j_:j_+size], crc)
crc_ = crc32c(data[j_:j_+size], crc_)
# found a crc?
else:
crc_, = struct.unpack('<I', data[j_:j_+4].ljust(4, b'\0'))
if crc != crc_:
crc__ = fromle32(data[j_:j_+4])
if crc_ != crc__:
break
# commit what we have
off = j_ + size
trunk = trunk_
off = trunkoff if trunkoff else j_ + size
crc = crc_
trunk_ = trunk__
weight = weight_
# evaluate trunks
if (tag & 0xc) != 0x4 and (
not trunk or trunk >= j_-d or wastrunk):
# new trunk?
if not wastrunk:
trunk__ = j_-d
lower_, upper_ = 0, 0
wastrunk = True
# keep track of weight
if tag & 0x8:
if tag & 0x4:
upper_ += w
else:
lower_ += w
else:
weight_ = lower_+upper_+w
wastrunk = False
# keep track of off for best matching trunk
if trunk and j_ + size > trunk:
trunkoff = j_ + size
if not tag & 0x8:
j_ += size
return Rbyd(block, limit, data, rev, off, trunk, weight)
return Rbyd(block, data, rev, off, trunk_, weight)
def lookup(self, id, tag):
if not self:
@@ -209,7 +223,7 @@ class Rbyd:
# descend down tree
j = self.trunk
while True:
_, alt, weight_, jump, delta = fromtag(self.data[j:])
_, alt, weight_, jump, d = fromtag(self.data[j:])
# found an alt?
if alt & 0x8:
@@ -224,7 +238,7 @@ class Rbyd:
else:
lower += weight_ if not alt & 0x4 else 0
upper -= weight_ if alt & 0x4 else 0
j = j + delta
j = j + d
# found tag
else:
id_ = upper-1
@@ -234,13 +248,13 @@ class Rbyd:
done = (id_, tag_) < (id, tag) or tag_ & 2
return (done, id_, tag_, w_,
j, delta, self.data[j+delta:j+delta+jump])
j, d, self.data[j+d:j+d+jump])
def __bool__(self):
return self.trunk is not None
return bool(self.trunk)
def __eq__(self, other):
return self.block == other.block and self.limit == other.limit
return self.block == other.block and self.trunk == other.trunk
def __ne__(self, other):
return not self.__eq__(other)
@@ -257,7 +271,9 @@ class Rbyd:
yield id, tag, w, j, d, data
def main(disk, block_size=None, trunk=0, limit=None, *,
def main(disk, root=0, *,
block_size=None,
trunk=None,
color='auto',
**args):
# figure out what color should be
@@ -268,11 +284,11 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
else:
color = False
# trunk may include a limit
if isinstance(trunk, tuple):
if limit is None:
limit = trunk[1]
trunk = trunk[0]
# root may encode a trunk
if isinstance(root, tuple):
if trunk is None:
trunk = root[1]
root = root[0]
# we seek around a bunch, so just keep the disk open
with open(disk, 'rb') as f:
@@ -281,18 +297,14 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
f.seek(0, os.SEEK_END)
block_size = f.tell()
# default limit to the block_size
if limit is None:
limit = block_size
# fetch the trunk
trunk = Rbyd.fetch(f, block_size, trunk, limit)
# fetch the root
btree = Rbyd.fetch(f, block_size, root, trunk)
print('btree 0x%x.%x, rev %d, weight %d' % (
trunk.block, trunk.limit, trunk.rev, trunk.weight))
btree.block, btree.trunk, btree.rev, btree.weight))
# look up an id, while keeping track of the search path
def lookup(id, depth=None):
rbyd = trunk
rbyd = btree
rid = id
depth_ = 1
path = []
@@ -337,9 +349,10 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
# is it another branch? continue down tree
if struct_tag == TAG_BRANCH and (
depth is None or depth_ < depth):
block, delta = fromleb128(struct_)
limit, _ = fromleb128(struct_[delta:])
rbyd = Rbyd.fetch(f, block_size, block, limit)
trunk, d1 = fromleb128(struct_)
block, d2 = fromleb128(struct_[d1:])
crc = fromle32(struct_[d1+d2:])
rbyd = Rbyd.fetch(f, block_size, block, trunk)
# corrupted? bail here so we can keep traversing the tree
if not rbyd:
@@ -371,7 +384,7 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
t_depth = max(t_depth, len(path))
t_width = 2*t_depth+2 if t_depth > 0 else 0
t_branches = [(0, trunk.weight)]
t_branches = [(0, btree.weight)]
def treerepr(id, w, leaf=True, depth=None):
branches_ = []
@@ -427,7 +440,7 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
# print header
w_width = 2*m.ceil(m.log10(max(1, trunk.weight)+1))+1
w_width = 2*m.ceil(m.log10(max(1, btree.weight)+1))+1
print('%-9s %*s%-*s %-22s %s' % (
'block',
t_width, '',
@@ -448,7 +461,7 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
# show human-readable representation
if name_tag:
print('%10s %s%*s %-22s %s' % (
'%04x.%04x:' % (rbyd.block, rbyd.limit)
'%04x.%04x:' % (rbyd.block, rbyd.trunk)
if prbyd is None or rbyd != prbyd
else '',
treerepr(id, w, True, depth) if args.get('tree') else '',
@@ -461,7 +474,7 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
for b in map(chr, name))))
prbyd = rbyd
print('%10s %s%*s %-22s %s' % (
'%04x.%04x:' % (rbyd.block, rbyd.limit)
'%04x.%04x:' % (rbyd.block, rbyd.trunk)
if prbyd is None or rbyd != prbyd
else '',
treerepr(id, w, not name_tag, depth) if args.get('tree') else '',
@@ -485,10 +498,9 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
w_width, '',
'%04x %08x %07x' % (name_tag, w, len(name)),
' %s' % ' '.join(
'%08x' % struct.unpack('<I',
'%08x' % fromle32(
rbyd.data[name_j+name_d+i*4
: name_j+name_d + min(i*4+4,len(name))]
.ljust(4, b'\0'))
: name_j+name_d + min(i*4+4,len(name))])
for i in range(min(m.ceil(len(name)/4), 3)))[:23]))
print('%9s %*s%*s %-22s%s' % (
'',
@@ -497,10 +509,9 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
'%04x %08x %07x' % (
struct_tag, w if not name_tag else 0, len(struct_)),
' %s' % ' '.join(
'%08x' % struct.unpack('<I',
'%08x' % fromle32(
rbyd.data[struct_j+struct_d+i*4
: struct_j+struct_d + min(i*4+4,len(struct_))]
.ljust(4, b'\0'))
: struct_j+struct_d + min(i*4+4,len(struct_))])
for i in range(min(m.ceil(len(struct_)/4), 3)))[:23]))
# show on-disk encoding of tags/data
@@ -550,7 +561,7 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
break
if args.get('inner') or args.get('tree'):
t_branches = [(0, trunk.weight)]
t_branches = [(0, btree.weight)]
changed = False
for i, (x, px) in enumerate(
it.zip_longest(path[:-1], ppath[:-1])):
@@ -576,10 +587,10 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
# corrupted? try to keep printing the tree
if not rbyd:
print('%04x.%04x: %s%s%s%s' % (
rbyd.block, rbyd.limit,
rbyd.block, rbyd.trunk,
treerepr(id, w) if args.get('tree') else '',
'\x1b[31m' if color else '',
'(corrupted rbyd 0x%x.%x)' % (rbyd.block, rbyd.limit),
'(corrupted rbyd 0x%x.%x)' % (rbyd.block, rbyd.trunk),
'\x1b[m' if color else ''))
prbyd = rbyd
@@ -616,18 +627,18 @@ if __name__ == "__main__":
'disk',
help="File containing the block device.")
parser.add_argument(
'trunk',
'root',
nargs='?',
type=blocklim,
help="Block address of the trunk of the tree.")
type=rbydaddr,
help="Block address of the root of the tree.")
parser.add_argument(
'-B', '--block-size',
type=lambda x: int(x, 0),
help="Block size in bytes.")
parser.add_argument(
'-L', '--limit',
'--trunk',
type=lambda x: int(x, 0),
help="Rbyd limit of the trunk of the tree (alias).")
help="Use this offset as the trunk of the tree.")
parser.add_argument(
'--color',
choices=['never', 'always', 'auto'],
scripts/dbgrbyd.py
+111 -102
View File
@@ -32,7 +32,7 @@ TAG_ALT = 0x0008
TAG_CRC = 0x0004
TAG_FCRC = 0x1004
def blocklim(s):
def rbydaddr(s):
if '.' in s:
s = s.strip()
b = 10
@@ -60,9 +60,10 @@ def crc32c(data, crc=0):
return 0xffffffff ^ crc
def fromle16(data):
if len(data) < 2:
return 0
return struct.unpack('<H', data[:2])[0]
return struct.unpack('<H', data[0:2].ljust(2, b'\0'))[0]
def fromle32(data):
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0]
def fromleb128(data):
word = 0
@@ -75,9 +76,9 @@ def fromleb128(data):
def fromtag(data):
tag = fromle16(data)
weight, delta = fromleb128(data[2:])
size, delta_ = fromleb128(data[2+delta:])
return tag&1, tag&~1, weight, size, 2+delta+delta_
weight, d = fromleb128(data[2:])
size, d_ = fromleb128(data[2+d:])
return tag&1, tag&~1, weight, size, 2+d+d_
def popc(x):
return bin(x).count('1')
@@ -143,7 +144,7 @@ def tagrepr(tag, w, size, off=None):
else:
return '0x%04x w%d %d' % (tag, w, size)
def show_log(block_size, data, rev, off, weight, *,
def show_log(data, block_size, rev, off, weight, *,
color=False,
**args):
crc = crc32c(data[0:4])
@@ -154,8 +155,8 @@ def show_log(block_size, data, rev, off, weight, *,
j_ = 4
while j_ < (block_size if args.get('all') else off):
j = j_
v, tag, w, size, delta = fromtag(data[j_:])
j_ += delta
v, tag, w, size, d = fromtag(data[j_:])
j_ += d
if not tag & 0x8:
j_ += size
@@ -251,8 +252,8 @@ def show_log(block_size, data, rev, off, weight, *,
j_ = 4
while j_ < (block_size if args.get('all') else off):
j = j_
v, tag, w, size, delta = fromtag(data[j_:])
j_ += delta
v, tag, w, size, d = fromtag(data[j_:])
j_ += d
if not tag & 0x8:
j_ += size
@@ -378,7 +379,11 @@ def show_log(block_size, data, rev, off, weight, *,
# print revision count
if args.get('raw'):
print('%8s: %s' % ('%04x' % 0, next(xxd(data[0:4]))))
print('%8s: %*s%*s %s' % (
'%04x' % 0,
lifetime_width, '',
w_width, '',
next(xxd(data[0:4]))))
# print tags
lower_, upper_ = 0, 0
@@ -388,12 +393,12 @@ def show_log(block_size, data, rev, off, weight, *,
notes = []
j = j_
v, tag, w, size, delta = fromtag(data[j_:])
v, tag, w, size, d = fromtag(data[j_:])
if v != (popc(crc) & 1):
notes.append('v!=%x' % (popc(crc) & 1))
tag &= ~1
crc = crc32c(data[j_:j_+delta], crc)
j_ += delta
crc = crc32c(data[j_:j_+d], crc)
j_ += d
# find trunk
if not wastrunk and (tag & 0xc) != 0x4:
@@ -415,7 +420,7 @@ def show_log(block_size, data, rev, off, weight, *,
crc = crc32c(data[j_:j_+size], crc)
# found a crc?
else:
crc_, = struct.unpack('<I', data[j_:j_+4].ljust(4, b'\0'))
crc_ = fromle32(data[j_:j_+4])
if crc != crc_:
notes.append('crc!=%08x' % crc)
j_ += size
@@ -433,7 +438,7 @@ def show_log(block_size, data, rev, off, weight, *,
'%-22s%s' % (
tagrepr(tag, w, size, j),
' %s' % next(xxd(
data[j+delta:j+delta+min(size, 8)], 8), '')
data[j+d:j+d+min(size, 8)], 8), '')
if not args.get('no_truncate')
and not tag & 0x8 else ''),
'\x1b[m' if color and j >= off else '',
@@ -453,9 +458,8 @@ def show_log(block_size, data, rev, off, weight, *,
'%-22s%s' % (
'%04x %08x %07x' % (tag, w, size),
' %s' % ' '.join(
'%08x' % struct.unpack('<I',
data[j+delta+i*4:j+delta+min(i*4+4,size)]
.ljust(4, b'\0'))
'%08x' % fromle32(
data[j+d+i*4:j+d+min(i*4+4,size)])
for i in range(min(m.ceil(size/4), 3)))[:23]
if not args.get('no_truncate')
and not tag & 0x8 else ''),
@@ -465,7 +469,7 @@ def show_log(block_size, data, rev, off, weight, *,
# show on-disk encoding of tags
if args.get('raw'):
for o, line in enumerate(xxd(data[j:j+delta])):
for o, line in enumerate(xxd(data[j:j+d])):
print('%s%8s: %*s%*s %s%s' % (
'\x1b[90m' if color and j >= off else '',
'%04x' % (j + o*16),
@@ -475,20 +479,20 @@ def show_log(block_size, data, rev, off, weight, *,
'\x1b[m' if color and j >= off else ''))
if args.get('raw') or args.get('no_truncate'):
if not tag & 0x8:
for o, line in enumerate(xxd(data[j+delta:j+delta+size])):
for o, line in enumerate(xxd(data[j+d:j+d+size])):
print('%s%8s: %*s%*s %s%s' % (
'\x1b[90m' if color and j >= off else '',
'%04x' % (j+delta + o*16),
'%04x' % (j+d + o*16),
lifetime_width, '',
w_width, '',
line,
'\x1b[m' if color and j >= off else ''))
def show_tree(block_size, data, rev, trunk, weight, *,
def show_tree(data, block_size, rev, trunk, weight, *,
color=False,
**args):
if trunk is None:
if not trunk:
return
# lookup a tag, returning also the search path for decoration
@@ -501,7 +505,7 @@ def show_tree(block_size, data, rev, trunk, weight, *,
# descend down tree
j = trunk
while True:
_, alt, w, jump, delta = fromtag(data[j:])
_, alt, w, jump, d = fromtag(data[j:])
# found an alt?
if alt & 0x8:
@@ -516,7 +520,7 @@ def show_tree(block_size, data, rev, trunk, weight, *,
if args.get('tree'):
# figure out which color
if alt & 0x2:
_, nalt, _, _, _ = fromtag(data[j+jump+delta:])
_, nalt, _, _, _ = fromtag(data[j+jump+d:])
if nalt & 0x2:
path.append((j+jump, j, True, 'y'))
else:
@@ -527,18 +531,18 @@ def show_tree(block_size, data, rev, trunk, weight, *,
else:
lower += w if not alt & 0x4 else 0
upper -= w if alt & 0x4 else 0
j = j + delta
j = j + d
if args.get('tree'):
# figure out which color
if alt & 0x2:
_, nalt, _, _, _ = fromtag(data[j:])
if nalt & 0x2:
path.append((j-delta, j, False, 'y'))
path.append((j-d, j, False, 'y'))
else:
path.append((j-delta, j, False, 'r'))
path.append((j-d, j, False, 'r'))
else:
path.append((j-delta, j, False, 'b'))
path.append((j-d, j, False, 'b'))
# found tag
else:
id_ = upper-1
@@ -547,7 +551,7 @@ def show_tree(block_size, data, rev, trunk, weight, *,
done = (id_, tag_) < (id, tag) or tag_ & 2
return done, id_, tag_, w_, j, delta, jump, path
return done, id_, tag_, w_, j, d, jump, path
# precompute tree
tree_width = 0
@@ -557,7 +561,7 @@ def show_tree(block_size, data, rev, trunk, weight, *,
id, tag = -1, 0
while True:
done, id, tag, w, j, delta, size, path = lookup(id, tag+0x10)
done, id, tag, w, j, d, size, path = lookup(id, tag+0x10)
# found end of tree?
if done:
break
@@ -692,7 +696,7 @@ def show_tree(block_size, data, rev, trunk, weight, *,
id, tag = -1, 0
while True:
done, id, tag, w, j, delta, size, path = lookup(id, tag+0x10)
done, id, tag, w, j, d, size, path = lookup(id, tag+0x10)
# found end of tree?
if done:
break
@@ -707,7 +711,7 @@ def show_tree(block_size, data, rev, trunk, weight, *,
if w > 0 else '',
tagrepr(tag, w, size, j),
' %s' % next(xxd(
data[j+delta:j+delta+min(size, 8)], 8), '')
data[j+d:j+d+min(size, 8)], 8), '')
if not args.get('no_truncate')
and not tag & 0x8 else '')))
@@ -720,16 +724,15 @@ def show_tree(block_size, data, rev, trunk, weight, *,
'%-22s%s' % (
'%04x %08x %07x' % (tag, 0xffffffff & id, size),
' %s' % ' '.join(
'%08x' % struct.unpack('<I',
data[j+delta+i*4:j+delta+min(i*4+4,size)]
.ljust(4, b'\0'))
'%08x' % fromle32(
data[j+d+i*4:j+d+min(i*4+4,size)])
for i in range(min(m.ceil(size/4), 3)))[:23]
if not args.get('no_truncate')
and not tag & 0x8 else '')))
# show on-disk encoding of tags
if args.get('raw'):
for o, line in enumerate(xxd(data[j:j+delta])):
for o, line in enumerate(xxd(data[j:j+d])):
print('%8s: %*s%*s %s' % (
'%04x' % (j + o*16),
tree_width, '',
@@ -737,16 +740,16 @@ def show_tree(block_size, data, rev, trunk, weight, *,
line))
if args.get('raw') or args.get('no_truncate'):
if not tag & 0x8:
for o, line in enumerate(xxd(data[j+delta:j+delta+size])):
for o, line in enumerate(xxd(data[j+d:j+d+size])):
print('%8s: %*s%*s %s' % (
'%04x' % (j+delta + o*16),
'%04x' % (j+d + o*16),
tree_width, '',
w_width, '',
line))
def main(disk, block_size=None, block1=0, block2=None, *,
limit=None,
def main(disk, block1=0, block2=None, *,
block_size=None,
trunk=None,
color='auto',
**args):
@@ -764,13 +767,13 @@ def main(disk, block_size=None, block1=0, block2=None, *,
f.seek(0, os.SEEK_END)
block_size = f.tell()
# blocks may also encode limits
# blocks may also encode trunks
blocks = [
block[0] if isinstance(block, tuple) else block
for block in [block1, block2]
if block is not None]
limits = [
limit if limit is not None
trunks = [
trunk if trunk is not None
else block[1] if isinstance(block, tuple)
else None
for block in [block1, block2]
@@ -778,68 +781,82 @@ def main(disk, block_size=None, block1=0, block2=None, *,
# read each block
datas = []
for block, limit in zip(blocks, limits):
for block in blocks:
f.seek(block * block_size)
datas.append(f.read(limit if limit is not None else block_size))
datas.append(f.read(block_size))
# first figure out which block as the most recent revision
def fetch(data):
rev, = struct.unpack('<I', data[0:4].ljust(4, b'\0'))
crc = crc32c(data[0:4])
def fetch(data, trunk):
rev = fromle32(data[0:4])
crc = 0
crc_ = crc32c(data[0:4])
off = 0
j_ = 4
trunk = None
trunk_ = None
trunk_ = 0
trunk__ = 0
weight = 0
lower_, upper_ = 0, 0
weight_ = 0
wastrunk = False
while j_ < len(data):
v, tag, w, size, delta = fromtag(data[j_:])
if v != (popc(crc) & 1):
trunkoff = None
while j_ < len(data) and (not trunk or off <= trunk):
v, tag, w, size, d = fromtag(data[j_:])
if v != (popc(crc_) & 1):
break
crc_ = crc32c(data[j_:j_+d], crc_)
j_ += d
if not tag & 0x8 and j_ + size > len(data):
break
crc = crc32c(data[j_:j_+delta], crc)
j_ += delta
# find trunk
if not wastrunk and (tag & 0xc) != 0x4:
trunk_ = j_ - delta
lower_, upper_ = 0, 0
wastrunk = not not tag & 0x8
# keep track of weight
if tag & 0x8:
if tag & 0x4:
upper_ += w
else:
lower_ += w
elif (tag & 0xc) == 0x0:
weight_ = lower_+upper_+w
# take care of crcs
if not tag & 0x8:
if (tag & 0xf00f) != TAG_CRC:
crc = crc32c(data[j_:j_+size], crc)
crc_ = crc32c(data[j_:j_+size], crc_)
# found a crc?
else:
crc_, = struct.unpack('<I', data[j_:j_+4].ljust(4, b'\0'))
if crc != crc_:
crc__ = fromle32(data[j_:j_+4])
if crc_ != crc__:
break
# commit what we have
off = j_ + size
trunk = trunk_
off = trunkoff if trunkoff else j_ + size
crc = crc_
trunk_ = trunk__
weight = weight_
# evaluate trunks
if (tag & 0xc) != 0x4 and (
not trunk or trunk >= j_-d or wastrunk):
# new trunk?
if not wastrunk:
trunk__ = j_-d
lower_, upper_ = 0, 0
wastrunk = True
# keep track of weight
if tag & 0x8:
if tag & 0x4:
upper_ += w
else:
lower_ += w
else:
weight_ = lower_+upper_+w
wastrunk = False
# keep track of off for best matching trunk
if trunk and j_ + size > trunk:
trunkoff = j_ + size
if not tag & 0x8:
j_ += size
return rev, off, trunk, weight
return rev, off, trunk_, weight
revs, offs, trunks, weights = [], [], [], []
revs, offs, trunks_, weights = [], [], [], []
i = 0
for data in datas:
rev, off, trunk_, weight = fetch(data)
for data, trunk in zip(datas, trunks):
rev, off, trunk_, weight = fetch(data, trunk)
revs.append(rev)
offs.append(off)
trunks.append(trunk_)
trunks_.append(trunk_)
weights.append(weight)
# compare with sequence arithmetic
@@ -847,25 +864,21 @@ def main(disk, block_size=None, block1=0, block2=None, *,
i = len(revs)-1
# print contents of the winning metadata block
block, limit, data, rev, off, trunk, weight = (
blocks[i], limits[i], datas[i], revs[i], offs[i],
trunk if trunk is not None else trunks[i],
weights[i])
block, data, rev, off, trunk, weight = (
blocks[i], datas[i], revs[i], offs[i], trunks_[i], weights[i])
print('rbyd 0x%x%s, rev %d, size %d, weight %d%s' % (
block, '.%x' % limit if limit is not None else '',
rev, off, weight,
' (was 0x%x%s, %d, %d, %d)' % (
blocks[~i], '.%x' % limits[~i] if limits[~i] is not None else '',
revs[~i], offs[~i], weights[~i])
print('rbyd 0x%x.%x, rev %d, size %d, weight %d%s' % (
block, trunk, rev, off, weight,
' (was 0x%x.%x, %d, %d, %d)' % (
blocks[~i], trunks_[~i], revs[~i], offs[~i], weights[~i])
if len(blocks) > 1 else ''))
if args.get('log'):
show_log(block_size, data, rev, off, weight,
show_log(data, block_size, rev, off, weight,
color=color,
**args)
else:
show_tree(block_size, data, rev, trunk, weight,
show_tree(data, block_size, rev, trunk, weight,
color=color,
**args)
@@ -885,21 +898,17 @@ if __name__ == "__main__":
parser.add_argument(
'block1',
nargs='?',
type=blocklim,
type=rbydaddr,
help="Block address of the first metadata block.")
parser.add_argument(
'block2',
nargs='?',
type=blocklim,
type=rbydaddr,
help="Block address of the second metadata block.")
parser.add_argument(
'-B', '--block-size',
type=lambda x: int(x, 0),
help="Block size in bytes.")
parser.add_argument(
'-L', '--limit',
type=lambda x: int(x, 0),
help="Use this offset as the rbyd limit.")
parser.add_argument(
'--trunk',
type=lambda x: int(x, 0),
tests/test_btree.toml
+412 -394
View File
File diff suppressed because it is too large Load Diff
tests/test_rbyd.toml
+161 -161
View File
File diff suppressed because it is too large Load Diff