Github/littlefs
1
Fork 0
Code Actions Activity

Added ability to bypass rbyd fetch during B-tree lookups

Browse Source 
This is an absurd optimization that stems from the observation that the
branch encoding for the inner-rbyds in a B-tree is enough information to
jump directly to the trunk of the rbyd without needing an lfsr_rbyd_fetch.

This results in a pretty ridiculous performance jump from O(m log_m(n/m))
to O(log(m) log_m(n/m)).

If the complexity analysis isn't impressive enough, look at some rough
benchmarking of read operations for 4KiB-block, 1K-entry B-trees:

   12KiB ^     ::  :. :: .: .: :. : .: :. : : .. : : . : .: : : :
         |    .:: .::.::.:: ::.::::::::::::.::::::::.::::::::::::.
         |    : :::':: ::'::'::':: :' :':: :'::::::::': ::::::': :
before   |  ::: ::' :' :' :: :' '' '  ' '' : : : '' ' ' '
         | :::            ''
         |:
      0B :'------------------------------------------------------>

  .17KiB ^               ............:::::::::::::::::::::::::::::
         |   .   .....:::::'''''''''  '         '          '
         |  .::::::::::::
after    |  :':''
         |.::
         .:'
      0B :------------------------------------------------------->
         0                                                      1K

In order for this to work, the branch encoding did need to be tweaked
slightly. Before it stored block+off, now it stores block+trunk where
"trunk" is the offset of the entry point into the rbyd tree. Both off
and trunk are enough info to know when to stop fetching, if necessary,
but trunk allows lookups to jump directly into the branches rbyd tree
without a fetch.

With the change to trunk, lfsr_rbyd_fetch has also be extended to allow
fetching of any internal trunks, not just the last trunk in the commit.
This is very useful for dbgrbyd.py, but doesn't currently have a use in
littlefs itself. But it's at least valuable to have the feature available
in case it does become useful.

Note that two cases still requires the slower O(m log_m(n/m)) lookup
with lfsr_rbyd_fetch:

1. Name lookups, since we currently use a linear-search O(m) to find names.

2. Validating B-tree rbyd's, which requires a linear fetch O(m) to
   validate the checksums. We will need to do this at least once
   after mount.

It's also worth mentioning this will likely have a large impact on B-tree
traversal speed. Which is huge as I am expecting B-tree traversal to be
the main bottleneck once garbage-collection (or its replacement) is
involved.
This commit is contained in:
Christopher Haster
2023-04-09 02:21:02 -05:00
parent ed8d8c0c24
commit a511696bad
7 changed files with 1304 additions and 1130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+483 -372
View File
File diff suppressed because it is too large Load Diff
lfs.h
+40 -20
View File
@@ -330,12 +330,16 @@ typedef struct lfs_cache {
uint8_t *buffer; uint8_t *buffer;
} lfs_cache_t; } lfs_cache_t;
// TODO do we get ram savings with a lfsr_rorbyd_t substruct? need to measure
typedef struct lfsr_rbyd { typedef struct lfsr_rbyd {
// note this lines up with weight in lfsr_btree_t
lfs_size_t weight;
lfs_block_t block; lfs_block_t block;
// off=0, trunk=0 => not yet committed
// off=0, trunk>0 => not yet fetched
// off=block_size => rbyd not erased/needs compaction // off=block_size => rbyd not erased/needs compaction
lfs_off_t off; lfs_off_t off;
lfs_off_t trunk; lfs_off_t trunk;
lfs_size_t weight;
uint32_t rev; uint32_t rev;
uint32_t crc; uint32_t crc;
} lfsr_rbyd_t; } lfsr_rbyd_t;
@@ -358,27 +362,43 @@ typedef struct lfsr_rbyd {
// - block addresses => 1 leb128 => 5 bytes (worst case) // - block addresses => 1 leb128 => 5 bytes (worst case)
#define LFSR_BTREE_INLINE_SIZE 5 #define LFSR_BTREE_INLINE_SIZE 5
typedef struct lfsr_branch { typedef union lfsr_btree {
lfs_block_t block; // note this lines up with weight in lfsr_rbyd_t
lfs_size_t limit; //
} lfsr_branch_t; // weight=0 => null btree
// weight<0 => inlined btree
typedef struct lfsr_btree { // weight>0 => normal btree
lfs_size_t weight; lfs_ssize_t weight;
// TODO do we need full tag actually? this fits in a byte? lfsr_rbyd_t root;
lfsr_tag_t tag; struct {
// how can we take advantage of byte packing with union alignment? lfs_ssize_t weight;
union { lfsr_tag_t tag;
struct { uint16_t len;
uint8_t size; uint8_t buf[LFSR_BTREE_INLINE_SIZE];
uint8_t buf[LFSR_BTREE_INLINE_SIZE]; } inlined;
} inlined;
// if we're not inlined, point to the trunk rbyd block of the btree
lfsr_branch_t trunk;
} u;
} lfsr_btree_t; } lfsr_btree_t;
//typedef struct lfsr_branch {
// lfs_block_t block;
// lfs_size_t limit;
//} lfsr_branch_t;
//
//typedef struct lfsr_btree {
// lfs_size_t weight;
// // TODO do we need full tag actually? this fits in a byte?
// lfsr_tag_t tag;
// // how can we take advantage of byte packing with union alignment?
// union {
// struct {
// uint8_t size;
// uint8_t buf[LFSR_BTREE_INLINE_SIZE];
// } inlined;
//
// // if we're not inlined, point to the trunk rbyd block of the btree
// lfsr_branch_t trunk;
// } u;
//} lfsr_btree_t;
typedef struct lfs_mdir { typedef struct lfs_mdir {
lfs_block_t pair[2]; lfs_block_t pair[2];
uint32_t rev; uint32_t rev;
lfs_util.h
+5
View File
@@ -146,6 +146,11 @@ static inline uint16_t lfs_min16(uint16_t a, uint16_t b) {
return (a < b) ? a : b; return (a < b) ? a : b;
} }
// Absolute value of signed numbers
static inline int32_t lfs_abs32(int32_t a) {
return a < 0 ? -a : a;
}
// TODO how many of these do we actually need // TODO how many of these do we actually need
// Swap two 16-bit numbers // Swap two 16-bit numbers
static inline void lfs_swap16(uint16_t *a, uint16_t *b) { static inline void lfs_swap16(uint16_t *a, uint16_t *b) {
scripts/dbgbtree.py
+92 -81
View File
@@ -22,7 +22,7 @@ TAG_ALT = 0x0008
TAG_CRC = 0x0004 TAG_CRC = 0x0004
TAG_FCRC = 0x1004 TAG_FCRC = 0x1004
def blocklim(s): def rbydaddr(s):
if '.' in s: if '.' in s:
s = s.strip() s = s.strip()
b = 10 b = 10
@@ -50,9 +50,10 @@ def crc32c(data, crc=0):
return 0xffffffff ^ crc return 0xffffffff ^ crc
def fromle16(data): def fromle16(data):
if len(data) < 2: return struct.unpack('<H', data[0:2].ljust(2, b'\0'))[0]
return 0
return struct.unpack('<H', data[:2])[0] def fromle32(data):
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0]
def fromleb128(data): def fromleb128(data):
word = 0 word = 0
@@ -65,9 +66,9 @@ def fromleb128(data):
def fromtag(data): def fromtag(data):
tag = fromle16(data) tag = fromle16(data)
weight, delta = fromleb128(data[2:]) weight, d = fromleb128(data[2:])
size, delta_ = fromleb128(data[2+delta:]) size, d_ = fromleb128(data[2+d:])
return tag&1, tag&~1, weight, size, 2+delta+delta_ return tag&1, tag&~1, weight, size, 2+d+d_
def popc(x): def popc(x):
return bin(x).count('1') return bin(x).count('1')
@@ -134,9 +135,8 @@ def tagrepr(tag, w, size, off=None):
return '0x%04x w%d %d' % (tag, w, size) return '0x%04x w%d %d' % (tag, w, size)
class Rbyd: class Rbyd:
def __init__(self, block, limit, data, rev, off, trunk, weight): def __init__(self, block, data, rev, off, trunk, weight):
self.block = block self.block = block
self.limit = limit
self.data = data self.data = data
self.rev = rev self.rev = rev
self.off = off self.off = off
@@ -144,60 +144,74 @@ class Rbyd:
self.weight = weight self.weight = weight
@classmethod @classmethod
def fetch(cls, f, block_size, block, limit): def fetch(cls, f, block_size, block, trunk):
# seek to the block # seek to the block
f.seek(block * block_size) f.seek(block * block_size)
data = f.read(limit) data = f.read(block_size)
# fetch the rbyd # fetch the rbyd
rev, = struct.unpack('<I', data[0:4].ljust(4, b'\0')) rev = fromle32(data[0:4])
crc = crc32c(data[0:4]) crc = 0
crc_ = crc32c(data[0:4])
off = 0 off = 0
j_ = 4 j_ = 4
trunk = None trunk_ = 0
trunk_ = None trunk__ = 0
weight = 0 weight = 0
lower_, upper_ = 0, 0 lower_, upper_ = 0, 0
weight_ = 0 weight_ = 0
wastrunk = False wastrunk = False
while j_ < limit: trunkoff = None
v, tag, w, size, delta = fromtag(data[j_:]) while j_ < len(data) and (not trunk or off <= trunk):
if v != (popc(crc) & 1): v, tag, w, size, d = fromtag(data[j_:])
if v != (popc(crc_) & 1):
break
crc_ = crc32c(data[j_:j_+d], crc_)
j_ += d
if not tag & 0x8 and j_ + size > len(data):
break break
crc = crc32c(data[j_:j_+delta], crc)
j_ += delta
# find trunk
if not wastrunk and (tag & 0xc) != 0x4:
trunk_ = j_ - delta
lower_, upper_ = 0, 0
wastrunk = not not tag & 0x8
# keep track of weight
if tag & 0x8:
if tag & 0x4:
upper_ += w
else:
lower_ += w
elif (tag & 0xc) == 0x0:
weight_ = lower_+upper_+w
# take care of crcs # take care of crcs
if not tag & 0x8: if not tag & 0x8:
if (tag & 0xf00f) != TAG_CRC: if (tag & 0xf00f) != TAG_CRC:
crc = crc32c(data[j_:j_+size], crc) crc_ = crc32c(data[j_:j_+size], crc_)
# found a crc? # found a crc?
else: else:
crc_, = struct.unpack('<I', data[j_:j_+4].ljust(4, b'\0')) crc__ = fromle32(data[j_:j_+4])
if crc != crc_: if crc_ != crc__:
break break
# commit what we have # commit what we have
off = j_ + size off = trunkoff if trunkoff else j_ + size
trunk = trunk_ crc = crc_
trunk_ = trunk__
weight = weight_ weight = weight_
# evaluate trunks
if (tag & 0xc) != 0x4 and (
not trunk or trunk >= j_-d or wastrunk):
# new trunk?
if not wastrunk:
trunk__ = j_-d
lower_, upper_ = 0, 0
wastrunk = True
# keep track of weight
if tag & 0x8:
if tag & 0x4:
upper_ += w
else:
lower_ += w
else:
weight_ = lower_+upper_+w
wastrunk = False
# keep track of off for best matching trunk
if trunk and j_ + size > trunk:
trunkoff = j_ + size
if not tag & 0x8:
j_ += size j_ += size
return Rbyd(block, limit, data, rev, off, trunk, weight) return Rbyd(block, data, rev, off, trunk_, weight)
def lookup(self, id, tag): def lookup(self, id, tag):
if not self: if not self:
@@ -209,7 +223,7 @@ class Rbyd:
# descend down tree # descend down tree
j = self.trunk j = self.trunk
while True: while True:
_, alt, weight_, jump, delta = fromtag(self.data[j:]) _, alt, weight_, jump, d = fromtag(self.data[j:])
# found an alt? # found an alt?
if alt & 0x8: if alt & 0x8:
@@ -224,7 +238,7 @@ class Rbyd:
else: else:
lower += weight_ if not alt & 0x4 else 0 lower += weight_ if not alt & 0x4 else 0
upper -= weight_ if alt & 0x4 else 0 upper -= weight_ if alt & 0x4 else 0
j = j + delta j = j + d
# found tag # found tag
else: else:
id_ = upper-1 id_ = upper-1
@@ -234,13 +248,13 @@ class Rbyd:
done = (id_, tag_) < (id, tag) or tag_ & 2 done = (id_, tag_) < (id, tag) or tag_ & 2
return (done, id_, tag_, w_, return (done, id_, tag_, w_,
j, delta, self.data[j+delta:j+delta+jump]) j, d, self.data[j+d:j+d+jump])
def __bool__(self): def __bool__(self):
return self.trunk is not None return bool(self.trunk)
def __eq__(self, other): def __eq__(self, other):
return self.block == other.block and self.limit == other.limit return self.block == other.block and self.trunk == other.trunk
def __ne__(self, other): def __ne__(self, other):
return not self.__eq__(other) return not self.__eq__(other)
@@ -257,7 +271,9 @@ class Rbyd:
yield id, tag, w, j, d, data yield id, tag, w, j, d, data
def main(disk, block_size=None, trunk=0, limit=None, *, def main(disk, root=0, *,
block_size=None,
trunk=None,
color='auto', color='auto',
**args): **args):
# figure out what color should be # figure out what color should be
@@ -268,11 +284,11 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
else: else:
color = False color = False
# trunk may include a limit # root may encode a trunk
if isinstance(trunk, tuple): if isinstance(root, tuple):
if limit is None: if trunk is None:
limit = trunk[1] trunk = root[1]
trunk = trunk[0] root = root[0]
# we seek around a bunch, so just keep the disk open # we seek around a bunch, so just keep the disk open
with open(disk, 'rb') as f: with open(disk, 'rb') as f:
@@ -281,18 +297,14 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
f.seek(0, os.SEEK_END) f.seek(0, os.SEEK_END)
block_size = f.tell() block_size = f.tell()
# default limit to the block_size # fetch the root
if limit is None: btree = Rbyd.fetch(f, block_size, root, trunk)
limit = block_size
# fetch the trunk
trunk = Rbyd.fetch(f, block_size, trunk, limit)
print('btree 0x%x.%x, rev %d, weight %d' % ( print('btree 0x%x.%x, rev %d, weight %d' % (
trunk.block, trunk.limit, trunk.rev, trunk.weight)) btree.block, btree.trunk, btree.rev, btree.weight))
# look up an id, while keeping track of the search path # look up an id, while keeping track of the search path
def lookup(id, depth=None): def lookup(id, depth=None):
rbyd = trunk rbyd = btree
rid = id rid = id
depth_ = 1 depth_ = 1
path = [] path = []
@@ -337,9 +349,10 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
# is it another branch? continue down tree # is it another branch? continue down tree
if struct_tag == TAG_BRANCH and ( if struct_tag == TAG_BRANCH and (
depth is None or depth_ < depth): depth is None or depth_ < depth):
block, delta = fromleb128(struct_) trunk, d1 = fromleb128(struct_)
limit, _ = fromleb128(struct_[delta:]) block, d2 = fromleb128(struct_[d1:])
rbyd = Rbyd.fetch(f, block_size, block, limit) crc = fromle32(struct_[d1+d2:])
rbyd = Rbyd.fetch(f, block_size, block, trunk)
# corrupted? bail here so we can keep traversing the tree # corrupted? bail here so we can keep traversing the tree
if not rbyd: if not rbyd:
@@ -371,7 +384,7 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
t_depth = max(t_depth, len(path)) t_depth = max(t_depth, len(path))
t_width = 2*t_depth+2 if t_depth > 0 else 0 t_width = 2*t_depth+2 if t_depth > 0 else 0
t_branches = [(0, trunk.weight)] t_branches = [(0, btree.weight)]
def treerepr(id, w, leaf=True, depth=None): def treerepr(id, w, leaf=True, depth=None):
branches_ = [] branches_ = []
@@ -427,7 +440,7 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
# print header # print header
w_width = 2*m.ceil(m.log10(max(1, trunk.weight)+1))+1 w_width = 2*m.ceil(m.log10(max(1, btree.weight)+1))+1
print('%-9s %*s%-*s %-22s %s' % ( print('%-9s %*s%-*s %-22s %s' % (
'block', 'block',
t_width, '', t_width, '',
@@ -448,7 +461,7 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
# show human-readable representation # show human-readable representation
if name_tag: if name_tag:
print('%10s %s%*s %-22s %s' % ( print('%10s %s%*s %-22s %s' % (
'%04x.%04x:' % (rbyd.block, rbyd.limit) '%04x.%04x:' % (rbyd.block, rbyd.trunk)
if prbyd is None or rbyd != prbyd if prbyd is None or rbyd != prbyd
else '', else '',
treerepr(id, w, True, depth) if args.get('tree') else '', treerepr(id, w, True, depth) if args.get('tree') else '',
@@ -461,7 +474,7 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
for b in map(chr, name)))) for b in map(chr, name))))
prbyd = rbyd prbyd = rbyd
print('%10s %s%*s %-22s %s' % ( print('%10s %s%*s %-22s %s' % (
'%04x.%04x:' % (rbyd.block, rbyd.limit) '%04x.%04x:' % (rbyd.block, rbyd.trunk)
if prbyd is None or rbyd != prbyd if prbyd is None or rbyd != prbyd
else '', else '',
treerepr(id, w, not name_tag, depth) if args.get('tree') else '', treerepr(id, w, not name_tag, depth) if args.get('tree') else '',
@@ -485,10 +498,9 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
w_width, '', w_width, '',
'%04x %08x %07x' % (name_tag, w, len(name)), '%04x %08x %07x' % (name_tag, w, len(name)),
' %s' % ' '.join( ' %s' % ' '.join(
'%08x' % struct.unpack('<I', '%08x' % fromle32(
rbyd.data[name_j+name_d+i*4 rbyd.data[name_j+name_d+i*4
: name_j+name_d + min(i*4+4,len(name))] : name_j+name_d + min(i*4+4,len(name))])
.ljust(4, b'\0'))
for i in range(min(m.ceil(len(name)/4), 3)))[:23])) for i in range(min(m.ceil(len(name)/4), 3)))[:23]))
print('%9s %*s%*s %-22s%s' % ( print('%9s %*s%*s %-22s%s' % (
'', '',
@@ -497,10 +509,9 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
'%04x %08x %07x' % ( '%04x %08x %07x' % (
struct_tag, w if not name_tag else 0, len(struct_)), struct_tag, w if not name_tag else 0, len(struct_)),
' %s' % ' '.join( ' %s' % ' '.join(
'%08x' % struct.unpack('<I', '%08x' % fromle32(
rbyd.data[struct_j+struct_d+i*4 rbyd.data[struct_j+struct_d+i*4
: struct_j+struct_d + min(i*4+4,len(struct_))] : struct_j+struct_d + min(i*4+4,len(struct_))])
.ljust(4, b'\0'))
for i in range(min(m.ceil(len(struct_)/4), 3)))[:23])) for i in range(min(m.ceil(len(struct_)/4), 3)))[:23]))
# show on-disk encoding of tags/data # show on-disk encoding of tags/data
@@ -550,7 +561,7 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
break break
if args.get('inner') or args.get('tree'): if args.get('inner') or args.get('tree'):
t_branches = [(0, trunk.weight)] t_branches = [(0, btree.weight)]
changed = False changed = False
for i, (x, px) in enumerate( for i, (x, px) in enumerate(
it.zip_longest(path[:-1], ppath[:-1])): it.zip_longest(path[:-1], ppath[:-1])):
@@ -576,10 +587,10 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
# corrupted? try to keep printing the tree # corrupted? try to keep printing the tree
if not rbyd: if not rbyd:
print('%04x.%04x: %s%s%s%s' % ( print('%04x.%04x: %s%s%s%s' % (
rbyd.block, rbyd.limit, rbyd.block, rbyd.trunk,
treerepr(id, w) if args.get('tree') else '', treerepr(id, w) if args.get('tree') else '',
'\x1b[31m' if color else '', '\x1b[31m' if color else '',
'(corrupted rbyd 0x%x.%x)' % (rbyd.block, rbyd.limit), '(corrupted rbyd 0x%x.%x)' % (rbyd.block, rbyd.trunk),
'\x1b[m' if color else '')) '\x1b[m' if color else ''))
prbyd = rbyd prbyd = rbyd
@@ -616,18 +627,18 @@ if __name__ == "__main__":
'disk', 'disk',
help="File containing the block device.") help="File containing the block device.")
parser.add_argument( parser.add_argument(
'trunk', 'root',
nargs='?', nargs='?',
type=blocklim, type=rbydaddr,
help="Block address of the trunk of the tree.") help="Block address of the root of the tree.")
parser.add_argument( parser.add_argument(
'-B', '--block-size', '-B', '--block-size',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Block size in bytes.") help="Block size in bytes.")
parser.add_argument( parser.add_argument(
'-L', '--limit', '--trunk',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Rbyd limit of the trunk of the tree (alias).") help="Use this offset as the trunk of the tree.")
parser.add_argument( parser.add_argument(
'--color', '--color',
choices=['never', 'always', 'auto'], choices=['never', 'always', 'auto'],
scripts/dbgrbyd.py
+111 -102
View File
@@ -32,7 +32,7 @@ TAG_ALT = 0x0008
TAG_CRC = 0x0004 TAG_CRC = 0x0004
TAG_FCRC = 0x1004 TAG_FCRC = 0x1004
def blocklim(s): def rbydaddr(s):
if '.' in s: if '.' in s:
s = s.strip() s = s.strip()
b = 10 b = 10
@@ -60,9 +60,10 @@ def crc32c(data, crc=0):
return 0xffffffff ^ crc return 0xffffffff ^ crc
def fromle16(data): def fromle16(data):
if len(data) < 2: return struct.unpack('<H', data[0:2].ljust(2, b'\0'))[0]
return 0
return struct.unpack('<H', data[:2])[0] def fromle32(data):
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0]
def fromleb128(data): def fromleb128(data):
word = 0 word = 0
@@ -75,9 +76,9 @@ def fromleb128(data):
def fromtag(data): def fromtag(data):
tag = fromle16(data) tag = fromle16(data)
weight, delta = fromleb128(data[2:]) weight, d = fromleb128(data[2:])
size, delta_ = fromleb128(data[2+delta:]) size, d_ = fromleb128(data[2+d:])
return tag&1, tag&~1, weight, size, 2+delta+delta_ return tag&1, tag&~1, weight, size, 2+d+d_
def popc(x): def popc(x):
return bin(x).count('1') return bin(x).count('1')
@@ -143,7 +144,7 @@ def tagrepr(tag, w, size, off=None):
else: else:
return '0x%04x w%d %d' % (tag, w, size) return '0x%04x w%d %d' % (tag, w, size)
def show_log(block_size, data, rev, off, weight, *, def show_log(data, block_size, rev, off, weight, *,
color=False, color=False,
**args): **args):
crc = crc32c(data[0:4]) crc = crc32c(data[0:4])
@@ -154,8 +155,8 @@ def show_log(block_size, data, rev, off, weight, *,
j_ = 4 j_ = 4
while j_ < (block_size if args.get('all') else off): while j_ < (block_size if args.get('all') else off):
j = j_ j = j_
v, tag, w, size, delta = fromtag(data[j_:]) v, tag, w, size, d = fromtag(data[j_:])
j_ += delta j_ += d
if not tag & 0x8: if not tag & 0x8:
j_ += size j_ += size
@@ -251,8 +252,8 @@ def show_log(block_size, data, rev, off, weight, *,
j_ = 4 j_ = 4
while j_ < (block_size if args.get('all') else off): while j_ < (block_size if args.get('all') else off):
j = j_ j = j_
v, tag, w, size, delta = fromtag(data[j_:]) v, tag, w, size, d = fromtag(data[j_:])
j_ += delta j_ += d
if not tag & 0x8: if not tag & 0x8:
j_ += size j_ += size
@@ -378,7 +379,11 @@ def show_log(block_size, data, rev, off, weight, *,
# print revision count # print revision count
if args.get('raw'): if args.get('raw'):
print('%8s: %s' % ('%04x' % 0, next(xxd(data[0:4])))) print('%8s: %*s%*s %s' % (
'%04x' % 0,
lifetime_width, '',
w_width, '',
next(xxd(data[0:4]))))
# print tags # print tags
lower_, upper_ = 0, 0 lower_, upper_ = 0, 0
@@ -388,12 +393,12 @@ def show_log(block_size, data, rev, off, weight, *,
notes = [] notes = []
j = j_ j = j_
v, tag, w, size, delta = fromtag(data[j_:]) v, tag, w, size, d = fromtag(data[j_:])
if v != (popc(crc) & 1): if v != (popc(crc) & 1):
notes.append('v!=%x' % (popc(crc) & 1)) notes.append('v!=%x' % (popc(crc) & 1))
tag &= ~1 tag &= ~1
crc = crc32c(data[j_:j_+delta], crc) crc = crc32c(data[j_:j_+d], crc)
j_ += delta j_ += d
# find trunk # find trunk
if not wastrunk and (tag & 0xc) != 0x4: if not wastrunk and (tag & 0xc) != 0x4:
@@ -415,7 +420,7 @@ def show_log(block_size, data, rev, off, weight, *,
crc = crc32c(data[j_:j_+size], crc) crc = crc32c(data[j_:j_+size], crc)
# found a crc? # found a crc?
else: else:
crc_, = struct.unpack('<I', data[j_:j_+4].ljust(4, b'\0')) crc_ = fromle32(data[j_:j_+4])
if crc != crc_: if crc != crc_:
notes.append('crc!=%08x' % crc) notes.append('crc!=%08x' % crc)
j_ += size j_ += size
@@ -433,7 +438,7 @@ def show_log(block_size, data, rev, off, weight, *,
'%-22s%s' % ( '%-22s%s' % (
tagrepr(tag, w, size, j), tagrepr(tag, w, size, j),
' %s' % next(xxd( ' %s' % next(xxd(
data[j+delta:j+delta+min(size, 8)], 8), '') data[j+d:j+d+min(size, 8)], 8), '')
if not args.get('no_truncate') if not args.get('no_truncate')
and not tag & 0x8 else ''), and not tag & 0x8 else ''),
'\x1b[m' if color and j >= off else '', '\x1b[m' if color and j >= off else '',
@@ -453,9 +458,8 @@ def show_log(block_size, data, rev, off, weight, *,
'%-22s%s' % ( '%-22s%s' % (
'%04x %08x %07x' % (tag, w, size), '%04x %08x %07x' % (tag, w, size),
' %s' % ' '.join( ' %s' % ' '.join(
'%08x' % struct.unpack('<I', '%08x' % fromle32(
data[j+delta+i*4:j+delta+min(i*4+4,size)] data[j+d+i*4:j+d+min(i*4+4,size)])
.ljust(4, b'\0'))
for i in range(min(m.ceil(size/4), 3)))[:23] for i in range(min(m.ceil(size/4), 3)))[:23]
if not args.get('no_truncate') if not args.get('no_truncate')
and not tag & 0x8 else ''), and not tag & 0x8 else ''),
@@ -465,7 +469,7 @@ def show_log(block_size, data, rev, off, weight, *,
# show on-disk encoding of tags # show on-disk encoding of tags
if args.get('raw'): if args.get('raw'):
for o, line in enumerate(xxd(data[j:j+delta])): for o, line in enumerate(xxd(data[j:j+d])):
print('%s%8s: %*s%*s %s%s' % ( print('%s%8s: %*s%*s %s%s' % (
'\x1b[90m' if color and j >= off else '', '\x1b[90m' if color and j >= off else '',
'%04x' % (j + o*16), '%04x' % (j + o*16),
@@ -475,20 +479,20 @@ def show_log(block_size, data, rev, off, weight, *,
'\x1b[m' if color and j >= off else '')) '\x1b[m' if color and j >= off else ''))
if args.get('raw') or args.get('no_truncate'): if args.get('raw') or args.get('no_truncate'):
if not tag & 0x8: if not tag & 0x8:
for o, line in enumerate(xxd(data[j+delta:j+delta+size])): for o, line in enumerate(xxd(data[j+d:j+d+size])):
print('%s%8s: %*s%*s %s%s' % ( print('%s%8s: %*s%*s %s%s' % (
'\x1b[90m' if color and j >= off else '', '\x1b[90m' if color and j >= off else '',
'%04x' % (j+delta + o*16), '%04x' % (j+d + o*16),
lifetime_width, '', lifetime_width, '',
w_width, '', w_width, '',
line, line,
'\x1b[m' if color and j >= off else '')) '\x1b[m' if color and j >= off else ''))
def show_tree(block_size, data, rev, trunk, weight, *, def show_tree(data, block_size, rev, trunk, weight, *,
color=False, color=False,
**args): **args):
if trunk is None: if not trunk:
return return
# lookup a tag, returning also the search path for decoration # lookup a tag, returning also the search path for decoration
@@ -501,7 +505,7 @@ def show_tree(block_size, data, rev, trunk, weight, *,
# descend down tree # descend down tree
j = trunk j = trunk
while True: while True:
_, alt, w, jump, delta = fromtag(data[j:]) _, alt, w, jump, d = fromtag(data[j:])
# found an alt? # found an alt?
if alt & 0x8: if alt & 0x8:
@@ -516,7 +520,7 @@ def show_tree(block_size, data, rev, trunk, weight, *,
if args.get('tree'): if args.get('tree'):
# figure out which color # figure out which color
if alt & 0x2: if alt & 0x2:
_, nalt, _, _, _ = fromtag(data[j+jump+delta:]) _, nalt, _, _, _ = fromtag(data[j+jump+d:])
if nalt & 0x2: if nalt & 0x2:
path.append((j+jump, j, True, 'y')) path.append((j+jump, j, True, 'y'))
else: else:
@@ -527,18 +531,18 @@ def show_tree(block_size, data, rev, trunk, weight, *,
else: else:
lower += w if not alt & 0x4 else 0 lower += w if not alt & 0x4 else 0
upper -= w if alt & 0x4 else 0 upper -= w if alt & 0x4 else 0
j = j + delta j = j + d
if args.get('tree'): if args.get('tree'):
# figure out which color # figure out which color
if alt & 0x2: if alt & 0x2:
_, nalt, _, _, _ = fromtag(data[j:]) _, nalt, _, _, _ = fromtag(data[j:])
if nalt & 0x2: if nalt & 0x2:
path.append((j-delta, j, False, 'y')) path.append((j-d, j, False, 'y'))
else: else:
path.append((j-delta, j, False, 'r')) path.append((j-d, j, False, 'r'))
else: else:
path.append((j-delta, j, False, 'b')) path.append((j-d, j, False, 'b'))
# found tag # found tag
else: else:
id_ = upper-1 id_ = upper-1
@@ -547,7 +551,7 @@ def show_tree(block_size, data, rev, trunk, weight, *,
done = (id_, tag_) < (id, tag) or tag_ & 2 done = (id_, tag_) < (id, tag) or tag_ & 2
return done, id_, tag_, w_, j, delta, jump, path return done, id_, tag_, w_, j, d, jump, path
# precompute tree # precompute tree
tree_width = 0 tree_width = 0
@@ -557,7 +561,7 @@ def show_tree(block_size, data, rev, trunk, weight, *,
id, tag = -1, 0 id, tag = -1, 0
while True: while True:
done, id, tag, w, j, delta, size, path = lookup(id, tag+0x10) done, id, tag, w, j, d, size, path = lookup(id, tag+0x10)
# found end of tree? # found end of tree?
if done: if done:
break break
@@ -692,7 +696,7 @@ def show_tree(block_size, data, rev, trunk, weight, *,
id, tag = -1, 0 id, tag = -1, 0
while True: while True:
done, id, tag, w, j, delta, size, path = lookup(id, tag+0x10) done, id, tag, w, j, d, size, path = lookup(id, tag+0x10)
# found end of tree? # found end of tree?
if done: if done:
break break
@@ -707,7 +711,7 @@ def show_tree(block_size, data, rev, trunk, weight, *,
if w > 0 else '', if w > 0 else '',
tagrepr(tag, w, size, j), tagrepr(tag, w, size, j),
' %s' % next(xxd( ' %s' % next(xxd(
data[j+delta:j+delta+min(size, 8)], 8), '') data[j+d:j+d+min(size, 8)], 8), '')
if not args.get('no_truncate') if not args.get('no_truncate')
and not tag & 0x8 else ''))) and not tag & 0x8 else '')))
@@ -720,16 +724,15 @@ def show_tree(block_size, data, rev, trunk, weight, *,
'%-22s%s' % ( '%-22s%s' % (
'%04x %08x %07x' % (tag, 0xffffffff & id, size), '%04x %08x %07x' % (tag, 0xffffffff & id, size),
' %s' % ' '.join( ' %s' % ' '.join(
'%08x' % struct.unpack('<I', '%08x' % fromle32(
data[j+delta+i*4:j+delta+min(i*4+4,size)] data[j+d+i*4:j+d+min(i*4+4,size)])
.ljust(4, b'\0'))
for i in range(min(m.ceil(size/4), 3)))[:23] for i in range(min(m.ceil(size/4), 3)))[:23]
if not args.get('no_truncate') if not args.get('no_truncate')
and not tag & 0x8 else ''))) and not tag & 0x8 else '')))
# show on-disk encoding of tags # show on-disk encoding of tags
if args.get('raw'): if args.get('raw'):
for o, line in enumerate(xxd(data[j:j+delta])): for o, line in enumerate(xxd(data[j:j+d])):
print('%8s: %*s%*s %s' % ( print('%8s: %*s%*s %s' % (
'%04x' % (j + o*16), '%04x' % (j + o*16),
tree_width, '', tree_width, '',
@@ -737,16 +740,16 @@ def show_tree(block_size, data, rev, trunk, weight, *,
line)) line))
if args.get('raw') or args.get('no_truncate'): if args.get('raw') or args.get('no_truncate'):
if not tag & 0x8: if not tag & 0x8:
for o, line in enumerate(xxd(data[j+delta:j+delta+size])): for o, line in enumerate(xxd(data[j+d:j+d+size])):
print('%8s: %*s%*s %s' % ( print('%8s: %*s%*s %s' % (
'%04x' % (j+delta + o*16), '%04x' % (j+d + o*16),
tree_width, '', tree_width, '',
w_width, '', w_width, '',
line)) line))
def main(disk, block_size=None, block1=0, block2=None, *, def main(disk, block1=0, block2=None, *,
limit=None, block_size=None,
trunk=None, trunk=None,
color='auto', color='auto',
**args): **args):
@@ -764,13 +767,13 @@ def main(disk, block_size=None, block1=0, block2=None, *,
f.seek(0, os.SEEK_END) f.seek(0, os.SEEK_END)
block_size = f.tell() block_size = f.tell()
# blocks may also encode limits # blocks may also encode trunks
blocks = [ blocks = [
block[0] if isinstance(block, tuple) else block block[0] if isinstance(block, tuple) else block
for block in [block1, block2] for block in [block1, block2]
if block is not None] if block is not None]
limits = [ trunks = [
limit if limit is not None trunk if trunk is not None
else block[1] if isinstance(block, tuple) else block[1] if isinstance(block, tuple)
else None else None
for block in [block1, block2] for block in [block1, block2]
@@ -778,68 +781,82 @@ def main(disk, block_size=None, block1=0, block2=None, *,
# read each block # read each block
datas = [] datas = []
for block, limit in zip(blocks, limits): for block in blocks:
f.seek(block * block_size) f.seek(block * block_size)
datas.append(f.read(limit if limit is not None else block_size)) datas.append(f.read(block_size))
# first figure out which block as the most recent revision # first figure out which block as the most recent revision
def fetch(data): def fetch(data, trunk):
rev, = struct.unpack('<I', data[0:4].ljust(4, b'\0')) rev = fromle32(data[0:4])
crc = crc32c(data[0:4]) crc = 0
crc_ = crc32c(data[0:4])
off = 0 off = 0
j_ = 4 j_ = 4
trunk = None trunk_ = 0
trunk_ = None trunk__ = 0
weight = 0 weight = 0
lower_, upper_ = 0, 0 lower_, upper_ = 0, 0
weight_ = 0 weight_ = 0
wastrunk = False wastrunk = False
while j_ < len(data): trunkoff = None
v, tag, w, size, delta = fromtag(data[j_:]) while j_ < len(data) and (not trunk or off <= trunk):
if v != (popc(crc) & 1): v, tag, w, size, d = fromtag(data[j_:])
if v != (popc(crc_) & 1):
break
crc_ = crc32c(data[j_:j_+d], crc_)
j_ += d
if not tag & 0x8 and j_ + size > len(data):
break break
crc = crc32c(data[j_:j_+delta], crc)
j_ += delta
# find trunk
if not wastrunk and (tag & 0xc) != 0x4:
trunk_ = j_ - delta
lower_, upper_ = 0, 0
wastrunk = not not tag & 0x8
# keep track of weight
if tag & 0x8:
if tag & 0x4:
upper_ += w
else:
lower_ += w
elif (tag & 0xc) == 0x0:
weight_ = lower_+upper_+w
# take care of crcs # take care of crcs
if not tag & 0x8: if not tag & 0x8:
if (tag & 0xf00f) != TAG_CRC: if (tag & 0xf00f) != TAG_CRC:
crc = crc32c(data[j_:j_+size], crc) crc_ = crc32c(data[j_:j_+size], crc_)
# found a crc? # found a crc?
else: else:
crc_, = struct.unpack('<I', data[j_:j_+4].ljust(4, b'\0')) crc__ = fromle32(data[j_:j_+4])
if crc != crc_: if crc_ != crc__:
break break
# commit what we have # commit what we have
off = j_ + size off = trunkoff if trunkoff else j_ + size
trunk = trunk_ crc = crc_
trunk_ = trunk__
weight = weight_ weight = weight_
# evaluate trunks
if (tag & 0xc) != 0x4 and (
not trunk or trunk >= j_-d or wastrunk):
# new trunk?
if not wastrunk:
trunk__ = j_-d
lower_, upper_ = 0, 0
wastrunk = True
# keep track of weight
if tag & 0x8:
if tag & 0x4:
upper_ += w
else:
lower_ += w
else:
weight_ = lower_+upper_+w
wastrunk = False
# keep track of off for best matching trunk
if trunk and j_ + size > trunk:
trunkoff = j_ + size
if not tag & 0x8:
j_ += size j_ += size
return rev, off, trunk, weight return rev, off, trunk_, weight
revs, offs, trunks, weights = [], [], [], [] revs, offs, trunks_, weights = [], [], [], []
i = 0 i = 0
for data in datas: for data, trunk in zip(datas, trunks):
rev, off, trunk_, weight = fetch(data) rev, off, trunk_, weight = fetch(data, trunk)
revs.append(rev) revs.append(rev)
offs.append(off) offs.append(off)
trunks.append(trunk_) trunks_.append(trunk_)
weights.append(weight) weights.append(weight)
# compare with sequence arithmetic # compare with sequence arithmetic
@@ -847,25 +864,21 @@ def main(disk, block_size=None, block1=0, block2=None, *,
i = len(revs)-1 i = len(revs)-1
# print contents of the winning metadata block # print contents of the winning metadata block
block, limit, data, rev, off, trunk, weight = ( block, data, rev, off, trunk, weight = (
blocks[i], limits[i], datas[i], revs[i], offs[i], blocks[i], datas[i], revs[i], offs[i], trunks_[i], weights[i])
trunk if trunk is not None else trunks[i],
weights[i])
print('rbyd 0x%x%s, rev %d, size %d, weight %d%s' % ( print('rbyd 0x%x.%x, rev %d, size %d, weight %d%s' % (
block, '.%x' % limit if limit is not None else '', block, trunk, rev, off, weight,
rev, off, weight, ' (was 0x%x.%x, %d, %d, %d)' % (
' (was 0x%x%s, %d, %d, %d)' % ( blocks[~i], trunks_[~i], revs[~i], offs[~i], weights[~i])
blocks[~i], '.%x' % limits[~i] if limits[~i] is not None else '',
revs[~i], offs[~i], weights[~i])
if len(blocks) > 1 else '')) if len(blocks) > 1 else ''))
if args.get('log'): if args.get('log'):
show_log(block_size, data, rev, off, weight, show_log(data, block_size, rev, off, weight,
color=color, color=color,
**args) **args)
else: else:
show_tree(block_size, data, rev, trunk, weight, show_tree(data, block_size, rev, trunk, weight,
color=color, color=color,
**args) **args)
@@ -885,21 +898,17 @@ if __name__ == "__main__":
parser.add_argument( parser.add_argument(
'block1', 'block1',
nargs='?', nargs='?',
type=blocklim, type=rbydaddr,
help="Block address of the first metadata block.") help="Block address of the first metadata block.")
parser.add_argument( parser.add_argument(
'block2', 'block2',
nargs='?', nargs='?',
type=blocklim, type=rbydaddr,
help="Block address of the second metadata block.") help="Block address of the second metadata block.")
parser.add_argument( parser.add_argument(
'-B', '--block-size', '-B', '--block-size',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
help="Block size in bytes.") help="Block size in bytes.")
parser.add_argument(
'-L', '--limit',
type=lambda x: int(x, 0),
help="Use this offset as the rbyd limit.")
parser.add_argument( parser.add_argument(
'--trunk', '--trunk',
type=lambda x: int(x, 0), type=lambda x: int(x, 0),
tests/test_btree.toml
+412 -394
View File
File diff suppressed because it is too large Load Diff
tests/test_rbyd.toml
+161 -161
View File
File diff suppressed because it is too large Load Diff