Changed rbyd/btree namelookups to only compare raw bytes
This is a simplification of the rbyd/btree layers, but implies
behavioral changes to the mtree/mdir layers.
Instead of ordering by leb128 did + name:
82 02 61 61 61 < 81 04 62 62 62
(0x102, "aaa") (0x201, "bbb")
We now order by the raw encoding, lexicographically:
82 02 61 61 61 > 81 04 62 62 62
(0x102, "aaa") (0x201, "bbb")
This may be unintuitive, but note:
1. Files _within_ a directory are still ordered, since they share a did
prefix.
2. We don't really care about the relative ordering of dids, just
that they are unique. Changing the ordering at this level does not
interfere with any of our did-related functions.
3. The only thing we may care about is that the root, did=0, is the
first mtree entry. This is still true. No leb128 encoding is < 0x00
even after encoding.
The motivation for this change is to allow for other named-btrees in the
system that may used non-did-prefixed names. At least one of these makes
sense for a sort of "content-tree" (cksum -> data block mapping).
As a plus, this change makes it possible to compare names and do btree
namelookups without needing to decode the leb128 prefix. Although I'm
struggling a bit to figure out exactly where this is useful...
One downside, this ordering only works if dids are always stored in
their canonical encoding, that is, the smallest leb128 encoding possible
for a given did. I think this is a reasonable requirement for just our
dids.
Another downside is this did add a decent chunk of code.
I did try limiting the changes to lfsr_data_namecmp, but it didn't have
much impact. I guess most of the cost comes from the reworked
lfsr_data_cmp function, which, to be fair, is quite a bit more
complicated now (it now supports limited data<=>data comparisons):
code stack
before: 34148 2896
namecmp: 34324 (+0.5%) 2896 (+0.0%)
after: 34340 (+0.6%) 2896 (+0.0%)
This commit is contained in:
Christopher Haster
@@ -1175,134 +1175,6 @@ class Rbyd:
|
||||
else:
|
||||
return True, -1, 0, None
|
||||
|
||||
# lookup by name
|
||||
def namelookup(self, did, name):
|
||||
# binary search
|
||||
best = (False, -1, 0, 0)
|
||||
lower = 0
|
||||
upper = self.weight
|
||||
while lower < upper:
|
||||
done, rid, tag, w, j, d, data, _ = self.lookup(
|
||||
lower + (upper-1-lower)//2, TAG_NAME)
|
||||
if done:
|
||||
break
|
||||
|
||||
# treat vestigial names as a catch-all
|
||||
if ((tag == TAG_NAME and rid-(w-1) == 0)
|
||||
or (tag & 0xff00) != TAG_NAME):
|
||||
did_ = 0
|
||||
name_ = b''
|
||||
else:
|
||||
did_, d = fromleb128(data)
|
||||
name_ = data[d:]
|
||||
|
||||
# bisect search space
|
||||
if (did_, name_) > (did, name):
|
||||
upper = rid-(w-1)
|
||||
elif (did_, name_) < (did, name):
|
||||
lower = rid + 1
|
||||
|
||||
# keep track of best match
|
||||
best = (False, rid, tag, w)
|
||||
else:
|
||||
# found a match
|
||||
return True, rid, tag, w
|
||||
|
||||
return best
|
||||
|
||||
# lookup by name with this rbyd as the btree root
|
||||
def btree_namelookup(self, f, block_size, did, name):
|
||||
rbyd = self
|
||||
bid = 0
|
||||
|
||||
while True:
|
||||
found, rid, tag, w = rbyd.namelookup(did, name)
|
||||
done, rid_, tag_, w_, j, d, data, _ = rbyd.lookup(rid, TAG_STRUCT)
|
||||
|
||||
# found another branch
|
||||
if tag_ & 0xfff == TAG_BRANCH:
|
||||
# update our bid
|
||||
bid += rid - (w-1)
|
||||
|
||||
block, trunk, cksum = frombranch(data)
|
||||
rbyd = Rbyd.fetch(f, block_size, block, trunk)
|
||||
|
||||
# found best match
|
||||
else:
|
||||
return bid + rid, tag_, w, data
|
||||
|
||||
# lookup by name with this rbyd as the mroot
|
||||
def mtree_namelookup(self, f, block_size, did, name):
|
||||
# have mtree?
|
||||
done, rid, tag, w, j, d, data, _ = self.lookup(-1, TAG_MTREE)
|
||||
if not done and rid == -1 and tag == TAG_MTREE:
|
||||
w, block, trunk, cksum = frombtree(data)
|
||||
mtree = Rbyd.fetch(f, block_size, block, trunk)
|
||||
# corrupted?
|
||||
if not mtree:
|
||||
return False, -1, 0, None, -1, 0, 0
|
||||
|
||||
# lookup our name in the mtree
|
||||
mbid, tag_, mw, data = mtree.btree_namelookup(
|
||||
f, block_size, did, name)
|
||||
if tag_ != TAG_MDIR:
|
||||
return False, -1, 0, None, -1, 0, 0
|
||||
|
||||
# fetch the mdir
|
||||
blocks = frommdir(data)
|
||||
mdir = Rbyd.fetch(f, block_size, blocks)
|
||||
|
||||
else:
|
||||
# have mdir?
|
||||
done, rid, tag, w, j, _, data, _ = self.lookup(-1, TAG_MDIR)
|
||||
if not done and rid == -1 and tag == TAG_MDIR:
|
||||
blocks = frommdir(data)
|
||||
mbid = 0
|
||||
mw = 0
|
||||
mdir = Rbyd.fetch(f, block_size, blocks)
|
||||
|
||||
else:
|
||||
# I guess we're inlined?
|
||||
mbid = -1
|
||||
mw = 0
|
||||
mdir = self
|
||||
|
||||
# lookup name in our mdir
|
||||
found, rid, tag, w = mdir.namelookup(did, name)
|
||||
return found, mbid, mw, mdir, rid, tag, w
|
||||
|
||||
# iterate through a directory assuming this is the mtree root
|
||||
def mtree_dir(self, f, block_size, did):
|
||||
# lookup the bookmark
|
||||
found, mbid, mw, mdir, rid, tag, w = self.mtree_namelookup(
|
||||
f, block_size, did, b'')
|
||||
# iterate through all files until the next bookmark
|
||||
while found:
|
||||
# lookup each rid
|
||||
done, rid, tag, w, j, d, data, _ = mdir.lookup(rid, TAG_NAME)
|
||||
if done:
|
||||
break
|
||||
|
||||
# parse out each name
|
||||
did_, d_ = fromleb128(data)
|
||||
name_ = data[d_:]
|
||||
|
||||
# end if we see another did
|
||||
if did_ != did:
|
||||
break
|
||||
|
||||
# yield what we've found
|
||||
yield name_, mbid, mw, mdir, rid, tag, w
|
||||
|
||||
rid += w
|
||||
if rid >= mdir.weight:
|
||||
rid -= mdir.weight
|
||||
mbid += 1
|
||||
|
||||
done, mbid, mw, mdir = self.mtree_lookup(f, block_size, mbid)
|
||||
if done:
|
||||
break
|
||||
|
||||
|
||||
def main(disk, mroots=None, *,
|
||||
block_size=None,
|
||||
|
||||
Reference in New Issue
Block a user