Github/littlefs
1
Fork 0
Code Actions Activity

Dropped separate BTREE/BRANCH encodings

Browse Source 
There is a bit of redundancy here, as we already know the weights of
btree's inner-branches from their parents. But in theory sharing the
same encoding for both the top level btree reference and inner-branches
should offer more chance for deduplication and hopefully less code.

This also moves some members around in the btree encoding so that the
redund blocks are at the beginning. This _might_ simplify decoding of
the variable-length redund blocks at some point.

Current btree encoding:

  .----+----+----+----.
  |       blocks    ...  redund leb128s (1-20 bytes)
  :                   :
  |----+----+----+----|
  |       trunk     ...  1 leb128 (1-5 bytes)
  |----+----+----+----|
  |       weight    ...  1 leb128 (1-5 bytes)
  |----+----+----+----|
  |       cksum       |  1 le32 (4 bytes)
  '----+----+----+----'

This also partially reverts some tag name changes:

- BNAME -> BRANCH
- DMARK -> BOOKMARK
This commit is contained in:
Christopher Haster
2023-08-22 13:20:37 -05:00
parent f70e7fe709
commit 256430213d
6 changed files with 114 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/dbgbtree.py
+11 -13
View File
@@ -14,8 +14,8 @@ TAG_SUPERCONFIG = 0x0004
TAG_GSTATE = 0x0100
TAG_GRM = 0x0100
TAG_NAME = 0x0200
TAG_BNAME = 0x0200
TAG_DMARK = 0x0201
TAG_BRANCH = 0x0200
TAG_BOOKMARK = 0x0201
TAG_REG = 0x0202
TAG_DIR = 0x0203
TAG_STRUCT = 0x0300
@@ -25,8 +25,7 @@ TAG_BTREE = 0x030c
TAG_MDIR = 0x0311
TAG_MTREE = 0x0314
TAG_MROOT = 0x0318
TAG_BRANCH = 0x031c
TAG_DID = 0x0320
TAG_DID = 0x031c
TAG_UATTR = 0x0400
TAG_SATTR = 0x0500
TAG_ALT = 0x4000
@@ -34,7 +33,6 @@ TAG_CKSUM = 0x2000
TAG_ECKSUM = 0x2100
# parse some rbyd addr encodings
# 0xa -> [0xa]
# 0xa.b -> ([0xa], b)
@@ -98,12 +96,13 @@ def fromtag(data):
size, d_ = fromleb128(data[2+d:])
return tag>>15, tag&0x7fff, weight, size, 2+d+d_
def frombranch(data):
def frombtree(data):
d = 0
block, d_ = fromleb128(data[d:]); d += d_
trunk, d_ = fromleb128(data[d:]); d += d_
w, d_ = fromleb128(data[d:]); d += d_
cksum = fromle32(data[d:]); d += 4
return block, trunk, cksum
return block, trunk, w, cksum
def popc(x):
return bin(x).count('1')
@@ -139,8 +138,8 @@ def tagrepr(tag, w, size, off=None):
size)
elif (tag & 0xff00) == TAG_NAME:
return '%s%s %d' % (
'bname' if tag == TAG_BNAME
else 'dmark' if tag == TAG_DMARK
'branch' if tag == TAG_BRANCH
else 'bookmark' if tag == TAG_BOOKMARK
else 'reg' if tag == TAG_REG
else 'dir' if tag == TAG_DIR
else 'name 0x%02x' % (tag & 0xff),
@@ -154,7 +153,6 @@ def tagrepr(tag, w, size, off=None):
else 'mdir' if tag == TAG_MDIR
else 'mtree' if tag == TAG_MTREE
else 'mroot' if tag == TAG_MROOT
else 'branch' if tag == TAG_BRANCH
else 'did' if tag == TAG_DID
else 'struct 0x%02x' % (tag & 0xff),
' w%d' % w if w else '',
@@ -544,7 +542,7 @@ def main(disk, roots=None, *,
rid_, w = rid__, w_
# catch any branches
if tag == TAG_BRANCH:
if tag == TAG_BTREE:
branch = (tag, j, d, data)
tags.append((tag, j, d, data))
@@ -556,7 +554,7 @@ def main(disk, roots=None, *,
if branch is not None and (
not depth or depth_ < depth):
tag, j, d, data = branch
block, trunk, cksum = frombranch(data)
block, trunk, _, cksum = frombtree(data)
rbyd = Rbyd.fetch(f, block_size, block, trunk)
# corrupted? bail here so we can keep traversing the tree
@@ -650,7 +648,7 @@ def main(disk, roots=None, *,
))
d_ += max(bdepths.get(d, 0), 1)
leaf = (bid-(w-1), d, rid-(w-1), TAG_BRANCH)
leaf = (bid-(w-1), d, rid-(w-1), TAG_BTREE)
# remap branches to leaves if we aren't showing inner branches
if not args.get('inner'):
scripts/dbglfs.py
+24 -31
View File
@@ -15,8 +15,8 @@ TAG_SUPERCONFIG = 0x0004
TAG_GSTATE = 0x0100
TAG_GRM = 0x0100
TAG_NAME = 0x0200
TAG_BNAME = 0x0200
TAG_DMARK = 0x0201
TAG_BRANCH = 0x0200
TAG_BOOKMARK = 0x0201
TAG_REG = 0x0202
TAG_DIR = 0x0203
TAG_STRUCT = 0x0300
@@ -26,8 +26,7 @@ TAG_BTREE = 0x030c
TAG_MDIR = 0x0311
TAG_MTREE = 0x0314
TAG_MROOT = 0x0318
TAG_BRANCH = 0x031c
TAG_DID = 0x0320
TAG_DID = 0x031c
TAG_UATTR = 0x0400
TAG_SATTR = 0x0500
TAG_ALT = 0x4000
@@ -107,18 +106,13 @@ def frommdir(data):
d += d_
return blocks
def frombranch(data):
def frombtree(data):
d = 0
block, d_ = fromleb128(data[d:]); d += d_
trunk, d_ = fromleb128(data[d:]); d += d_
cksum = fromle32(data[d:]); d += 4
return block, trunk, cksum
def frombtree(data):
d = 0
w, d_ = fromleb128(data[d:]); d += d_
block, trunk, cksum = frombranch(data[d:])
return w, block, trunk, cksum
cksum = fromle32(data[d:]); d += 4
return block, trunk, w, cksum
def popc(x):
return bin(x).count('1')
@@ -154,8 +148,8 @@ def tagrepr(tag, w, size, off=None):
size)
elif (tag & 0xff00) == TAG_NAME:
return '%s%s %d' % (
'bname' if tag == TAG_BNAME
else 'dmark' if tag == TAG_DMARK
'branch' if tag == TAG_BRANCH
else 'bookmark' if tag == TAG_BOOKMARK
else 'reg' if tag == TAG_REG
else 'dir' if tag == TAG_DIR
else 'name 0x%02x' % (tag & 0xff),
@@ -169,7 +163,6 @@ def tagrepr(tag, w, size, off=None):
else 'mdir' if tag == TAG_MDIR
else 'mtree' if tag == TAG_MTREE
else 'mroot' if tag == TAG_MROOT
else 'branch' if tag == TAG_BRANCH
else 'did' if tag == TAG_DID
else 'struct 0x%02x' % (tag & 0xff),
' w%d' % w if w else '',
@@ -436,7 +429,7 @@ class Rbyd:
rid_, w = rid__, w_
# catch any branches
if tag == TAG_BRANCH:
if tag == TAG_BTREE:
branch = (tag, j, d, data)
tags.append((tag, j, d, data))
@@ -448,7 +441,7 @@ class Rbyd:
if branch is not None and (
not depth or depth_ < depth):
tag, j, d, data = branch
block, trunk, cksum = frombranch(data)
block, trunk, _, cksum = frombtree(data)
rbyd = Rbyd.fetch(f, block_size, block, trunk)
# corrupted? bail here so we can keep traversing the tree
@@ -465,7 +458,7 @@ class Rbyd:
# have mtree?
done, rid, tag, w, j, d, data, _ = self.lookup(-1, TAG_MTREE)
if not done and rid == -1 and tag == TAG_MTREE:
w, block, trunk, cksum = frombtree(data)
block, trunk, w, cksum = frombtree(data)
mtree = Rbyd.fetch(f, block_size, block, trunk)
# corrupted?
if not mtree:
@@ -516,7 +509,7 @@ class Rbyd:
break
# treat vestigial names as a catch-all
if ((tag == TAG_BNAME and rid-(w-1) == 0)
if ((tag == TAG_BRANCH and rid-(w-1) == 0)
or (tag & 0xff00) != TAG_NAME):
did_ = 0
name_ = b''
@@ -548,11 +541,11 @@ class Rbyd:
done, rid_, tag_, w_, j, d, data, _ = rbyd.lookup(rid, TAG_STRUCT)
# found another branch
if tag_ == TAG_BRANCH:
if tag_ == TAG_BTREE:
# update our bid
bid += rid - (w-1)
block, trunk, cksum = frombranch(data)
block, trunk, _, cksum = frombtree(data)
rbyd = Rbyd.fetch(f, block_size, block, trunk)
# found best match
@@ -564,7 +557,7 @@ class Rbyd:
# have mtree?
done, rid, tag, w, j, d, data, _ = self.lookup(-1, TAG_MTREE)
if not done and rid == -1 and tag == TAG_MTREE:
w, block, trunk, cksum = frombtree(data)
block, trunk, w, cksum = frombtree(data)
mtree = Rbyd.fetch(f, block_size, block, trunk)
# corrupted?
if not mtree:
@@ -716,7 +709,7 @@ def grepr(tag, data):
return 'gstate 0x%02x %d' % (tag, len(data))
def frepr(mdir, rid, tag):
if tag == TAG_DMARK:
if tag == TAG_BOOKMARK:
# read the did
did = '?'
done, rid_, tag_, w_, j, d, data, _ = mdir.lookup(rid, tag)
@@ -795,7 +788,7 @@ def main(disk, mroots=None, *,
did, d = fromleb128(data)
dir_dids.append(
(did, data[d:], -1, mroot, rid, tag, w))
elif tag == TAG_DMARK:
elif tag == TAG_BOOKMARK:
did, d = fromleb128(data)
bookmark_dids.append(
(did, data[d:], -1, mroot, rid, tag, w))
@@ -829,7 +822,7 @@ def main(disk, mroots=None, *,
did, d = fromleb128(data)
dir_dids.append((
did, data[d:], 0, mdir, rid, tag, w))
elif tag == TAG_DMARK:
elif tag == TAG_BOOKMARK:
did, d = fromleb128(data)
bookmark_dids.append((
did, data[d:], 0, mdir, rid, tag, w))
@@ -838,7 +831,7 @@ def main(disk, mroots=None, *,
mtree = None
done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MTREE)
if not done and rid == -1 and tag == TAG_MTREE:
w, block, trunk, cksum = frombtree(data)
block, trunk, w, cksum = frombtree(data)
mtree = Rbyd.fetch(f, block_size, block, trunk)
mweight = w
@@ -880,7 +873,7 @@ def main(disk, mroots=None, *,
did, d = fromleb128(data)
dir_dids.append((
did, data[d:], mid, mdir_, rid, tag, w))
elif tag == TAG_DMARK:
elif tag == TAG_BOOKMARK:
did, d = fromleb128(data)
bookmark_dids.append((
did, data[d:], mid, mdir_, rid, tag, w))
@@ -1077,7 +1070,7 @@ def main(disk, mroots=None, *,
f, block_size, did):
if not args.get('all'):
# skip bookmarks
if tag == TAG_DMARK:
if tag == TAG_BOOKMARK:
continue
# skip grmed entries
if (max(mid, 0), rid) in gstate.grm:
@@ -1112,7 +1105,7 @@ def main(disk, mroots=None, *,
if did_ not in grmed_bookmark_dids:
notes.append('missing bookmark')
# orphaned?
if tag == TAG_DMARK:
if tag == TAG_BOOKMARK:
done, rid_, tag_, w_, j, d, data, _ = mdir.lookup(
rid, tag)
if not done and rid_ == rid and tag_ == tag:
@@ -1122,7 +1115,7 @@ def main(disk, mroots=None, *,
# print human readable dtree entry
print('%s%12s %*s %-*s %s%s%s' % (
'\x1b[90m' if color and (grmed or tag == TAG_DMARK)
'\x1b[90m' if color and (grmed or tag == TAG_BOOKMARK)
else '',
'{%s}:' % ','.join('%04x' % block
for block in it.chain([mdir.block],
@@ -1140,7 +1133,7 @@ def main(disk, mroots=None, *,
', '.join(notes),
'\x1b[m' if color and not grmed else '')
if notes else '',
'\x1b[m' if color and (grmed or tag == TAG_DMARK)
'\x1b[m' if color and (grmed or tag == TAG_BOOKMARK)
else ''))
pmid = mid
scripts/dbgmtree.py
+13 -20
View File
@@ -14,8 +14,8 @@ TAG_SUPERCONFIG = 0x0004
TAG_GSTATE = 0x0100
TAG_GRM = 0x0100
TAG_NAME = 0x0200
TAG_BNAME = 0x0200
TAG_DMARK = 0x0201
TAG_BRANCH = 0x0200
TAG_BOOKMARK = 0x0201
TAG_REG = 0x0202
TAG_DIR = 0x0203
TAG_STRUCT = 0x0300
@@ -25,8 +25,7 @@ TAG_BTREE = 0x030c
TAG_MDIR = 0x0311
TAG_MTREE = 0x0314
TAG_MROOT = 0x0318
TAG_BRANCH = 0x031c
TAG_DID = 0x0320
TAG_DID = 0x031c
TAG_UATTR = 0x0400
TAG_SATTR = 0x0500
TAG_ALT = 0x4000
@@ -106,18 +105,13 @@ def frommdir(data):
d += d_
return blocks
def frombranch(data):
def frombtree(data):
d = 0
block, d_ = fromleb128(data[d:]); d += d_
trunk, d_ = fromleb128(data[d:]); d += d_
cksum = fromle32(data[d:]); d += 4
return block, trunk, cksum
def frombtree(data):
d = 0
w, d_ = fromleb128(data[d:]); d += d_
block, trunk, cksum = frombranch(data[d:])
return w, block, trunk, cksum
cksum = fromle32(data[d:]); d += 4
return block, trunk, w, cksum
def popc(x):
return bin(x).count('1')
@@ -153,8 +147,8 @@ def tagrepr(tag, w, size, off=None):
size)
elif (tag & 0xff00) == TAG_NAME:
return '%s%s %d' % (
'bname' if tag == TAG_BNAME
else 'dmark' if tag == TAG_DMARK
'branch' if tag == TAG_BRANCH
else 'bookmark' if tag == TAG_BOOKMARK
else 'reg' if tag == TAG_REG
else 'dir' if tag == TAG_DIR
else 'name 0x%02x' % (tag & 0xff),
@@ -168,7 +162,6 @@ def tagrepr(tag, w, size, off=None):
else 'mdir' if tag == TAG_MDIR
else 'mtree' if tag == TAG_MTREE
else 'mroot' if tag == TAG_MROOT
else 'branch' if tag == TAG_BRANCH
else 'did' if tag == TAG_DID
else 'struct 0x%02x' % (tag & 0xff),
' w%d' % w if w else '',
@@ -527,7 +520,7 @@ class Rbyd:
rid_, w = rid__, w_
# catch any branches
if tag == TAG_BRANCH:
if tag == TAG_BTREE:
branch = (tag, j, d, data)
tags.append((tag, j, d, data))
@@ -539,7 +532,7 @@ class Rbyd:
if branch is not None and (
not depth or depth_ < depth):
tag, j, d, data = branch
block, trunk, cksum = frombranch(data)
block, trunk, _, cksum = frombtree(data)
rbyd = Rbyd.fetch(f, block_size, block, trunk)
# corrupted? bail here so we can keep traversing the tree
@@ -634,7 +627,7 @@ class Rbyd:
))
d_ += max(bdepths.get(d, 0), 1)
leaf = (bid-(w-1), d, rid-(w-1), TAG_BRANCH)
leaf = (bid-(w-1), d, rid-(w-1), TAG_BTREE)
# remap branches to leaves if we aren't showing inner branches
if not inner:
@@ -813,7 +806,7 @@ def main(disk, mroots=None, *,
if not args.get('depth') or mdepth < args.get('depth'):
done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MTREE)
if not done and rid == -1 and tag == TAG_MTREE:
w, block, trunk, cksum = frombtree(data)
block, trunk, w, cksum = frombtree(data)
mtree = Rbyd.fetch(f, block_size, block, trunk)
mweight = w
@@ -1482,7 +1475,7 @@ def main(disk, mroots=None, *,
if not args.get('depth') or mdepth < args.get('depth'):
done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MTREE)
if not done and rid == -1 and tag == TAG_MTREE:
w, block, trunk, cksum = frombtree(data)
block, trunk, w, cksum = frombtree(data)
mtree = Rbyd.fetch(f, block_size, block, trunk)
# traverse entries
scripts/dbgrbyd.py
+5 -7
View File
@@ -23,8 +23,8 @@ TAG_SUPERCONFIG = 0x0004
TAG_GSTATE = 0x0100
TAG_GRM = 0x0100
TAG_NAME = 0x0200
TAG_BNAME = 0x0200
TAG_DMARK = 0x0201
TAG_BRANCH = 0x0200
TAG_BOOKMARK = 0x0201
TAG_REG = 0x0202
TAG_DIR = 0x0203
TAG_STRUCT = 0x0300
@@ -34,8 +34,7 @@ TAG_BTREE = 0x030c
TAG_MDIR = 0x0311
TAG_MTREE = 0x0314
TAG_MROOT = 0x0318
TAG_BRANCH = 0x031c
TAG_DID = 0x0320
TAG_DID = 0x031c
TAG_UATTR = 0x0400
TAG_SATTR = 0x0500
TAG_ALT = 0x4000
@@ -140,8 +139,8 @@ def tagrepr(tag, w, size, off=None):
size)
elif (tag & 0xff00) == TAG_NAME:
return '%s%s %d' % (
'bname' if tag == TAG_BNAME
else 'dmark' if tag == TAG_DMARK
'branch' if tag == TAG_BRANCH
else 'bookmark' if tag == TAG_BOOKMARK
else 'reg' if tag == TAG_REG
else 'dir' if tag == TAG_DIR
else 'name 0x%02x' % (tag & 0xff),
@@ -155,7 +154,6 @@ def tagrepr(tag, w, size, off=None):
else 'mdir' if tag == TAG_MDIR
else 'mtree' if tag == TAG_MTREE
else 'mroot' if tag == TAG_MROOT
else 'branch' if tag == TAG_BRANCH
else 'did' if tag == TAG_DID
else 'struct 0x%02x' % (tag & 0xff),
' w%d' % w if w else '',