Github/littlefs
1
Fork 0
Code Actions Activity

Moved bits around in tag encoding to allow leb128 custom attributes

Browse Source 
Yet another tag encoding, but hopefully narrowing in on a good long term
design. This change trades a subtype bit for the ability to extend
subtypes indefinitely via leb128 in the future.

The immediate benefit is ~unlimited custom attributes, though I'm not
sure how to make this configurable yet. Extended custom attributes may
have a significant impact on alt tag sizes, so it may be worth
defaulting to only 8-bit custom attributes still.

Tag encoding:

   vmmmtttt 0TTTTTTT 0wwwwwww 0sssssss
   ^--^---^--------^--------^--------^- valid bit
      '---|--------|--------|--------|- 3-bit mode
          '--------|--------|--------|- 4-bit suptype
                   '--------|--------|- leb128 subtype
                            '--------|- leb128 weight
                                     '- leb128 size/jump

This limits subtypes to 7-bits, but this seems very reasonable at the
moment.

This also seems to limit custom attributes to 7-bits, but we can use two
separate suptypes to bring this back up to 8-bits. I was planning to do
this anyways to have separate "user-attributes" and "system-attributes",
so this actually fits in really well.
This commit is contained in:
Christopher Haster
2023-06-16 01:41:42 -05:00
parent 4ff7c1f771
commit 2113d877d6
4 changed files with 404 additions and 405 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/dbgrbyd.py
+99 -107
View File
@@ -17,23 +17,23 @@ COLORS = [
]
TAG_UNR = 0x0002
TAG_MAGIC = 0x0030
TAG_CONFIG = 0x0040
TAG_MROOT = 0x0110
TAG_NAME = 0x1000
TAG_BRANCH = 0x1000
TAG_REG = 0x1010
TAG_DIR = 0x1020
TAG_STRUCT = 0x3000
TAG_INLINED = 0x3000
TAG_BLOCK = 0x3100
TAG_MDIR = 0x3200
TAG_BTREE = 0x3300
TAG_UATTR = 0x4000
TAG_ALT = 0x0008
TAG_CRC = 0x0004
TAG_FCRC = 0x1004
TAG_UNR = 0x1000
TAG_SUPERMAGIC = 0x0003
TAG_SUPERCONFIG = 0x0004
TAG_MROOT = 0x0304
TAG_NAME = 0x0100
TAG_BRANCH = 0x0100
TAG_REG = 0x0101
TAG_DIR = 0x0102
TAG_STRUCT = 0x0300
TAG_INLINED = 0x0300
TAG_BLOCK = 0x0302
TAG_BTREE = 0x0303
TAG_MDIR = 0x0305
TAG_UATTR = 0x0400
TAG_ALT = 0x4000
TAG_CRC = 0x2000
TAG_FCRC = 0x2100
# parse some rbyd addr encodings
# 0xa -> [0xa]
@@ -79,9 +79,6 @@ def crc32c(data, crc=0):
crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78)
return 0xffffffff ^ crc
def fromle16(data):
return struct.unpack('<H', data[0:2].ljust(2, b'\0'))[0]
def fromle32(data):
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0]
@@ -95,10 +92,10 @@ def fromleb128(data):
return word, len(data)
def fromtag(data):
tag = fromle16(data)
tag = (data[0] << 8) | data[1]
weight, d = fromleb128(data[2:])
size, d_ = fromleb128(data[2+d:])
return tag&1, tag&~1, weight, size, 2+d+d_
return tag>>15, tag&0x7fff, weight, size, 2+d+d_
def popc(x):
return bin(x).count('1')
@@ -114,64 +111,60 @@ def xxd(data, width=16, crc=False):
for b in map(chr, data[i:i+width])))
def tagrepr(tag, w, size, off=None):
if (tag & 0xfffe) == TAG_UNR:
if (tag & 0x7fff) == TAG_UNR:
return 'unr%s%s' % (
' w%d' % w if w else '',
' %d' % size if size else '')
elif (tag & 0xfffc) == TAG_MAGIC:
return '%smagic%s %d' % (
'rm' if tag & 0x2 else '',
elif (tag & 0x6fff) == TAG_SUPERMAGIC:
return '%ssupermagic%s%s' % (
'rm' if tag & 0x1000 else '',
' w%d' % w if w else '',
size)
elif (tag & 0xfffc) == TAG_CONFIG:
return '%sconfig%s %d' % (
'rm' if tag & 0x2 else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6fff) == TAG_SUPERCONFIG:
return '%ssuperconfig%s%s' % (
'rm' if tag & 0x1000 else '',
' w%d' % w if w else '',
size)
elif (tag & 0xfffc) == TAG_MROOT:
return '%smroot%s %d' % (
'rm' if tag & 0x2 else '',
' w%d' % w if w else '',
size)
elif (tag & 0xf00c) == TAG_NAME:
return '%s%s%s %d' % (
'rm' if tag & 0x2 else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_NAME:
return '%s%s%s%s' % (
'rm' if tag & 0x1000 else '',
'branch' if (tag & 0xfffe) == TAG_BRANCH
else 'reg' if (tag & 0xfffe) == TAG_REG
else 'dir' if (tag & 0xfffe) == TAG_DIR
else 'name 0x%02x' % ((tag & 0x0ff0) >> 4),
' w%d' % w if w else '',
size)
elif (tag & 0xf00c) == TAG_STRUCT:
return '%s%s%s %d' % (
'rm' if tag & 0x2 else '',
'inlined' if (tag & 0xfffe) == TAG_INLINED
else 'block' if (tag & 0xfffe) == TAG_BLOCK
else 'mdir' if (tag & 0xfffe) == TAG_MDIR
else 'btree' if (tag & 0xfffe) == TAG_BTREE
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_STRUCT:
return '%s%s%s%s' % (
'rm' if tag & 0x1000 else '',
'inlined' if (tag & 0x6fff) == TAG_INLINED
else 'block' if (tag & 0x6fff) == TAG_BLOCK
else 'btree' if (tag & 0x6fff) == TAG_BTREE
else 'mdir' if (tag & 0x6fff) == TAG_MROOT
else 'mdir' if (tag & 0x6fff) == TAG_MDIR
else 'struct 0x%02x' % ((tag & 0x0ff0) >> 4),
' w%d' % w if w else '',
size)
elif (tag & 0xf00c) == TAG_UATTR:
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_UATTR:
return '%suattr 0x%02x%s%s' % (
'rm' if tag & 0x2 else '',
(tag & 0x0ff0) >> 4,
'rm' if tag & 0x1000 else '',
tag & 0xff,
' w%d' % w if w else '',
' %d' % size if not tag & 0x2 or size else '')
elif (tag & 0xf00e) == TAG_CRC:
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x7f00) == TAG_CRC:
return 'crc%x%s %d' % (
1 if tag & 0x10 else 0,
1 if tag & 0x1 else 0,
' 0x%x' % w if w > 0 else '',
size)
elif (tag & 0xfffe) == TAG_FCRC:
elif (tag & 0x7fff) == TAG_FCRC:
return 'fcrc%s %d' % (
' 0x%x' % w if w > 0 else '',
size)
elif tag & 0x8:
elif tag & 0x4000:
return 'alt%s%s 0x%x w%d %s' % (
'r' if tag & 0x2 else 'b',
'gt' if tag & 0x4 else 'le',
tag & 0xfff0,
'r' if tag & 0x1000 else 'b',
'gt' if tag & 0x2000 else 'le',
tag & 0x0fff,
w,
'0x%x' % (0xffffffff & (off-size))
if off is not None
@@ -192,14 +185,14 @@ def dbg_log(data, block_size, rev, off, weight, *,
j = j_
v, tag, w, size, d = fromtag(data[j_:])
j_ += d
if not tag & 0x8:
if not tag & 0x4000:
j_ += size
if tag & 0x8:
if tag & 0x4000:
# figure out which alt color
if tag & 0x2:
if tag & 0x1000:
_, ntag, _, _, _ = fromtag(data[j_:])
if ntag & 0x2:
if ntag & 0x1000:
jumps.append((j, j-size, 0, 'y'))
else:
jumps.append((j, j-size, 0, 'r'))
@@ -289,21 +282,21 @@ def dbg_log(data, block_size, rev, off, weight, *,
j = j_
v, tag, w, size, d = fromtag(data[j_:])
j_ += d
if not tag & 0x8:
if not tag & 0x4000:
j_ += size
# find trunk
if not wastrunk and (tag & 0xc) != 0x4:
if not wastrunk and (tag & 0x6000) != 0x2000:
lower_, upper_ = 0, 0
wastrunk = not not tag & 0x8
wastrunk = not not tag & 0x4000
# keep track of weight
if tag & 0x8:
if tag & 0x4:
if tag & 0x4000:
if tag & 0x2000:
upper_ += w
else:
lower_ += w
elif (tag & 0xc) == 0x0:
elif (tag & 0x6000) == 0x2000:
delta = (lower_+upper_+w) - weight_
weight_ = lower_+upper_+w
id = lower_+w-1
@@ -346,7 +339,7 @@ def dbg_log(data, block_size, rev, off, weight, *,
weights = weights_
lifetimes = lifetimes_
if not tag & 0x2 and id >= 0:
if not tag & 0x1000 and id >= 0:
# attach tag to lifetime
i, id_ = index(weights, id)
if i < len(weights):
@@ -431,27 +424,26 @@ def dbg_log(data, block_size, rev, off, weight, *,
v, tag, w, size, d = fromtag(data[j_:])
if v != (popc(crc) & 1):
notes.append('v!=%x' % (popc(crc) & 1))
tag &= ~1
crc = crc32c(data[j_:j_+d], crc)
j_ += d
# find trunk
if not wastrunk and (tag & 0xc) != 0x4:
if not wastrunk and (tag & 0x6000) != 0x2000:
lower_, upper_ = 0, 0
wastrunk = not not tag & 0x8
wastrunk = not not tag & 0x4000
# calculate id from alt weights
if tag & 0x8:
if tag & 0x4:
if tag & 0x4000:
if tag & 0x2000:
upper_ += w
else:
lower_ += w
elif (tag & 0xc) == 0x0:
elif (tag & 0x6000) == 0x0:
weight_ = lower_+upper_+w
id = lower_+w-1
if not tag & 0x8:
if (tag & 0xf00f) != TAG_CRC:
if not tag & 0x4000:
if (tag & 0x7f00) != TAG_CRC:
crc = crc32c(data[j_:j_+size], crc)
# found a crc?
else:
@@ -467,7 +459,7 @@ def dbg_log(data, block_size, rev, off, weight, *,
'\x1b[m' if color and j >= off else '',
lifetime_width, lifetimerepr(j) if args.get('lifetimes') else '',
'\x1b[90m' if color and j >= off else '',
w_width, '' if (tag & 0xc) != 0x0
w_width, '' if (tag & 0x6000) != 0x0
else '%d-%d' % (id-(w-1), id) if w > 1
else id,
'%-22s%s' % (
@@ -475,7 +467,7 @@ def dbg_log(data, block_size, rev, off, weight, *,
' %s' % next(xxd(
data[j+d:j+d+min(size, 8)], 8), '')
if not args.get('no_truncate')
and not tag & 0x8 else ''),
and not tag & 0x4000 else ''),
'\x1b[m' if color and j >= off else '',
' (%s)' % ', '.join(notes) if notes
else ' %s' % jumprepr(j)
@@ -497,7 +489,7 @@ def dbg_log(data, block_size, rev, off, weight, *,
data[j+d+i*4:j+d+min(i*4+4,size)])
for i in range(min(m.ceil(size/4), 3)))[:23]
if not args.get('no_truncate')
and not tag & 0x8 else ''),
and not tag & 0x4000 else ''),
crc,
popc(crc) & 1,
'\x1b[m' if color and j >= off else ''))
@@ -513,7 +505,7 @@ def dbg_log(data, block_size, rev, off, weight, *,
line,
'\x1b[m' if color and j >= off else ''))
if args.get('raw') or args.get('no_truncate'):
if not tag & 0x8:
if not tag & 0x4000:
for o, line in enumerate(xxd(data[j+d:j+d+size])):
print('%s%8s: %*s%*s %s%s' % (
'\x1b[90m' if color and j >= off else '',
@@ -543,19 +535,19 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
_, alt, w, jump, d = fromtag(data[j:])
# found an alt?
if alt & 0x8:
if alt & 0x4000:
# follow?
if ((id, tag & ~0xf) > (upper-w-1, alt & ~0xf)
if alt & 0x4
else ((id, tag & ~0xf) <= (lower+w, alt & ~0xf))):
lower += upper-lower-1-w if alt & 0x4 else 0
upper -= upper-lower-1-w if not alt & 0x4 else 0
if ((id, tag & 0xfff) > (upper-w-1, alt & 0xfff)
if alt & 0x2000
else ((id, tag & 0xfff) <= (lower+w, alt & 0xfff))):
lower += upper-lower-1-w if alt & 0x2000 else 0
upper -= upper-lower-1-w if not alt & 0x2000 else 0
j = j - jump
# figure out which color
if alt & 0x2:
if alt & 0x1000:
_, nalt, _, _, _ = fromtag(data[j+jump+d:])
if nalt & 0x2:
if nalt & 0x1000:
path.append((j+jump, j, True, 'y'))
else:
path.append((j+jump, j, True, 'r'))
@@ -564,14 +556,14 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
# stay on path
else:
lower += w if not alt & 0x4 else 0
upper -= w if alt & 0x4 else 0
lower += w if not alt & 0x2000 else 0
upper -= w if alt & 0x2000 else 0
j = j + d
# figure out which color
if alt & 0x2:
if alt & 0x1000:
_, nalt, _, _, _ = fromtag(data[j:])
if nalt & 0x2:
if nalt & 0x1000:
path.append((j-d, j, False, 'y'))
else:
path.append((j-d, j, False, 'r'))
@@ -584,7 +576,7 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
tag_ = alt
w_ = id_-lower
done = (id_, tag_) < (id, tag) or tag_ & 2
done = (id_, tag_) < (id, tag) or tag_ & 0x1000
return done, id_, tag_, w_, j, d, jump, path
@@ -596,7 +588,7 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
id, tag = -1, 0
while True:
done, id, tag, w, j, d, size, path = lookup(id, tag+0x10)
done, id, tag, w, j, d, size, path = lookup(id, tag+0x1)
# found end of tree?
if done:
break
@@ -742,7 +734,7 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
id, tag = -1, 0
while True:
done, id, tag, w, j, d, size, path = lookup(id, tag+0x10)
done, id, tag, w, j, d, size, path = lookup(id, tag+0x1)
# found end of tree?
if done:
break
@@ -759,7 +751,7 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
' %s' % next(xxd(
data[j+d:j+d+min(size, 8)], 8), '')
if not args.get('no_truncate')
and not tag & 0x8 else '')))
and not tag & 0x4000 else '')))
# show in-device representation
if args.get('device'):
@@ -774,7 +766,7 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
data[j+d+i*4:j+d+min(i*4+4,size)])
for i in range(min(m.ceil(size/4), 3)))[:23]
if not args.get('no_truncate')
and not tag & 0x8 else '')))
and not tag & 0x4000 else '')))
# show on-disk encoding of tags
if args.get('raw'):
@@ -785,7 +777,7 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
w_width, '',
line))
if args.get('raw') or args.get('no_truncate'):
if not tag & 0x8:
if not tag & 0x4000:
for o, line in enumerate(xxd(data[j+d:j+d+size])):
print('%8s: %*s%*s %s' % (
'%04x' % (j+d + o*16),
@@ -853,12 +845,12 @@ def main(disk, blocks=None, *,
break
crc_ = crc32c(data[j_:j_+d], crc_)
j_ += d
if not tag & 0x8 and j_ + size > len(data):
if not tag & 0x4000 and j_ + size > len(data):
break
# take care of crcs
if not tag & 0x8:
if (tag & 0xf00f) != TAG_CRC:
if not tag & 0x4000:
if (tag & 0x7f00) != TAG_CRC:
crc_ = crc32c(data[j_:j_+size], crc_)
# found a crc?
else:
@@ -872,7 +864,7 @@ def main(disk, blocks=None, *,
weight = weight_
# evaluate trunks
if (tag & 0xc) != 0x4 and (
if (tag & 0x6000) != 0x2000 and (
not trunk or trunk >= j_-d or wastrunk):
# new trunk?
if not wastrunk:
@@ -881,8 +873,8 @@ def main(disk, blocks=None, *,
wastrunk = True
# keep track of weight
if tag & 0x8:
if tag & 0x4:
if tag & 0x4000:
if tag & 0x2000:
upper_ += w
else:
lower_ += w
@@ -893,7 +885,7 @@ def main(disk, blocks=None, *,
if trunk and j_ + size > trunk:
trunkoff = j_ + size
if not tag & 0x8:
if not tag & 0x4000:
j_ += size
return rev, off, trunk_, weight