Github/littlefs
1
Fork 0
Code Actions Activity

Middle of a rewrite for 3-leb encoding, but rbyd appends and creates both work

Browse Source 
If we combine rbyd ids and B-tree weights, we need 32-bit ids since this
will eventually need to cover the full range of a file. This simply
doesn't fit into a single word anymore, unless littlefs uses 64-bit tags.
Generally not a great idea for a filesystem targeting even 8-bit
microcontrollers.

So here is a tag encoding that uses 3 leb128 words. This will likely
have more code cost and slightly more disk usage (we can no longer fit
tags into 2 bytes), though with most tags being alt pointers (O(m log m)
vs O(m)), this may not be that significant.

Note that we try to keep tags limited to 14-bits to avoid an extra leb128 byte,
which would likely affect all alt pointers. To pull this off we do away
with the subtype/suptype distinction, limiting in-tree tag types to
10-bits encoded on a per-suptype basis:

  in-tree tags:
                       ttttttt ttt00rv
                                 ^--^^- 10-bit type
                                    '|- removed bit
                                     '- valid bit
  iiii iiiiiii iiiiiii iiiiiii iiiiiii
                                     ^- n-bit id
       lllllll lllllll lllllll lllllll
                                     ^- m-bit length

  out-of-tree tags:
                       ttttttt ttt010v
                                 ^---^- 10-bit type
                                     '- valid bit
                               0000000
       lllllll lllllll lllllll lllllll
                                     ^- m-bit length

  alt tags:
                       kkkkkkk kkk1dcv
                                 ^-^^^- 10-bit key
                                   '||- direction bit
                                    '|- color bit
                                     '- valid bit
  wwww wwwwwww wwwwwww wwwwwww wwwwwww
                                     ^- n-bit weight
       jjjjjjj jjjjjjj jjjjjjj jjjjjjj
                                     ^- m-bit jump

The real pain is that with separate integers for id and tag, it no
longer makes sense to combine these into one big weight field. This
requires a significant rewrite.
This commit is contained in:
Christopher Haster
2023-01-24 01:38:26 -06:00
parent cdc3a486d6
commit 08f5d9ddf4
5 changed files with 4772 additions and 3954 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+973 -655
View File
File diff suppressed because it is too large Load Diff
lfs.h
+9 -3
View File
@@ -44,6 +44,11 @@ typedef int32_t lfs_soff_t;
typedef uint32_t lfs_block_t; typedef uint32_t lfs_block_t;
typedef uint16_t lfsr_tag_t;
typedef int16_t lfsr_stag_t;
typedef uint32_t lfsr_id_t;
typedef int32_t lfsr_sid_t;
// Maximum name size in bytes, may be redefined to reduce the size of the // Maximum name size in bytes, may be redefined to reduce the size of the
// info struct. Limited to <= 1022. Stored in superblock and must be // info struct. Limited to <= 1022. Stored in superblock and must be
// respected by other littlefs drivers. // respected by other littlefs drivers.
@@ -331,11 +336,12 @@ typedef struct lfs_cache {
typedef struct lfsr_rbyd { typedef struct lfsr_rbyd {
lfs_block_t block; lfs_block_t block;
lfs_off_t trunk;
lfs_off_t off;
uint32_t rev; uint32_t rev;
lfs_off_t off;
uint32_t crc; uint32_t crc;
uint16_t count; lfs_off_t trunk;
lfs_size_t weight;
// TODO can we get rid of erased? use sign bit of off maybe?
bool erased; bool erased;
} lfsr_rbyd_t; } lfsr_rbyd_t;
lfs_util.h
+21 -1
View File
@@ -114,13 +114,33 @@ static inline uint32_t lfs_min(uint32_t a, uint32_t b) {
return (a < b) ? a : b; return (a < b) ? a : b;
} }
// TODO how many of these do we actually need
// Swap two 16-bit numbers
static inline void lfs_swap16(uint16_t *a, uint16_t *b) {
uint16_t t = *a;
*a = *b;
*b = t;
}
static inline void lfs_swaps16(int16_t *a, int16_t *b) {
int16_t t = *a;
*a = *b;
*b = t;
}
// Swap two 32-bit numbers // Swap two 32-bit numbers
static inline void lfs_swap(uint32_t *a, uint32_t *b) { static inline void lfs_swap32(uint32_t *a, uint32_t *b) {
uint32_t t = *a; uint32_t t = *a;
*a = *b; *a = *b;
*b = t; *b = t;
} }
static inline void lfs_swaps32(int32_t *a, int32_t *b) {
int32_t t = *a;
*a = *b;
*b = t;
}
// Align to nearest multiple of a size // Align to nearest multiple of a size
static inline uint32_t lfs_aligndown(uint32_t a, uint32_t alignment) { static inline uint32_t lfs_aligndown(uint32_t a, uint32_t alignment) {
return a - (a % alignment); return a - (a % alignment);
scripts/dbgrbyd.py
+113 -121
View File
@@ -33,9 +33,10 @@ def fromleb128(data):
return word, len(data) return word, len(data)
def fromtag(data): def fromtag(data):
tag, delta1 = fromleb128(data) tag, delta = fromleb128(data)
size, delta2 = fromleb128(data[delta1:]) id, delta_ = fromleb128(data[delta:])
return tag & 1, tag >> 1, size, delta1+delta2 size, delta__ = fromleb128(data[delta+delta_:])
return tag&1, tag&~1, id-1, size, delta+delta_+delta__
def popc(x): def popc(x):
return bin(x).count('1') return bin(x).count('1')
@@ -50,50 +51,41 @@ def xxd(data, width=16, crc=False):
b if b >= ' ' and b <= '~' else '.' b if b >= ' ' and b <= '~' else '.'
for b in map(chr, data[i:i+width]))) for b in map(chr, data[i:i+width])))
def tagrepr(tag, size, off=None): def tagrepr(tag, id, size, off=None):
type = tag & 0x7fff if (tag & ~0x3f0) == 0x0400:
suptype = tag & 0x7807 return 'mk%s id%d %d' % (
subtype = (tag >> 3) & 0xff 'branch' if ((tag & 0x3f0) >> 4) == 0x00
xsuptype = tag & 0x7e else 'reg' if ((tag & 0x3f0) >> 4) == 0x01
xsubtype = (tag >> 7) & 0xff else 'dir' if ((tag & 0x3f0) >> 4) == 0x02
id = ((tag >> 15) & 0xffff) - 1
if suptype == 0x0800:
return 'mk%s id%d%s' % (
'reg' if subtype == 0
else ' 0x%02x' % subtype, else ' 0x%02x' % subtype,
id, id,
' %d' % size if not tag & 0x1 else '') size)
elif suptype == 0x0801: elif (tag & ~0xff2) == 0x2000:
return 'rm%s id%d%s' % (
' 0x%02x' % subtype if subtype else '',
id,
' %d' % size if not tag & 0x1 else '')
elif (suptype & ~0x1) == 0x1000:
return '%suattr 0x%02x%s%s' % ( return '%suattr 0x%02x%s%s' % (
'rm' if suptype & 0x1 else '', 'rm' if tag & 0x1 else '',
subtype, (tag & 0xff0) >> 4,
' id%d' % id if id != -1 else '', ' id%d' % id if id != -1 else '',
' %d' % size if not tag & 0x1 else '') ' %d' % size if not tag & 0x1 else '')
elif xsuptype == 0x0002: elif (tag & ~0x10) == 0x24:
return 'crc%x%s %d' % ( return 'crc%x%s %d' % (
tag & 0x1, 1 if tag & 0x10 else 0,
' 0x%02x' % xsubtype if xsubtype else '', ' 0x%02x' % id if id != -1 else '',
size) size)
elif xsuptype == 0x000a: elif tag == 0x44:
return 'fcrc%s %d' % ( return 'fcrc%s %d' % (
' 0x%02x' % xsubtype if xsubtype else '', ' 0x%02x' % id if id != -1 else '',
size) size)
elif suptype & 0x4: elif tag & 0x8:
return 'alt%s%s 0x%x %s' % ( return 'alt%s%s 0x%x w%d %s' % (
'r' if suptype & 0x1 else 'b', 'r' if tag & 0x2 else 'b',
'gt' if suptype & 0x2 else 'lt', 'gt' if tag & 0x4 else 'le',
tag & ~0x7, tag & 0x3ff0,
id+1,
'0x%x' % (0xffffffff & (off-size)) '0x%x' % (0xffffffff & (off-size))
if off is not None if off is not None
else '-%d' % off) else '-%d' % off)
else: else:
return '0x%02x id%d %d' % (type, id, size) return '0x%04x id%d %d' % (tag, id-1, size)
def show_log(block_size, data, rev, off, *, def show_log(block_size, data, rev, off, *,
color=False, color=False,
@@ -106,16 +98,16 @@ def show_log(block_size, data, rev, off, *,
j_ = 4 j_ = 4
while j_ < (block_size if args.get('all') else off): while j_ < (block_size if args.get('all') else off):
j = j_ j = j_
v, tag, size, delta = fromtag(data[j_:]) v, tag, id, size, delta = fromtag(data[j_:])
j_ += delta j_ += delta
if not tag & 0x4: if not tag & 0x8:
j_ += size j_ += size
if tag & 0x4: if tag & 0x8:
# figure out which alt color # figure out which alt color
if tag & 0x1: if tag & 0x2:
_, ntag, _, _ = fromtag(data[j_:]) _, ntag, _, _, _ = fromtag(data[j_:])
if ntag & 0x1: if ntag & 0x2:
jumps.append((j, j-size, 0, 'y')) jumps.append((j, j-size, 0, 'y'))
else: else:
jumps.append((j, j-size, 0, 'r')) jumps.append((j, j-size, 0, 'r'))
@@ -245,14 +237,15 @@ def show_log(block_size, data, rev, off, *,
notes = [] notes = []
j = j_ j = j_
v, tag, size, delta = fromtag(data[j_:]) v, tag, id, size, delta = fromtag(data[j_:])
if v != popc(crc) & 1: if v != (popc(crc) & 1):
notes.append('v!=%x' % (popc(crc) & 1)) notes.append('v!=%x' % (popc(crc) & 1))
tag &= ~1
crc = crc32c(data[j_:j_+delta], crc) crc = crc32c(data[j_:j_+delta], crc)
j_ += delta j_ += delta
if not tag & 0x4: if not tag & 0x8:
if (tag & 0x007e) != 0x0002: if (tag & ~0x10) != 0x24:
crc = crc32c(data[j_:j_+size], crc) crc = crc32c(data[j_:j_+size], crc)
# found a crc? # found a crc?
else: else:
@@ -261,73 +254,72 @@ def show_log(block_size, data, rev, off, *,
notes.append('crc!=%08x' % crc) notes.append('crc!=%08x' % crc)
j_ += size j_ += size
# adjust count? # # adjust count?
if args.get('lifetimes'): # if args.get('lifetimes'):
if (tag & 0x7807) == 0x0800: # if (tag & 0x7807) == 0x0800:
count += 1 # count += 1
elif ((tag & 0x7807) == 0x0801 # elif ((tag & 0x7807) == 0x0801
and ((tag >> 15) & 0xffff)-1 < len(ids)): # and ((tag >> 15) & 0xffff)-1 < len(ids)):
count -= 1 # count -= 1
if not args.get('in_tree') or (tag & 0x6) == 0: # show human-readable tag representation
# show human-readable tag representation print('%s%08x:%s %s%s%-57s%s%s' % (
print('%s%08x:%s %s%s%-57s%s%s' % ( '\x1b[90m' if color and j >= off else '',
'\x1b[90m' if color and j >= off else '', j,
j, '\x1b[m' if color and j >= off else '',
'\x1b[m' if color and j >= off else '', lifetimerepr(j) if args.get('lifetimes') else '',
lifetimerepr(j) if args.get('lifetimes') else '', '\x1b[90m' if color and j >= off else '',
'\x1b[90m' if color and j >= off else '', '%-22s%s' % (
'%-22s%s' % ( tagrepr(tag, id, size, j),
tagrepr(tag, size, j), ' %s' % next(xxd(
' %s' % next(xxd( data[j+delta:j+delta+min(size, 8)], 8), '')
data[j+delta:j+delta+min(size, 8)], 8), '') if not args.get('no_truncate')
if not args.get('no_truncate') and not tag & 0x8 else ''),
and not tag & 0x4 else ''), '\x1b[m' if color and j >= off else '',
'\x1b[m' if color and j >= off else '', ' (%s)' % ', '.join(notes) if notes
' (%s)' % ', '.join(notes) if notes else ' %s' % ''.join(
else ' %s' % ''.join( ('\x1b[33my\x1b[m' if color else 'y')
('\x1b[33my\x1b[m' if color else 'y') if alts[i] & 0x2
if alts[i] & 0x1 and i+1 < len(alts)
and i+1 < len(alts) and alts[i+1] & 0x2
and alts[i+1] & 0x1 else ('\x1b[31mr\x1b[m' if color else 'r')
else ('\x1b[31mr\x1b[m' if color else 'r') if alts[i] & 0x2
if alts[i] & 0x1 else ('\x1b[90mb\x1b[m' if color else 'b')
else ('\x1b[90mb\x1b[m' if color else 'b') for i in range(len(alts)-1, -1, -1))
for i in range(len(alts)-1, -1, -1)) if args.get('rbyd') and (tag & 0x7) == 0
if args.get('rbyd') and (tag & 0x7) == 0 else ' %s' % jumprepr(j)
else ' %s' % jumprepr(j) if args.get('jumps')
if args.get('jumps') else ''))
else ''))
if not args.get('in_tree') or (tag & 0x6) != 2: if args.get('raw'):
if args.get('raw'): # show on-disk encoding of tags
# show on-disk encoding of tags for o, line in enumerate(xxd(data[j:j+delta])):
for o, line in enumerate(xxd(data[j:j+delta])): print('%s%8s: %s%s' % (
print('%s%8s: %s%s' % (
'\x1b[90m' if color and j >= off else '',
'%04x' % (j + o*16),
line,
'\x1b[m' if color and j >= off else ''))
# show in-device representation, including some extra
# crc/parity info
if args.get('device'):
print('%s%8s %s%-47s %08x %x%s' % (
'\x1b[90m' if color and j >= off else '', '\x1b[90m' if color and j >= off else '',
'', '%04x' % (j + o*16),
lifetimerepr(0) if args.get('lifetimes') else '', line,
'%-22s%s' % (
'%08x %08x' % (tag, size),
' %s' % ' '.join(
'%08x' % struct.unpack('<I',
data[j+delta+i*4:j+delta+i*4+4])
for i in range(min(size//4, 3)))[:23]
if not tag & 0x4 else ''),
crc,
popc(crc) & 1,
'\x1b[m' if color and j >= off else '')) '\x1b[m' if color and j >= off else ''))
if not tag & 0x4 and (not args.get('in_tree') or (tag & 0x6) != 2): # show in-device representation, including some extra
# crc/parity info
if args.get('device'):
print('%s%8s %s%-47s %08x %x%s' % (
'\x1b[90m' if color and j >= off else '',
'',
lifetimerepr(0) if args.get('lifetimes') else '',
'%-22s%s' % (
'%04x %08x %07x' % (tag, 0xffffffff & id, size),
' %s' % ' '.join(
'%08x' % struct.unpack('<I',
data[j+delta+i*4:j+delta+min(i*4+4,size)]
.ljust(4, b'\0'))
for i in range(min(m.ceil(size/4), 3)))[:23]
if not tag & 0x8 else ''),
crc,
popc(crc) & 1,
'\x1b[m' if color and j >= off else ''))
if not tag & 0x8:
# show on-disk encoding of data # show on-disk encoding of data
if args.get('raw') or args.get('no_truncate'): if args.get('raw') or args.get('no_truncate'):
for o, line in enumerate(xxd(data[j+delta:j+delta+size])): for o, line in enumerate(xxd(data[j+delta:j+delta+size])):
@@ -338,7 +330,7 @@ def show_log(block_size, data, rev, off, *,
'\x1b[m' if color and j >= off else '')) '\x1b[m' if color and j >= off else ''))
if args.get('rbyd'): if args.get('rbyd'):
if tag & 0x4: if tag & 0x8:
alts.append(tag) alts.append(tag)
else: else:
alts = [] alts = []
@@ -590,24 +582,24 @@ def main(disk, block_size, block1, block2=None, *,
count_ = 0 count_ = 0
wastrunk = False wastrunk = False
while j_ < block_size: while j_ < block_size:
v, tag, size, delta = fromtag(data[j_:]) v, tag, id, size, delta = fromtag(data[j_:])
if v != popc(crc) & 1: if v != (popc(crc) & 1):
break break
crc = crc32c(data[j_:j_+delta], crc) crc = crc32c(data[j_:j_+delta], crc)
j_ += delta j_ += delta
if not wastrunk and (tag & 0x6) != 0x2: if not wastrunk and (tag & 0xc) != 0x4:
trunk_ = j_ - delta trunk_ = j_ - delta
wastrunk = True wastrunk = True
if not tag & 0x4: if not tag & 0x8:
if (tag & 0x007e) != 0x0002: if (tag & ~0x10) != 0x24:
crc = crc32c(data[j_:j_+size], crc) crc = crc32c(data[j_:j_+size], crc)
# keep track of id count # # keep track of id count
if (tag & 0x7807) == 0x0800: # if (tag & 0x7807) == 0x0800:
count_ += 1 # count_ += 1
elif (tag & 0x7807) == 0x0801: # elif (tag & 0x7807) == 0x0801:
count_ = max(count_ - 1, 0) # count_ = max(count_ - 1, 0)
# found a crc? # found a crc?
else: else:
crc_, = struct.unpack('<I', data[j_:j_+4].ljust(4, b'\0')) crc_, = struct.unpack('<I', data[j_:j_+4].ljust(4, b'\0'))
@@ -698,10 +690,10 @@ if __name__ == "__main__":
'-l', '--log', '-l', '--log',
action='store_true', action='store_true',
help="Show the raw tags as they appear in the log.") help="Show the raw tags as they appear in the log.")
parser.add_argument( # parser.add_argument(
'-i', '--in-tree', # '-i', '--in-tree',
action='store_true', # action='store_true',
help="Only show tags in the tree.") # help="Only show tags in the tree.")
parser.add_argument( parser.add_argument(
'-r', '--raw', '-r', '--raw',
action='store_true', action='store_true',
tests/test_rbyd.toml
+3656 -3174
View File
File diff suppressed because it is too large Load Diff