From 900ea807ae71fbaa7eec4cbe56f563cdc74c3574 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 11 Sep 2023 01:59:19 -0500 Subject: [PATCH] Changed to a shifted mid=bid.rid representation for debugging This only matters for developers, not users, but it still helps a lot to get debug representations right. Since the exact mid encoding depends on the block_size in an unintuitive manner, it's tricky to render in a debug-friendly way that is useful both with and without tools. Previously, I avoided shifting the bid representation, since this would be closer to the value in the device, but this hides the actual structure of the mtree. Now the bid is shifted, showing the underlying mtree/mdir structure, at the cost of needing to know the number of mbits to encode the mid back into an integer. So for example, on a device with 4KiB blocks, or 8 mbits: mid=1 mid=258 mid=515 Becomes: mid=0.1 mid=1.2 mid=2.3 This continues to make the mbits a more fundamental part of littlefs, but that's probably just how that's going to be. --- lfs.c | 51 +++++++++++++++++++------------------------ scripts/dbglfs.py | 51 ++++++++++++++++++++++--------------------- scripts/dbgmtree.py | 19 ++++++++++------ tests/test_mtree.toml | 42 +++++++++++++++++------------------ 4 files changed, 81 insertions(+), 82 deletions(-) diff --git a/lfs.c b/lfs.c index 2aac81bc..c056ffa4 100644 --- a/lfs.c +++ b/lfs.c @@ -5289,12 +5289,11 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, // adjust our sibling's mid after committing attrs msibling_.mid += lfsr_mweight(lfs); - LFS_DEBUG("Splitting mdir %"PRId32".%"PRId32" " + LFS_DEBUG("Splitting mdir %"PRId32" " "0x{%"PRIx32",%"PRIx32"} " "-> 0x{%"PRIx32",%"PRIx32"}, " "0x{%"PRIx32",%"PRIx32"}", - mid & lfsr_mbidmask(lfs), - mid & lfsr_mridmask(lfs), + mid >> lfs->mbits, mdir->u.m.blocks[0], mdir->u.m.blocks[1], mdir_.u.m.blocks[0], mdir_.u.m.blocks[1], msibling_.u.m.blocks[0], msibling_.u.m.blocks[1]); @@ -5304,33 +5303,29 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, // both siblings reduced to zero if (mdir_.u.m.weight == 0 && msibling_.u.m.weight == 0) { - LFS_DEBUG("Dropping mdir %"PRId32".%"PRId32" " + LFS_DEBUG("Dropping mdir %"PRId32" " "0x{%"PRIx32",%"PRIx32"}", - mdir_.mid & lfsr_mbidmask(lfs), - mdir_.mid & lfsr_mridmask(lfs), + mdir_.mid >> lfs->mbits, mdir_.u.m.blocks[0], mdir_.u.m.blocks[1]); - LFS_DEBUG("Dropping mdir %"PRId32".%"PRId32" " + LFS_DEBUG("Dropping mdir %"PRId32" " "0x{%"PRIx32",%"PRIx32"}", - msibling_.mid & lfsr_mbidmask(lfs), - msibling_.mid & lfsr_mridmask(lfs), + msibling_.mid >> lfs->mbits, msibling_.u.m.blocks[0], msibling_.u.m.blocks[1]); goto drop; // one sibling reduced to zero } else if (msibling_.u.m.weight == 0) { - LFS_DEBUG("Dropping mdir %"PRId32".%"PRId32" " + LFS_DEBUG("Dropping mdir %"PRId32" " "0x{%"PRIx32",%"PRIx32"}", - msibling_.mid & lfsr_mbidmask(lfs), - msibling_.mid & lfsr_mridmask(lfs), + msibling_.mid >> lfs->mbits, msibling_.u.m.blocks[0], msibling_.u.m.blocks[1]); goto relocate; // other sibling reduced to zero } else if (mdir_.u.m.weight == 0) { - LFS_DEBUG("Dropping mdir %"PRId32".%"PRId32" " + LFS_DEBUG("Dropping mdir %"PRId32" " "0x{%"PRIx32",%"PRIx32"}", - mdir_.mid & lfsr_mbidmask(lfs), - mdir_.mid & lfsr_mridmask(lfs), + mdir_.mid >> lfs->mbits, mdir_.u.m.blocks[0], mdir_.u.m.blocks[1]); mdir_.u.m = msibling_.u.m; msibling_.u.m.weight = 0; @@ -5380,10 +5375,9 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, // mdir reduced to zero? need to drop? } else if (err == LFS_ERR_NOENT) { - LFS_DEBUG("Dropping mdir %"PRId32".%"PRId32" " + LFS_DEBUG("Dropping mdir %"PRId32" " "0x{%"PRIx32",%"PRIx32"}", - mid & lfsr_mbidmask(lfs), - mid & lfsr_mridmask(lfs), + mid >> lfs->mbits, mdir->u.m.blocks[0], mdir->u.m.blocks[1]); // consume gstate so we don't lose any info @@ -5407,10 +5401,9 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, // need to relocate? } else if (lfsr_mdir_cmp(mdir, &mdir_) != 0 && !(mid == -1 || lfsr_mtree_isinlined(lfs))) { - LFS_DEBUG("Relocating mdir %"PRId32".%"PRId32" " + LFS_DEBUG("Relocating mdir %"PRId32" " "0x{%"PRIx32",%"PRIx32"} -> 0x{%"PRIx32",%"PRIx32"}", - mid & lfsr_mbidmask(lfs), - mid & lfsr_mridmask(lfs), + mid >> lfs->mbits, mdir->u.m.blocks[0], mdir->u.m.blocks[1], mdir_.u.m.blocks[0], mdir_.u.m.blocks[1]); @@ -6618,13 +6611,13 @@ static int lfsr_mountinited(lfs_t *lfs) { if (lfsr_grm_count(&lfs->grm) == 2) { LFS_DEBUG("Found pending grm " "%"PRId32".%"PRId32" %"PRId32".%"PRId32, - lfs->grm.rms[0] & lfsr_mbidmask(lfs), + lfs->grm.rms[0] >> lfs->mbits, lfs->grm.rms[0] & lfsr_mridmask(lfs), - lfs->grm.rms[0] & lfsr_mbidmask(lfs), - lfs->grm.rms[0] & lfsr_mridmask(lfs)); + lfs->grm.rms[1] >> lfs->mbits, + lfs->grm.rms[1] & lfsr_mridmask(lfs)); } else if (lfsr_grm_count(&lfs->grm) == 1) { LFS_DEBUG("Found pending grm %"PRId32".%"PRId32, - lfs->grm.rms[0] & lfsr_mbidmask(lfs), + lfs->grm.rms[0] >> lfs->mbits, lfs->grm.rms[0] & lfsr_mridmask(lfs)); } } @@ -7448,13 +7441,13 @@ static int lfsr_fs_preparemutation(lfs_t *lfs) { if (lfsr_grm_count(&lfs->grm) == 2) { LFS_DEBUG("Fixing pending grm " "%"PRId32".%"PRId32" %"PRId32".%"PRId32, - lfs->grm.rms[0] & lfsr_mbidmask(lfs), + lfs->grm.rms[0] >> lfs->mbits, lfs->grm.rms[0] & lfsr_mridmask(lfs), - lfs->grm.rms[0] & lfsr_mbidmask(lfs), - lfs->grm.rms[0] & lfsr_mridmask(lfs)); + lfs->grm.rms[1] >> lfs->mbits, + lfs->grm.rms[1] & lfsr_mridmask(lfs)); } else if (lfsr_grm_count(&lfs->grm) == 1) { LFS_DEBUG("Fixing pending grm %"PRId32".%"PRId32, - lfs->grm.rms[0] & lfsr_mbidmask(lfs), + lfs->grm.rms[0] >> lfs->mbits, lfs->grm.rms[0] & lfsr_mridmask(lfs)); } diff --git a/scripts/dbglfs.py b/scripts/dbglfs.py index ea505814..45224566 100755 --- a/scripts/dbglfs.py +++ b/scripts/dbglfs.py @@ -561,13 +561,13 @@ class Rbyd: mtree = Rbyd.fetch(f, block_size, block, trunk) # corrupted? if not mtree: - return False, -1, None, -1, 0, 0 + return False, -1, 0, None, -1, 0, 0 # lookup our name in the mtree mbid, tag_, mw, data = mtree.btree_namelookup( f, block_size, did, name) if tag_ != TAG_MDIR: - return False, -1, None, -1, 0, 0 + return False, -1, 0, None, -1, 0, 0 # fetch the mdir blocks = frommdir(data) @@ -653,10 +653,10 @@ def superconfig(mroot): # collect gstate class GState: - def __init__(self, block_size): + def __init__(self, mbits): self.gstate = {} self.gdelta = {} - self.block_size = block_size + self.mbits = mbits def xor(self, mbid, mw, mdir): tag = TAG_GSTATE-0x1 @@ -682,10 +682,6 @@ class GState: if TAG_GRM not in self.gstate: return [] - # derive the mbid/mrid masks from the block_size - mridmask = (1 << m.ceil(m.log2(self.block_size // 16)))-1 - mbidmask = ~mridmask - data = self.gstate[TAG_GRM] d = 0 count, d_ = fromleb128(data[d:]); d += d_ @@ -693,26 +689,27 @@ class GState: if count <= 2: for _ in range(count): mid, d_ = fromleb128(data[d:]); d += d_ - rms.append((mid & mbidmask, mid & mridmask)) + rms.append(( + mid & ~((1 << self.mbits)-1), + mid & ((1 << self.mbits)-1))) return rms -def grepr(tag, data, block_size): +def grepr(tag, data, mbits): if tag == TAG_GRM: - # derive the mbid/mrid masks from the block_size - mridmask = (1 << m.ceil(m.log2(block_size // 16)))-1 - mbidmask = ~mridmask - d = 0 count, d_ = fromleb128(data[d:]); d += d_ rms = [] if count <= 2: for _ in range(count): mid, d_ = fromleb128(data[d:]); d += d_ - rms.append((mid & mbidmask, mid & mridmask)) + rms.append(( + mid & ~((1 << mbits)-1), + mid & ((1 << mbits)-1))) return 'grm %s' % ( 'none' if count == 0 - else ' '.join('%d.%d' % (mbid, rid) for mbid, rid in rms) - if count <= 2 + else ' '.join('%d.%d' % (mbid >> mbits, rid) + for mbid, rid in rms) + if count <= 2 else '0x%x' % count) else: return 'gstate 0x%02x %d' % (tag, len(data)) @@ -762,6 +759,10 @@ def main(disk, mroots=None, *, f.seek(0, os.SEEK_END) block_size = f.tell() + # determine the number of mbits from the block_size, this is just + # for printing purposes + mbits = m.ceil(m.log2(block_size // 16)) + # before we print, we need to do a pass for a few things: # - find the actual mroot # - find the total weight @@ -772,7 +773,7 @@ def main(disk, mroots=None, *, mweight = 0 rweight = 0 corrupted = False - gstate = GState(block_size) + gstate = GState(mbits) config = {} dir_dids = [(0, b'', -1, 0, None, -1, TAG_DID, 0)] bookmark_dids = [] @@ -937,13 +938,13 @@ def main(disk, mroots=None, *, #### actual debugging begins here # print some information about the filesystem - print('littlefs v%s.%s %s, rev %d, weight %d' % ( + print('littlefs v%s.%s %s, rev %d, weight %d.%d' % ( config.get('major_version', ('?',))[0], config.get('minor_version', ('?',))[0], - mroot.addr(), mroot.rev, mweight)) + mroot.addr(), mroot.rev, mweight >> mbits, 1 << mbits)) # print header - w_width = (m.ceil(m.log10(max(1, mweight)+1)) + w_width = (m.ceil(m.log10(max(1, mweight >> mbits)+1)) + 2*m.ceil(m.log10(max(1, rweight)+1)) + 2) if dtree: @@ -998,7 +999,7 @@ def main(disk, mroots=None, *, print('%12s %-*s %s' % ( 'gstate:' if i == 0 else '', w_width + 23, - grepr(tag, data, block_size), + grepr(tag, data, mbits), next(xxd(data, 8), '') if not args.get('no_truncate') else '')) @@ -1033,7 +1034,7 @@ def main(disk, mroots=None, *, ','.join('%04x' % block for block in it.chain([mdir.block], mdir.redund_blocks)), - w_width, mbid-max(mw-1, 0), + w_width, mbid >> mbits, tagrepr(tag, 0, len(data)), next(xxd(data, 8), '') if not args.get('no_truncate') else '', @@ -1136,8 +1137,8 @@ def main(disk, mroots=None, *, mdir.redund_blocks)) if mbid != pmbid else '', w_width, '%d.%d-%d' % ( - mbid-max(mw-1, 0), rid-(w-1), rid) - if w > 1 else '%d.%d' % (mbid-max(mw-1, 0), rid) + mbid >> mbits, rid-(w-1), rid) + if w > 1 else '%d.%d' % (mbid >> mbits, rid) if w > 0 else '', f_width, '%s%s' % ( prefixes[0+(i==len(dir)-1)], diff --git a/scripts/dbgmtree.py b/scripts/dbgmtree.py index 9b3149c2..af8c4a85 100755 --- a/scripts/dbgmtree.py +++ b/scripts/dbgmtree.py @@ -761,6 +761,10 @@ def main(disk, mroots=None, *, f.seek(0, os.SEEK_END) block_size = f.tell() + # determine the number of mbits from the block_size, this is just + # for printing purposes + mbits = m.ceil(m.log2(block_size // 16)) + # before we print, we need to do a pass for a few things: # - find the actual mroot # - find the total weight @@ -1303,8 +1307,8 @@ def main(disk, mroots=None, *, if args.get('tree') or args.get('btree') else '', '%*s %-22s%s' % ( w_width, '%d.%d-%d' % ( - mbid-max(mw-1, 0), rid-(w-1), rid) - if w > 1 else '%d.%d' % (mbid-max(mw-1, 0), rid) + mbid >> mbits, rid-(w-1), rid) + if w > 1 else '%d.%d' % (mbid >> mbits, rid) if w > 0 or i == 0 else '', tagrepr(tag, w, len(data), j), ' %s' % next(xxd(data, 8), '') @@ -1360,8 +1364,9 @@ def main(disk, mroots=None, *, treerepr(bid, w, bd, rid, 0, tag) if args.get('tree') or args.get('btree') else '', w_width, '' if i != 0 - else '%d-%d' % (bid-(w-1), bid) if w > 1 - else bid if w > 0 + else '%d-%d' % ((bid-(w-1)) >> mbits, bid >> mbits) + if (w >> mbits) > 1 + else bid >> mbits if w > 0 else '', tagrepr(tag, w if i == 0 else 0, len(data), None), # note we render names a bit different here @@ -1401,11 +1406,11 @@ def main(disk, mroots=None, *, #### actual debugging begins here # print some information about the mtree - print('mtree %s, rev %d, weight %d' % ( - mroot.addr(), mroot.rev, mweight)) + print('mtree %s, rev %d, weight %d.%d' % ( + mroot.addr(), mroot.rev, mweight >> mbits, 1 << mbits)) # print header - w_width = (m.ceil(m.log10(max(1, mweight)+1)) + w_width = (m.ceil(m.log10(max(1, mweight >> mbits)+1)) + 2*m.ceil(m.log10(max(1, rweight)+1)) + 2) print('%-11s %*s%-*s %-22s %s' % ( diff --git a/tests/test_mtree.toml b/tests/test_mtree.toml index b812605e..fbe35d64 100644 --- a/tests/test_mtree.toml +++ b/tests/test_mtree.toml @@ -3469,7 +3469,7 @@ code = ''' if (tag_ == LFSR_TAG_BTREE) { lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.b.buffer; printf("traversal: %d.%d 0x%x btree 0x%x.%x\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, branch->block, branch->trunk); @@ -3479,7 +3479,7 @@ code = ''' } else if (tag_ == LFSR_TAG_MDIR) { lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.b.buffer; printf("traversal: %d.%d 0x%x mdir 0x{%x,%x}\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, mdir->u.m.blocks[0], mdir->u.m.blocks[1]); @@ -3490,7 +3490,7 @@ code = ''' } else { // this shouldn't happen printf("traversal: %d.%d 0x%x %d\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, lfsr_data_size(&data_)); @@ -3593,7 +3593,7 @@ code = ''' if (tag_ == LFSR_TAG_BTREE) { lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.b.buffer; printf("traversal: %d.%d 0x%x btree 0x%x.%x\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, branch->block, branch->trunk); @@ -3603,7 +3603,7 @@ code = ''' } else if (tag_ == LFSR_TAG_MDIR) { lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.b.buffer; printf("traversal: %d.%d 0x%x mdir 0x{%x,%x}\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, mdir->u.m.blocks[0], mdir->u.m.blocks[1]); @@ -3614,7 +3614,7 @@ code = ''' } else { // this shouldn't happen printf("traversal: %d.%d 0x%x %d\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, lfsr_data_size(&data_)); @@ -3728,7 +3728,7 @@ code = ''' if (tag_ == LFSR_TAG_BTREE) { lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.b.buffer; printf("traversal: %d.%d 0x%x btree 0x%x.%x\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, branch->block, branch->trunk); @@ -3738,7 +3738,7 @@ code = ''' } else if (tag_ == LFSR_TAG_MDIR) { lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.b.buffer; printf("traversal: %d.%d 0x%x mdir 0x{%x,%x}\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, mdir->u.m.blocks[0], mdir->u.m.blocks[1]); @@ -3749,7 +3749,7 @@ code = ''' } else { // this shouldn't happen printf("traversal: %d.%d 0x%x %d\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, lfsr_data_size(&data_)); @@ -3855,7 +3855,7 @@ code = ''' if (tag_ == LFSR_TAG_BTREE) { lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.b.buffer; printf("traversal: %d.%d 0x%x btree 0x%x.%x\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, branch->block, branch->trunk); @@ -3865,7 +3865,7 @@ code = ''' } else if (tag_ == LFSR_TAG_MDIR) { lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.b.buffer; printf("traversal: %d.%d 0x%x mdir 0x{%x,%x}\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, mdir->u.m.blocks[0], mdir->u.m.blocks[1]); @@ -3876,7 +3876,7 @@ code = ''' } else { // this shouldn't happen printf("traversal: %d.%d 0x%x %d\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, lfsr_data_size(&data_)); @@ -3993,7 +3993,7 @@ code = ''' if (tag_ == LFSR_TAG_BTREE) { lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.b.buffer; printf("traversal: %d.%d 0x%x btree 0x%x.%x\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, branch->block, branch->trunk); @@ -4003,7 +4003,7 @@ code = ''' } else if (tag_ == LFSR_TAG_MDIR) { lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.b.buffer; printf("traversal: %d.%d 0x%x mdir 0x{%x,%x}\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, mdir->u.m.blocks[0], mdir->u.m.blocks[1]); @@ -4014,7 +4014,7 @@ code = ''' } else { // this shouldn't happen printf("traversal: %d.%d 0x%x %d\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, lfsr_data_size(&data_)); @@ -4163,7 +4163,7 @@ code = ''' if (tag_ == LFSR_TAG_BTREE) { lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.b.buffer; printf("traversal: %d.%d 0x%x btree 0x%x.%x\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, branch->block, branch->trunk); @@ -4173,7 +4173,7 @@ code = ''' } else if (tag_ == LFSR_TAG_MDIR) { lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.b.buffer; printf("traversal: %d.%d 0x%x mdir 0x{%x,%x}\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, mdir->u.m.blocks[0], mdir->u.m.blocks[1]); @@ -4184,7 +4184,7 @@ code = ''' } else { // this shouldn't happen printf("traversal: %d.%d 0x%x %d\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, lfsr_data_size(&data_)); @@ -4272,21 +4272,21 @@ code = ''' if (tag_ == LFSR_TAG_BTREE) { lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.b.buffer; printf("traversal: %d.%d 0x%x btree 0x%x.%x\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, branch->block, branch->trunk); } else if (tag_ == LFSR_TAG_MDIR) { lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.b.buffer; printf("traversal: %d.%d 0x%x mdir 0x{%x,%x}\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, mdir->u.m.blocks[0], mdir->u.m.blocks[1]); } else { // this shouldn't happen printf("traversal: %d.%d 0x%x %d\n", - mid_ & lfsr_mbidmask(&lfs), + mid_ >> lfs.mbits, mid_ & lfsr_mridmask(&lfs), tag_, lfsr_data_size(&data_));