From 97e2786545be746fb3329a0bdf4b9c5c4e17ae27 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 5 Apr 2025 02:23:52 -0500 Subject: [PATCH] scripts: Synced dbgbmapd3.py Lfs class changes - Added Lfs.traverse for full filesystem traversal - Added Rbyd.shrub flag so we can tell if an Rbyd is a shrub - Removed redundant leaves from paths in leaf iters --- scripts/dbgbmapd3.py | 4 -- scripts/dbgbtree.py | 11 +++++- scripts/dbglfs.py | 91 ++++++++++++++++++++++++++++++++++++++++++-- scripts/dbgmtree.py | 22 ++++++++--- scripts/dbgrbyd.py | 3 ++ 5 files changed, 117 insertions(+), 14 deletions(-) diff --git a/scripts/dbgbmapd3.py b/scripts/dbgbmapd3.py index 6dd5f61f..c1c2f66f 100755 --- a/scripts/dbgbmapd3.py +++ b/scripts/dbgbmapd3.py @@ -586,7 +586,6 @@ class Ralt: return hash((self.tag, self.weight, self.jump)) -# TODO sync # our core rbyd type class Rbyd: def __init__(self, blocks, trunk, weight, rev, eoff, cksum, data, *, @@ -1018,7 +1017,6 @@ class Rbyd: return best -# TODO sync # our rbyd btree type class Btree: def __init__(self, bd, rbyd): @@ -1648,7 +1646,6 @@ class Mdir: return Mid(self.mid, rid), name_ -# TODO sync # the mtree, the skeletal structure of littlefs class Mtree: def __init__(self, bd, mrootchain, mtree, *, @@ -2797,7 +2794,6 @@ class Gstate: locals()[g.__name__.lower()] = ft.cached_property(_parser(g)) -# TODO sync # high-level littlefs representation class Lfs: def __init__(self, bd, mtree, config=None, gstate=None, cksum=None, *, diff --git a/scripts/dbgbtree.py b/scripts/dbgbtree.py index de05a4fa..e953b140 100755 --- a/scripts/dbgbtree.py +++ b/scripts/dbgbtree.py @@ -471,6 +471,7 @@ class Ralt: # our core rbyd type class Rbyd: def __init__(self, blocks, trunk, weight, rev, eoff, cksum, data, *, + shrub=False, gcksumdelta=None, corrupt=False): if isinstance(blocks, int): @@ -484,6 +485,7 @@ class Rbyd: self.cksum = cksum self.data = data + self.shrub = shrub self.gcksumdelta = gcksumdelta self.corrupt = corrupt @@ -672,6 +674,7 @@ class Rbyd: # this helps avoid race conditions with cksums and stuff shrub = cls._fetch(rbyd.data, rbyd.block, trunk) shrub.blocks = rbyd.blocks + shrub.shrub = True return shrub def lookupnext(self, rid, tag=None, *, @@ -930,6 +933,10 @@ class Btree: def cksum(self): return self.rbyd.cksum + @property + def shrub(self): + return self.rbyd.shrub + def addr(self): return self.rbyd.addr() @@ -1112,7 +1119,9 @@ class Btree: break if path: - yield bid-rid + (rbyd.weight-1), rbyd, path_[:-1] + yield (bid-rid + (rbyd.weight-1), rbyd, + # path tail is usually redundant unless corrupt + path_[:-1] if rbyd else path_) else: yield bid-rid + (rbyd.weight-1), rbyd bid += rbyd.weight - rid + 1 diff --git a/scripts/dbglfs.py b/scripts/dbglfs.py index 5ebc99f2..18253472 100755 --- a/scripts/dbglfs.py +++ b/scripts/dbglfs.py @@ -520,6 +520,7 @@ class Ralt: # our core rbyd type class Rbyd: def __init__(self, blocks, trunk, weight, rev, eoff, cksum, data, *, + shrub=False, gcksumdelta=None, corrupt=False): if isinstance(blocks, int): @@ -533,6 +534,7 @@ class Rbyd: self.cksum = cksum self.data = data + self.shrub = shrub self.gcksumdelta = gcksumdelta self.corrupt = corrupt @@ -721,6 +723,7 @@ class Rbyd: # this helps avoid race conditions with cksums and stuff shrub = cls._fetch(rbyd.data, rbyd.block, trunk) shrub.blocks = rbyd.blocks + shrub.shrub = True return shrub def lookupnext(self, rid, tag=None, *, @@ -979,6 +982,10 @@ class Btree: def cksum(self): return self.rbyd.cksum + @property + def shrub(self): + return self.rbyd.shrub + def addr(self): return self.rbyd.addr() @@ -1161,7 +1168,9 @@ class Btree: break if path: - yield bid-rid + (rbyd.weight-1), rbyd, path_[:-1] + yield (bid-rid + (rbyd.weight-1), rbyd, + # path tail is usually redundant unless corrupt + path_[:-1] if rbyd else path_) else: yield bid-rid + (rbyd.weight-1), rbyd bid += rbyd.weight - rid + 1 @@ -1926,7 +1935,9 @@ class Mtree: else: bid, rbyd, rid = mdir if path: - yield (bid-rid + (rbyd.weight-1), rbyd), path_[:-1] + yield ((bid-rid + (rbyd.weight-1), rbyd), + # path tail is usually redundant unless corrupt + path_[:-1] if rbyd else path_) else: yield (bid-rid + (rbyd.weight-1), rbyd) mid = self.mid(bid-rid + (rbyd.weight-1) + 1) @@ -3111,6 +3122,70 @@ class Lfs: file.orphaned = True yield file + # traverse the filesystem + def traverse(self, *, + mtree_only=False, + shrubs=False, + fragments=False, + path=False): + # traverse the mtree + for r in self.mtree.traverse( + path=path): + if path: + mdir, path_ = r + else: + mdir = r + + # mdir? + if isinstance(mdir, Mdir): + if path: + yield mdir, path_ + else: + yield mdir + + # btree node? we only care about the rbyd for simplicity + else: + bid, rbyd = mdir + if path: + yield rbyd, path_ + else: + yield rbyd + + # traverse file bshrubs/btrees + if not mtree_only and isinstance(mdir, Mdir): + for mid, name in mdir.mids(): + file = self._open(mid, mdir, name.tag, name) + for r in file.traverse( + path=path): + if path: + pos, data, path__ = r + path__ = [(mid, mdir, name)]+path__ + else: + pos, data = r + + # inlined data? we usually ignore these + if isinstance(data, Rattr): + if fragments: + if path: + yield data, path_+path__ + else: + yield data + # block pointer? + elif isinstance(data, Bptr): + if path: + yield data, path_+path__ + else: + yield data + # bshrub/btree node? we only care about the rbyd + # for simplicity, we also usually ignore shrubs + # since these live the the parent mdir + else: + if shrubs or not data.shrub: + if path: + yield data, path_+path__ + else: + yield data + # common file operations, note Reg extends this for regular files class File: tag = None @@ -3216,6 +3291,13 @@ class Lfs: def _lookupleaf(self, pos, *, path=False, depth=None): + # no bshrub? + if self.bshrub is None: + if path: + return None, None, [] + else: + return None, None + # lookup data in our bshrub r = self.bshrub.lookupleaf(pos, path=path or depth, @@ -3228,7 +3310,7 @@ class Lfs: if path: return None, None, path_ else: - return None, None, *path_ + return None, None # corrupt btree node? if not rbyd: @@ -3317,7 +3399,8 @@ class Lfs: bid, rbyd, rid = data if path: yield (pos, (bid-rid + (rbyd.weight-1), rbyd), - path_[:-1]) + # path tail is usually redundant unless corrupt + path_[:-1] if rbyd else path_) else: yield pos, (bid-rid + (rbyd.weight-1), rbyd) pos += rbyd.weight diff --git a/scripts/dbgmtree.py b/scripts/dbgmtree.py index 418ddd7f..6013991d 100755 --- a/scripts/dbgmtree.py +++ b/scripts/dbgmtree.py @@ -486,6 +486,7 @@ class Ralt: # our core rbyd type class Rbyd: def __init__(self, blocks, trunk, weight, rev, eoff, cksum, data, *, + shrub=False, gcksumdelta=None, corrupt=False): if isinstance(blocks, int): @@ -499,6 +500,7 @@ class Rbyd: self.cksum = cksum self.data = data + self.shrub = shrub self.gcksumdelta = gcksumdelta self.corrupt = corrupt @@ -687,6 +689,7 @@ class Rbyd: # this helps avoid race conditions with cksums and stuff shrub = cls._fetch(rbyd.data, rbyd.block, trunk) shrub.blocks = rbyd.blocks + shrub.shrub = True return shrub def lookupnext(self, rid, tag=None, *, @@ -945,6 +948,10 @@ class Btree: def cksum(self): return self.rbyd.cksum + @property + def shrub(self): + return self.rbyd.shrub + def addr(self): return self.rbyd.addr() @@ -1127,7 +1134,9 @@ class Btree: break if path: - yield bid-rid + (rbyd.weight-1), rbyd, path_[:-1] + yield (bid-rid + (rbyd.weight-1), rbyd, + # path tail is usually redundant unless corrupt + path_[:-1] if rbyd else path_) else: yield bid-rid + (rbyd.weight-1), rbyd bid += rbyd.weight - rid + 1 @@ -1892,7 +1901,9 @@ class Mtree: else: bid, rbyd, rid = mdir if path: - yield (bid-rid + (rbyd.weight-1), rbyd), path_[:-1] + yield ((bid-rid + (rbyd.weight-1), rbyd), + # path tail is usually redundant unless corrupt + path_[:-1] if rbyd else path_) else: yield (bid-rid + (rbyd.weight-1), rbyd) mid = self.mid(bid-rid + (rbyd.weight-1) + 1) @@ -2107,9 +2118,10 @@ class Mtree: else: bid_, rbyd_, rid_, name_ = r if bid_ is None: - return None, path_ - else: - return None + if path: + return None, path_ + else: + return None # corrupt btree node? if not rbyd_: diff --git a/scripts/dbgrbyd.py b/scripts/dbgrbyd.py index 68d57860..192fa2fc 100755 --- a/scripts/dbgrbyd.py +++ b/scripts/dbgrbyd.py @@ -454,6 +454,7 @@ class Ralt: # our core rbyd type class Rbyd: def __init__(self, blocks, trunk, weight, rev, eoff, cksum, data, *, + shrub=False, gcksumdelta=None, corrupt=False): if isinstance(blocks, int): @@ -467,6 +468,7 @@ class Rbyd: self.cksum = cksum self.data = data + self.shrub = shrub self.gcksumdelta = gcksumdelta self.corrupt = corrupt @@ -655,6 +657,7 @@ class Rbyd: # this helps avoid race conditions with cksums and stuff shrub = cls._fetch(rbyd.data, rbyd.block, trunk) shrub.blocks = rbyd.blocks + shrub.shrub = True return shrub def lookupnext(self, rid, tag=None, *,