Github/littlefs
1
Fork 0
Code Actions Activity

Added dbgbmap.py, tweaked tracebd.py to match

Browse Source 
dbgbmap.py parses littlefs's mtree/btrees and displays that status of
every block in use:

  $ ./scripts/dbgbmap.py disk -B4096x256 -Z -H8 -W64
  bd 4096x256,   7.8% mdir,  10.2% btree,  78.1% data
  mmddbbddddddmmddddmmdd--bbbbddddddddddddddbbdddd--ddddddmmdddddd
  mmddddbbddbbddddddddddddddddbbddddbbddddddmmddbbdddddddddddddddd
  bbdddddddddddd--ddddddddddddddddbbddddmmmmddddddddddddmmmmdddddd
  ddddddddddbbdddddddddd--ddddddddddddddmmddddddddddddddddddddmmdd
  ddddddbbddddddddbb--ddddddddddddddddddddbb--mmmmddbbdddddddddddd
  ddddddddddddddddddddbbddbbdddddddddddddddddddddddddddddddddddddd
  dddddddddd--ddddbbddddddddmmbbdd--ddddddddddddddbbmmddddbbdddddd
  ddmmddddddddddmmddddddddmmddddbbbbdddddddd--ddbbddddddmmdd--ddbb

  (ok, it looks a bit better with colors)

dbgbmap.py matches the layout and has the same options as tracebd.py,
allowing the combination of both to provide valuable insight into what
exactly littlefs is doing.

This required a bit of tweaking of tracebd.py to get right, mostly
around conflicting order-based arguments. This also reworks the internal
Bmap class to be more resilient to out-of-window ops, and adds an
optional informative header.
This commit is contained in:
Christopher Haster
2023-10-30 15:15:58 -05:00
parent 3e46139a45
commit 4ecf4cc654
6 changed files with 2098 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/dbgbmap.py
Executable
+1835
View File
File diff suppressed because it is too large Load Diff
scripts/dbgbtree.py
+5 -5
View File
@@ -119,6 +119,9 @@ def crc32c(data, crc=0):
crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78)
return 0xffffffff ^ crc
def popc(x):
return bin(x).count('1')
def fromle32(data):
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0]
@@ -145,9 +148,6 @@ def frombranch(data):
cksum = fromle32(data[d:]); d += 4
return block, trunk, cksum
def popc(x):
return bin(x).count('1')
def xxd(data, width=16):
for i in range(0, len(data), width):
yield '%-*s %-*s' % (
@@ -994,8 +994,8 @@ def main(disk, roots=None, *,
# show the branch
dbg_branch(bid, w, rbyd, rid, tags, len(path)-1)
if args.get('error_on_corrupt') and corrupted:
sys.exit(2)
if args.get('error_on_corrupt') and corrupted:
sys.exit(2)
if __name__ == "__main__":
scripts/dbglfs.py
+24 -29
View File
@@ -120,6 +120,9 @@ def crc32c(data, crc=0):
crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78)
return 0xffffffff ^ crc
def popc(x):
return bin(x).count('1')
def fromle32(data):
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0]
@@ -148,6 +151,12 @@ def frommdir(data):
d += d_
return blocks
def fromshrub(data):
d = 0
weight, d_ = fromleb128(data[d:]); d += d_
trunk, d_ = fromleb128(data[d:]); d += d_
return weight, trunk
def frombranch(data):
d = 0
block, d_ = fromleb128(data[d:]); d += d_
@@ -161,8 +170,12 @@ def frombtree(data):
block, trunk, cksum = frombranch(data[d:])
return w, block, trunk, cksum
def popc(x):
return bin(x).count('1')
def frombptr(data):
d = 0
size, d_ = fromleb128(data[d:]); d += d_
block, d_ = fromleb128(data[d:]); d += d_
off, d_ = fromleb128(data[d:]); d += d_
return size, block, off
def xxd(data, width=16):
for i in range(0, len(data), width):
@@ -1263,28 +1276,19 @@ def frepr(mdir, rid, tag):
# shrub?
done, rid_, tag_, w_, j, d, data, _ = mdir.lookup(rid, TAG_TRUNK)
if not done and rid_ == rid and tag_ == TAG_TRUNK:
d = 0
weight, d_ = fromleb128(data[d:]); d += d_
trunk, d_ = fromleb128(data[d:]); d += d_
weight, trunk = fromshrub(data)
size = max(size, weight)
structs.append('trunk 0x%x.%x %d' % (mdir.block, trunk, weight))
# direct block?
done, rid_, tag_, w_, j, d, data, _ = mdir.lookup(rid, TAG_BLOCK)
if not done and rid_ == rid and tag_ == TAG_BLOCK:
d = 0
size_, d_ = fromleb128(data[d:]); d += d_
block, d_ = fromleb128(data[d:]); d += d_
off, d_ = fromleb128(data[d:]); d += d_
size_, block, off = frombptr(data)
size = max(size, size_)
structs.append('block 0x%x.%x %d' % (block, off, size_))
# indirect btree?
done, rid_, tag_, w_, j, d, data, _ = mdir.lookup(rid, TAG_BTREE)
if not done and rid_ == rid and tag_ == TAG_BTREE:
d = 0
weight, d_ = fromleb128(data[d:]); d += d_
block, d_ = fromleb128(data[d:]); d += d_
trunk, d_ = fromleb128(data[d:]); d += d_
cksum = fromle32(data[d:]); d += 4
weight, block, trunk, cksum = frombtree(data)
size = max(size, weight)
structs.append('btree 0x%x.%x %d' % (block, trunk, weight))
return 'reg %s' % ', '.join(it.chain(['%d' % size], structs))
@@ -1311,9 +1315,7 @@ def dbg_fstruct(f, block_size, mdir, rid, tag, j, d, data, *,
w = len(data)
# shrub?
elif tag == TAG_TRUNK:
d = 0
weight, d_ = fromleb128(data[d:]); d += d_
trunk, d_ = fromleb128(data[d:]); d += d_
weight, trunk = fromshrub(data)
btree = Rbyd.fetch(f, block_size, mdir.block, trunk)
w = weight
# direct block?
@@ -1325,18 +1327,11 @@ def dbg_fstruct(f, block_size, mdir, rid, tag, j, d, data, *,
mdir.eoff,
j,
0)
d = 0
size, d_ = fromleb128(data[d:]); d += d_
block, d_ = fromleb128(data[d:]); d += d_
off, d_ = fromleb128(data[d:]); d += d_
size, block, off = frombptr(data)
w = size
# indirect btree?
elif tag == TAG_BTREE:
d = 0
weight, d_ = fromleb128(data[d:]); d += d_
block, d_ = fromleb128(data[d:]); d += d_
trunk, d_ = fromleb128(data[d:]); d += d_
cksum = fromle32(data[d:]); d += 4
weight, block, trunk, cksum = frombtree(data)
btree = Rbyd.fetch(f, block_size, block, trunk)
w = weight
@@ -2202,8 +2197,8 @@ def main(disk, mroots=None, *,
rec_dir(0, args.get('depth') or m.inf)
if args.get('error_on_corrupt') and corrupted:
sys.exit(2)
if args.get('error_on_corrupt') and corrupted:
sys.exit(2)
if __name__ == "__main__":
@@ -2305,7 +2300,7 @@ if __name__ == "__main__":
parser.add_argument(
'-e', '--error-on-corrupt',
action='store_true',
help="Error if B-tree is corrupt.")
help="Error if the filesystem is corrupt.")
sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None}))
scripts/dbgmtree.py
+6 -6
View File
@@ -119,6 +119,9 @@ def crc32c(data, crc=0):
crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78)
return 0xffffffff ^ crc
def popc(x):
return bin(x).count('1')
def fromle32(data):
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0]
@@ -160,9 +163,6 @@ def frombtree(data):
block, trunk, cksum = frombranch(data[d:])
return w, block, trunk, cksum
def popc(x):
return bin(x).count('1')
def xxd(data, width=16):
for i in range(0, len(data), width):
yield '%-*s %-*s' % (
@@ -1646,8 +1646,8 @@ def main(disk, mroots=None, *,
# force next btree entry to be shown
prbyd = None
if args.get('error_on_corrupt') and corrupted:
sys.exit(2)
if args.get('error_on_corrupt') and corrupted:
sys.exit(2)
if __name__ == "__main__":
@@ -1715,7 +1715,7 @@ if __name__ == "__main__":
parser.add_argument(
'-e', '--error-on-corrupt',
action='store_true',
help="Error if B-tree is corrupt.")
help="Error if the filesystem is corrupt.")
sys.exit(main(**{k: v
for k, v in vars(parser.parse_intermixed_args()).items()
if v is not None}))
scripts/dbgrbyd.py
+3 -3
View File
@@ -128,6 +128,9 @@ def crc32c(data, crc=0):
crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78)
return 0xffffffff ^ crc
def popc(x):
return bin(x).count('1')
def fromle32(data):
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0]
@@ -147,9 +150,6 @@ def fromtag(data):
size, d_ = fromleb128(data[2+d:])
return tag>>15, tag&0x7fff, weight, size, 2+d+d_
def popc(x):
return bin(x).count('1')
def xxd(data, width=16):
for i in range(0, len(data), width):
yield '%-*s %-*s' % (
scripts/tracebd.py
+225 -137
View File
@@ -22,10 +22,10 @@ import time
CHARS = 'rpe-'
COLORS = ['32', '35', '34', '']
COLORS = ['32', '35', '34', '90']
WEAR_CHARS = '-123456789'
WEAR_COLORS = ['', '', '', '', '', '', '', '35', '35', '1;31']
WEAR_COLORS = ['90', '', '', '', '', '', '', '35', '35', '1;31']
CHARS_DOTS = " .':"
CHARS_BRAILLE = (
@@ -390,25 +390,52 @@ class Pixel(int):
return c
class Bd:
class Bmap:
def __init__(self, *,
block_size=1,
block_count=1,
block_window=None,
off_window=None,
width=None,
height=1,
pixels=None):
# default width to block_window or block_size
if width is None:
width = block_count
if block_window is not None:
width = len(block_window)
else:
width = block_count
# allocate pixels if not provided
if pixels is None:
self.pixels = [Pixel() for _ in range(width*height)]
else:
self.pixels = pixels
pixels = [Pixel() for _ in range(width*height)]
self.pixels = pixels
self.block_size = block_size
self.block_count = block_count
self.block_window = block_window
self.off_window = off_window
self.width = width
self.height = height
@property
def _block_window(self):
if self.block_window is None:
return range(0, self.block_count)
else:
return self.block_window
@property
def _off_window(self):
if self.off_window is None:
return range(0, self.block_size)
else:
return self.off_window
@property
def _window(self):
return len(self._off_window)*len(self._block_window)
def _op(self, f, block=None, off=None, size=None):
if block is None:
range_ = range(len(self.pixels))
@@ -418,21 +445,31 @@ class Bd:
elif size is None:
off, size = 0, off
# update our geometry?
if off+size > self.block_size or block >= self.block_count:
self.resize(
block_size=max(self.block_size, off+size),
block_count=max(self.block_count, block+1))
# map into our window
if block not in self._block_window:
return
block -= self._block_window.start
size = (max(self._off_window.start,
min(self._off_window.stop, off+size))
- max(self._off_window.start,
min(self._off_window.stop, off)))
off = (max(self._off_window.start,
min(self._off_window.stop, off))
- self._off_window.start)
if size == 0:
return
# map to our block space
start = block*self.block_size + off
stop = block*self.block_size + off+size
start = ((start*len(self.pixels))
// (self.block_size*self.block_count))
stop = ((stop*len(self.pixels))
// (self.block_size*self.block_count))
stop = max(stop, start+1)
range_ = range(start, stop)
range_ = range(
block*len(self._off_window) + off,
block*len(self._off_window) + off+size)
range_ = range(
(range_.start*len(self.pixels)) // self._window,
(range_.stop*len(self.pixels)) // self._window)
range_ = range(
range_.start,
max(range_.stop, range_.start+1))
# apply the op
for i in range_:
@@ -450,14 +487,6 @@ class Bd:
def clear(self, block=None, off=None, size=None):
self._op(Pixel.clear, block, off, size)
def copy(self):
return Bd(
pixels=self.pixels.copy(),
block_size=self.block_size,
block_count=self.block_count,
width=self.width,
height=self.height)
def resize(self, *,
block_size=None,
block_count=None,
@@ -477,21 +506,25 @@ class Bd:
return
# transform our pixels
self.block_size = block_size
self.block_count = block_count
pixels = []
for x in range(width*height):
# map into our old bd space
start = (x*(block_size*block_count)) // (width*height)
stop = ((x+1)*(block_size*block_count)) // (width*height)
stop = max(stop, start+1)
range_ = range(
(x*self._window) // (width*height),
((x+1)*self._window) // (width*height))
range_ = range(
range_.start,
max(range_.stop, range_.start+1))
# aggregate state
pixels.append(ft.reduce(
Pixel.__or__,
self.pixels[start:stop],
self.pixels[range_.start:range_.stop],
Pixel()))
self.block_size = block_size
self.block_count = block_count
self.width = width
self.height = height
self.pixels = pixels
@@ -610,17 +643,19 @@ class Bd:
def main(path='-', block=None, *,
off=None,
size=None,
def main(path='-', *,
block_size=None,
block_count=None,
block_cycles=None,
block=None,
off=None,
size=None,
read=False,
prog=False,
erase=False,
wear=False,
reset=False,
no_header=False,
color='auto',
dots=False,
braille=False,
@@ -671,52 +706,64 @@ def main(path='-', block=None, *,
if block_count is None:
block_count = block_count_
# allow ranges for blocks/offs
# try to simplify the block/off/size arguments a bit
if not isinstance(block, tuple):
block = (block,)
if any(isinstance(block, list) and len(block) > 1 for block in block):
block = block,
if isinstance(off, tuple) and len(off) == 1:
off, = off
if isinstance(size, tuple) and len(size) == 1:
if off is None:
off, = size
size = None
if any(isinstance(b, list) and len(b) > 1 for b in block):
print("error: More than one block address?")
sys.exit(-1)
block = tuple(
block[0] if isinstance(block, list) else block
for block in block)
if not isinstance(off, tuple):
off = (off,)
if not isinstance(size, tuple):
size = (size,)
if isinstance(block[0], list):
block = (block[0][0], *block[1:])
if len(block) > 1 and isinstance(block[1], list):
block = (block[0], block[1][0])
if isinstance(block[0], tuple):
block, off_ = (block[0][0], *block[1:]), block[0][1]
if off is None:
off = off_
if len(block) > 1 and isinstance(block[1], tuple):
block = (block[0], block[1][0])
if len(block) == 1:
block, = block
block_start = (
block[0][0] if isinstance(block[0], tuple)
else block[0] if block[0] is not None
else 0)
block_stop = (
block[1][0] if len(block) > 1 and isinstance(block[1], tuple)
else block[1] if len(block) > 1 and block[1] is not None
else block_start+1 if len(block) == 1 and block[0] is not None
else block_count)
off_start = (
off[0] if off[0] is not None
else block[0][1] if isinstance(block[0], tuple)
else block[1][1] if len(block) > 1 and isinstance(block[1], tuple)
else size[0] if len(size) > 1 and size[0] is not None
else 0)
off_stop = (
off_start + size[0] if len(size) == 1 and size[0] is not None
else off[1] if len(off) > 1 and off[1] is not None
else size[1] if len(size) > 1 and size[1] is not None
else block_size)
if isinstance(off, tuple):
off, size_ = off[0], off[1] - off[0]
if size is None:
size = size_
if isinstance(size, tuple):
off_, size = off[0], off[1] - off[0]
if off is None:
off = off_
# create a block device representation
bd = Bd()
# is a block window specified?
block_window = None
if block is not None:
if isinstance(block, tuple):
block_window = range(*block)
else:
block_window = range(block, block+1)
def resize(*, block_size=None, block_count=None):
nonlocal bd
off_window = None
if off is not None or size is not None:
off_ = off if off is not None else 0
size_ = size if size is not None else 1
off_window = range(off_, off_+size_)
# size may be overriden by cli args
if off_stop is not None:
block_size = off_stop-off_start
if block_stop is not None:
block_count = block_stop-block_start
# create our block device representation
bmap = Bmap(
block_size=block_size,
block_count=block_count,
block_window=block_window,
off_window=off_window)
def resize():
nonlocal bmap
# figure out best width/height
if width is None:
@@ -733,17 +780,22 @@ def main(path='-', block=None, *,
else:
height_ = shutil.get_terminal_size((80, 5))[1]
bd.resize(
block_size=block_size,
block_count=block_count,
# scale if we're printing with dots or braille
width=2*width_ if braille else width_,
height=max(1,
4*height_ if braille
else 2*height_ if dots
else height_))
# terminal size changed?
if width_ != bmap.width or height_ != bmap.height:
bmap.resize(
# scale if we're printing with dots or braille
width=2*width_ if braille else width_,
height=max(1,
4*height_ if braille
else 2*height_ if dots
else height_))
resize()
# keep track of some extra info
readed = 0
proged = 0
erased = 0
# parse a line of trace output
pattern = re.compile(
'^(?P<file>[^:]*):(?P<line>[0-9]+):trace:.*?bd_(?:'
@@ -771,7 +823,10 @@ def main(path='-', block=None, *,
'|' '(?P<sync>sync)\('
'\s*(?P<sync_ctx>\w+)' '\s*\)' ')\s*$')
def parse(line):
nonlocal bd
nonlocal bmap
nonlocal readed
nonlocal proged
nonlocal erased
# string searching is much faster than the regex here, and this
# actually has a big impact given how much trace output comes
@@ -787,13 +842,20 @@ def main(path='-', block=None, *,
block_size = int(m.group('block_size'), 0)
block_count = int(m.group('block_count'), 0)
resize(block_size=block_size, block_count=block_count)
if reset:
bd = Bd(
block_size=bd.block_size,
block_count=bd.block_count,
width=bd.width,
height=bd.height)
bmap = Bmap(
block_size=block_size,
block_count=block_count,
block_window=bmap.block_window,
off_window=bmap.off_window,
width=bmap.width,
height=bmap.height)
else:
if (block_size != bmap.block_size
or block_count != bmap.block_count):
bmap.resize(
block_size=block_size,
block_count=block_count)
return True
elif m.group('read') and read:
@@ -801,17 +863,13 @@ def main(path='-', block=None, *,
off = int(m.group('read_off'), 0)
size = int(m.group('read_size'), 0)
if ((block_stop is not None and block >= block_stop)
or block < block_start
or (off_stop is not None and off >= off_stop)
or off+size <= off_start):
return False
block -= block_start
size = ((min(off+size, off_stop)
if off_stop is not None else off+size)
- max(off, off_start))
if block >= bmap.block_count or off+size > bmap.block_size:
bmap.resize(
block_size=max(off+size, bmap.block_size),
block_count=max(block+1, bmap.block_count))
bd.read(block, off, size)
bmap.read(block, off, size)
readed += size
return True
elif m.group('prog') and prog:
@@ -819,35 +877,26 @@ def main(path='-', block=None, *,
off = int(m.group('prog_off'), 0)
size = int(m.group('prog_size'), 0)
if ((block_stop is not None and block >= block_stop)
or block < block_start
or (off_stop is not None and off >= off_stop)
or off+size <= off_start):
return False
block -= block_start
size = ((min(off+size, off_stop)
if off_stop is not None else off+size)
- max(off, off_start))
if block >= bmap.block_count or off+size > bmap.block_size:
bmap.resize(
block_size=max(off+size, bmap.block_size),
block_count=max(block+1, bmap.block_count))
bd.prog(block, off, size)
bmap.prog(block, off, size)
proged += size
return True
elif m.group('erase') and (erase or wear):
block = int(m.group('erase_block'), 0)
off = 0
size = int(m.group('erase_size'), 0)
if ((block_stop is not None and block >= block_stop)
or block < block_start
or (off_stop is not None and off >= off_stop)
or off+size <= off_start):
return False
block -= block_start
size = ((min(off+size, off_stop)
if off_stop is not None else off+size)
- max(off, off_start))
if block >= bmap.block_count or size > bmap.block_size:
bmap.resize(
block_size=max(off+size, bmap.block_size),
block_count=max(block+1, bmap.block_count))
bd.erase(block, off, size)
bmap.erase(block, size)
erased += size
return True
else:
@@ -855,17 +904,49 @@ def main(path='-', block=None, *,
# print trace output
def draw(f):
nonlocal readed
nonlocal proged
nonlocal erased
def writeln(s=''):
f.write(s)
f.write('\n')
f.writeln = writeln
# print some information about read/prog/erases
if not no_header:
# compute total ops
total = readed+proged+erased
# compute stddev of wear using our bmap, this is a bit different
# from read/prog/erase which ignores any bmap window, but it's
# what we have
if wear:
mean = (sum(p.wear for p in bmap.pixels)
/ max(len(bmap.pixels), 1))
stddev = m.sqrt(sum((p.wear - mean)**2 for p in bmap.pixels)
/ max(len(bmap.pixels), 1))
worst = max((p.wear for p in bmap.pixels), default=0)
f.writeln('bd %dx%d%s%s%s%s' % (
block_size, block_count,
', %6s read' % ('%.1f%%' % (100*readed / max(total, 1)))
if read else '',
', %6s prog' % ('%.1f%%' % (100*proged / max(total, 1)))
if prog else '',
', %6s erase' % ('%.1f%%' % (100*erased / max(total, 1)))
if erase else '',
', %13s wear' % ('%.1fσ (%.1f%%)' % (
worst / max(stddev, 1),
100*stddev / max(worst, 1)))
if wear else ''))
# don't forget we've scaled this for braille/dots!
for row in range(
m.ceil(bd.height/4) if braille
else m.ceil(bd.height/2) if dots
else bd.height):
line = bd.draw(row,
m.ceil(bmap.height/4) if braille
else m.ceil(bmap.height/2) if dots
else bmap.height):
line = bmap.draw(row,
read=read,
prog=prog,
erase=erase,
@@ -880,7 +961,10 @@ def main(path='-', block=None, *,
if line:
f.writeln(line)
bd.clear()
bmap.clear()
readed = 0
proged = 0
erased = 0
resize()
@@ -888,7 +972,7 @@ def main(path='-', block=None, *,
if cat:
ring = sys.stdout
else:
ring = LinesIO(lines)
ring = LinesIO(lines + (1 if not no_header else 0))
# if sleep print in background thread to avoid getting stuck in a read call
event = th.Event()
@@ -945,20 +1029,13 @@ if __name__ == "__main__":
import sys
import argparse
parser = argparse.ArgumentParser(
description="Display operations on block devices based on "
description="Render operations on block devices based on "
"trace output.",
allow_abbrev=False)
parser.add_argument(
'path',
nargs='?',
help="Path to read from.")
parser.add_argument(
'block',
nargs='?',
type=lambda x: tuple(
rbydaddr(x) if x.strip() else None
for x in x.split(',')),
help="Optional block to show, may be a range.")
parser.add_argument(
'-B', '--block-size',
type=bdgeom,
@@ -971,6 +1048,13 @@ if __name__ == "__main__":
'-C', '--block-cycles',
type=lambda x: int(x, 0),
help="Assumed maximum number of erase cycles when measuring wear.")
parser.add_argument(
'-@', '--block',
nargs='?',
type=lambda x: tuple(
rbydaddr(x) if x.strip() else None
for x in x.split(',')),
help="Optional block to show, may be a range.")
parser.add_argument(
'--off',
type=lambda x: tuple(
@@ -1003,6 +1087,10 @@ if __name__ == "__main__":
'-R', '--reset',
action='store_true',
help="Reset wear on block device initialization.")
parser.add_argument(
'-N', '--no-header',
action='store_true',
help="Don't show the header.")
parser.add_argument(
'--color',
choices=['never', 'always', 'auto'],