scripts: Added simple mroot cycle detectors to dbg scripts

These work by keeping a set of all seen mroots as we descend down the
mroot chain. Simple, but it works.

The downside of this approach is that the mroot set grows unbounded, but
it's unlikely we'll ever have enough mroots in a system for this to
really matter.

This fixes scripts like dbgbmap.py getting stuck on intentional mroot
cycles created for testing. It's not a problem for a foreground script
to get stuck in an infinite loop, since you can just kill it, but a
background script getting stuck at 100% CPU is a bit more annoying.
This commit is contained in:
Christopher Haster
2024-11-07 02:14:54 -06:00
parent b08c66e387
commit e3fdc3dbd7
9 changed files with 173 additions and 126 deletions
+6 -6
View File
@@ -27,10 +27,10 @@ def bdgeom(s):
return int(s, b)
# parse some rbyd addr encodings
# 0xa -> [0xa]
# 0xa.c -> [(0xa, 0xc)]
# 0x{a,b} -> [0xa, 0xb]
# 0x{a,b}.c -> [(0xa, 0xc), (0xb, 0xc)]
# 0xa -> (0xa,)
# 0xa.c -> ((0xa, 0xc),)
# 0x{a,b} -> (0xa, 0xb)
# 0x{a,b}.c -> ((0xa, 0xc), (0xb, 0xc))
def rbydaddr(s):
s = s.strip()
b = 10
@@ -61,7 +61,7 @@ def rbydaddr(s):
else:
addr.append(int(s, b))
return addr
return tuple(addr)
def xxd(data, width=16):
for i in range(0, len(data), width):
@@ -95,7 +95,7 @@ def main(disk, block=None, *,
# flatten block, default to block 0
if not block:
block = [0]
block = (0,)
if len(block) > 1:
print("error: more than one block address?",
+30 -22
View File
@@ -92,10 +92,10 @@ def bdgeom(s):
return int(s, b)
# parse some rbyd addr encodings
# 0xa -> [0xa]
# 0xa.c -> [(0xa, 0xc)]
# 0x{a,b} -> [0xa, 0xb]
# 0x{a,b}.c -> [(0xa, 0xc), (0xb, 0xc)]
# 0xa -> (0xa,)
# 0xa.c -> ((0xa, 0xc),)
# 0x{a,b} -> (0xa, 0xb)
# 0x{a,b}.c -> ((0xa, 0xc), (0xb, 0xc))
def rbydaddr(s):
s = s.strip()
b = 10
@@ -126,7 +126,7 @@ def rbydaddr(s):
else:
addr.append(int(s, b))
return addr
return tuple(addr)
def crc32c(data, crc=0):
crc ^= 0xffffffff
@@ -168,7 +168,7 @@ def frommdir(data):
block, d_ = fromleb128(data[d:])
blocks.append(block)
d += d_
return blocks
return tuple(blocks)
def fromshrub(data):
d = 0
@@ -591,33 +591,34 @@ class Bmap:
# our core rbyd type
class Rbyd:
def __init__(self, block, data, rev, eoff, trunk, weight, cksum):
self.block = block
def __init__(self, blocks, data, rev, eoff, trunk, weight, cksum):
if isinstance(blocks, int):
blocks = (blocks,)
self.blocks = tuple(blocks)
self.data = data
self.rev = rev
self.eoff = eoff
self.trunk = trunk
self.weight = weight
self.cksum = cksum
self.redund_blocks = []
@property
def blocks(self):
return (self.block, *self.redund_blocks)
def block(self):
return self.blocks[0]
def addr(self):
if not self.redund_blocks:
if len(self.blocks) == 1:
return '0x%x.%x' % (self.block, self.trunk)
else:
return '0x{%x,%s}.%x' % (
self.block,
','.join('%x' % block for block in self.redund_blocks),
return '0x{%s}.%x' % (
','.join('%x' % block for block in self.blocks),
self.trunk)
@classmethod
def fetch(cls, f, block_size, blocks, trunk=None):
if isinstance(blocks, int):
blocks = [blocks]
blocks = (blocks,)
if len(blocks) > 1:
# fetch all blocks
@@ -635,9 +636,9 @@ class Rbyd:
i = i_
# keep track of the other blocks
rbyd = rbyds[i]
rbyd.redund_blocks = [
rbyd.blocks += tuple(
rbyds[(i+1+j) % len(rbyds)].block
for j in range(len(rbyds)-1)]
for j in range(len(rbyds)-1))
return rbyd
else:
# block may encode a trunk
@@ -982,13 +983,13 @@ def main(disk, mroots=None, *,
off, = size
size = None
if any(isinstance(b, list) and len(b) > 1 for b in block):
if any(isinstance(b, tuple) and len(b) > 1 for b in block):
print("error: more than one block address?",
file=sys.stderr)
sys.exit(-1)
if isinstance(block[0], list):
if isinstance(block[0], tuple):
block = (block[0][0], *block[1:])
if len(block) > 1 and isinstance(block[1], list):
if len(block) > 1 and isinstance(block[1], tuple):
block = (block[0], block[1][0])
if isinstance(block[0], tuple):
block, off_ = (block[0][0], *block[1:]), block[0][1]
@@ -1058,7 +1059,7 @@ def main(disk, mroots=None, *,
# flatten mroots, default to 0x{0,1}
if not mroots:
mroots = [[0,1]]
mroots = [(0,1)]
mroots = [block for mroots_ in mroots for block in mroots_]
# we seek around a bunch, so just keep the disk open
@@ -1087,11 +1088,18 @@ def main(disk, mroots=None, *,
btrees__ = []
mroot = Rbyd.fetch(f, block_size, mroots)
mdepth = 1
mseen = set()
while True:
# corrupted?
if not mroot:
corrupted = True
break
# cycle detected
elif mroot.blocks in mseen:
corrupted = True
break
mseen.add(mroot.blocks)
# mark mroots in our bmap
for block in mroot.blocks:
+21 -16
View File
@@ -75,10 +75,10 @@ def bdgeom(s):
return int(s, b)
# parse some rbyd addr encodings
# 0xa -> [0xa]
# 0xa.c -> [(0xa, 0xc)]
# 0x{a,b} -> [0xa, 0xb]
# 0x{a,b}.c -> [(0xa, 0xc), (0xb, 0xc)]
# 0xa -> (0xa,)
# 0xa.c -> ((0xa, 0xc),)
# 0x{a,b} -> (0xa, 0xb)
# 0x{a,b}.c -> ((0xa, 0xc), (0xb, 0xc))
def rbydaddr(s):
s = s.strip()
b = 10
@@ -109,7 +109,7 @@ def rbydaddr(s):
else:
addr.append(int(s, b))
return addr
return tuple(addr)
def crc32c(data, crc=0):
crc ^= 0xffffffff
@@ -263,29 +263,34 @@ TBranch = co.namedtuple('TBranch', 'a, b, d, c')
# our core rbyd type
class Rbyd:
def __init__(self, block, data, rev, eoff, trunk, weight, cksum):
self.block = block
def __init__(self, blocks, data, rev, eoff, trunk, weight, cksum):
if isinstance(blocks, int):
blocks = (blocks,)
self.blocks = tuple(blocks)
self.data = data
self.rev = rev
self.eoff = eoff
self.trunk = trunk
self.weight = weight
self.cksum = cksum
self.redund_blocks = []
@property
def block(self):
return self.blocks[0]
def addr(self):
if not self.redund_blocks:
if len(self.blocks) == 1:
return '0x%x.%x' % (self.block, self.trunk)
else:
return '0x{%x,%s}.%x' % (
self.block,
','.join('%x' % block for block in self.redund_blocks),
return '0x{%s}.%x' % (
','.join('%x' % block for block in self.blocks),
self.trunk)
@classmethod
def fetch(cls, f, block_size, blocks, trunk=None):
if isinstance(blocks, int):
blocks = [blocks]
blocks = (blocks,)
if len(blocks) > 1:
# fetch all blocks
@@ -303,9 +308,9 @@ class Rbyd:
i = i_
# keep track of the other blocks
rbyd = rbyds[i]
rbyd.redund_blocks = [
rbyd.blocks += tuple(
rbyds[(i+1+j) % len(rbyds)].block
for j in range(len(rbyds)-1)]
for j in range(len(rbyds)-1))
return rbyd
else:
# block may encode a trunk
@@ -605,7 +610,7 @@ def main(disk, roots=None, *,
# flatten roots, default to block 0
if not roots:
roots = [[0]]
roots = [(0,)]
roots = [block for roots_ in roots for block in roots_]
# we seek around a bunch, so just keep the disk open
+6 -6
View File
@@ -27,10 +27,10 @@ def bdgeom(s):
return int(s, b)
# parse some rbyd addr encodings
# 0xa -> [0xa]
# 0xa.c -> [(0xa, 0xc)]
# 0x{a,b} -> [0xa, 0xb]
# 0x{a,b}.c -> [(0xa, 0xc), (0xb, 0xc)]
# 0xa -> (0xa,)
# 0xa.c -> ((0xa, 0xc),)
# 0x{a,b} -> (0xa, 0xb)
# 0x{a,b}.c -> ((0xa, 0xc), (0xb, 0xc))
def rbydaddr(s):
s = s.strip()
b = 10
@@ -61,7 +61,7 @@ def rbydaddr(s):
else:
addr.append(int(s, b))
return addr
return tuple(addr)
def xxd(data, width=16):
for i in range(0, len(data), width):
@@ -94,7 +94,7 @@ def main(disk, blocks=None, *,
# flatten block, default to block 0
if not blocks:
blocks = [[0]]
blocks = [(0,)]
blocks = [block for blocks_ in blocks for block in blocks_]
with open(disk, 'rb') as f:
+32 -26
View File
@@ -76,10 +76,10 @@ def bdgeom(s):
return int(s, b)
# parse some rbyd addr encodings
# 0xa -> [0xa]
# 0xa.c -> [(0xa, 0xc)]
# 0x{a,b} -> [0xa, 0xb]
# 0x{a,b}.c -> [(0xa, 0xc), (0xb, 0xc)]
# 0xa -> (0xa,)
# 0xa.c -> ((0xa, 0xc),)
# 0x{a,b} -> (0xa, 0xb)
# 0x{a,b}.c -> ((0xa, 0xc), (0xb, 0xc))
def rbydaddr(s):
s = s.strip()
b = 10
@@ -110,7 +110,7 @@ def rbydaddr(s):
else:
addr.append(int(s, b))
return addr
return tuple(addr)
def crc32c(data, crc=0):
crc ^= 0xffffffff
@@ -152,7 +152,7 @@ def frommdir(data):
block, d_ = fromleb128(data[d:])
blocks.append(block)
d += d_
return blocks
return tuple(blocks)
def fromshrub(data):
d = 0
@@ -294,29 +294,34 @@ TBranch = co.namedtuple('TBranch', 'a, b, d, c')
# our core rbyd type
class Rbyd:
def __init__(self, block, data, rev, eoff, trunk, weight, cksum):
self.block = block
def __init__(self, blocks, data, rev, eoff, trunk, weight, cksum):
if isinstance(blocks, int):
blocks = (blocks,)
self.blocks = tuple(blocks)
self.data = data
self.rev = rev
self.eoff = eoff
self.trunk = trunk
self.weight = weight
self.cksum = cksum
self.redund_blocks = []
@property
def block(self):
return self.blocks[0]
def addr(self):
if not self.redund_blocks:
if len(self.blocks) == 1:
return '0x%x.%x' % (self.block, self.trunk)
else:
return '0x{%x,%s}.%x' % (
self.block,
','.join('%x' % block for block in self.redund_blocks),
return '0x{%s}.%x' % (
','.join('%x' % block for block in self.blocks),
self.trunk)
@classmethod
def fetch(cls, f, block_size, blocks, trunk=None):
if isinstance(blocks, int):
blocks = [blocks]
blocks = (blocks,)
if len(blocks) > 1:
# fetch all blocks
@@ -334,9 +339,9 @@ class Rbyd:
i = i_
# keep track of the other blocks
rbyd = rbyds[i]
rbyd.redund_blocks = [
rbyd.blocks += tuple(
rbyds[(i+1+j) % len(rbyds)].block
for j in range(len(rbyds)-1)]
for j in range(len(rbyds)-1))
return rbyd
else:
# block may encode a trunk
@@ -1722,7 +1727,7 @@ def main(disk, mroots=None, *,
# flatten mroots, default to 0x{0,1}
if not mroots:
mroots = [[0,1]]
mroots = [(0,1)]
mroots = [block for mroots_ in mroots for block in mroots_]
# we seek around a bunch, so just keep the disk open
@@ -1753,11 +1758,18 @@ def main(disk, mroots=None, *,
mroot = Rbyd.fetch(f, block_size, mroots)
mdepth = 1
mseen = set()
while True:
# corrupted?
if not mroot:
corrupted = True
break
# cycle detected?
elif mroot.blocks in mseen:
corrupted = True
break
mseen.add(mroot.blocks)
rweight = max(rweight, mroot.weight)
# yes we get gstate from all mroots
@@ -1932,9 +1944,7 @@ def main(disk, mroots=None, *,
for i, (repr_, tag, j, data) in enumerate(config.repr()):
print('%12s %*s %-*s %s' % (
'{%s}:' % ','.join('%04x' % block
for block in it.chain(
[mroot.block],
mroot.redund_blocks))
for block in mroot.blocks)
if i == 0 else '',
2*w_width+1, '%d.%d' % (-1, -1)
if i == 0 else '',
@@ -1978,9 +1988,7 @@ def main(disk, mroots=None, *,
print('%s%12s %*s %-*s %s%s' % (
'\x1b[90m' if color else '',
'{%s}:' % ','.join('%04x' % block
for block in it.chain(
[mdir.block],
mdir.redund_blocks)),
for block in mdir.blocks),
2*w_width+1, '%d.%d' % (
mbid//mleaf_weight, -1),
21+w_width, tagrepr(tag, 0, len(data)),
@@ -2075,9 +2083,7 @@ def main(disk, mroots=None, *,
or tag == TAG_ORPHAN)
else '',
'{%s}:' % ','.join('%04x' % block
for block in it.chain(
[mdir.block],
mdir.redund_blocks))
for block in mdir.blocks)
if mbid != pmbid else '',
2*w_width+1, '%d.%d-%d' % (
mbid//mleaf_weight, rid-(w-1), rid)
+58 -30
View File
@@ -75,10 +75,10 @@ def bdgeom(s):
return int(s, b)
# parse some rbyd addr encodings
# 0xa -> [0xa]
# 0xa.c -> [(0xa, 0xc)]
# 0x{a,b} -> [0xa, 0xb]
# 0x{a,b}.c -> [(0xa, 0xc), (0xb, 0xc)]
# 0xa -> (0xa,)
# 0xa.c -> ((0xa, 0xc),)
# 0x{a,b} -> (0xa, 0xb)
# 0x{a,b}.c -> ((0xa, 0xc), (0xb, 0xc))
def rbydaddr(s):
s = s.strip()
b = 10
@@ -109,7 +109,7 @@ def rbydaddr(s):
else:
addr.append(int(s, b))
return addr
return tuple(addr)
def crc32c(data, crc=0):
crc ^= 0xffffffff
@@ -151,7 +151,7 @@ def frommdir(data):
block, d_ = fromleb128(data[d:])
blocks.append(block)
d += d_
return blocks
return tuple(blocks)
def frombranch(data):
d = 0
@@ -278,29 +278,34 @@ TBranch = co.namedtuple('TBranch', 'a, b, d, c')
# our core rbyd type
class Rbyd:
def __init__(self, block, data, rev, eoff, trunk, weight, cksum):
self.block = block
def __init__(self, blocks, data, rev, eoff, trunk, weight, cksum):
if isinstance(blocks, int):
blocks = (blocks,)
self.blocks = tuple(blocks)
self.data = data
self.rev = rev
self.eoff = eoff
self.trunk = trunk
self.weight = weight
self.cksum = cksum
self.redund_blocks = []
@property
def block(self):
return self.blocks[0]
def addr(self):
if not self.redund_blocks:
if len(self.blocks) == 1:
return '0x%x.%x' % (self.block, self.trunk)
else:
return '0x{%x,%s}.%x' % (
self.block,
','.join('%x' % block for block in self.redund_blocks),
return '0x{%s}.%x' % (
','.join('%x' % block for block in self.blocks),
self.trunk)
@classmethod
def fetch(cls, f, block_size, blocks, trunk=None):
if isinstance(blocks, int):
blocks = [blocks]
blocks = (blocks,)
if len(blocks) > 1:
# fetch all blocks
@@ -318,9 +323,9 @@ class Rbyd:
i = i_
# keep track of the other blocks
rbyd = rbyds[i]
rbyd.redund_blocks = [
rbyd.blocks += tuple(
rbyds[(i+1+j) % len(rbyds)].block
for j in range(len(rbyds)-1)]
for j in range(len(rbyds)-1))
return rbyd
else:
# block may encode a trunk
@@ -872,8 +877,8 @@ def main(disk, mroots=None, *,
# flatten mroots, default to 0x{0,1}
if not mroots:
mroots = [[0,1]]
mroots = [block for mroots_ in mroots for block in mroots_]
mroots = [(0,1)]
mroots = tuple(block for mroots_ in mroots for block in mroots_)
# we seek around a bunch, so just keep the disk open
with open(disk, 'rb') as f:
@@ -894,10 +899,16 @@ def main(disk, mroots=None, *,
mroot = Rbyd.fetch(f, block_size, mroots)
mdepth = 1
mseen = set()
while True:
# corrupted?
if not mroot:
break
# cycle detected?
elif mroot.blocks in mseen:
break
mseen.add(mroot.blocks)
rweight = max(rweight, mroot.weight)
@@ -976,10 +987,16 @@ def main(disk, mroots=None, *,
d_ = 0
mroot_ = Rbyd.fetch(f, block_size, mroots)
mdepth_ = 1
mseen_ = set()
for d in it.count():
# corrupted?
if not mroot_:
break
# cycle detected?
elif mroot_.blocks in mseen_:
break
mseen_.add(mroot_.blocks)
# compute the mroots rbyd-tree
rtree, rdepth = mroot_.tree(rbyd=args.get('rbyd'))
@@ -1238,10 +1255,16 @@ def main(disk, mroots=None, *,
tree = set()
mroot_ = Rbyd.fetch(f, block_size, mroots)
mdepth_ = 1
mseen_ = set()
for d in it.count():
# corrupted?
if not mroot_:
break
# cycle detected?
elif mroot_.blocks in mseen_:
break
mseen_.add(mroot_.blocks)
# connect branch to our first tag
if d > 0:
@@ -1427,9 +1450,7 @@ def main(disk, mroots=None, *,
# show human-readable tag representation
print('%12s %s%s' % (
'{%s}:' % ','.join('%04x' % block
for block in it.chain(
[mdir.block],
mdir.redund_blocks))
for block in mdir.blocks)
if i == 0 else '',
treerepr(mbid-max(mw-1, 0), 0, md, 0, rid, tag)
if args.get('tree')
@@ -1537,23 +1558,34 @@ def main(disk, mroots=None, *,
corrupted = False
mroot = Rbyd.fetch(f, block_size, mroots)
mdepth = 1
mseen = set()
for d in it.count():
# corrupted?
if not mroot:
print('{%s}: %s%s%s' % (
','.join('%04x' % block
for block in it.chain(
[mroot.block],
mroot.redund_blocks)),
for block in mroot.blocks),
'\x1b[31m' if color else '',
'(corrupted mroot %s)' % mroot.addr(),
'\x1b[m' if color else ''))
corrupted = True
break
# cycle detected?
elif mroot.blocks in mseen:
print('{%s}: %s%s%s' % (
','.join('%04x' % block
for block in mroot.blocks),
'\x1b[31m' if color else '',
'(mroot cycle detected %s)' % mroot.addr(),
'\x1b[m' if color else ''))
corrupted = True
break
else:
# show the mdir
dbg_mdir(mroot, -1, 0, d)
mseen.add(mroot.blocks)
# stop here?
if args.get('depth') and mdepth >= args.get('depth'):
break
@@ -1578,9 +1610,7 @@ def main(disk, mroots=None, *,
if not mdir:
print('{%s}: %s%s%s' % (
','.join('%04x' % block
for block in it.chain(
[mdir.block],
mdir.redund_blocks)),
for block in mdir.blocks),
'\x1b[31m' if color else '',
'(corrupted mdir %s)' % mdir.addr(),
'\x1b[m' if color else ''))
@@ -1677,9 +1707,7 @@ def main(disk, mroots=None, *,
if not mdir_:
print('{%s}: %*s%s%s%s' % (
','.join('%04x' % block
for block in it.chain(
[mdir_.block],
mdir_.redund_blocks)),
for block in mdir_.blocks),
t_width, '',
'\x1b[31m' if color else '',
'(corrupted mdir %s)' % mdir_.addr(),
+6 -6
View File
@@ -85,10 +85,10 @@ def bdgeom(s):
return int(s, b)
# parse some rbyd addr encodings
# 0xa -> [0xa]
# 0xa.c -> [(0xa, 0xc)]
# 0x{a,b} -> [0xa, 0xb]
# 0x{a,b}.c -> [(0xa, 0xc), (0xb, 0xc)]
# 0xa -> (0xa,)
# 0xa.c -> ((0xa, 0xc),)
# 0x{a,b} -> (0xa, 0xb)
# 0x{a,b}.c -> ((0xa, 0xc), (0xb, 0xc))
def rbydaddr(s):
s = s.strip()
b = 10
@@ -119,7 +119,7 @@ def rbydaddr(s):
else:
addr.append(int(s, b))
return addr
return tuple(addr)
def crc32c(data, crc=0):
crc ^= 0xffffffff
@@ -910,7 +910,7 @@ def main(disk, blocks=None, *,
# flatten blocks, default to block 0
if not blocks:
blocks = [[0]]
blocks = [(0,)]
blocks = [block for blocks_ in blocks for block in blocks_]
with open(disk, 'rb') as f:
+6 -6
View File
@@ -73,10 +73,10 @@ def bdgeom(s):
return int(s, b)
# parse some rbyd addr encodings
# 0xa -> [0xa]
# 0xa.c -> [(0xa, 0xc)]
# 0x{a,b} -> [0xa, 0xb]
# 0x{a,b}.c -> [(0xa, 0xc), (0xb, 0xc)]
# 0xa -> (0xa,)
# 0xa.c -> ((0xa, 0xc),)
# 0x{a,b} -> (0xa, 0xb)
# 0x{a,b}.c -> ((0xa, 0xc), (0xb, 0xc))
def rbydaddr(s):
s = s.strip()
b = 10
@@ -107,7 +107,7 @@ def rbydaddr(s):
else:
addr.append(int(s, b))
return addr
return tuple(addr)
def fromleb128(data):
word = 0
@@ -293,7 +293,7 @@ def main(tags, *,
# flatten block, default to block 0
if not blocks:
blocks = [[0]]
blocks = [(0,)]
blocks = [block for blocks_ in blocks for block in blocks_]
with open(disk, 'rb') as f:
+8 -8
View File
@@ -73,10 +73,10 @@ def bdgeom(s):
return int(s, b)
# parse some rbyd addr encodings
# 0xa -> [0xa]
# 0xa.c -> [(0xa, 0xc)]
# 0x{a,b} -> [0xa, 0xb]
# 0x{a,b}.c -> [(0xa, 0xc), (0xb, 0xc)]
# 0xa -> (0xa,)
# 0xa.c -> ((0xa, 0xc),)
# 0x{a,b} -> (0xa, 0xb)
# 0x{a,b}.c -> ((0xa, 0xc), (0xb, 0xc))
def rbydaddr(s):
s = s.strip()
b = 10
@@ -107,7 +107,7 @@ def rbydaddr(s):
else:
addr.append(int(s, b))
return addr
return tuple(addr)
class RingIO:
@@ -722,13 +722,13 @@ def main(path='-', *,
off, = size
size = None
if any(isinstance(b, list) and len(b) > 1 for b in block):
if any(isinstance(b, tuple) and len(b) > 1 for b in block):
print("error: more than one block address?",
file=sys.stderr)
sys.exit(-1)
if isinstance(block[0], list):
if isinstance(block[0], tuple):
block = (block[0][0], *block[1:])
if len(block) > 1 and isinstance(block[1], list):
if len(block) > 1 and isinstance(block[1], tuple):
block = (block[0], block[1][0])
if isinstance(block[0], tuple):
block, off_ = (block[0][0], *block[1:]), block[0][1]