Github/littlefs
1
Fork 0
Code Actions Activity

Changed most references to crc/csum -> cksum

Browse Source 
The reason for this is to move away from the idea that littlefs is
strictly bound to CRCs and make the code more welcoming to other
checksum types, such as SHA256, etc.

Of course, changing the name doesn't really do anything. littlefs
actually _is_ strictly bound to CRCs in a couple ways that other
filesystems aren't. These would need to have workarounds for other
checksum types:

- We leverage the parity-preserving nature of (some) CRCs to not have
  to also calculate the parity of metadata in rbyd commits.

- We leverage the linearity of CRCs to retroactively flip the
  perturb bit in the cksum tag without needing to recalculate the
  checksum. Though the fact we need to do this is because of how we
  use parity above, so this may just not be needed for non-CRC
  checksums.

- The plans for global-CRCs (not yet implemented) rely heavily on the
  mathematical properties of CRC polynomials. This doesn't mean
  global-CRCs can't work with other checksums, you would just need to
  find a different type of polynomial.
This commit is contained in:
Christopher Haster
2023-08-07 13:32:38 -05:00
parent dc3b7d435e
commit 7031d6e1b3
7 changed files with 359 additions and 359 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/dbgbtree.py
+21 -21
View File
@@ -29,8 +29,8 @@ TAG_DID = 0x0307
TAG_UATTR = 0x0400
TAG_SATTR = 0x0500
TAG_ALT = 0x4000
TAG_CRC = 0x2000
TAG_FCRC = 0x2100
TAG_CKSUM = 0x2000
TAG_FCKSUM = 0x2100
@@ -98,16 +98,16 @@ def fromtag(data):
return tag>>15, tag&0x7fff, weight, size, 2+d+d_
def frombtree(data):
crc = fromle32(data)
cksum = fromle32(data)
w, d1 = fromleb128(data[4:])
trunk, d2 = fromleb128(data[4+d1:])
block, d3 = fromleb128(data[4+d1+d2:])
return w, trunk, block, crc
return w, trunk, block, cksum
def popc(x):
return bin(x).count('1')
def xxd(data, width=16, crc=False):
def xxd(data, width=16):
for i in range(0, len(data), width):
yield '%-*s %-*s' % (
3*width,
@@ -167,13 +167,13 @@ def tagrepr(tag, w, size, off=None):
tag & 0xff,
' w%d' % w if w else '',
size)
elif (tag & 0xff00) == TAG_CRC:
return 'crc%x%s %d' % (
elif (tag & 0xff00) == TAG_CKSUM:
return 'cksum%x%s %d' % (
1 if tag & 0x1 else 0,
' 0x%x' % w if w > 0 else '',
size)
elif tag == TAG_FCRC:
return 'fcrc%s %d' % (
elif tag == TAG_FCKSUM:
return 'fcksum%s %d' % (
' 0x%x' % w if w > 0 else '',
size)
elif tag & 0x4000:
@@ -249,8 +249,8 @@ class Rbyd:
# fetch the rbyd
rev = fromle32(data[0:4])
crc = 0
crc_ = crc32c(data[0:4])
cksum = 0
cksum_ = crc32c(data[0:4])
off = 0
j_ = 4
trunk_ = 0
@@ -262,25 +262,25 @@ class Rbyd:
trunkoff = None
while j_ < len(data) and (not trunk or off <= trunk):
v, tag, w, size, d = fromtag(data[j_:])
if v != (popc(crc_) & 1):
if v != (popc(cksum_) & 1):
break
crc_ = crc32c(data[j_:j_+d], crc_)
cksum_ = crc32c(data[j_:j_+d], cksum_)
j_ += d
if not tag & 0x4000 and j_ + size > len(data):
break
# take care of crcs
# take care of cksums
if not tag & 0x4000:
if (tag & 0xff00) != TAG_CRC:
crc_ = crc32c(data[j_:j_+size], crc_)
# found a crc?
if (tag & 0xff00) != TAG_CKSUM:
cksum_ = crc32c(data[j_:j_+size], cksum_)
# found a cksum?
else:
crc__ = fromle32(data[j_:j_+4])
if crc_ != crc__:
cksum__ = fromle32(data[j_:j_+4])
if cksum_ != cksum__:
break
# commit what we have
off = trunkoff if trunkoff else j_ + size
crc = crc_
cksum = cksum_
trunk_ = trunk__
weight = weight_
@@ -554,7 +554,7 @@ def main(disk, roots=None, *,
if branch is not None and (
not depth or depth_ < depth):
tag, j, d, data = branch
w_, trunk, block, crc = frombtree(data)
w_, trunk, block, cksum = frombtree(data)
rbyd = Rbyd.fetch(f, block_size, block, trunk)
# corrupted? bail here so we can keep traversing the tree
scripts/dbglfs.py
+25 -25
View File
@@ -30,8 +30,8 @@ TAG_DID = 0x0307
TAG_UATTR = 0x0400
TAG_SATTR = 0x0500
TAG_ALT = 0x4000
TAG_CRC = 0x2000
TAG_FCRC = 0x2100
TAG_CKSUM = 0x2000
TAG_FCKSUM = 0x2100
# parse some rbyd addr encodings
@@ -107,16 +107,16 @@ def frommdir(data):
return blocks
def frombtree(data):
crc = fromle32(data)
cksum = fromle32(data)
w, d1 = fromleb128(data[4:])
trunk, d2 = fromleb128(data[4+d1:])
block, d3 = fromleb128(data[4+d1+d2:])
return w, trunk, block, crc
return w, trunk, block, cksum
def popc(x):
return bin(x).count('1')
def xxd(data, width=16, crc=False):
def xxd(data, width=16):
for i in range(0, len(data), width):
yield '%-*s %-*s' % (
3*width,
@@ -176,13 +176,13 @@ def tagrepr(tag, w, size, off=None):
tag & 0xff,
' w%d' % w if w else '',
size)
elif (tag & 0xff00) == TAG_CRC:
return 'crc%x%s %d' % (
elif (tag & 0xff00) == TAG_CKSUM:
return 'cksum%x%s %d' % (
1 if tag & 0x1 else 0,
' 0x%x' % w if w > 0 else '',
size)
elif tag == TAG_FCRC:
return 'fcrc%s %d' % (
elif tag == TAG_FCKSUM:
return 'fcksum%s %d' % (
' 0x%x' % w if w > 0 else '',
size)
elif tag & 0x4000:
@@ -255,8 +255,8 @@ class Rbyd:
# fetch the rbyd
rev = fromle32(data[0:4])
crc = 0
crc_ = crc32c(data[0:4])
cksum = 0
cksum_ = crc32c(data[0:4])
off = 0
j_ = 4
trunk_ = 0
@@ -268,25 +268,25 @@ class Rbyd:
trunkoff = None
while j_ < len(data) and (not trunk or off <= trunk):
v, tag, w, size, d = fromtag(data[j_:])
if v != (popc(crc_) & 1):
if v != (popc(cksum_) & 1):
break
crc_ = crc32c(data[j_:j_+d], crc_)
cksum_ = crc32c(data[j_:j_+d], cksum_)
j_ += d
if not tag & 0x4000 and j_ + size > len(data):
break
# take care of crcs
# take care of cksums
if not tag & 0x4000:
if (tag & 0xff00) != TAG_CRC:
crc_ = crc32c(data[j_:j_+size], crc_)
# found a crc?
if (tag & 0xff00) != TAG_CKSUM:
cksum_ = crc32c(data[j_:j_+size], cksum_)
# found a cksum?
else:
crc__ = fromle32(data[j_:j_+4])
if crc_ != crc__:
cksum__ = fromle32(data[j_:j_+4])
if cksum_ != cksum__:
break
# commit what we have
off = trunkoff if trunkoff else j_ + size
crc = crc_
cksum = cksum_
trunk_ = trunk__
weight = weight_
@@ -440,7 +440,7 @@ class Rbyd:
if branch is not None and (
not depth or depth_ < depth):
tag, j, d, data = branch
w_, trunk, block, crc = frombtree(data)
w_, trunk, block, cksum = frombtree(data)
rbyd = Rbyd.fetch(f, block_size, block, trunk)
# corrupted? bail here so we can keep traversing the tree
@@ -457,7 +457,7 @@ class Rbyd:
# have mtree?
done, rid, tag, w, j, d, data, _ = self.lookup(-1, TAG_MTREE)
if not done and rid == -1 and tag == TAG_MTREE:
w, trunk, block, crc = frombtree(data)
w, trunk, block, cksum = frombtree(data)
mtree = Rbyd.fetch(f, block_size, block, trunk)
# corrupted?
if not mtree:
@@ -544,7 +544,7 @@ class Rbyd:
# update our bid
bid += rid - (w-1)
w_, trunk, block, crc = frombtree(data)
w_, trunk, block, cksum = frombtree(data)
rbyd = Rbyd.fetch(f, block_size, block, trunk)
# found best match
@@ -556,7 +556,7 @@ class Rbyd:
# have mtree?
done, rid, tag, w, j, d, data, _ = self.lookup(-1, TAG_MTREE)
if not done and rid == -1 and tag == TAG_MTREE:
w, trunk, block, crc = frombtree(data)
w, trunk, block, cksum = frombtree(data)
mtree = Rbyd.fetch(f, block_size, block, trunk)
# corrupted?
if not mtree:
@@ -830,7 +830,7 @@ def main(disk, mroots=None, *,
mtree = None
done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MTREE)
if not done and rid == -1 and tag == TAG_MTREE:
w, trunk, block, crc = frombtree(data)
w, trunk, block, cksum = frombtree(data)
mtree = Rbyd.fetch(f, block_size, block, trunk)
mweight = w
scripts/dbgmtree.py
+23 -23
View File
@@ -29,8 +29,8 @@ TAG_DID = 0x0307
TAG_UATTR = 0x0400
TAG_SATTR = 0x0500
TAG_ALT = 0x4000
TAG_CRC = 0x2000
TAG_FCRC = 0x2100
TAG_CKSUM = 0x2000
TAG_FCKSUM = 0x2100
# parse some rbyd addr encodings
@@ -106,16 +106,16 @@ def frommdir(data):
return blocks
def frombtree(data):
crc = fromle32(data)
cksum = fromle32(data)
w, d1 = fromleb128(data[4:])
trunk, d2 = fromleb128(data[4+d1:])
block, d3 = fromleb128(data[4+d1+d2:])
return w, trunk, block, crc
return w, trunk, block, cksum
def popc(x):
return bin(x).count('1')
def xxd(data, width=16, crc=False):
def xxd(data, width=16):
for i in range(0, len(data), width):
yield '%-*s %-*s' % (
3*width,
@@ -175,13 +175,13 @@ def tagrepr(tag, w, size, off=None):
tag & 0xff,
' w%d' % w if w else '',
size)
elif (tag & 0xff00) == TAG_CRC:
return 'crc%x%s %d' % (
elif (tag & 0xff00) == TAG_CKSUM:
return 'cksum%x%s %d' % (
1 if tag & 0x1 else 0,
' 0x%x' % w if w > 0 else '',
size)
elif tag == TAG_FCRC:
return 'fcrc%s %d' % (
elif tag == TAG_FCKSUM:
return 'fcksum%s %d' % (
' 0x%x' % w if w > 0 else '',
size)
elif tag & 0x4000:
@@ -257,8 +257,8 @@ class Rbyd:
# fetch the rbyd
rev = fromle32(data[0:4])
crc = 0
crc_ = crc32c(data[0:4])
cksum = 0
cksum_ = crc32c(data[0:4])
off = 0
j_ = 4
trunk_ = 0
@@ -270,25 +270,25 @@ class Rbyd:
trunkoff = None
while j_ < len(data) and (not trunk or off <= trunk):
v, tag, w, size, d = fromtag(data[j_:])
if v != (popc(crc_) & 1):
if v != (popc(cksum_) & 1):
break
crc_ = crc32c(data[j_:j_+d], crc_)
cksum_ = crc32c(data[j_:j_+d], cksum_)
j_ += d
if not tag & 0x4000 and j_ + size > len(data):
break
# take care of crcs
# take care of cksums
if not tag & 0x4000:
if (tag & 0xff00) != TAG_CRC:
crc_ = crc32c(data[j_:j_+size], crc_)
# found a crc?
if (tag & 0xff00) != TAG_CKSUM:
cksum_ = crc32c(data[j_:j_+size], cksum_)
# found a cksum?
else:
crc__ = fromle32(data[j_:j_+4])
if crc_ != crc__:
cksum__ = fromle32(data[j_:j_+4])
if cksum_ != cksum__:
break
# commit what we have
off = trunkoff if trunkoff else j_ + size
crc = crc_
cksum = cksum_
trunk_ = trunk__
weight = weight_
@@ -531,7 +531,7 @@ class Rbyd:
if branch is not None and (
not depth or depth_ < depth):
tag, j, d, data = branch
w_, trunk, block, crc = frombtree(data)
w_, trunk, block, cksum = frombtree(data)
rbyd = Rbyd.fetch(f, block_size, block, trunk)
# corrupted? bail here so we can keep traversing the tree
@@ -805,7 +805,7 @@ def main(disk, mroots=None, *,
if not args.get('depth') or mdepth < args.get('depth'):
done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MTREE)
if not done and rid == -1 and tag == TAG_MTREE:
w, trunk, block, crc = frombtree(data)
w, trunk, block, cksum = frombtree(data)
mtree = Rbyd.fetch(f, block_size, block, trunk)
mweight = w
@@ -1474,7 +1474,7 @@ def main(disk, mroots=None, *,
if not args.get('depth') or mdepth < args.get('depth'):
done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MTREE)
if not done and rid == -1 and tag == TAG_MTREE:
w, trunk, block, crc = frombtree(data)
w, trunk, block, cksum = frombtree(data)
mtree = Rbyd.fetch(f, block_size, block, trunk)
# traverse entries
scripts/dbgrbyd.py
+32 -32
View File
@@ -38,8 +38,8 @@ TAG_DID = 0x0307
TAG_UATTR = 0x0400
TAG_SATTR = 0x0500
TAG_ALT = 0x4000
TAG_CRC = 0x2000
TAG_FCRC = 0x2100
TAG_CKSUM = 0x2000
TAG_FCKSUM = 0x2100
@@ -109,7 +109,7 @@ def fromtag(data):
def popc(x):
return bin(x).count('1')
def xxd(data, width=16, crc=False):
def xxd(data, width=16):
for i in range(0, len(data), width):
yield '%-*s %-*s' % (
3*width,
@@ -169,13 +169,13 @@ def tagrepr(tag, w, size, off=None):
tag & 0xff,
' w%d' % w if w else '',
size)
elif (tag & 0xff00) == TAG_CRC:
return 'crc%x%s %d' % (
elif (tag & 0xff00) == TAG_CKSUM:
return 'cksum%x%s %d' % (
1 if tag & 0x1 else 0,
' 0x%x' % w if w > 0 else '',
size)
elif tag == TAG_FCRC:
return 'fcrc%s %d' % (
elif tag == TAG_FCKSUM:
return 'fcksum%s %d' % (
' 0x%x' % w if w > 0 else '',
size)
elif tag & 0x4000:
@@ -194,7 +194,7 @@ def tagrepr(tag, w, size, off=None):
def dbg_log(data, block_size, rev, off, weight, *,
color=False,
**args):
crc = crc32c(data[0:4])
cksum = crc32c(data[0:4])
# preprocess jumps
if args.get('jumps'):
@@ -444,20 +444,20 @@ def dbg_log(data, block_size, rev, off, weight, *,
j = j_
v, tag, w, size, d = fromtag(data[j_:])
if v != (popc(crc) & 1):
notes.append('v!=%x' % (popc(crc) & 1))
crc = crc32c(data[j_:j_+d], crc)
if v != (popc(cksum) & 1):
notes.append('v!=%x' % (popc(cksum) & 1))
cksum = crc32c(data[j_:j_+d], cksum)
j_ += d
# take care of crcs
# take care of cksums
if not tag & 0x4000:
if (tag & 0xff00) != TAG_CRC:
crc = crc32c(data[j_:j_+size], crc)
# found a crc?
if (tag & 0xff00) != TAG_CKSUM:
cksum = crc32c(data[j_:j_+size], cksum)
# found a cksum?
else:
crc_ = fromle32(data[j_:j_+4])
if crc != crc_:
notes.append('crc!=%08x' % crc)
cksum_ = fromle32(data[j_:j_+4])
if cksum != cksum_:
notes.append('cksum!=%08x' % cksum)
j_ += size
# evaluate trunks
@@ -499,7 +499,7 @@ def dbg_log(data, block_size, rev, off, weight, *,
else ''))
# show in-device representation, including some extra
# crc/parity info
# cksum/parity info
if args.get('device'):
print('%s%8s %*s%*s %-47s %08x %x%s' % (
'\x1b[90m' if color and j >= off else '',
@@ -514,8 +514,8 @@ def dbg_log(data, block_size, rev, off, weight, *,
for i in range(min(m.ceil(size/4), 3)))[:23]
if not args.get('no_truncate')
and not tag & 0x4000 else ''),
crc,
popc(crc) & 1,
cksum,
popc(cksum) & 1,
'\x1b[m' if color and j >= off else ''))
# show on-disk encoding of tags
@@ -852,8 +852,8 @@ def main(disk, blocks=None, *,
# first figure out which block as the most recent revision
def fetch(data, trunk):
rev = fromle32(data[0:4])
crc = 0
crc_ = crc32c(data[0:4])
cksum = 0
cksum_ = crc32c(data[0:4])
off = 0
j_ = 4
trunk_ = 0
@@ -865,25 +865,25 @@ def main(disk, blocks=None, *,
trunkoff = None
while j_ < len(data) and (not trunk or off <= trunk):
v, tag, w, size, d = fromtag(data[j_:])
if v != (popc(crc_) & 1):
if v != (popc(cksum_) & 1):
break
crc_ = crc32c(data[j_:j_+d], crc_)
cksum_ = crc32c(data[j_:j_+d], cksum_)
j_ += d
if not tag & 0x4000 and j_ + size > len(data):
break
# take care of crcs
# take care of cksums
if not tag & 0x4000:
if (tag & 0xff00) != TAG_CRC:
crc_ = crc32c(data[j_:j_+size], crc_)
# found a crc?
if (tag & 0xff00) != TAG_CKSUM:
cksum_ = crc32c(data[j_:j_+size], cksum_)
# found a cksum?
else:
crc__ = fromle32(data[j_:j_+4])
if crc_ != crc__:
cksum__ = fromle32(data[j_:j_+4])
if cksum_ != cksum__:
break
# commit what we have
off = trunkoff if trunkoff else j_ + size
crc = crc_
cksum = cksum_
trunk_ = trunk__
weight = weight_