From 1044c9d2b7855bba652ed55ee87c6dbf15182650 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 2 Aug 2024 01:15:32 -0500 Subject: [PATCH] Adopted odd-parity-zero rbyd perturb scheme I've been scratching my head over our rbyd perturb scheme. It's gotten rather clunky with needing to xor valid bits and whatnot. But it's tricky with needing erased-state to be included in parity bits, while at the same time excluded from our canonical checksum. If only there was some way to flip the checksums parity without changing its value... Enter the crc32c odd-parity zero: 0xfca42daf! This bends the definition of zero a bit, but it is one of two numbers in our crc32c-ring with a very interesting property: crc32c(m) == crc32c(m xor 0xfca42daf) xor 0xfca42daf // odd-p zero crc32c(m) == crc32c(m xor 0x00000000) xor 0x00000000 // even-p zero Recall that crc32c's polynomial, 0x11edc6f41, is composed of two polynomials: 0x3, the parity polynomial, and 0xf5b4253f, a maximally sized irreducible polynomial. Because our polynomial breaks down into two smaller polynomials, our crc32c space turns out to not be a field, but rather a ring containing two smaller sub-fields. Because these sub-fields are defined by their polynomials, one is the 31-bit crc defined by the polynomial 0xf5b4253f, while the other is the current parity. We can move in the parity sub-field without changing our position in the 31-bit crc sub-field by xoring with a number that is one in the parity sub-field, but zero in the 31-bit crc sub-field. This number happens to be 0xf5b4253f (0xfca42daf bit-reversed)! (crcs being bit-reversed will never not be annoying) So long story short, xoring any crc32c with 0xfca42daf will change its parity but not its value. --- An that's basically our new perturb scheme. If we need to perturb, xor with 0xfca42daf to change the parity, and after calculating/validating the checksum, xor with 0xfca42daf to get our canonical checksum. Isn't that neat! There was one small hiccup: At first I assumed you could continue including the valid bits in the checksum, which would have been nice for bulk checksumming. But this doesn't work because while valid bits cancel out so the parity doesn't change, changing valid bits _does_ change the underlying 31-bit crc, poisoning our checksum and making everything a mess. So we still need to mask out valid bits, which is a bit annoying. But then I stumbled on the funny realization that by masking our valid bits, we accidentally end up with a fully functional parity scheme. Because valid bits _don't_ include the previous valid bit, we can figure out the parity for not only the entire commit, but also each individual tag: 80 03 00 08 6c 69 74 74 6c 65 66 73 80 ^'----------------.---------------' ^ | | | v + parity = v' Or more simply: 80 03 00 08 6c 69 74 74 6c 65 66 73 80 '----------------.----------------' ^ | | parity = v' Double neat! Some other notes: - By keeping the commit checksum perturbed, but not the canonical checksum, the perturb state is self-validating. We no longer need to explicitly check the previous-perturb-bit (q) to avoid the perturb hole we ran into previously. I'm still keeping the previous-perturb-bit (q) around, since it's useful for debugging. We still need to know the perturb state internally at all times in order to xor out the canonical checksum correctly anyways. - Thanks to all of our perturb iterations, we now know how to remove the valid bits from the checksum easily: cksum ^= 0x00000080 & (tag >> 8) This makes the whole omitting-valid-bits thing less of a pain point. - It wasn't actually worth it to perturb the checksum when building commits, vs manually flipping each valid bit, as this would have made our internal appendattr API really weird. At least the perturbed checksum made fetch a bit simpler. Not sure exactly how to draw this with our perturb scheme diagrams, maybe something like this? .---+---+---+---. \ \ \ \ |v| tag | | | | | +---+---+---+---+ | | | | | commit | | | | | | | +-. | | | +---+---+---+---+ / | | | | |v|qp-------------->p>p-->p . +---+---+---+---+ | . . . | cksum | | . . . +---+---+---+---+ | . . . | padding | | . . . | | | . . . +---+---+---+---+ | | | | |v------------------' | | | +---+---+---+---+ | | | | commit | +-. | +- rbyd | | | | | | cksum +---+---+---+---+ / | +-. / |v----------------------' | | +-------+---+---+ / | | cksum ----------------' +---+---+---+---+ | padding | | | +---+---+---+---+ | erased | | | . . . . --- Code changes were minimal, saving a tiny bit of code: code stack before: 36368 2664 after: 36352 (-0.0%) 2672 (+0.3%) There was a stack bump in lfsr_bd_readtag, but as far as I can tell it's just compiler noise? I poked around a bit but couldn't figure out why it changed... --- lfs.c | 125 +++++++++++++++++++++++--------------------- lfs_util.h | 4 ++ scripts/dbgbmap.py | 16 ++---- scripts/dbgbtree.py | 16 ++---- scripts/dbglfs.py | 16 ++---- scripts/dbgmtree.py | 16 ++---- scripts/dbgrbyd.py | 35 +++++-------- 7 files changed, 103 insertions(+), 125 deletions(-) diff --git a/lfs.c b/lfs.c index 300c8f59..3bc3f8ec 100644 --- a/lfs.c +++ b/lfs.c @@ -1146,20 +1146,27 @@ static lfs_ssize_t lfsr_bd_readtag(lfs_t *lfs, return err; } + // check the valid bit? + if (cksum) { + // on-disk, the tag's valid bit must reflect the parity of the + // preceding data + // + // fortunately crc32cs are parity-preserving, so this is the + // same as the parity of the checksum + if ((tag_buf[0] >> 7) != lfs_parity(*cksum)) { + return LFS_ERR_CORRUPT; + } + } + + // clear the valid bit once checked, we exclude these from the + // next checksum + tag_buf[0] &= ~0x80; + lfsr_tag_t tag = ((lfsr_tag_t)tag_buf[0] << 8) | ((lfsr_tag_t)tag_buf[1] << 0); lfs_ssize_t d = 2; - if (cksum) { - // on-disk, the tags valid bit must reflect the parity of the - // preceding data, fortunately for crc32c, this is the same as the - // parity of the crc - if ((tag >> 15) != lfs_parity(*cksum)) { - return LFS_ERR_CORRUPT; - } - } - lfsr_rid_t weight; lfs_ssize_t d_ = lfs_fromleb128(&weight, &tag_buf[d], tag_dsize-d); if (d_ < 0) { @@ -1187,16 +1194,15 @@ static lfs_ssize_t lfsr_bd_readtag(lfs_t *lfs, *cksum = lfs_crc32c(*cksum, tag_buf, d); } - // save what we found, clearing the valid bit from the tag, note we - // checked this earlier - *tag_ = tag & 0x7fff; + // save what we found + *tag_ = tag; *weight_ = weight; *size_ = size; return d; } static lfs_ssize_t lfsr_bd_progtag(lfs_t *lfs, - lfs_block_t block, lfs_size_t off, + lfs_block_t block, lfs_size_t off, bool perturb, lfsr_tag_t tag, lfsr_rid_t weight, lfs_size_t size, uint32_t *cksum, bool align) { // we set the valid bit here @@ -1208,10 +1214,12 @@ static lfs_ssize_t lfsr_bd_progtag(lfs_t *lfs, // size should not exceed 28-bits LFS_ASSERT(size <= 0x0fffffff); - // set the valid bit to the parity of the current cksum - if (cksum) { - tag |= (lfsr_tag_t)lfs_parity(*cksum) << 15; - } + // set the valid bit to the parity of the current checksum, inverted + // if the perturb bit is set, and exclude from the next checksum + LFS_ASSERT(cksum); + bool v = lfs_parity(*cksum) ^ perturb; + tag |= (lfsr_tag_t)v << 15; + *cksum ^= (uint32_t)v << 7; // encode into a be16 and pair of leb128s uint8_t tag_buf[LFSR_TAG_DSIZE]; @@ -2112,7 +2120,7 @@ static int lfsr_bptr_ck(lfs_t *lfs, const lfsr_bptr_t *bptr) { /// Red-black-yellow Dhara tree operations /// #define LFSR_RBYD_ISSHRUB 0x80000000 -#define LFSR_RBYD_PERTURB 0x80000000 +#define LFSR_RBYD_ISPERTURB 0x80000000 // helper functions static inline bool lfsr_rbyd_isshrub(const lfsr_rbyd_t *rbyd) { @@ -2127,12 +2135,12 @@ static inline bool lfsr_rbyd_isfetched(const lfsr_rbyd_t *rbyd) { return !lfsr_rbyd_trunk(rbyd) || rbyd->eoff; } -static inline bool lfsr_rbyd_perturb(const lfsr_rbyd_t *rbyd) { - return rbyd->eoff & LFSR_RBYD_PERTURB; +static inline bool lfsr_rbyd_isperturb(const lfsr_rbyd_t *rbyd) { + return rbyd->eoff & LFSR_RBYD_ISPERTURB; } static inline lfs_size_t lfsr_rbyd_eoff(const lfsr_rbyd_t *rbyd) { - return rbyd->eoff & ~LFSR_RBYD_PERTURB; + return rbyd->eoff & ~LFSR_RBYD_ISPERTURB; } static inline int lfsr_rbyd_cmp( @@ -2197,11 +2205,6 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, // scan tags, checking valid bits, cksums, etc while (off < lfs->cfg->block_size && (!trunk || lfsr_rbyd_eoff(rbyd) <= trunk)) { - // perturb? - if (lfsr_rbyd_perturb(rbyd)) { - cksum_ ^= 0x00000080; - } - // read next tag lfsr_tag_t tag; lfsr_rid_t weight__; @@ -2251,13 +2254,7 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, // is an end-of-commit cksum } else { - // check perturb bit - if (lfsr_rbyd_perturb(rbyd) != lfsr_tag_q(tag)) { - // uh oh, perturb bits don't match - break; - } - - // check cksum + // check checksum uint32_t cksum__ = 0; err = lfsr_bd_read(lfs, block, off_, -1, &cksum__, sizeof(uint32_t)); @@ -2270,10 +2267,13 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, cksum__ = lfs_fromle32_(&cksum__); if (cksum_ != cksum__) { - // uh oh, cksums don't match + // uh oh, checksums don't match break; } + // if checksums match, perturb bits should also match + LFS_ASSERT(lfsr_tag_q(tag) == lfsr_rbyd_isperturb(rbyd)); + // save what we've found so far rbyd->eoff = ((lfs_size_t)lfsr_tag_p(tag) @@ -2284,8 +2284,10 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, rbyd->weight = weight; ecksum = ecksum_; - // revert to canonical checksum - cksum_ = cksum; + // revert to canonical checksum and perturb if necessary + cksum_ = cksum ^ ((lfsr_rbyd_isperturb(rbyd)) + ? LFS_CRC32C_ODDZERO + : LFS_CRC32C_EVENZERO); ecksum_.cksize = -1; } } @@ -2311,8 +2313,12 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, // end of trunk? if (!lfsr_tag_isalt(tag)) { - // update canonical checksum - cksum = cksum_; + // update canonical checksum, xoring out any perturb + // state, we don't want erased-state affecting our + // canonical checksum + cksum = cksum_ ^ ((lfsr_rbyd_isperturb(rbyd)) + ? LFS_CRC32C_ODDZERO + : LFS_CRC32C_EVENZERO); // update trunk and weight, unless we are a shrub trunk if (!lfsr_tag_isshrub(tag) || trunk__ == trunk) { trunk_ = trunk__; @@ -2350,7 +2356,7 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, return err; } - if (((e >> 7)^lfsr_rbyd_perturb(rbyd)) != lfs_parity(rbyd->cksum)) { + if (((e >> 7)^lfsr_rbyd_isperturb(rbyd)) != lfs_parity(rbyd->cksum)) { // check that erased-state matches our checksum, if this fails // most likely a write was interrupted uint32_t ecksum_ = 0; @@ -2597,13 +2603,8 @@ static int lfsr_rbyd_appendtag(lfs_t *lfs, lfsr_rbyd_t *rbyd, return LFS_ERR_RANGE; } - // perturb? - if (lfsr_rbyd_perturb(rbyd)) { - rbyd->cksum ^= 0x00000080; - } - lfs_ssize_t d = lfsr_bd_progtag(lfs, - rbyd->blocks[0], lfsr_rbyd_eoff(rbyd), + rbyd->blocks[0], lfsr_rbyd_eoff(rbyd), lfsr_rbyd_isperturb(rbyd), tag, weight, size, &rbyd->cksum, false); if (d < 0) { @@ -3415,7 +3416,7 @@ static int lfsr_rbyd_appendcksum(lfs_t *lfs, lfsr_rbyd_t *rbyd) { // we don't want the next commit to appear as valid, so we // intentionally perturb the commit if this happens, this is - // equivalent to inverting all tag's valid bits + // roughly equivalent to inverting all tags' valid bits perturb = ((e >> 7) == lfs_parity(cksum)); // calculate the erased-state checksum @@ -3449,23 +3450,20 @@ static int lfsr_rbyd_appendcksum(lfs_t *lfs, lfsr_rbyd_t *rbyd) { return LFS_ERR_RANGE; } - // perturb? - if (lfsr_rbyd_perturb(rbyd)) { - rbyd->cksum ^= 0x00000080; - } - - // build end-of-commit cksum + // build the end-of-commit checksum tag // // note padding-size depends on leb-encoding depends on padding-size // depends leb-encoding depends on... to get around this catch-22 we // just always write a fully-expanded leb128 encoding + // + bool v = lfs_parity(rbyd->cksum) ^ lfsr_rbyd_isperturb(rbyd); uint8_t cksum_buf[2+1+4+4]; cksum_buf[0] = (uint8_t)(LFSR_TAG_CKSUM >> 8) // set the valid bit to the cksum parity - | ((uint8_t)lfs_parity(rbyd->cksum) << 7); + | ((uint8_t)v << 7); cksum_buf[1] = (uint8_t)(LFSR_TAG_CKSUM >> 0) // include the current perturb bit - | ((uint8_t)lfsr_rbyd_perturb(rbyd) << 1) + | ((uint8_t)lfsr_rbyd_isperturb(rbyd) << 1) // set the perturb bit so next commit is invalid | ((uint8_t)perturb << 0); cksum_buf[2] = 0; @@ -3476,9 +3474,19 @@ static int lfsr_rbyd_appendcksum(lfs_t *lfs, lfsr_rbyd_t *rbyd) { cksum_buf[5] = 0x80 | (0x7f & (padding >> 14)); cksum_buf[6] = 0x00 | (0x7f & (padding >> 21)); - // calculate checksum - rbyd->cksum = lfs_crc32c(rbyd->cksum, cksum_buf, 2+1+4); - lfs_tole32_(rbyd->cksum, &cksum_buf[2+1+4]); + // exclude the valid bit + uint32_t cksum_ = rbyd->cksum ^ ((uint32_t)v << 7); + // calculate the commit checksum + cksum_ = lfs_crc32c(cksum_, cksum_buf, 2+1+4); + // and perturb, perturbing the commit checksum avoids a perturb hole + // after the last valid bit without needing to manually validate q + // + // note the odd-parity zero preserves our position in the crc32c + // ring while only changing the parity + cksum_ ^= (lfsr_rbyd_isperturb(rbyd)) + ? LFS_CRC32C_ODDZERO + : LFS_CRC32C_EVENZERO; + lfs_tole32_(cksum_, &cksum_buf[2+1+4]); // prog, when this lands on disk commit is committed err = lfsr_bd_prog(lfs, rbyd->blocks[0], lfsr_rbyd_eoff(rbyd), @@ -3496,8 +3504,7 @@ static int lfsr_rbyd_appendcksum(lfs_t *lfs, lfsr_rbyd_t *rbyd) { // update the eoff and perturb rbyd->eoff - = ((lfs_size_t)perturb - << (8*sizeof(lfs_size_t)-1)) + = ((lfs_size_t)perturb << (8*sizeof(lfs_size_t)-1)) | off_; // revert to canonical checksum rbyd->cksum = cksum; diff --git a/lfs_util.h b/lfs_util.h index c393e606..3d7307d9 100644 --- a/lfs_util.h +++ b/lfs_util.h @@ -559,6 +559,10 @@ static inline size_t lfs_strcspn(const char *a, const char *cs) { //// Calculate CRC-32 with polynomial = 0x04c11db7 //uint32_t lfs_crc(uint32_t crc, const void *buffer, size_t size); +// Odd-parity and even-parity zeros in our crc32c ring +#define LFS_CRC32C_ODDZERO 0xfca42daf +#define LFS_CRC32C_EVENZERO 0x00000000 + // Calculate crc32c incrementally // // polynomial = 0x11edc6f41 diff --git a/scripts/dbgbmap.py b/scripts/dbgbmap.py index f3fb584d..da51a3c0 100755 --- a/scripts/dbgbmap.py +++ b/scripts/dbgbmap.py @@ -664,14 +664,11 @@ class Rbyd: weight_ = 0 weight__ = 0 while j_ < len(data) and (not trunk or eoff <= trunk): - # perturb? - if perturb: - cksum__ ^= 0x00000080 - # read next tag v, tag, w, size, d = fromtag(data[j_:]) if v != parity(cksum__): break + cksum__ ^= 0x00000080 if v else 0 cksum__ = crc32c(data[j_:j_+d], cksum__) j_ += d if not tag & TAG_ALT and j_ + size > len(data): @@ -683,9 +680,6 @@ class Rbyd: cksum__ = crc32c(data[j_:j_+size], cksum__) # found a cksum? else: - # check perturb bit - if perturb != bool(tag & TAG_Q): - break # check cksum cksum___ = fromle32(data[j_:j_+4]) if cksum__ != cksum___: @@ -697,8 +691,8 @@ class Rbyd: weight = weight_ # update perturb bit perturb = tag & TAG_P - # revert to data cksum - cksum__ = cksum_ + # revert to data cksum and perturb + cksum__ = cksum_ ^ (0xfca42daf if perturb else 0) # evaluate trunks if (tag & 0xf000) != TAG_CKSUM and ( @@ -713,8 +707,8 @@ class Rbyd: # end of trunk? if not tag & TAG_ALT: - # update canonical checksum - cksum_ = cksum__ + # update canonical checksum, xoring out any perturb state + cksum_ = cksum__ ^ (0xfca42daf if perturb else 0) # update trunk/weight unless we found a shrub or an # explicit trunk (which may be a shrub) is requested if not tag & TAG_SHRUB or trunk___ == trunk: diff --git a/scripts/dbgbtree.py b/scripts/dbgbtree.py index ded2b8b0..dddb7bca 100755 --- a/scripts/dbgbtree.py +++ b/scripts/dbgbtree.py @@ -337,14 +337,11 @@ class Rbyd: weight_ = 0 weight__ = 0 while j_ < len(data) and (not trunk or eoff <= trunk): - # perturb? - if perturb: - cksum__ ^= 0x00000080 - # read next tag v, tag, w, size, d = fromtag(data[j_:]) if v != parity(cksum__): break + cksum__ ^= 0x00000080 if v else 0 cksum__ = crc32c(data[j_:j_+d], cksum__) j_ += d if not tag & TAG_ALT and j_ + size > len(data): @@ -356,9 +353,6 @@ class Rbyd: cksum__ = crc32c(data[j_:j_+size], cksum__) # found a cksum? else: - # check perturb bit - if perturb != bool(tag & TAG_Q): - break # check cksum cksum___ = fromle32(data[j_:j_+4]) if cksum__ != cksum___: @@ -370,8 +364,8 @@ class Rbyd: weight = weight_ # update perturb bit perturb = tag & TAG_P - # revert to data cksum - cksum__ = cksum_ + # revert to data cksum and perturb + cksum__ = cksum_ ^ (0xfca42daf if perturb else 0) # evaluate trunks if (tag & 0xf000) != TAG_CKSUM and ( @@ -386,8 +380,8 @@ class Rbyd: # end of trunk? if not tag & TAG_ALT: - # update canonical checksum - cksum_ = cksum__ + # update canonical checksum, xoring out any perturb state + cksum_ = cksum__ ^ (0xfca42daf if perturb else 0) # update trunk/weight unless we found a shrub or an # explicit trunk (which may be a shrub) is requested if not tag & TAG_SHRUB or trunk___ == trunk: diff --git a/scripts/dbglfs.py b/scripts/dbglfs.py index 9a26c0b8..126ebde0 100755 --- a/scripts/dbglfs.py +++ b/scripts/dbglfs.py @@ -368,14 +368,11 @@ class Rbyd: weight_ = 0 weight__ = 0 while j_ < len(data) and (not trunk or eoff <= trunk): - # perturb? - if perturb: - cksum__ ^= 0x00000080 - # read next tag v, tag, w, size, d = fromtag(data[j_:]) if v != parity(cksum__): break + cksum__ ^= 0x00000080 if v else 0 cksum__ = crc32c(data[j_:j_+d], cksum__) j_ += d if not tag & TAG_ALT and j_ + size > len(data): @@ -387,9 +384,6 @@ class Rbyd: cksum__ = crc32c(data[j_:j_+size], cksum__) # found a cksum? else: - # check perturb bit - if perturb != bool(tag & TAG_Q): - break # check cksum cksum___ = fromle32(data[j_:j_+4]) if cksum__ != cksum___: @@ -401,8 +395,8 @@ class Rbyd: weight = weight_ # update perturb bit perturb = tag & TAG_P - # revert to data cksum - cksum__ = cksum_ + # revert to data cksum and perturb + cksum__ = cksum_ ^ (0xfca42daf if perturb else 0) # evaluate trunks if (tag & 0xf000) != TAG_CKSUM and ( @@ -417,8 +411,8 @@ class Rbyd: # end of trunk? if not tag & TAG_ALT: - # update canonical checksum - cksum_ = cksum__ + # update canonical checksum, xoring out any perturb state + cksum_ = cksum__ ^ (0xfca42daf if perturb else 0) # update trunk/weight unless we found a shrub or an # explicit trunk (which may be a shrub) is requested if not tag & TAG_SHRUB or trunk___ == trunk: diff --git a/scripts/dbgmtree.py b/scripts/dbgmtree.py index 05d533a9..8103d2d4 100755 --- a/scripts/dbgmtree.py +++ b/scripts/dbgmtree.py @@ -352,14 +352,11 @@ class Rbyd: weight_ = 0 weight__ = 0 while j_ < len(data) and (not trunk or eoff <= trunk): - # perturb? - if perturb: - cksum__ ^= 0x00000080 - # read next tag v, tag, w, size, d = fromtag(data[j_:]) if v != parity(cksum__): break + cksum__ ^= 0x00000080 if v else 0 cksum__ = crc32c(data[j_:j_+d], cksum__) j_ += d if not tag & TAG_ALT and j_ + size > len(data): @@ -371,9 +368,6 @@ class Rbyd: cksum__ = crc32c(data[j_:j_+size], cksum__) # found a cksum? else: - # check perturb bit - if perturb != bool(tag & TAG_Q): - break # check cksum cksum___ = fromle32(data[j_:j_+4]) if cksum__ != cksum___: @@ -385,8 +379,8 @@ class Rbyd: weight = weight_ # update perturb bit perturb = tag & TAG_P - # revert to data cksum - cksum__ = cksum_ + # revert to data cksum and perturb + cksum__ = cksum_ ^ (0xfca42daf if perturb else 0) # evaluate trunks if (tag & 0xf000) != TAG_CKSUM and ( @@ -401,8 +395,8 @@ class Rbyd: # end of trunk? if not tag & TAG_ALT: - # update canonical checksum - cksum_ = cksum__ + # update canonical checksum, xoring out any perturb state + cksum_ = cksum__ ^ (0xfca42daf if perturb else 0) # update trunk/weight unless we found a shrub or an # explicit trunk (which may be a shrub) is requested if not tag & TAG_SHRUB or trunk___ == trunk: diff --git a/scripts/dbgrbyd.py b/scripts/dbgrbyd.py index cd6d1463..4134b048 100755 --- a/scripts/dbgrbyd.py +++ b/scripts/dbgrbyd.py @@ -537,15 +537,12 @@ def dbg_log(data, block_size, rev, eoff, weight, *, while j_ < (block_size if args.get('all') else eoff): notes = [] - # perturb? - if perturb: - cksum_ ^= 0x00000080 - # read next tag j = j_ v, tag, w, size, d = fromtag(data[j_:]) if v != parity(cksum_): notes.append('v!=%x' % parity(cksum_)) + cksum_ ^= 0x00000080 if v else 0 cksum_ = crc32c(data[j_:j_+d], cksum_) j_ += d @@ -555,17 +552,16 @@ def dbg_log(data, block_size, rev, eoff, weight, *, cksum_ = crc32c(data[j_:j_+size], cksum_) # found a cksum? else: - # check perturb bit - if perturb != bool(tag & TAG_Q): - notes.append('q!=%x' % perturb) # check cksum cksum__ = fromle32(data[j_:j_+4]) if cksum_ != cksum__: notes.append('cksum!=%08x' % cksum__) + if perturb != bool(tag & TAG_Q): + notes.append('q!=%x' % perturb) # update perturb bit perturb = tag & TAG_P - # revert to data cksum - cksum_ = cksum + # revert to data cksum and perturb + cksum_ = cksum ^ (0xfca42daf if perturb else 0) j_ += size # evaluate trunks @@ -579,9 +575,10 @@ def dbg_log(data, block_size, rev, eoff, weight, *, else: upper_ += w + # end of trunk? if not tag & TAG_ALT: - # update canonical checksum - cksum = cksum_ + # update canonical checksum, xoring out any perturb state + cksum = cksum_ ^ (0xfca42daf if perturb else 0) # derive the current tag's rid from alt weights rid = lower_ + w-1 trunk_ = 0 @@ -948,14 +945,11 @@ def main(disk, blocks=None, *, weight_ = 0 weight__ = 0 while j_ < len(data) and (not trunk or eoff <= trunk): - # perturb? - if perturb: - cksum__ ^= 0x00000080 - # read next tag v, tag, w, size, d = fromtag(data[j_:]) if v != parity(cksum__): break + cksum__ ^= 0x00000080 if v else 0 cksum__ = crc32c(data[j_:j_+d], cksum__) j_ += d if not tag & TAG_ALT and j_ + size > len(data): @@ -967,9 +961,6 @@ def main(disk, blocks=None, *, cksum__ = crc32c(data[j_:j_+size], cksum__) # found a cksum? else: - # check perturb bit - if perturb != bool(tag & TAG_Q): - break # check cksum cksum___ = fromle32(data[j_:j_+4]) if cksum__ != cksum___: @@ -981,8 +972,8 @@ def main(disk, blocks=None, *, weight = weight_ # update perturb bit perturb = tag & TAG_P - # revert to data cksum - cksum__ = cksum_ + # revert to data cksum and perturb + cksum__ = cksum_ ^ (0xfca42daf if perturb else 0) # evaluate trunks if (tag & 0xf000) != TAG_CKSUM and ( @@ -997,8 +988,8 @@ def main(disk, blocks=None, *, # end of trunk? if not tag & TAG_ALT: - # update canonical checksum - cksum_ = cksum__ + # update canonical checksum, xoring out any perturb state + cksum_ = cksum__ ^ (0xfca42daf if perturb else 0) # update trunk/weight unless we found a shrub or an # explicit trunk (which may be a shrub) is requested if not tag & TAG_SHRUB or trunk___ == trunk: