diff --git a/lfs.c b/lfs.c index a1310032..b9e3e847 100644 --- a/lfs.c +++ b/lfs.c @@ -830,7 +830,9 @@ enum lfsr_tag { // checksum tags LFSR_TAG_CKSUM = 0x3000, - LFSR_TAG_PERTURB = 0x3100, + LFSR_TAG_Q = 0x0000, + LFSR_TAG_P = 0x0001, + LFSR_TAG_NOISE = 0x3100, LFSR_TAG_ECKSUM = 0x3200, // in-device only tags, these should never get written to disk @@ -900,6 +902,10 @@ static inline bool lfsr_tag_istrunk(lfsr_tag_t tag) { return lfsr_tag_mode(tag) != LFSR_TAG_CKSUM; } +static inline bool lfsr_tag_perturb(lfsr_tag_t tag) { + return tag & LFSR_TAG_P; +} + static inline bool lfsr_tag_isinternal(lfsr_tag_t tag) { return tag & LFSR_TAG_INTERNAL; } @@ -1111,7 +1117,7 @@ static inline bool lfsr_tag_diverging2( // total: <=11 bytes #define LFSR_TAG_DSIZE (2+5+4) -static lfs_ssize_t lfsr_bd_readtag_(lfs_t *lfs, +static lfs_ssize_t lfsr_bd_readtag(lfs_t *lfs, lfs_block_t block, lfs_size_t off, lfs_size_t hint, lfsr_tag_t *tag_, lfsr_rid_t *weight_, lfs_size_t *size_, uint32_t *cksum_) { @@ -1133,6 +1139,15 @@ static lfs_ssize_t lfsr_bd_readtag_(lfs_t *lfs, | ((lfsr_tag_t)tag_buf[1] << 0); lfs_ssize_t d = 2; + if (cksum_) { + // on-disk, the tags valid bit must reflect the parity of the + // preceding data, fortunately for crc32c, this is the same as the + // parity of the crc + if ((tag >> 15) != lfs_parity(*cksum_)) { + return LFS_ERR_CORRUPT; + } + } + lfsr_rid_t weight; lfs_ssize_t d_ = lfs_fromleb128(&weight, &tag_buf[d], tag_dsize-d); if (d_ < 0) { @@ -1157,39 +1172,23 @@ static lfs_ssize_t lfsr_bd_readtag_(lfs_t *lfs, // optional checksum if (cksum_) { - // ignore the valid bit when calculating checksums - *cksum_ ^= tag_buf[0] & 0x80; *cksum_ = lfs_crc32c(*cksum_, tag_buf, d); } - // save what we found - *tag_ = tag; + // save what we found, clearing the valid bit from the tag, note we + // checked this earlier + *tag_ = tag & 0x7fff; *weight_ = weight; *size_ = size; return d; } -// clear the valid bit, since most readtag calls don't care -static lfs_ssize_t lfsr_bd_readtag(lfs_t *lfs, - lfs_block_t block, lfs_size_t off, lfs_size_t hint, - lfsr_tag_t *tag_, lfsr_rid_t *weight_, lfs_size_t *size_, - uint32_t *cksum_) { - lfs_ssize_t d = lfsr_bd_readtag_(lfs, block, off, hint, - tag_, weight_, size_, cksum_); - if (d < 0) { - return d; - } - - if (tag_) { - *tag_ &= 0x7fff; - } - return d; -} - static lfs_ssize_t lfsr_bd_progtag(lfs_t *lfs, lfs_block_t block, lfs_size_t off, lfsr_tag_t tag, lfsr_rid_t weight, lfs_size_t size, uint32_t *cksum_, bool align) { + // we set the valid bit here + LFS_ASSERT(!(tag & 0x8000)); // bit 7 is reserved for future subtype extensions LFS_ASSERT(!(tag & 0x80)); // weight should not exceed 31-bits @@ -1197,6 +1196,11 @@ static lfs_ssize_t lfsr_bd_progtag(lfs_t *lfs, // size should not exceed 28-bits LFS_ASSERT(size <= 0x0fffffff); + // set the valid bit to the parity of the current cksum + if (cksum_) { + tag |= (lfsr_tag_t)lfs_parity(*cksum_) << 15; + } + // encode into a be16 and pair of leb128s uint8_t tag_buf[LFSR_TAG_DSIZE]; tag_buf[0] = (uint8_t)(tag >> 8); @@ -1215,10 +1219,6 @@ static lfs_ssize_t lfsr_bd_progtag(lfs_t *lfs, } d += d_; - // ignore the valid bit when calculating checksums - if (cksum_ && !align) { - *cksum_ ^= tag_buf[0] & 0x80; - } int err = lfsr_bd_prog(lfs, block, off, tag_buf, d, cksum_, align); if (err) { @@ -2081,7 +2081,7 @@ static void lfs_alloc_ckpoint(lfs_t *lfs); /// Red-black-yellow Dhara tree operations /// #define LFSR_RBYD_ISSHRUB 0x80000000 -#define LFSR_RBYD_PARITY 0x80000000 +#define LFSR_RBYD_PERTURB 0x80000000 // helper functions static inline bool lfsr_rbyd_isshrub(const lfsr_rbyd_t *rbyd) { @@ -2096,12 +2096,12 @@ static inline bool lfsr_rbyd_isfetched(const lfsr_rbyd_t *rbyd) { return !lfsr_rbyd_trunk(rbyd) || rbyd->eoff; } -static inline bool lfsr_rbyd_parity(const lfsr_rbyd_t *rbyd) { - return rbyd->eoff >> (8*sizeof(lfs_size_t)-1); +static inline bool lfsr_rbyd_perturb(const lfsr_rbyd_t *rbyd) { + return rbyd->eoff & LFSR_RBYD_PERTURB; } static inline lfs_size_t lfsr_rbyd_eoff(const lfsr_rbyd_t *rbyd) { - return rbyd->eoff & ~LFSR_RBYD_PARITY; + return rbyd->eoff & ~LFSR_RBYD_PERTURB; } static inline int lfsr_rbyd_cmp( @@ -2146,7 +2146,6 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, // temporary state until we validate a cksum uint32_t cksum_ = cksum; - bool parity_ = lfs_parity(cksum); lfs_size_t off = sizeof(uint32_t); lfs_size_t trunk_ = 0; lfs_size_t trunk__ = 0; @@ -2155,16 +2154,22 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, // assume unerased until proven otherwise lfsr_ecksum_t ecksum = {.cksize=-1}; + lfsr_ecksum_t ecksum_ = {.cksize=-1}; // scan tags, checking valid bits, cksums, etc while (off < lfs->cfg->block_size && (!trunk || lfsr_rbyd_eoff(rbyd) <= trunk)) { + // perturb? + if (lfsr_rbyd_perturb(rbyd)) { + cksum_ ^= 0x00000080; + } + + // read next tag lfsr_tag_t tag; lfsr_rid_t weight__; lfs_size_t size; - uint32_t cksum__ = cksum_; - lfs_ssize_t d = lfsr_bd_readtag_(lfs, block, off, -1, - &tag, &weight__, &size, &cksum__); + lfs_ssize_t d = lfsr_bd_readtag(lfs, block, off, -1, + &tag, &weight__, &size, &cksum_); if (d < 0) { if (d == LFS_ERR_CORRUPT) { break; @@ -2173,14 +2178,6 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, } lfs_size_t off_ = off + d; - // parity mismatch? - if ((tag >> 15) != parity_) { - break; - } - tag &= 0x7fff; - parity_ ^= lfs_parity(cksum_ ^ cksum__); - cksum_ = cksum__; - // tag goes out of range? if (!lfsr_tag_isalt(tag) && off_ + size > lfs->cfg->block_size) { break; @@ -2191,37 +2188,32 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, // not an end-of-commit cksum if (lfsr_tag_suptype(tag) != LFSR_TAG_CKSUM) { // cksum the entry, hopefully leaving it in the cache - uint32_t cksum__ = cksum_; err = lfsr_bd_cksum(lfs, block, off_, -1, size, - &cksum__); + &cksum_); if (err) { if (err == LFS_ERR_CORRUPT) { break; } return err; } - parity_ ^= lfs_parity(cksum_ ^ cksum__); - cksum_ = cksum__; // found an ecksum? save for later if (tag == LFSR_TAG_ECKSUM) { err = lfsr_data_readecksum(lfs, &LFSR_DATA_DISK(block, off_, lfs->cfg->block_size - off_), - &ecksum); - if (err && err != LFS_ERR_CORRUPT) { + &ecksum_); + if (err) { + if (err == LFS_ERR_CORRUPT) { + break; + } return err; } - - // TODO ignore?? why not break? - // ignore malformed ecksums - if (err == LFS_ERR_CORRUPT) { - ecksum.cksize = -1; - } } // is an end-of-commit cksum } else { + // check cksum uint32_t cksum__ = 0; err = lfsr_bd_read(lfs, block, off_, -1, &cksum__, sizeof(uint32_t)); @@ -2240,14 +2232,17 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, // save what we've found so far rbyd->eoff - = ((lfs_size_t)parity_ << (8*sizeof(lfs_size_t)-1)) + = ((lfs_size_t)lfsr_tag_perturb(tag) + << (8*sizeof(lfs_size_t)-1)) | (off_ + size); rbyd->cksum = cksum; rbyd->trunk = (LFSR_RBYD_ISSHRUB & rbyd->trunk) | trunk_; rbyd->weight = weight; + ecksum = ecksum_; // revert to data checksum cksum_ = cksum; + ecksum_.cksize = -1; } } @@ -2298,9 +2293,10 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, // did we end on a valid commit? we may have erased-state bool erased = false; - if (lfsr_rbyd_eoff(rbyd) < lfs->cfg->block_size - && lfsr_rbyd_eoff(rbyd) % lfs->cfg->prog_size == 0 - && ecksum.cksize != -1) { + if (ecksum.cksize != -1 + && lfsr_rbyd_eoff(rbyd)+ecksum.cksize <= lfs->cfg->block_size + && lfsr_rbyd_eoff(rbyd) % lfs->cfg->prog_size == 0) { + // the next valid bit must _not_ match, or a commit was attempted uint8_t e = 0; err = lfsr_bd_read(lfs, rbyd->blocks[0], lfsr_rbyd_eoff(rbyd), ecksum.cksize, @@ -2309,8 +2305,7 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, return err; } - // the next valid bit must _not_ match, or a commit was attempted - if ((e >> 7) != lfsr_rbyd_parity(rbyd)) { + if (((e >> 7)^lfsr_rbyd_perturb(rbyd)) != lfs_parity(rbyd->cksum)) { // check that erased-state matches our checksum, if this fails // most likely a write was interrupted uint32_t ecksum_ = 0; @@ -2540,20 +2535,15 @@ static int lfsr_rbyd_appendrev(lfs_t *lfs, lfsr_rbyd_t *rbyd, uint32_t rev) { uint8_t rev_buf[sizeof(uint32_t)]; lfs_tole32_(rev, &rev_buf); - uint32_t cksum_ = rbyd->cksum; - int err = lfsr_bd_prog(lfs, rbyd->blocks[0], lfsr_rbyd_eoff(rbyd), + int err = lfsr_bd_prog(lfs, + rbyd->blocks[0], lfsr_rbyd_eoff(rbyd), &rev_buf, sizeof(uint32_t), - &cksum_, false); + &rbyd->cksum, false); if (err) { return err; } - // update eoff, xor cksum parity - rbyd->eoff - += ((lfs_size_t)lfs_parity(rbyd->cksum ^ cksum_) - << (8*sizeof(lfs_size_t)-1)) - + sizeof(uint32_t); - rbyd->cksum = cksum_; + rbyd->eoff += sizeof(uint32_t); return 0; } @@ -2566,24 +2556,20 @@ static int lfsr_rbyd_appendtag(lfs_t *lfs, lfsr_rbyd_t *rbyd, return LFS_ERR_RANGE; } - // include the previous tag parity - tag ^= (lfsr_tag_t)lfsr_rbyd_parity(rbyd) << 15; + // perturb? + if (lfsr_rbyd_perturb(rbyd)) { + rbyd->cksum ^= 0x00000080; + } - uint32_t cksum_ = rbyd->cksum; lfs_ssize_t d = lfsr_bd_progtag(lfs, rbyd->blocks[0], lfsr_rbyd_eoff(rbyd), tag, weight, size, - &cksum_, false); + &rbyd->cksum, false); if (d < 0) { return d; } - // update eoff, xor cksum parity - rbyd->eoff - += ((lfs_size_t)lfs_parity(rbyd->cksum ^ cksum_) - << (8*sizeof(lfs_size_t)-1)) - + d; - rbyd->cksum = cksum_; + rbyd->eoff += d; return 0; } @@ -2595,20 +2581,15 @@ static int lfsr_rbyd_appendcat(lfs_t *lfs, lfsr_rbyd_t *rbyd, return LFS_ERR_RANGE; } - uint32_t cksum_ = rbyd->cksum; - int err = lfsr_bd_progcat(lfs, rbyd->blocks[0], lfsr_rbyd_eoff(rbyd), + int err = lfsr_bd_progcat(lfs, + rbyd->blocks[0], lfsr_rbyd_eoff(rbyd), cat, count, - &cksum_, false); + &rbyd->cksum, false); if (err) { return err; } - // update eoff, xor cksum parity - rbyd->eoff - += ((lfs_size_t)lfs_parity(rbyd->cksum ^ cksum_) - << (8*sizeof(lfs_size_t)-1)) - + lfsr_cat_size(cat, count); - rbyd->cksum = cksum_; + rbyd->eoff += lfsr_cat_size(cat, count); return 0; } @@ -3379,9 +3360,10 @@ static int lfsr_rbyd_appendcksum(lfs_t *lfs, lfsr_rbyd_t *rbyd) { lfs->cfg->prog_size); // space for ecksum? - uint8_t e = 0; + bool perturb = false; if (off_ < lfs->cfg->block_size) { - // read the leading byte in case we need to perturb the next tag + // read the leading byte in case we need to perturb the next commit + uint8_t e = 0; err = lfsr_bd_read(lfs, rbyd->blocks[0], off_, lfs->cfg->prog_size, &e, 1); @@ -3389,6 +3371,11 @@ static int lfsr_rbyd_appendcksum(lfs_t *lfs, lfsr_rbyd_t *rbyd) { return err; } + // we don't want the next commit to appear as valid, so we + // intentionally perturb the commit if this happens, this is + // equivalent to inverting all tag's valid bits + perturb = ((e >> 7) == lfs_parity(cksum)); + // calculate the erased-state checksum lfsr_ecksum_t ecksum; ecksum.cksize = lfs->cfg->prog_size; @@ -3421,16 +3408,23 @@ static int lfsr_rbyd_appendcksum(lfs_t *lfs, lfsr_rbyd_t *rbyd) { return LFS_ERR_RANGE; } + // perturb? + if (lfsr_rbyd_perturb(rbyd)) { + rbyd->cksum ^= 0x00000080; + } + // build end-of-commit cksum // // note padding-size depends on leb-encoding depends on padding-size // depends leb-encoding depends on... to get around this catch-22 we // just always write a fully-expanded leb128 encoding uint8_t cksum_buf[2+1+4+4]; - cksum_buf[0] = (uint8_t)(LFSR_TAG_CKSUM >> 8); + cksum_buf[0] = (uint8_t)(LFSR_TAG_CKSUM >> 8) + // set the valid bit to the cksum parity + | ((uint8_t)lfs_parity(rbyd->cksum) << 7); cksum_buf[1] = (uint8_t)(LFSR_TAG_CKSUM >> 0) - // include tag parity in the perturb bits - | ((uint8_t)lfsr_rbyd_parity(rbyd) << 1); + // set the perturb bit so next commit is invalid + | perturb; cksum_buf[2] = 0; lfs_size_t padding = off_ - (lfsr_rbyd_eoff(rbyd) + 2+1+4); @@ -3439,20 +3433,11 @@ static int lfsr_rbyd_appendcksum(lfs_t *lfs, lfsr_rbyd_t *rbyd) { cksum_buf[5] = 0x80 | (0x7f & (padding >> 14)); cksum_buf[6] = 0x00 | (0x7f & (padding >> 21)); - // calculate checksum before tag parity - uint32_t cksum_ = lfs_crc32c(rbyd->cksum, cksum_buf, 2+1+4); - // xor in the tag parity - cksum_buf[0] ^= (uint8_t)lfsr_rbyd_parity(rbyd) << 7; - // find the new parity - bool parity_ = lfsr_rbyd_parity(rbyd) ^ lfs_parity(rbyd->cksum ^ cksum_); - // and intentionally perturb the commit so the next tag appears invalid - if ((e >> 7) == parity_) { - cksum_buf[1] ^= 0x01; - cksum_ ^= 0xef306b19; - parity_ ^= 0x1; - } - lfs_tole32_(cksum_, &cksum_buf[2+1+4]); + // calculate checksum + rbyd->cksum = lfs_crc32c(rbyd->cksum, cksum_buf, 2+1+4); + lfs_tole32_(rbyd->cksum, &cksum_buf[2+1+4]); + // prog, when this lands on disk commit is committed err = lfsr_bd_prog(lfs, rbyd->blocks[0], lfsr_rbyd_eoff(rbyd), cksum_buf, 2+1+4+4, NULL, false); @@ -3466,9 +3451,10 @@ static int lfsr_rbyd_appendcksum(lfs_t *lfs, lfsr_rbyd_t *rbyd) { return err; } - // update the eoff and parity + // update the eoff and perturb rbyd->eoff - = ((lfs_size_t)parity_ << (8*sizeof(lfs_size_t)-1)) + = ((lfs_size_t)perturb + << (8*sizeof(lfs_size_t)-1)) | off_; // revert to data checksum rbyd->cksum = cksum; diff --git a/lfs.h b/lfs.h index 26107da9..fcc45bcb 100644 --- a/lfs.h +++ b/lfs.h @@ -384,7 +384,7 @@ typedef struct lfsr_rbyd { // sign(trunk)=0 => normal rbyd // sign(trunk)=1 => shrub rbyd lfs_size_t trunk; - // sign(eoff) => commit parity + // sign(eoff) => perturb bit // eoff=0, trunk=0 => not yet committed // eoff=0, trunk>0 => not yet fetched // eoff>=block_size => rbyd not erased/needs compaction diff --git a/scripts/dbgbmap.py b/scripts/dbgbmap.py index eb8f1d47..2a2b5423 100755 --- a/scripts/dbgbmap.py +++ b/scripts/dbgbmap.py @@ -40,12 +40,16 @@ TAG_BRANCH = 0x032c TAG_UATTR = 0x0400 TAG_SATTR = 0x0600 TAG_SHRUB = 0x1000 -TAG_CKSUM = 0x3000 -TAG_PERTURB = 0x3100 -TAG_ECKSUM = 0x3200 TAG_ALT = 0x4000 +TAG_B = 0x0000 TAG_R = 0x2000 +TAG_LE = 0x0000 TAG_GT = 0x1000 +TAG_CKSUM = 0x3000 +TAG_Q = 0x0000 +TAG_P = 0x0001 +TAG_NOISE = 0x3100 +TAG_ECKSUM = 0x3200 CHARS = 'mbd-' @@ -649,7 +653,7 @@ class Rbyd: cksum = 0 cksum_ = crc32c(data[0:4]) cksum__ = cksum_ - parity__ = parity(cksum_) + perturb = False eoff = 0 eoff_ = None j_ = 4 @@ -660,13 +664,15 @@ class Rbyd: weight_ = 0 weight__ = 0 while j_ < len(data) and (not trunk or eoff <= trunk): + # perturb? + if perturb: + cksum__ ^= 0x00000080 + + # read next tag v, tag, w, size, d = fromtag(data[j_:]) - if v != parity__: + if v != parity(cksum__): break - parity__ ^= parity(cksum__) - cksum__ = crc32c([data[j_] & ~0x80], cksum__) - cksum__ = crc32c(data[j_+1:j_+d], cksum__) - parity__ ^= parity(cksum__) + cksum__ = crc32c(data[j_:j_+d], cksum__) j_ += d if not tag & TAG_ALT and j_ + size > len(data): break @@ -674,9 +680,7 @@ class Rbyd: # take care of cksums if not tag & TAG_ALT: if (tag & 0xff00) != TAG_CKSUM: - parity__ ^= parity(cksum__) cksum__ = crc32c(data[j_:j_+size], cksum__) - parity__ ^= parity(cksum__) # found a cksum? else: cksum___ = fromle32(data[j_:j_+4]) @@ -687,6 +691,8 @@ class Rbyd: cksum = cksum_ trunk_ = trunk__ weight = weight_ + # update perturb bit + perturb = tag & TAG_P # revert to data cksum cksum__ = cksum_ diff --git a/scripts/dbgbtree.py b/scripts/dbgbtree.py index 52e9b74f..33501353 100755 --- a/scripts/dbgbtree.py +++ b/scripts/dbgbtree.py @@ -38,12 +38,16 @@ TAG_BRANCH = 0x032c TAG_UATTR = 0x0400 TAG_SATTR = 0x0600 TAG_SHRUB = 0x1000 -TAG_CKSUM = 0x3000 -TAG_PERTURB = 0x3100 -TAG_ECKSUM = 0x3200 TAG_ALT = 0x4000 +TAG_B = 0x0000 TAG_R = 0x2000 +TAG_LE = 0x0000 TAG_GT = 0x1000 +TAG_CKSUM = 0x3000 +TAG_Q = 0x0000 +TAG_P = 0x0001 +TAG_NOISE = 0x3100 +TAG_ECKSUM = 0x3200 # some ways of block geometry representations @@ -221,21 +225,6 @@ def tagrepr(tag, w=None, size=None, off=None): ((tag & 0x100) >> 1) | (tag & 0xff), ' w%d' % w if w else '', ' %s' % size if size is not None else '') - elif (tag & 0x7f00) == TAG_CKSUM: - return 'cksum 0x%02x%s%s' % ( - tag & 0xff, - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') - elif (tag & 0x7f00) == TAG_PERTURB: - return 'perturb%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') - elif (tag & 0x7f00) == TAG_ECKSUM: - return 'ecksum%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') elif tag & TAG_ALT: return 'alt%s%s%s%s%s' % ( 'r' if tag & TAG_R else 'b', @@ -249,6 +238,23 @@ def tagrepr(tag, w=None, size=None, off=None): if size and off is not None else ' -%d' % size if size else '') + elif (tag & 0x7f00) == TAG_CKSUM: + return 'cksum%s%s%s' % ( + 'p' if tag & 0xff == TAG_P + else 'q' if tag & 0xff == TAG_Q + else ' 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') + elif (tag & 0x7f00) == TAG_NOISE: + return 'noise%s%s%s' % ( + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') + elif (tag & 0x7f00) == TAG_ECKSUM: + return 'ecksum%s%s%s' % ( + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') else: return '0x%04x%s%s' % ( tag, @@ -320,7 +326,7 @@ class Rbyd: cksum = 0 cksum_ = crc32c(data[0:4]) cksum__ = cksum_ - parity__ = parity(cksum_) + perturb = False eoff = 0 eoff_ = None j_ = 4 @@ -331,13 +337,15 @@ class Rbyd: weight_ = 0 weight__ = 0 while j_ < len(data) and (not trunk or eoff <= trunk): + # perturb? + if perturb: + cksum__ ^= 0x00000080 + + # read next tag v, tag, w, size, d = fromtag(data[j_:]) - if v != parity__: + if v != parity(cksum__): break - parity__ ^= parity(cksum__) - cksum__ = crc32c([data[j_] & ~0x80], cksum__) - cksum__ = crc32c(data[j_+1:j_+d], cksum__) - parity__ ^= parity(cksum__) + cksum__ = crc32c(data[j_:j_+d], cksum__) j_ += d if not tag & TAG_ALT and j_ + size > len(data): break @@ -345,9 +353,7 @@ class Rbyd: # take care of cksums if not tag & TAG_ALT: if (tag & 0xff00) != TAG_CKSUM: - parity__ ^= parity(cksum__) cksum__ = crc32c(data[j_:j_+size], cksum__) - parity__ ^= parity(cksum__) # found a cksum? else: cksum___ = fromle32(data[j_:j_+4]) @@ -358,6 +364,8 @@ class Rbyd: cksum = cksum_ trunk_ = trunk__ weight = weight_ + # update perturb bit + perturb = tag & TAG_P # revert to data cksum cksum__ = cksum_ diff --git a/scripts/dbglfs.py b/scripts/dbglfs.py index cde560aa..09b2bee9 100755 --- a/scripts/dbglfs.py +++ b/scripts/dbglfs.py @@ -39,12 +39,16 @@ TAG_BRANCH = 0x032c TAG_UATTR = 0x0400 TAG_SATTR = 0x0600 TAG_SHRUB = 0x1000 -TAG_CKSUM = 0x3000 -TAG_PERTURB = 0x3100 -TAG_ECKSUM = 0x3200 TAG_ALT = 0x4000 +TAG_B = 0x0000 TAG_R = 0x2000 +TAG_LE = 0x0000 TAG_GT = 0x1000 +TAG_CKSUM = 0x3000 +TAG_Q = 0x0000 +TAG_P = 0x0001 +TAG_NOISE = 0x3100 +TAG_ECKSUM = 0x3200 # some ways of block geometry representations @@ -252,21 +256,6 @@ def tagrepr(tag, w=None, size=None, off=None): ((tag & 0x100) >> 1) | (tag & 0xff), ' w%d' % w if w else '', ' %s' % size if size is not None else '') - elif (tag & 0x7f00) == TAG_CKSUM: - return 'cksum 0x%02x%s%s' % ( - tag & 0xff, - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') - elif (tag & 0x7f00) == TAG_PERTURB: - return 'perturb%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') - elif (tag & 0x7f00) == TAG_ECKSUM: - return 'ecksum%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') elif tag & TAG_ALT: return 'alt%s%s%s%s%s' % ( 'r' if tag & TAG_R else 'b', @@ -280,6 +269,23 @@ def tagrepr(tag, w=None, size=None, off=None): if size and off is not None else ' -%d' % size if size else '') + elif (tag & 0x7f00) == TAG_CKSUM: + return 'cksum%s%s%s' % ( + 'p' if tag & 0xff == TAG_P + else 'q' if tag & 0xff == TAG_Q + else ' 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') + elif (tag & 0x7f00) == TAG_NOISE: + return 'noise%s%s%s' % ( + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') + elif (tag & 0x7f00) == TAG_ECKSUM: + return 'ecksum%s%s%s' % ( + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') else: return '0x%04x%s%s' % ( tag, @@ -351,7 +357,7 @@ class Rbyd: cksum = 0 cksum_ = crc32c(data[0:4]) cksum__ = cksum_ - parity__ = parity(cksum_) + perturb = False eoff = 0 eoff_ = None j_ = 4 @@ -362,13 +368,15 @@ class Rbyd: weight_ = 0 weight__ = 0 while j_ < len(data) and (not trunk or eoff <= trunk): + # perturb? + if perturb: + cksum__ ^= 0x00000080 + + # read next tag v, tag, w, size, d = fromtag(data[j_:]) - if v != parity__: + if v != parity(cksum__): break - parity__ ^= parity(cksum__) - cksum__ = crc32c([data[j_] & ~0x80], cksum__) - cksum__ = crc32c(data[j_+1:j_+d], cksum__) - parity__ ^= parity(cksum__) + cksum__ = crc32c(data[j_:j_+d], cksum__) j_ += d if not tag & TAG_ALT and j_ + size > len(data): break @@ -376,9 +384,7 @@ class Rbyd: # take care of cksums if not tag & TAG_ALT: if (tag & 0xff00) != TAG_CKSUM: - parity__ ^= parity(cksum__) cksum__ = crc32c(data[j_:j_+size], cksum__) - parity__ ^= parity(cksum__) # found a cksum? else: cksum___ = fromle32(data[j_:j_+4]) @@ -389,6 +395,8 @@ class Rbyd: cksum = cksum_ trunk_ = trunk__ weight = weight_ + # update perturb bit + perturb = tag & TAG_P # revert to data cksum cksum__ = cksum_ diff --git a/scripts/dbgmtree.py b/scripts/dbgmtree.py index 1db7ab2c..2c69aa1c 100755 --- a/scripts/dbgmtree.py +++ b/scripts/dbgmtree.py @@ -38,12 +38,16 @@ TAG_BRANCH = 0x032c TAG_UATTR = 0x0400 TAG_SATTR = 0x0600 TAG_SHRUB = 0x1000 -TAG_CKSUM = 0x3000 -TAG_PERTURB = 0x3100 -TAG_ECKSUM = 0x3200 TAG_ALT = 0x4000 +TAG_B = 0x0000 TAG_R = 0x2000 +TAG_LE = 0x0000 TAG_GT = 0x1000 +TAG_CKSUM = 0x3000 +TAG_Q = 0x0000 +TAG_P = 0x0001 +TAG_NOISE = 0x3100 +TAG_ECKSUM = 0x3200 # some ways of block geometry representations @@ -236,21 +240,6 @@ def tagrepr(tag, w=None, size=None, off=None): ((tag & 0x100) >> 1) | (tag & 0xff), ' w%d' % w if w else '', ' %s' % size if size is not None else '') - elif (tag & 0x7f00) == TAG_CKSUM: - return 'cksum 0x%02x%s%s' % ( - tag & 0xff, - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') - elif (tag & 0x7f00) == TAG_PERTURB: - return 'perturb%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') - elif (tag & 0x7f00) == TAG_ECKSUM: - return 'ecksum%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') elif tag & TAG_ALT: return 'alt%s%s%s%s%s' % ( 'r' if tag & TAG_R else 'b', @@ -264,6 +253,23 @@ def tagrepr(tag, w=None, size=None, off=None): if size and off is not None else ' -%d' % size if size else '') + elif (tag & 0x7f00) == TAG_CKSUM: + return 'cksum%s%s%s' % ( + 'p' if tag & 0xff == TAG_P + else 'q' if tag & 0xff == TAG_Q + else ' 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') + elif (tag & 0x7f00) == TAG_NOISE: + return 'noise%s%s%s' % ( + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') + elif (tag & 0x7f00) == TAG_ECKSUM: + return 'ecksum%s%s%s' % ( + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') else: return '0x%04x%s%s' % ( tag, @@ -335,7 +341,7 @@ class Rbyd: cksum = 0 cksum_ = crc32c(data[0:4]) cksum__ = cksum_ - parity__ = parity(cksum_) + perturb = False eoff = 0 eoff_ = None j_ = 4 @@ -346,13 +352,15 @@ class Rbyd: weight_ = 0 weight__ = 0 while j_ < len(data) and (not trunk or eoff <= trunk): + # perturb? + if perturb: + cksum__ ^= 0x00000080 + + # read next tag v, tag, w, size, d = fromtag(data[j_:]) - if v != parity__: + if v != parity(cksum__): break - parity__ ^= parity(cksum__) - cksum__ = crc32c([data[j_] & ~0x80], cksum__) - cksum__ = crc32c(data[j_+1:j_+d], cksum__) - parity__ ^= parity(cksum__) + cksum__ = crc32c(data[j_:j_+d], cksum__) j_ += d if not tag & TAG_ALT and j_ + size > len(data): break @@ -360,9 +368,7 @@ class Rbyd: # take care of cksums if not tag & TAG_ALT: if (tag & 0xff00) != TAG_CKSUM: - parity__ ^= parity(cksum__) cksum__ = crc32c(data[j_:j_+size], cksum__) - parity__ ^= parity(cksum__) # found a cksum? else: cksum___ = fromle32(data[j_:j_+4]) @@ -373,6 +379,8 @@ class Rbyd: cksum = cksum_ trunk_ = trunk__ weight = weight_ + # update perturb bit + perturb = tag & TAG_P # revert to data cksum cksum__ = cksum_ diff --git a/scripts/dbgrbyd.py b/scripts/dbgrbyd.py index 6832dcd0..580a3448 100755 --- a/scripts/dbgrbyd.py +++ b/scripts/dbgrbyd.py @@ -47,12 +47,16 @@ TAG_BRANCH = 0x032c TAG_UATTR = 0x0400 TAG_SATTR = 0x0600 TAG_SHRUB = 0x1000 -TAG_CKSUM = 0x3000 -TAG_PERTURB = 0x3100 -TAG_ECKSUM = 0x3200 TAG_ALT = 0x4000 +TAG_B = 0x0000 TAG_R = 0x2000 +TAG_LE = 0x0000 TAG_GT = 0x1000 +TAG_CKSUM = 0x3000 +TAG_Q = 0x0000 +TAG_P = 0x0001 +TAG_NOISE = 0x3100 +TAG_ECKSUM = 0x3200 # some ways of block geometry representations @@ -129,9 +133,6 @@ def popc(x): def parity(x): return popc(x) & 1 -def parity(x): - return popc(x) & 1 - def fromle32(data): return struct.unpack('> 1) | (tag & 0xff), ' w%d' % w if w else '', ' %s' % size if size is not None else '') - elif (tag & 0x7f00) == TAG_CKSUM: - return 'cksum 0x%02x%s%s' % ( - tag & 0xff, - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') - elif (tag & 0x7f00) == TAG_PERTURB: - return 'perturb%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') - elif (tag & 0x7f00) == TAG_ECKSUM: - return 'ecksum%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') elif tag & TAG_ALT: return 'alt%s%s%s%s%s' % ( 'r' if tag & TAG_R else 'b', @@ -254,6 +240,23 @@ def tagrepr(tag, w=None, size=None, off=None): if size and off is not None else ' -%d' % size if size else '') + elif (tag & 0x7f00) == TAG_CKSUM: + return 'cksum%s%s%s' % ( + 'p' if tag & 0xff == TAG_P + else 'q' if tag & 0xff == TAG_Q + else ' 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') + elif (tag & 0x7f00) == TAG_NOISE: + return 'noise%s%s%s' % ( + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') + elif (tag & 0x7f00) == TAG_ECKSUM: + return 'ecksum%s%s%s' % ( + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') else: return '0x%04x%s%s' % ( tag, @@ -527,34 +530,36 @@ def dbg_log(data, block_size, rev, eoff, weight, *, # print tags cksum = crc32c(data[0:4]) cksum_ = cksum - parity_ = parity(cksum) + perturb = False lower_, upper_ = 0, 0 trunk_ = 0 j_ = 4 while j_ < (block_size if args.get('all') else eoff): notes = [] + # perturb? + if perturb: + cksum_ ^= 0x00000080 + + # read next tag j = j_ v, tag, w, size, d = fromtag(data[j_:]) - if v != parity_: - notes.append('v!=%x' % parity_) - parity_ ^= parity(cksum_) - cksum_ = crc32c([data[j_] & ~0x80], cksum_) - cksum_ = crc32c(data[j_+1:j_+d], cksum_) - parity_ ^= parity(cksum_) + if v != parity(cksum_): + notes.append('v!=%x' % parity(cksum_)) + cksum_ = crc32c(data[j_:j_+d], cksum_) j_ += d # take care of cksums if not tag & TAG_ALT: if (tag & 0xff00) != TAG_CKSUM: - parity_ ^= parity(cksum_) cksum_ = crc32c(data[j_:j_+size], cksum_) - parity_ ^= parity(cksum_) # found a cksum? else: cksum__ = fromle32(data[j_:j_+4]) if cksum_ != cksum__: - notes.append('cksum!=%08x' % cksum_) + notes.append('cksum!=%08x' % cksum__) + # update perturb bit + perturb = tag & TAG_P # revert to data cksum cksum_ = cksum j_ += size @@ -928,7 +933,7 @@ def main(disk, blocks=None, *, cksum = 0 cksum_ = crc32c(data[0:4]) cksum__ = cksum_ - parity__ = parity(cksum_) + perturb = False eoff = 0 eoff_ = None j_ = 4 @@ -939,13 +944,15 @@ def main(disk, blocks=None, *, weight_ = 0 weight__ = 0 while j_ < len(data) and (not trunk or eoff <= trunk): + # perturb? + if perturb: + cksum__ ^= 0x00000080 + + # read next tag v, tag, w, size, d = fromtag(data[j_:]) - if v != parity__: + if v != parity(cksum__): break - parity__ ^= parity(cksum__) - cksum__ = crc32c([data[j_] & ~0x80], cksum__) - cksum__ = crc32c(data[j_+1:j_+d], cksum__) - parity__ ^= parity(cksum__) + cksum__ = crc32c(data[j_:j_+d], cksum__) j_ += d if not tag & TAG_ALT and j_ + size > len(data): break @@ -953,9 +960,7 @@ def main(disk, blocks=None, *, # take care of cksums if not tag & TAG_ALT: if (tag & 0xff00) != TAG_CKSUM: - parity__ ^= parity(cksum__) cksum__ = crc32c(data[j_:j_+size], cksum__) - parity__ ^= parity(cksum__) # found a cksum? else: cksum___ = fromle32(data[j_:j_+4]) @@ -966,6 +971,8 @@ def main(disk, blocks=None, *, cksum = cksum_ trunk_ = trunk__ weight = weight_ + # update perturb bit + perturb = tag & TAG_P # revert to data cksum cksum__ = cksum_ diff --git a/scripts/dbgtag.py b/scripts/dbgtag.py index 6ff5e11b..c77f9db0 100755 --- a/scripts/dbgtag.py +++ b/scripts/dbgtag.py @@ -36,12 +36,16 @@ TAG_BRANCH = 0x032c TAG_UATTR = 0x0400 TAG_SATTR = 0x0600 TAG_SHRUB = 0x1000 -TAG_CKSUM = 0x3000 -TAG_PERTURB = 0x3100 -TAG_ECKSUM = 0x3200 TAG_ALT = 0x4000 +TAG_B = 0x0000 TAG_R = 0x2000 +TAG_LE = 0x0000 TAG_GT = 0x1000 +TAG_CKSUM = 0x3000 +TAG_Q = 0x0000 +TAG_P = 0x0001 +TAG_NOISE = 0x3100 +TAG_ECKSUM = 0x3200 # some ways of block geometry representations @@ -178,21 +182,6 @@ def tagrepr(tag, w=None, size=None, off=None): ((tag & 0x100) >> 1) | (tag & 0xff), ' w%d' % w if w else '', ' %s' % size if size is not None else '') - elif (tag & 0x7f00) == TAG_CKSUM: - return 'cksum 0x%02x%s%s' % ( - tag & 0xff, - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') - elif (tag & 0x7f00) == TAG_PERTURB: - return 'perturb%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') - elif (tag & 0x7f00) == TAG_ECKSUM: - return 'ecksum%s%s%s' % ( - ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', - ' w%d' % w if w else '', - ' %s' % size if size is not None else '') elif tag & TAG_ALT: return 'alt%s%s%s%s%s' % ( 'r' if tag & TAG_R else 'b', @@ -206,6 +195,23 @@ def tagrepr(tag, w=None, size=None, off=None): if size and off is not None else ' -%d' % size if size else '') + elif (tag & 0x7f00) == TAG_CKSUM: + return 'cksum%s%s%s' % ( + 'p' if tag & 0xff == TAG_P + else 'q' if tag & 0xff == TAG_Q + else ' 0x%02x' % (tag & 0xff), + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') + elif (tag & 0x7f00) == TAG_NOISE: + return 'noise%s%s%s' % ( + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') + elif (tag & 0x7f00) == TAG_ECKSUM: + return 'ecksum%s%s%s' % ( + ' 0x%02x' % (tag & 0xff) if tag & 0xff else '', + ' w%d' % w if w else '', + ' %s' % size if size is not None else '') else: return '0x%04x%s%s' % ( tag,