From 0ccf2833211f3cd8526142a98c546699669bb0b1 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 5 Apr 2023 14:23:35 -0500 Subject: [PATCH] Changed in-tree tags to store their weights Sorting weights instead of ids just had a number of benefits, suggesting this is a better design: - Calculating the id and delta of each rbyd trunk is surprisingly easier - id is now just lower+w-1, and no extra conditions are needed for unr tags, which just have a weight of zero. - Removes ambiguity around which id unr tags should be assigned to, especially unrs that delete ids. - No more +-1 weirdness when encoding/decoding tag ids - the weight can be written as-is and -1 ids are infered from their weight and position in the tree (lower+w-1 = 0+0-1 = -1). - Weights compress better under leb128 encoding, since they are usually quite small. --- lfs.c | 108 ++++++------- scripts/dbgbtree.py | 363 +++++++++++++++++++++++--------------------- scripts/dbgrbyd.py | 166 +++++++++++--------- 3 files changed, 337 insertions(+), 300 deletions(-) diff --git a/lfs.c b/lfs.c index 6aa5320f..f8bb68b2 100644 --- a/lfs.c +++ b/lfs.c @@ -611,23 +611,23 @@ static inline lfsr_tag_t lfsr_tag_key(lfsr_tag_t tag) { return tag & ~0xf; } -static inline bool lfsr_tag_follow(lfsr_tag_t alt, lfs_ssize_t weight, +static inline bool lfsr_tag_follow(lfsr_tag_t alt, lfs_size_t weight, lfs_ssize_t lower, lfs_ssize_t upper, lfsr_tag_t tag, lfs_ssize_t id) { if (lfsr_tag_isgt(alt)) { - return id > upper - weight - 1 - || (id == upper - weight - 1 + return id > upper - (lfs_ssize_t)weight - 1 + || (id == upper - (lfs_ssize_t)weight - 1 && lfsr_tag_key(tag) > lfsr_tag_key(alt)); } else { - return id < lower + weight - || (id == lower + weight + return id < lower + (lfs_ssize_t)weight + || (id == lower + (lfs_ssize_t)weight && lfsr_tag_key(tag) <= lfsr_tag_key(alt)); } } static inline bool lfsr_tag_follow2( - lfsr_tag_t alt, lfs_ssize_t weight, - lfsr_tag_t alt2, lfs_ssize_t weight2, + lfsr_tag_t alt, lfs_size_t weight, + lfsr_tag_t alt2, lfs_size_t weight2, lfs_ssize_t lower, lfs_ssize_t upper, lfsr_tag_t tag, lfs_ssize_t id) { if (lfsr_tag_isred(alt2) && lfsr_tag_isparallel(alt, alt2)) { @@ -657,13 +657,13 @@ static inline bool lfsr_tag_prune2( } } -static inline void lfsr_tag_flip(lfsr_tag_t *alt, lfs_ssize_t *weight, +static inline void lfsr_tag_flip(lfsr_tag_t *alt, lfs_size_t *weight, lfs_ssize_t lower, lfs_ssize_t upper) { *alt = *alt ^ 0x4; *weight = (upper-lower) - *weight - 1; } -static inline void lfsr_tag_flip2(lfsr_tag_t *alt, lfs_ssize_t *weight, +static inline void lfsr_tag_flip2(lfsr_tag_t *alt, lfs_size_t *weight, lfsr_tag_t alt2, lfs_size_t weight2, lfs_ssize_t lower, lfs_ssize_t upper) { if (lfsr_tag_isred(alt2)) { @@ -674,7 +674,7 @@ static inline void lfsr_tag_flip2(lfsr_tag_t *alt, lfs_ssize_t *weight, } static inline void lfsr_tag_trim( - lfsr_tag_t alt, lfs_ssize_t weight, + lfsr_tag_t alt, lfs_size_t weight, lfs_ssize_t *lower_id, lfs_ssize_t *upper_id, lfsr_tag_t *lower_tag, lfsr_tag_t *upper_tag) { if (lfsr_tag_isgt(alt)) { @@ -691,8 +691,8 @@ static inline void lfsr_tag_trim( } static inline void lfsr_tag_trim2( - lfsr_tag_t alt, lfs_ssize_t weight, - lfsr_tag_t alt2, lfs_ssize_t weight2, + lfsr_tag_t alt, lfs_size_t weight, + lfsr_tag_t alt2, lfs_size_t weight2, lfs_ssize_t *lower_id, lfs_ssize_t *upper_id, lfsr_tag_t *lower_tag, lfsr_tag_t *upper_tag) { if (lfsr_tag_isred(alt2)) { @@ -1264,7 +1264,7 @@ static int lfsr_rbyd_alloc(lfs_t *lfs, lfsr_rbyd_t *rbyd, uint32_t rev) { static lfs_ssize_t lfsr_rbyd_readtag(lfs_t *lfs, const lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_size_t hint, lfs_block_t block, lfs_off_t off, - lfsr_tag_t *tag, lfs_ssize_t *id, lfs_size_t *size, uint32_t *crc) { + lfsr_tag_t *tag, lfs_size_t *weight, lfs_size_t *size, uint32_t *crc) { // // needed to quiet an uninitialized warning, zeroing tag on error is // // probably a good idea anyways // *tag = 0; @@ -1303,15 +1303,15 @@ static lfs_ssize_t lfsr_rbyd_readtag(lfs_t *lfs, uint16_t tag_ = lfs_fromle16_(&buffer[0]); - lfs_size_t id_; + lfs_size_t weight_; ssize_t delta = 2; - lfs_ssize_t delta_ = lfs_fromleb128(&id_, &buffer[delta], 5); + lfs_ssize_t delta_ = lfs_fromleb128(&weight_, &buffer[delta], 5); if (delta_ < 0) { return delta_; } delta += delta_; - if (id_ > 0x7fffffff) { + if (weight_ > 0x7fffffff) { return LFS_ERR_CORRUPT; } @@ -1335,11 +1335,11 @@ static lfs_ssize_t lfsr_rbyd_readtag(lfs_t *lfs, // - clear the valid bit from tag, we checked this earlier // - adjust id so reserved id is -1, so we don't have mixed zero/one indexed // - if (!lfsr_tag_isalt(tag_)) { - id_ -= 1; - } +// if (!lfsr_tag_isalt(tag_)) { +// id_ -= 1; +// } *tag = tag_ & ~0x1; - *id = id_; + *weight = weight_; *size = size_; return delta; @@ -1389,11 +1389,11 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, // scan tags, checking valid bits, crcs, etc while (off < limit) { lfsr_tag_t tag; - lfs_ssize_t id; + lfs_size_t w; lfs_size_t size; lfs_ssize_t delta = lfsr_rbyd_readtag(lfs, NULL, &lfs->rcache, limit-off, - block, off, &tag, &id, &size, &crc); + block, off, &tag, &w, &size, &crc); if (delta < 0) { if (delta == LFS_ERR_INVAL || delta == LFS_ERR_CORRUPT) { maybeerased = maybeerased && delta == LFS_ERR_INVAL; @@ -1419,18 +1419,16 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, // on a bad crc if (lfsr_tag_isalt(tag)) { if (lfsr_tag_isgt(tag)) { - upper += id; + upper += w; } else { - lower += id; + lower += w; } } else if (lfsr_tag_istrunk(tag)) { // note we need to include our id's weight in our weight // calculation, but only when our id is included in the tree - lfs_ssize_t delta = (lower+upper) - weight; - if (!lfsr_tag_isrm(tag)) { - delta += id+1-lower; - } + lfs_ssize_t delta = (lower+upper+w) - weight; + weight = lower+upper+ w; // adjust any pending finds if (find && find->predicted_id >= (lfs_ssize_t)lower) { @@ -1438,13 +1436,11 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, if (delta < 0 && find->predicted_id < (lfs_ssize_t)lower + -delta) { find->predicted_tag = 0; - find->predicted_id = lower-1; // TODO id-1? + find->predicted_id = lower-1; // TODO is this correct for insertion behavior? } else { find->predicted_id += delta; } } - - weight += delta; } // we mostly just skip non-data tags here @@ -1510,6 +1506,7 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, } // found match? + lfs_ssize_t id = lower+w-1; if (cmp == LFS_CMP_EQ) { find->predicted_tag = tag; find->predicted_id = id; @@ -1614,7 +1611,7 @@ static int lfsr_rbyd_lookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd, // descend down tree while (true) { lfsr_tag_t alt; - lfs_ssize_t weight; + lfs_size_t weight; lfs_off_t jump; lfs_ssize_t delta = lfsr_rbyd_readtag(lfs, &lfs->pcache, &lfs->rcache, 0, @@ -1851,25 +1848,25 @@ static int lfsr_rbyd_progdata(lfs_t *lfs, lfsr_rbyd_t *rbyd_, } static int lfsr_rbyd_progtag(lfs_t *lfs, lfsr_rbyd_t *rbyd_, - lfsr_tag_t tag, lfs_ssize_t id, lfs_size_t size, uint32_t *crc) { + lfsr_tag_t tag, lfs_size_t weight, lfs_size_t size, uint32_t *crc) { // check for underflow issues - LFS_ASSERT((lfs_size_t)(id+1) < 0x80000000); + LFS_ASSERT(weight < 0x80000000); LFS_ASSERT(size < 0x80000000); // make sure to include the parity of the current crc tag |= lfs_popc(rbyd_->crc) & 1; - // change ids to on-disk representation - if (!lfsr_tag_isalt(tag)) { - id += 1; - } +// // change ids to on-disk representation +// if (!lfsr_tag_isalt(tag)) { +// id += 1; +// } // compress into an le16 and pair of leb128s uint8_t buf[LFSR_TAG_DSIZE]; lfs_tole16_(tag, &buf[0]); lfs_size_t delta = 2; - ssize_t delta_ = lfs_toleb128(id, &buf[delta], 5); + ssize_t delta_ = lfs_toleb128(weight, &buf[delta], 5); if (delta_ < 0) { return delta_; } @@ -1891,7 +1888,7 @@ static int lfsr_rbyd_progtag(lfs_t *lfs, lfsr_rbyd_t *rbyd_, static int lfsr_rbyd_p_flush(lfs_t *lfs, lfsr_rbyd_t *rbyd_, lfsr_tag_t p_alts[static 3], - lfs_ssize_t p_weights[static 3], + lfs_size_t p_weights[static 3], lfs_off_t p_jumps[static 3], unsigned count) { // write out some number of alt pointers in our queue @@ -1899,7 +1896,7 @@ static int lfsr_rbyd_p_flush(lfs_t *lfs, lfsr_rbyd_t *rbyd_, if (p_alts[3-1-i]) { // change to a relative jump at the last minute lfsr_tag_t alt = p_alts[3-1-i]; - lfs_ssize_t weight = p_weights[3-1-i]; + lfs_size_t weight = p_weights[3-1-i]; lfs_off_t jump = rbyd_->off - p_jumps[3-1-i]; int err = lfsr_rbyd_progtag(lfs, rbyd_, @@ -1915,7 +1912,7 @@ static int lfsr_rbyd_p_flush(lfs_t *lfs, lfsr_rbyd_t *rbyd_, static inline int lfsr_rbyd_p_push(lfs_t *lfs, lfsr_rbyd_t *rbyd_, lfsr_tag_t p_alts[static 3], - lfs_ssize_t p_weights[static 3], + lfs_size_t p_weights[static 3], lfs_off_t p_jumps[static 3], lfsr_tag_t alt, lfs_ssize_t weight, lfs_off_t jump) { int err = lfsr_rbyd_p_flush(lfs, rbyd_, p_alts, p_weights, p_jumps, 1); @@ -1935,10 +1932,10 @@ static inline int lfsr_rbyd_p_push(lfs_t *lfs, lfsr_rbyd_t *rbyd_, static inline void lfsr_rbyd_p_pop( lfsr_tag_t p_alts[static 3], - lfs_ssize_t p_weights[static 3], + lfs_size_t p_weights[static 3], lfs_off_t p_jumps[static 3]) { memmove(p_alts, p_alts+1, 2*sizeof(lfsr_tag_t)); - memmove(p_weights, p_weights+1, 2*sizeof(lfs_ssize_t)); + memmove(p_weights, p_weights+1, 2*sizeof(lfs_size_t)); memmove(p_jumps, p_jumps+1, 2*sizeof(lfs_off_t)); p_alts[2] = 0; p_weights[2] = 0; @@ -1947,7 +1944,7 @@ static inline void lfsr_rbyd_p_pop( static void lfsr_rbyd_p_red( lfsr_tag_t p_alts[static 3], - lfs_ssize_t p_weights[static 3], + lfs_size_t p_weights[static 3], lfs_off_t p_jumps[static 3]) { // propagate a red edge upwards p_alts[0] = lfsr_tag_setblack(p_alts[0]); @@ -1961,7 +1958,7 @@ static void lfsr_rbyd_p_red( // no reorder needed } else if (lfsr_tag_isparallel(p_alts[0], p_alts[2])) { lfsr_tag_t alt_ = p_alts[1]; - lfs_ssize_t weight_ = p_weights[1]; + lfs_size_t weight_ = p_weights[1]; lfs_off_t jump_ = p_jumps[1]; p_alts[1] = lfsr_tag_setred(p_alts[0]); p_weights[1] = p_weights[0]; @@ -1971,7 +1968,7 @@ static void lfsr_rbyd_p_red( p_jumps[0] = jump_; } else if (lfsr_tag_isparallel(p_alts[0], p_alts[1])) { lfsr_tag_t alt_ = p_alts[2]; - lfs_ssize_t weight_ = p_weights[2]; + lfs_size_t weight_ = p_weights[2]; lfs_off_t jump_ = p_jumps[2]; p_alts[2] = lfsr_tag_setred(p_alts[1]); p_weights[2] = p_weights[1]; @@ -2103,7 +2100,7 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd, // queue of pending alts we can emulate rotations with lfsr_tag_t p_alts[3] = {0, 0, 0}; - lfs_ssize_t p_weights[3] = {0, 0, 0}; + lfs_size_t p_weights[3] = {0, 0, 0}; lfs_off_t p_jumps[3] = {0, 0, 0}; lfs_off_t graft = 0; @@ -2111,7 +2108,7 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd, while (true) { // read the alt pointer lfsr_tag_t alt; - lfs_ssize_t weight; + lfs_size_t weight; lfs_off_t jump; lfs_ssize_t d = lfsr_rbyd_readtag(lfs, &lfs->pcache, &lfs->rcache, 0, @@ -2213,7 +2210,7 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd, lfs_swap32(&jump, &branch_); lfs_swap16(&p_alts[0], &alt); - lfs_sswap32(&p_weights[0], &weight); + lfs_swap32(&p_weights[0], &weight); lfs_swap32(&p_jumps[0], &jump); alt = lfsr_tag_setblack(alt); @@ -2278,7 +2275,7 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd, lower_id, upper_id, tag_, id_)) { lfs_swap16(&p_alts[0], &alt); - lfs_sswap32(&p_weights[0], &weight); + lfs_swap32(&p_weights[0], &weight); lfs_swap32(&p_jumps[0], &jump); p_alts[0] = lfsr_tag_setred(p_alts[0]); alt = lfsr_tag_setblack(alt); @@ -2383,10 +2380,12 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd, // if removed make our tag unreachable alt = LFSR_TAG_ALT(B, GT, 0); weight = upper_id - lower_id - 1 + delta; + upper_id -= weight; } else { // split less than alt = LFSR_TAG_ALT(R, LE, tag_); weight = id_ - lower_id; + lower_id += weight; } } else if (id_ > id @@ -2397,10 +2396,12 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd, // if removed make our tag unreachable alt = LFSR_TAG_ALT(B, GT, 0); weight = upper_id - lower_id - 1 + delta; + upper_id -= weight; } else { // split greater than alt = LFSR_TAG_ALT(R, GT, tag); weight = upper_id - id - 1; + upper_id -= weight; } } @@ -2433,7 +2434,8 @@ leaf:; // note we always need something after the alts! without something between // alts we may not be able to find the trunk of our tree err = lfsr_rbyd_progtag(lfs, rbyd, - lfsr_tag_setnomk(tag), id + delta, lfsr_data_len(data), &rbyd->crc); + lfsr_tag_setnomk(tag), upper_id - lower_id - 1 + delta, + lfsr_data_len(data), &rbyd->crc); if (err) { rbyd->erased = false; return err; @@ -2576,7 +2578,7 @@ static int lfsr_rbyd_commit(lfs_t *lfs, lfsr_rbyd_t *rbyd, uint8_t fbuf[LFSR_FCRC_DSIZE]; lfs_size_t fcrc_delta = lfsr_fcrc_todisk(&fcrc, fbuf); err = lfsr_rbyd_progtag(lfs, &rbyd_, - LFSR_TAG_FCRC, -1, fcrc_delta, &rbyd_.crc); + LFSR_TAG_FCRC, 0, fcrc_delta, &rbyd_.crc); if (err) { rbyd_.erased = false; return err; diff --git a/scripts/dbgbtree.py b/scripts/dbgbtree.py index 648d0ac1..5efc92ee 100755 --- a/scripts/dbgbtree.py +++ b/scripts/dbgbtree.py @@ -65,9 +65,9 @@ def fromleb128(data): def fromtag(data): tag = fromle16(data) - id, delta = fromleb128(data[2:]) + weight, delta = fromleb128(data[2:]) size, delta_ = fromleb128(data[2+delta:]) - return tag&1, tag&~1, id if tag&0x8 else id-1, size, 2+delta+delta_ + return tag&1, tag&~1, weight, size, 2+delta+delta_ def popc(x): return bin(x).count('1') @@ -82,56 +82,56 @@ def xxd(data, width=16, crc=False): b if b >= ' ' and b <= '~' else '.' for b in map(chr, data[i:i+width]))) -def tagrepr(tag, id, size, off=None): +def tagrepr(tag, w, size, off=None): if (tag & 0xfffe) == TAG_UNR: - return 'unr id%d%s' % ( - id, + return 'unr%s%s' % ( + ' w%d' % w if w else '', ' %d' % size if size else '') elif (tag & 0xf00c) == TAG_NAME: - return '%s%s id%d %d' % ( + return '%s%s%s %d' % ( 'rm' if tag & 0x2 else '', 'bname' if (tag & 0xfffe) == TAG_BNAME else 'reg' if (tag & 0xfffe) == TAG_REG else 'dir' if (tag & 0xfffe) == TAG_DIR else 'name 0x%02x' % ((tag & 0x0ff0) >> 4), - id, + ' w%d' % w if w else '', size) elif (tag & 0xf00c) == TAG_STRUCT: - return '%s%s id%d %d' % ( + return '%s%s%s %d' % ( 'rm' if tag & 0x2 else '', 'inlined' if (tag & 0xfffe) == TAG_INLINED else 'block' if (tag & 0xfffe) == TAG_BLOCK else 'branch' if (tag & 0xfffe) == TAG_BRANCH else 'btree' if (tag & 0xfffe) == TAG_BTREE else 'struct 0x%02x' % ((tag & 0x0ff0) >> 4), - id, + ' w%d' % w if w else '', size) elif (tag & 0xf00c) == TAG_UATTR: return '%suattr 0x%02x%s%s' % ( 'rm' if tag & 0x2 else '', (tag & 0x0ff0) >> 4, - ' id%d' % id if id != -1 else '', + ' w%d' % w if w else '', ' %d' % size if not tag & 0x2 or size else '') elif (tag & 0xf00e) == TAG_CRC: return 'crc%x%s %d' % ( 1 if tag & 0x10 else 0, - ' 0x%02x' % id if id != -1 else '', + ' 0x%x' % w if w > 0 else '', size) elif (tag & 0xfffe) == TAG_FCRC: return 'fcrc%s %d' % ( - ' 0x%02x' % id if id != -1 else '', + ' 0x%x' % w if w > 0 else '', size) elif tag & 0x8: return 'alt%s%s 0x%x w%d %s' % ( 'r' if tag & 0x2 else 'b', 'gt' if tag & 0x4 else 'le', tag & 0xfff0, - id, + w, '0x%x' % (0xffffffff & (off-size)) if off is not None else '-%d' % off) else: - return '0x%04x id%d %d' % (tag, id, size) + return '0x%04x w%d %d' % (tag, w, size) class Rbyd: def __init__(self, block, limit, data, rev, off, trunk, weight): @@ -161,7 +161,7 @@ class Rbyd: weight_ = 0 wastrunk = False while j_ < limit: - v, tag, id, size, delta = fromtag(data[j_:]) + v, tag, w, size, delta = fromtag(data[j_:]) if v != (popc(crc) & 1): break crc = crc32c(data[j_:j_+delta], crc) @@ -176,13 +176,11 @@ class Rbyd: # keep track of weight if tag & 0x8: if tag & 0x4: - upper_ += id + upper_ += w else: - lower_ += id + lower_ += w elif (tag & 0xc) == 0x0: - weight_ = lower_+upper_ - if not tag & 0x2: - weight_ += id+1-lower_ + weight_ = lower_+upper_+w # take care of crcs if not tag & 0x8: @@ -301,9 +299,9 @@ def main(disk, block_size=None, trunk=0, limit=None, *, # corrupted? return a corrupted block once if not rbyd: - return (id > 0, id, 0, rbyd, -1, 0, - 0, 0, b'', - 0, 0, b'', + return (id > 0, id, 0, rbyd, -1, + (0, 0, 0, b''), + (0, 0, 0, b''), path) while True: @@ -311,49 +309,51 @@ def main(disk, block_size=None, trunk=0, limit=None, *, (done, name_tag, rid_, w, name_j, name_d, name) = rbyd.lookup(0, rid) if done: - return (True, id, 0, rbyd, -1, 0, - 0, 0, b'', - 0, 0, b'', + return (True, id, 0, rbyd, -1, + (0, 0, 0, b''), + (0, 0, 0, b''), path) if name_tag & 0xf00f == TAG_NAME: # then lookup struct - (done, tag, _, _, + (done, struct_tag, _, _, struct_j, struct_d, struct_) = rbyd.lookup( TAG_STRUCT, rid_) if done: - return (True, id, 0, rbyd, -1, 0, - 0, 0, b'', - 0, 0, b'', + return (True, id, 0, rbyd, -1, + (0, 0, 0, b''), + (0, 0, 0, b''), path) else: tag = name_tag - struct_j, struct_d, struct_ = name_j, name_d, name - name_j, name_d, name = name_j, 0, b'' + struct_tag, struct_j, struct_d, struct_ = ( + name_tag, name_j, name_d, name) + name_tag, name_j, name_d, name = 0, 0, 0, b'' - path.append((id + (rid_-rid), w, rbyd, rid_, tag, - name_j, name_d, name, - struct_j, struct_d, struct_)) + path.append((id + (rid_-rid), w, rbyd, rid_, + (name_tag, name_j, name_d, name), + (struct_tag, struct_j, struct_d, struct_))) # is it another branch? continue down tree - if tag == TAG_BRANCH and (depth is None or depth_ < depth): + if struct_tag == TAG_BRANCH and ( + depth is None or depth_ < depth): block, delta = fromleb128(struct_) limit, _ = fromleb128(struct_[delta:]) rbyd = Rbyd.fetch(f, block_size, block, limit) # corrupted? bail here so we can keep traversing the tree if not rbyd: - return (False, id + (rid_-rid), w, rbyd, -1, 0, - 0, 0, b'', - 0, 0, b'', + return (False, id + (rid_-rid), w, rbyd, -1, + (0, 0, 0, b''), + (0, 0, 0, b''), path) rid -= (rid_-(w-1)) depth_ += 1 else: - return (False, id + (rid_-rid), w, rbyd, rid_, tag, - name_j, name_d, name, - struct_j, struct_d, struct_, + return (False, id + (rid_-rid), w, rbyd, rid_, + (name_tag, name_j, name_d, name), + (struct_tag, struct_j, struct_d, struct_), path) # if we're printing the tree, first find the max depth so we know how @@ -363,9 +363,7 @@ def main(disk, block_size=None, trunk=0, limit=None, *, t_depth = 0 id = -1 while True: - (done, id, w, rbyd, rid, tag, - name_j, name_d, name, - struct_j, struct_d, struct_, + (done, id, w, rbyd, rid, _, _, path) = (lookup(id+1, depth=args.get('depth'))) if done: break @@ -375,23 +373,28 @@ def main(disk, block_size=None, trunk=0, limit=None, *, t_width = 2*t_depth+2 if t_depth > 0 else 0 t_branches = [(0, trunk.weight)] - def t_repr(id, w, d=None): + def t_repr(id, w, leaf=True, depth=None): branches_ = [] for i in range(len(t_branches)): - if d is not None and d == i-1: - branches_.append('+') - elif i+1 < len(t_branches): - if (id-(w-1) == t_branches[i+1][0] + if depth is not None and depth == i-1: + branches_.append('+' if leaf else '|') + break + + if i+1 < len(t_branches): + if (leaf + and id-(w-1) == t_branches[i+1][0] and t_branches[i][0] == t_branches[i+1][0] and (not args.get('inner') - or (i == 0 and d == 0))): + or (i == 0 and depth == 0))): branches_.append('+-') - elif (id-(w-1) == t_branches[i+1][0] + elif (leaf + and id-(w-1) == t_branches[i+1][0] and t_branches[i][1] == t_branches[i+1][1] - and (not args.get('inner') or d == i)): + and (not args.get('inner') or depth == i)): branches_.append('\'-') - elif (id-(w-1) == t_branches[i+1][0] - and (not args.get('inner') or d == i)): + elif (leaf + and id-(w-1) == t_branches[i+1][0] + and (not args.get('inner') or depth == i)): branches_.append('|-') elif (id-(w-1) >= t_branches[i][0] and id-(w-1) < t_branches[i][1] @@ -400,14 +403,22 @@ def main(disk, block_size=None, trunk=0, limit=None, *, else: branches_.append(' ') else: - if (id-(w-1) == t_branches[i][0] + if (leaf + and id-(w-1) == t_branches[i][0] and (not args.get('inner') or i == 0)): branches_.append('+-%s> ' % ('-'*2*(t_depth-i-1))) - elif id == t_branches[i][1]-1: + elif (leaf + and id == t_branches[i][1]-1): branches_.append('\'-%s> ' % ('-'*2*(t_depth-i-1))) - elif (id >= t_branches[i][0] + elif (leaf + and id >= t_branches[i][0] and id-(w-1) < t_branches[i][1]): branches_.append('|-%s> ' % ('-'*2*(t_depth-i-1))) + elif (id >= t_branches[i][0] + and id < t_branches[i][1]-1): + branches_.append('|') + else: + branches_.append(' ') return '%s%-*s%s' % ( '\x1b[90m' if color else '', @@ -416,24 +427,123 @@ def main(disk, block_size=None, trunk=0, limit=None, *, # print header w_width = 2*m.ceil(m.log10(max(1, trunk.weight)+1))+1 - print('%-9s %*s%-*s %-8s %-22s %s' % ( + print('%-9s %*s%-*s %-22s %s' % ( 'block', t_width, '', w_width, 'ids', - 'name', 'tag', 'data (truncated)' if not args.get('no_truncate') else '')) + # prbyd here means the last rendered rbyd, we update + # in show_entry to always print interleaved addresses + prbyd = None + def show_entry(id, w, rbyd, rid, + name_tag, name_j, name_d, name, + struct_tag, struct_j, struct_d, struct_, + depth=None): + nonlocal prbyd + + # show human-readable representation + if name_tag: + print('%10s %s%*s %-22s %s' % ( + '%04x.%04x:' % (rbyd.block, rbyd.limit) + if prbyd is None or rbyd != prbyd + else '', + t_repr(id, w, True, depth) if args.get('tree') else '', + w_width, '%d-%d' % (id-(w-1), id) if w > 1 + else id if w > 0 + else '', + tagrepr(name_tag, w, len(name), None), + ''.join( + b if b >= ' ' and b <= '~' else '.' + for b in map(chr, name)))) + prbyd = rbyd + print('%10s %s%*s %-22s %s' % ( + '%04x.%04x:' % (rbyd.block, rbyd.limit) + if prbyd is None or rbyd != prbyd + else '', + t_repr(id, w, not name_tag, depth) if args.get('tree') else '', + w_width, '' if name_tag + else '%d-%d' % (id-(w-1), id) if w > 1 + else id if w > 0 + else '', + tagrepr(struct_tag, + w if not name_tag else 0, + len(struct_), None), + next(xxd(struct_, 8), '') + if not args.get('no_truncate') else '')) + prbyd = rbyd + + # show in-device representation + if args.get('device'): + if name_tag: + print('%9s %*s%*s %-22s%s' % ( + '', + t_width, '', + w_width, '', + '%04x %08x %07x' % (name_tag, w, len(name)), + ' %s' % ' '.join( + '%08x' % struct.unpack(' 1 else id_ - if w_ > 0 else '', - ''.join( - b if b >= ' ' and b <= '~' else '.' - for b in map(chr, name_)), - tagrepr(tag_, rid_, len(struct__), None), - next(xxd(struct__, 8), '') - if not args.get('no_truncate') else '')) - - # show in-device representation - if args.get('device'): - print('%9s %*s%*s %8s %-22s%s' % ( - '', - t_width, '', - w_width, '', - '', - '%04x %08x %07x' % ( - tag_, 0xffffffff & rid_, len(struct__)), - ' %s' % ' '.join( - '%08x' % struct.unpack(' 1 else id - if w > 0 else '', - ''.join( - b if b >= ' ' and b <= '~' else '.' - for b in map(chr, name)), - tagrepr(tag, rid, len(struct_), None), - next(xxd(struct_, 8), '') - if not args.get('no_truncate') else '')) + # show the entry + show_entry(id, w, rbyd, rid, + name_tag, name_j, name_d, name, + struct_tag, struct_j, struct_d, struct_) - # show in-device representation - if args.get('device'): - print('%9s %*s%*s %8s %-22s%s' % ( - '', - t_width, '', - w_width, '', - '', - '%04x %08x %07x' % ( - tag, 0xffffffff & rid, len(struct_)), - ' %s' % ' '.join( - '%08x' % struct.unpack('= ' ' and b <= '~' else '.' for b in map(chr, data[i:i+width]))) -def tagrepr(tag, id, size, off=None): +def tagrepr(tag, w, size, off=None): if (tag & 0xfffe) == TAG_UNR: - return 'unr id%d%s' % ( - id, + return 'unr%s%s' % ( + ' w%d' % w if w else '', ' %d' % size if size else '') elif (tag & 0xf00c) == TAG_NAME: - return '%s%s id%d %d' % ( + return '%s%s%s %d' % ( 'rm' if tag & 0x2 else '', 'bname' if (tag & 0xfffe) == TAG_BNAME else 'reg' if (tag & 0xfffe) == TAG_REG else 'dir' if (tag & 0xfffe) == TAG_DIR else 'name 0x%02x' % ((tag & 0x0ff0) >> 4), - id, + ' w%d' % w if w else '', size) elif (tag & 0xf00c) == TAG_STRUCT: - return '%s%s id%d %d' % ( + return '%s%s%s %d' % ( 'rm' if tag & 0x2 else '', 'inlined' if (tag & 0xfffe) == TAG_INLINED else 'block' if (tag & 0xfffe) == TAG_BLOCK else 'branch' if (tag & 0xfffe) == TAG_BRANCH else 'btree' if (tag & 0xfffe) == TAG_BTREE else 'struct 0x%02x' % ((tag & 0x0ff0) >> 4), - id, + ' w%d' % w if w else '', size) elif (tag & 0xf00c) == TAG_UATTR: return '%suattr 0x%02x%s%s' % ( 'rm' if tag & 0x2 else '', (tag & 0x0ff0) >> 4, - ' id%d' % id if id != -1 else '', + ' w%d' % w if w else '', ' %d' % size if not tag & 0x2 or size else '') elif (tag & 0xf00e) == TAG_CRC: return 'crc%x%s %d' % ( 1 if tag & 0x10 else 0, - ' 0x%02x' % id if id != -1 else '', + ' 0x%x' % w if w > 0 else '', size) elif (tag & 0xfffe) == TAG_FCRC: return 'fcrc%s %d' % ( - ' 0x%02x' % id if id != -1 else '', + ' 0x%x' % w if w > 0 else '', size) elif tag & 0x8: return 'alt%s%s 0x%x w%d %s' % ( 'r' if tag & 0x2 else 'b', 'gt' if tag & 0x4 else 'le', tag & 0xfff0, - id, + w, '0x%x' % (0xffffffff & (off-size)) if off is not None else '-%d' % off) else: - return '0x%04x id%d %d' % (tag, id, size) + return '0x%04x w%d %d' % (tag, w, size) -def show_log(block_size, data, rev, off, *, +def show_log(block_size, data, rev, off, weight, *, color=False, **args): crc = crc32c(data[0:4]) @@ -153,7 +153,7 @@ def show_log(block_size, data, rev, off, *, j_ = 4 while j_ < (block_size if args.get('all') else off): j = j_ - v, tag, id, size, delta = fromtag(data[j_:]) + v, tag, w, size, delta = fromtag(data[j_:]) j_ += delta if not tag & 0x8: j_ += size @@ -250,7 +250,7 @@ def show_log(block_size, data, rev, off, *, j_ = 4 while j_ < (block_size if args.get('all') else off): j = j_ - v, tag, id, size, delta = fromtag(data[j_:]) + v, tag, w, size, delta = fromtag(data[j_:]) j_ += delta if not tag & 0x8: j_ += size @@ -263,14 +263,13 @@ def show_log(block_size, data, rev, off, *, # keep track of weight if tag & 0x8: if tag & 0x4: - upper_ += id + upper_ += w else: - lower_ += id + lower_ += w elif (tag & 0xc) == 0x0: - delta = (lower_+upper_) - weight_ - if not tag & 0x2: - delta += id+1-lower_ - weight_ += delta + delta = (lower_+upper_+w) - weight_ + weight_ = lower_+upper_+w + id = lower_+w-1 # note we ignore out-of-bounds here for debugging if delta > 0: @@ -310,7 +309,7 @@ def show_log(block_size, data, rev, off, *, weights = weights_ lifetimes = lifetimes_ - if not tag & 0x2: + if not tag & 0x2 and id >= 0: # attach tag to lifetime i, id_ = index(weights, id) if i < len(weights): @@ -366,9 +365,11 @@ def show_log(block_size, data, rev, off, *, lifetime_width - sum(len(r) for r in reprs), '') # print header - print('%-8s %*s%-22s %s' % ( + w_width = 2*m.ceil(m.log10(max(1, weight)+1))+1 + print('%-8s %*s%-*s %-22s %s' % ( 'off', lifetime_width, '', + w_width, 'ids', 'tag', 'data (truncated)' if not args.get('no_truncate') else '')) @@ -378,18 +379,35 @@ def show_log(block_size, data, rev, off, *, print('%8s: %s' % ('%04x' % 0, next(xxd(data[0:4])))) # print tags + lower_, upper_ = 0, 0 + wastrunk = False j_ = 4 while j_ < (block_size if args.get('all') else off): notes = [] j = j_ - v, tag, id, size, delta = fromtag(data[j_:]) + v, tag, w, size, delta = fromtag(data[j_:]) if v != (popc(crc) & 1): notes.append('v!=%x' % (popc(crc) & 1)) tag &= ~1 crc = crc32c(data[j_:j_+delta], crc) j_ += delta + # find trunk + if not wastrunk and (tag & 0xc) != 0x4: + lower_, upper_ = 0, 0 + wastrunk = not not tag & 0x8 + + # calculate id from alt weights + if tag & 0x8: + if tag & 0x4: + upper_ += w + else: + lower_ += w + elif (tag & 0xc) == 0x0: + weight_ = lower_+upper_+w + id = lower_+w-1 + if not tag & 0x8: if (tag & 0xf00f) != TAG_CRC: crc = crc32c(data[j_:j_+size], crc) @@ -401,14 +419,17 @@ def show_log(block_size, data, rev, off, *, j_ += size # show human-readable tag representation - print('%s%08x:%s %s%s%-57s%s%s' % ( + print('%s%08x:%s %*s%s%*s %-57s%s%s' % ( '\x1b[90m' if color and j >= off else '', j, '\x1b[m' if color and j >= off else '', - lifetimerepr(j) if args.get('lifetimes') else '', + lifetime_width, lifetimerepr(j) if args.get('lifetimes') else '', '\x1b[90m' if color and j >= off else '', + w_width, '' if (tag & 0xc) != 0x0 + else '%d-%d' % (id-(w-1), id) if w > 1 + else id, '%-22s%s' % ( - tagrepr(tag, id, size, j), + tagrepr(tag, w, size, j), ' %s' % next(xxd( data[j+delta:j+delta+min(size, 8)], 8), '') if not args.get('no_truncate') @@ -419,24 +440,16 @@ def show_log(block_size, data, rev, off, *, if args.get('jumps') else '')) - if args.get('raw'): - # show on-disk encoding of tags - for o, line in enumerate(xxd(data[j:j+delta])): - print('%s%8s: %s%s' % ( - '\x1b[90m' if color and j >= off else '', - '%04x' % (j + o*16), - line, - '\x1b[m' if color and j >= off else '')) - # show in-device representation, including some extra # crc/parity info if args.get('device'): - print('%s%8s %*s%-47s %08x %x%s' % ( + print('%s%8s %*s%*s %-47s %08x %x%s' % ( '\x1b[90m' if color and j >= off else '', '', lifetime_width, '', + w_width, '', '%-22s%s' % ( - '%04x %08x %07x' % (tag, 0xffffffff & id, size), + '%04x %08x %07x' % (tag, w, size), ' %s' % ' '.join( '%08x' % struct.unpack('= off else '')) - if not tag & 0x8: - # show on-disk encoding of data - if args.get('raw') or args.get('no_truncate'): + # show on-disk encoding of tags + if args.get('raw'): + for o, line in enumerate(xxd(data[j:j+delta])): + print('%s%8s: %*s%*s %s%s' % ( + '\x1b[90m' if color and j >= off else '', + '%04x' % (j + o*16), + lifetime_width, '', + w_width, '', + line, + '\x1b[m' if color and j >= off else '')) + if args.get('raw') or args.get('no_truncate'): + if not tag & 0x8: for o, line in enumerate(xxd(data[j+delta:j+delta+size])): - print('%s%8s: %s%s' % ( + print('%s%8s: %*s%*s %s%s' % ( '\x1b[90m' if color and j >= off else '', '%04x' % (j+delta + o*16), + lifetime_width, '', + w_width, '', line, '\x1b[m' if color and j >= off else '')) @@ -475,16 +499,16 @@ def show_tree(block_size, data, rev, trunk, weight, *, # descend down tree j = trunk while True: - _, alt, weight_, jump, delta = fromtag(data[j:]) + _, alt, w, jump, delta = fromtag(data[j:]) # found an alt? if alt & 0x8: # follow? - if ((id, tag & ~0xf) > (upper-weight_-1, alt & ~0xf) + if ((id, tag & ~0xf) > (upper-w-1, alt & ~0xf) if alt & 0x4 - else ((id, tag & ~0xf) <= (lower+weight_, alt & ~0xf))): - lower += upper-lower-1-weight_ if alt & 0x4 else 0 - upper -= upper-lower-1-weight_ if not alt & 0x4 else 0 + else ((id, tag & ~0xf) <= (lower+w, alt & ~0xf))): + lower += upper-lower-1-w if alt & 0x4 else 0 + upper -= upper-lower-1-w if not alt & 0x4 else 0 j = j - jump if args.get('tree'): @@ -499,8 +523,8 @@ def show_tree(block_size, data, rev, trunk, weight, *, path.append((j+jump, j, 'b')) # stay on path else: - lower += weight_ if not alt & 0x4 else 0 - upper -= weight_ if alt & 0x4 else 0 + lower += w if not alt & 0x4 else 0 + upper -= w if alt & 0x4 else 0 j = j + delta if args.get('tree'): @@ -638,14 +662,14 @@ def show_tree(block_size, data, rev, trunk, weight, *, break # show human-readable tag representation - print('%08x:%s %*s %-57s' % ( + print('%08x:%s %-57s' % ( j, treerepr(j) if args.get('tree') else '', - w_width, '%d-%d' % (id-(w-1), id) - if w > 1 else id - if w > 0 else '', - '%-22s%s' % ( - tagrepr(tag, id, size, j), + '%*s %-22s%s' % ( + w_width, '%d-%d' % (id-(w-1), id) + if w > 1 else id + if w > 0 else '', + tagrepr(tag, w, size, j), ' %s' % next(xxd( data[j+delta:j+delta+min(size, 8)], 8), '') if not args.get('no_truncate') @@ -667,19 +691,21 @@ def show_tree(block_size, data, rev, trunk, weight, *, if not args.get('no_truncate') and not tag & 0x8 else ''))) + # show on-disk encoding of tags if args.get('raw'): - # show on-disk encoding of tags for o, line in enumerate(xxd(data[j:j+delta])): - print('%8s: %s' % ( + print('%8s: %*s%*s %s' % ( '%04x' % (j + o*16), + tree_width, '', + w_width, '', line)) - - if not tag & 0x8: - # show on-disk encoding of data - if args.get('raw') or args.get('no_truncate'): + if args.get('raw') or args.get('no_truncate'): + if not tag & 0x8: for o, line in enumerate(xxd(data[j+delta:j+delta+size])): - print('%8s: %s' % ( + print('%8s: %*s%*s %s' % ( '%04x' % (j+delta + o*16), + tree_width, '', + w_width, '', line)) @@ -733,7 +759,7 @@ def main(disk, block_size=None, block1=0, block2=None, *, weight_ = 0 wastrunk = False while j_ < len(data): - v, tag, id, size, delta = fromtag(data[j_:]) + v, tag, w, size, delta = fromtag(data[j_:]) if v != (popc(crc) & 1): break crc = crc32c(data[j_:j_+delta], crc) @@ -748,13 +774,11 @@ def main(disk, block_size=None, block1=0, block2=None, *, # keep track of weight if tag & 0x8: if tag & 0x4: - upper_ += id + upper_ += w else: - lower_ += id + lower_ += w elif (tag & 0xc) == 0x0: - weight_ = lower_+upper_ - if not tag & 0x2: - weight_ += id+1-lower_ + weight_ = lower_+upper_+w # take care of crcs if not tag & 0x8: @@ -801,7 +825,7 @@ def main(disk, block_size=None, block1=0, block2=None, *, if len(blocks) > 1 else '')) if args.get('log'): - show_log(block_size, data, rev, off, + show_log(block_size, data, rev, off, weight, color=color, **args) else: