From 7180b70c9c983d59403e1b2cff676036fedc3efb Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 17 Jun 2023 18:11:45 -0500 Subject: [PATCH] Allowed "alta" (altbgt 0) to terminate rbyd trunks, dropped rm bit This replaces unr with null on disk, though note both the rm bit and unr are used in-device still, they just don't get written to disk. This removes the need for the rm bit on disk. Since we no longer need to figure out what's been removed during fetch, we can save this bit for both internal and future on-disk use. Special handling of alta allows us to avoid emitting an unr tag (now null) if the current trunk is truly unreachable. This is minor now, but important for a theoretical rbyd rebalance operation (planned), which brings the rbyd overhead down from ~3x to ~2x. These changes give us two ways to terminate trunks without a tag: 1. With an alta, if the current trunk is unreachable: altbgt 0x403 w0 0x7b altbgt 0x402 w0 0x29 alta w0 0x4 2. With a null, if the current trunk is reachable, either for code convenience or because emitting an alta is impossible (an empty rbyd for example): altbgt 0x403 w0 0x7b altbgt 0x402 w0 0x29 altbgt 0x401 w0 0x4 null --- lfs.c | 163 ++++++++++++++++++++++-------------------- scripts/dbgbtree.py | 97 ++++++++++++------------- scripts/dbgmtree.py | 97 ++++++++++++------------- scripts/dbgrbyd.py | 168 +++++++++++++++++++++++--------------------- 4 files changed, 275 insertions(+), 250 deletions(-) diff --git a/lfs.c b/lfs.c index 0164ace3..dbffe0f6 100644 --- a/lfs.c +++ b/lfs.c @@ -584,7 +584,8 @@ static int lfsr_bd_erase(lfs_t *lfs, lfs_block_t block) { // 16-bit metadata tags enum lfsr_tag_type { - LFSR_TAG_UNR = 0x1000, + LFSR_TAG_NULL = 0x0000, + LFSR_TAG_UNR = 0x1000, // in-device only LFSR_TAG_MKUNR = 0x3000, // in-device only LFSR_TAG_SUPERMAGIC = 0x0003, @@ -618,6 +619,7 @@ enum lfsr_tag_type { LFSR_TAG_ALTRLE = 0x5000, LFSR_TAG_ALTBGT = 0x6000, LFSR_TAG_ALTRGT = 0x7000, + LFSR_TAG_ALTA = 0x6000, LFSR_TAG_CRC = 0x2000, LFSR_TAG_FCRC = 0x2100, @@ -691,12 +693,16 @@ static inline lfsr_tag_t lfsr_tag_setrm(lfsr_tag_t tag) { return tag | 0x1000; } -static inline bool lfsr_tag_istrunk(lfsr_tag_t tag) { +static inline bool lfsr_tag_isalt(lfsr_tag_t tag) { + return tag & 0x4000; +} + +static inline bool lfsr_tag_istrunkstart(lfsr_tag_t tag) { return (tag & 0x6000) != 0x2000; } -static inline bool lfsr_tag_isalt(lfsr_tag_t tag) { - return tag & 0x4000; +static inline bool lfsr_tag_istrunkend(lfsr_tag_t tag) { + return !lfsr_tag_isalt(tag) || tag == LFSR_TAG_ALTA; } static inline lfsr_tag_t lfsr_tag_next(lfsr_tag_t tag) { @@ -704,24 +710,33 @@ static inline lfsr_tag_t lfsr_tag_next(lfsr_tag_t tag) { } // lfsr_rbyd_append specific flags -static inline bool lfsr_tag_isupper(lfsr_tag_t tag) { +static inline bool lfsr_tag_hasdiverged(lfsr_tag_t tag) { return tag & 0x2000; } -static inline bool lfsr_tag_islower(lfsr_tag_t tag) { - return !lfsr_tag_isupper(tag); +static inline bool lfsr_tag_isdivergedupper(lfsr_tag_t tag) { + return lfsr_tag_hasdiverged(tag) && (tag & 0x1000); } -static inline lfsr_tag_t lfsr_tag_setupper(lfsr_tag_t tag) { - return tag | 0x2000; -} - -static inline bool lfsr_tag_hasdiverged(lfsr_tag_t tag) { - return tag & 0x4000; +static inline bool lfsr_tag_isdivergedlower(lfsr_tag_t tag) { + return lfsr_tag_hasdiverged(tag) && !(tag & 0x1000); } static inline lfsr_tag_t lfsr_tag_setdiverged(lfsr_tag_t tag) { - return tag | 0x4000; + return tag | 0x2000; +} + +static inline lfsr_tag_t lfsr_tag_setdivergedvalid( + lfsr_tag_t tag, lfsr_tag_t tag_) { + return (tag & 0x3000) | tag_; +} + +static inline lfsr_tag_t lfsr_tag_setdivergedupper(lfsr_tag_t tag) { + return tag | 0x3000; +} + +static inline lfsr_tag_t lfsr_tag_cleardiverged(lfsr_tag_t tag) { + return tag & ~0x3000; } // alt operations @@ -1586,9 +1601,8 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, lfs_off_t off = sizeof(uint32_t); lfs_off_t trunk_ = 0; bool wastrunk = false; - lfs_size_t lower = 0; - lfs_size_t upper = 0; lfs_size_t weight = 0; + lfs_size_t weight_ = 0; // assume unerased until proven otherwise lfsr_fcrc_t fcrc; @@ -1687,35 +1701,30 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, } // found a trunk of a tree? - if (lfsr_tag_istrunk(tag) && (!trunk || trunk >= off-d || wastrunk)) { + if (lfsr_tag_istrunkstart(tag) + && (!trunk || trunk >= off-d || wastrunk)) { + // start of trunk? if (!wastrunk) { + wastrunk = true; // save trunk entry point trunk_ = off-d; - // reset weights - lower = 0; - upper = 0; - wastrunk = true; + // reset weight + weight_ = 0; } - // derive the new weight of the tree from alt pointers + // derive weight of the tree from alt pointers // // NOTE we can't check for overflow/underflow here because we // may be overeagerly parsing an invalid commit, it's ok for // this to overflow/underflow as long as we throw it out later // on a bad crc - if (lfsr_tag_isalt(tag)) { - if (lfsr_tag_isgt(tag)) { - upper += w; - } else { - lower += w; - } + weight_ += w; - } else { - // update current weight - weight = lower+upper+w; - - // any non-alt terminates the current trunk + // end of trunk? + if (lfsr_tag_istrunkend(tag)) { wastrunk = false; + // update current weight + weight = weight_; } } @@ -1801,9 +1810,10 @@ static int lfsr_rbyd_lookupnext(lfs_t *lfs, const lfsr_rbyd_t *rbyd, lfsr_tag_t tag__ = alt; // not what we're looking for? - if (id__ < id - || (id__ == id && lfsr_tag_key(tag__) < lfsr_tag_key(tag)) - || lfsr_tag_isrm(tag__)) { + if (!tag__ + || id__ < id + || (id__ == id && lfsr_tag_key(tag__) + < lfsr_tag_key(tag))) { return LFS_ERR_NOENT; } @@ -2043,7 +2053,7 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd, } // mark as invalid until found tag_ = lfsr_tag_setinvalid(tag_); - other_tag_ = lfsr_tag_setinvalid(lfsr_tag_setupper(other_tag_)); + other_tag_ = lfsr_tag_setinvalid(lfsr_tag_setdivergedupper(other_tag_)); // keep track of bounds as we descend down the tree // @@ -2284,7 +2294,7 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd, // prune inner alts if our tags diverged if (lfsr_tag_hasdiverged(tag_) - && lfsr_tag_isupper(tag_) != lfsr_tag_isgt(alt)) { + && lfsr_tag_isdivergedupper(tag_) != lfsr_tag_isgt(alt)) { continue; } @@ -2301,10 +2311,9 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd, // update the found tag/id // // note we: - // - clear valid bit (0x8000) - // - preserve diverged bit (0x4000) - // - preserve isupper tag (0x2000) - tag_ = lfsr_tag_setvalid(alt | (tag_ & 0x6000)); + // - clear valid bit, marking the tag as found + // - preserve diverged state + tag_ = lfsr_tag_setdivergedvalid(tag_, alt); id_ = upper_id-1; // done? @@ -2333,14 +2342,15 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd, LFS_ASSERT(lfsr_tag_isvalid(tag_)); LFS_ASSERT(!lfsr_tag_hasdiverged(tag_) || lfsr_tag_isvalid(other_tag_)); - if (lfsr_tag_hasdiverged(tag_) && lfsr_tag_islower(tag_)) { + if (lfsr_tag_isdivergedlower(tag_)) { // finished on lower path - tag_ = other_tag_; + tag_ = lfsr_tag_cleardiverged(other_tag_); id_ = other_id_; branch = other_branch; upper_id = other_upper_id; - } else if (lfsr_tag_hasdiverged(tag_) && lfsr_tag_isupper(tag_)) { + } else if (lfsr_tag_isdivergedupper(tag_)) { // finished on upper path + tag_ = lfsr_tag_cleardiverged(tag_); lower_id = other_lower_id; } @@ -2350,17 +2360,14 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd, // always finds the next biggest tag lfsr_tag_t alt = 0; lfs_size_t weight = 0; - if (lfsr_tag_isrm(tag_)) { - // found an old removed tag, no split needed, just prune the - // removed tag - - } else if (id_ < id-lfs_smax32(-delta, 0) + // note if tag_ is null, we found a removed tag that we should just prune + if (tag_ && (id_ < id-lfs_smax32(-delta, 0) || (id_ == id-lfs_smax32(-delta, 0) && ((lfsr_tag_ismk(tag) && delta > 0) - || lfsr_tag_key(tag_) < lfsr_tag_key(tag)))) { + || lfsr_tag_key(tag_) < lfsr_tag_key(tag))))) { if (lfsr_tag_isrm(tag)) { - // if removed make our tag unreachable - alt = LFSR_TAG_ALT(B, GT, 0); + // if removed, make our tag unreachable + alt = LFSR_TAG_ALTA; weight = upper_id - lower_id - 1 + delta; upper_id -= weight; } else { @@ -2370,13 +2377,13 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd, lower_id += weight; } - } else if (id_ > id + } else if (tag_ && (id_ > id || (id_ == id && ((lfsr_tag_ismk(tag) && delta > 0) - || lfsr_tag_key(tag_) > lfsr_tag_key(tag)))) { + || lfsr_tag_key(tag_) > lfsr_tag_key(tag))))) { if (lfsr_tag_isrm(tag)) { - // if removed make our tag unreachable - alt = LFSR_TAG_ALT(B, GT, 0); + // if removed, make our tag unreachable + alt = LFSR_TAG_ALTA; weight = upper_id - lower_id - 1 + delta; upper_id -= weight; } else { @@ -2385,6 +2392,12 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd, weight = upper_id - id - 1; upper_id -= weight; } + + } else { + if (lfsr_tag_isrm(tag)) { + // if removed, replace our tag with a null tag + tag = LFSR_TAG_NULL; + } } if (alt) { @@ -2409,26 +2422,26 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd, } leaf:; - // write the actual tag - // - // note we always need something after the alts! without something between - // alts we may not be able to find the trunk of our tree - lfs_ssize_t d = lfsr_bd_progtag(lfs, rbyd->block, rbyd->off, - lfsr_tag_setnomk(tag), upper_id - lower_id - 1 + delta, - lfsr_data_size(data), - &rbyd->crc); - if (d < 0) { - err = d; - goto failed; - } - rbyd->off += d; + if (!lfsr_tag_isrm(tag)) { + // write the actual tag + lfs_ssize_t d = lfsr_bd_progtag(lfs, rbyd->block, rbyd->off, + lfsr_tag_setnomk(tag), upper_id - lower_id - 1 + delta, + lfsr_data_size(data), + &rbyd->crc); + if (d < 0) { + err = d; + goto failed; + } + rbyd->off += d; - // don't forget the data! - err = lfsr_bd_progdata(lfs, rbyd->block, rbyd->off, data, &rbyd->crc); - if (err) { - goto failed; + // don't forget the data! + err = lfsr_bd_progdata(lfs, rbyd->block, rbyd->off, data, + &rbyd->crc); + if (err) { + goto failed; + } + rbyd->off += lfsr_data_size(data); } - rbyd->off += lfsr_data_size(data); return 0; diff --git a/scripts/dbgbtree.py b/scripts/dbgbtree.py index 080203f9..e50cddaf 100755 --- a/scripts/dbgbtree.py +++ b/scripts/dbgbtree.py @@ -8,7 +8,7 @@ import os import struct -TAG_UNR = 0x1000 +TAG_NULL = 0x0000 TAG_SUPERMAGIC = 0x0003 TAG_SUPERCONFIG = 0x0004 TAG_MROOT = 0x0304 @@ -23,6 +23,7 @@ TAG_BTREE = 0x0303 TAG_MDIR = 0x0305 TAG_UATTR = 0x0400 TAG_ALT = 0x4000 +TAG_ALTA = 0x6000 TAG_CRC = 0x2000 TAG_FCRC = 0x2100 @@ -110,55 +111,56 @@ def xxd(data, width=16, crc=False): for b in map(chr, data[i:i+width]))) def tagrepr(tag, w, size, off=None): - if (tag & 0x7fff) == TAG_UNR: - return 'unr%s%s' % ( + if tag == TAG_NULL: + return 'null%s%s' % ( ' w%d' % w if w else '', ' %d' % size if size else '') - elif (tag & 0x6fff) == TAG_SUPERMAGIC: - return '%ssupermagic%s%s' % ( - 'rm' if tag & 0x1000 else '', + elif tag == TAG_SUPERMAGIC: + return 'supermagic%s %d' % ( ' w%d' % w if w else '', - ' %d' % size if not tag & 0x1000 or size else '') - elif (tag & 0x6fff) == TAG_SUPERCONFIG: - return '%ssuperconfig%s%s' % ( - 'rm' if tag & 0x1000 else '', + size) + elif tag == TAG_SUPERCONFIG: + return 'superconfig%s %d' % ( ' w%d' % w if w else '', - ' %d' % size if not tag & 0x1000 or size else '') - elif (tag & 0x6f00) == TAG_NAME: - return '%s%s%s%s' % ( - 'rm' if tag & 0x1000 else '', - 'branch' if (tag & 0xfffe) == TAG_BRANCH - else 'reg' if (tag & 0xfffe) == TAG_REG - else 'dir' if (tag & 0xfffe) == TAG_DIR - else 'name 0x%02x' % ((tag & 0x0ff0) >> 4), + size) + elif (tag & 0xff00) == TAG_NAME: + return '%s%s %d' % ( + 'branch' if tag == TAG_BRANCH + else 'reg' if tag == TAG_REG + else 'dir' if tag == TAG_DIR + else 'name 0x%02x' % (tag & 0xff), ' w%d' % w if w else '', - ' %d' % size if not tag & 0x1000 or size else '') - elif (tag & 0x6f00) == TAG_STRUCT: - return '%s%s%s%s' % ( - 'rm' if tag & 0x1000 else '', - 'inlined' if (tag & 0x6fff) == TAG_INLINED - else 'block' if (tag & 0x6fff) == TAG_BLOCK - else 'btree' if (tag & 0x6fff) == TAG_BTREE - else 'mdir' if (tag & 0x6fff) == TAG_MROOT - else 'mdir' if (tag & 0x6fff) == TAG_MDIR - else 'struct 0x%02x' % ((tag & 0x0ff0) >> 4), + size) + elif (tag & 0xff00) == TAG_STRUCT: + return '%s%s %d' % ( + 'inlined' if tag == TAG_INLINED + else 'block' if tag == TAG_BLOCK + else 'btree' if tag == TAG_BTREE + else 'mdir' if tag == TAG_MROOT + else 'mdir' if tag == TAG_MDIR + else 'struct 0x%02x' % (tag & 0xff), ' w%d' % w if w else '', - ' %d' % size if not tag & 0x1000 or size else '') - elif (tag & 0x6f00) == TAG_UATTR: - return '%suattr 0x%02x%s%s' % ( - 'rm' if tag & 0x1000 else '', + size) + elif (tag & 0xff00) == TAG_UATTR: + return 'uattr 0x%02x%s %d' % ( tag & 0xff, ' w%d' % w if w else '', - ' %d' % size if not tag & 0x1000 or size else '') - elif (tag & 0x7f00) == TAG_CRC: + size) + elif (tag & 0xff00) == TAG_CRC: return 'crc%x%s %d' % ( 1 if tag & 0x1 else 0, ' 0x%x' % w if w > 0 else '', size) - elif (tag & 0x7fff) == TAG_FCRC: + elif tag == TAG_FCRC: return 'fcrc%s %d' % ( ' 0x%x' % w if w > 0 else '', size) + elif tag == TAG_ALTA: + return 'alta w%d %s' % ( + w, + '0x%x' % (0xffffffff & (off-size)) + if off is not None + else '-%d' % off) elif tag & 0x4000: return 'alt%s%s 0x%x w%d %s' % ( 'r' if tag & 0x1000 else 'b', @@ -238,8 +240,8 @@ class Rbyd: trunk_ = 0 trunk__ = 0 weight = 0 - lower_, upper_ = 0, 0 weight_ = 0 + weight__ = 0 wastrunk = False trunkoff = None while j_ < len(data) and (not trunk or off <= trunk): @@ -253,7 +255,7 @@ class Rbyd: # take care of crcs if not tag & 0x4000: - if (tag & 0x7f00) != TAG_CRC: + if (tag & 0xff00) != TAG_CRC: crc_ = crc32c(data[j_:j_+size], crc_) # found a crc? else: @@ -267,23 +269,22 @@ class Rbyd: weight = weight_ # evaluate trunks - if (tag & 0x6000) != 0x2000 and ( + if (tag & 0xe000) != 0x2000 and ( not trunk or trunk >= j_-d or wastrunk): # new trunk? if not wastrunk: - trunk__ = j_-d - lower_, upper_ = 0, 0 wastrunk = True + trunk__ = j_-d + weight__ = 0 # keep track of weight - if tag & 0x4000: - if tag & 0x2000: - upper_ += w - else: - lower_ += w - else: - weight_ = lower_+upper_+w + weight__ += w + + # end of trunk? + if not tag & 0x4000 or tag == TAG_ALTA: wastrunk = False + # update weight + weight_ = weight__ # keep track of off for best matching trunk if trunk and j_ + size > trunk: trunkoff = j_ + size @@ -349,7 +350,7 @@ class Rbyd: tag_ = alt w_ = id_-lower - done = (id_, tag_) < (id, tag) or tag_ & 0x1000 + done = not tag_ or (id_, tag_) < (id, tag) return done, id_, tag_, w_, j, d, self.data[j+d:j+d+jump], path diff --git a/scripts/dbgmtree.py b/scripts/dbgmtree.py index cc0a6cab..79ce3868 100755 --- a/scripts/dbgmtree.py +++ b/scripts/dbgmtree.py @@ -8,7 +8,7 @@ import os import struct -TAG_UNR = 0x1000 +TAG_NULL = 0x0000 TAG_SUPERMAGIC = 0x0003 TAG_SUPERCONFIG = 0x0004 TAG_MROOT = 0x0304 @@ -23,6 +23,7 @@ TAG_BTREE = 0x0303 TAG_MDIR = 0x0305 TAG_UATTR = 0x0400 TAG_ALT = 0x4000 +TAG_ALTA = 0x6000 TAG_CRC = 0x2000 TAG_FCRC = 0x2100 @@ -119,55 +120,56 @@ def xxd(data, width=16, crc=False): for b in map(chr, data[i:i+width]))) def tagrepr(tag, w, size, off=None): - if (tag & 0x7fff) == TAG_UNR: - return 'unr%s%s' % ( + if tag == TAG_NULL: + return 'null%s%s' % ( ' w%d' % w if w else '', ' %d' % size if size else '') - elif (tag & 0x6fff) == TAG_SUPERMAGIC: - return '%ssupermagic%s%s' % ( - 'rm' if tag & 0x1000 else '', + elif tag == TAG_SUPERMAGIC: + return 'supermagic%s %d' % ( ' w%d' % w if w else '', - ' %d' % size if not tag & 0x1000 or size else '') - elif (tag & 0x6fff) == TAG_SUPERCONFIG: - return '%ssuperconfig%s%s' % ( - 'rm' if tag & 0x1000 else '', + size) + elif tag == TAG_SUPERCONFIG: + return 'superconfig%s %d' % ( ' w%d' % w if w else '', - ' %d' % size if not tag & 0x1000 or size else '') - elif (tag & 0x6f00) == TAG_NAME: - return '%s%s%s%s' % ( - 'rm' if tag & 0x1000 else '', - 'branch' if (tag & 0xfffe) == TAG_BRANCH - else 'reg' if (tag & 0xfffe) == TAG_REG - else 'dir' if (tag & 0xfffe) == TAG_DIR - else 'name 0x%02x' % ((tag & 0x0ff0) >> 4), + size) + elif (tag & 0xff00) == TAG_NAME: + return '%s%s %d' % ( + 'branch' if tag == TAG_BRANCH + else 'reg' if tag == TAG_REG + else 'dir' if tag == TAG_DIR + else 'name 0x%02x' % (tag & 0xff), ' w%d' % w if w else '', - ' %d' % size if not tag & 0x1000 or size else '') - elif (tag & 0x6f00) == TAG_STRUCT: - return '%s%s%s%s' % ( - 'rm' if tag & 0x1000 else '', - 'inlined' if (tag & 0x6fff) == TAG_INLINED - else 'block' if (tag & 0x6fff) == TAG_BLOCK - else 'btree' if (tag & 0x6fff) == TAG_BTREE - else 'mdir' if (tag & 0x6fff) == TAG_MROOT - else 'mdir' if (tag & 0x6fff) == TAG_MDIR - else 'struct 0x%02x' % ((tag & 0x0ff0) >> 4), + size) + elif (tag & 0xff00) == TAG_STRUCT: + return '%s%s %d' % ( + 'inlined' if tag == TAG_INLINED + else 'block' if tag == TAG_BLOCK + else 'btree' if tag == TAG_BTREE + else 'mdir' if tag == TAG_MROOT + else 'mdir' if tag == TAG_MDIR + else 'struct 0x%02x' % (tag & 0xff), ' w%d' % w if w else '', - ' %d' % size if not tag & 0x1000 or size else '') - elif (tag & 0x6f00) == TAG_UATTR: - return '%suattr 0x%02x%s%s' % ( - 'rm' if tag & 0x1000 else '', + size) + elif (tag & 0xff00) == TAG_UATTR: + return 'uattr 0x%02x%s %d' % ( tag & 0xff, ' w%d' % w if w else '', - ' %d' % size if not tag & 0x1000 or size else '') - elif (tag & 0x7f00) == TAG_CRC: + size) + elif (tag & 0xff00) == TAG_CRC: return 'crc%x%s %d' % ( 1 if tag & 0x1 else 0, ' 0x%x' % w if w > 0 else '', size) - elif (tag & 0x7fff) == TAG_FCRC: + elif tag == TAG_FCRC: return 'fcrc%s %d' % ( ' 0x%x' % w if w > 0 else '', size) + elif tag == TAG_ALTA: + return 'alta w%d %s' % ( + w, + '0x%x' % (0xffffffff & (off-size)) + if off is not None + else '-%d' % off) elif tag & 0x4000: return 'alt%s%s 0x%x w%d %s' % ( 'r' if tag & 0x1000 else 'b', @@ -247,8 +249,8 @@ class Rbyd: trunk_ = 0 trunk__ = 0 weight = 0 - lower_, upper_ = 0, 0 weight_ = 0 + weight__ = 0 wastrunk = False trunkoff = None while j_ < len(data) and (not trunk or off <= trunk): @@ -262,7 +264,7 @@ class Rbyd: # take care of crcs if not tag & 0x4000: - if (tag & 0x7f00) != TAG_CRC: + if (tag & 0xff00) != TAG_CRC: crc_ = crc32c(data[j_:j_+size], crc_) # found a crc? else: @@ -276,23 +278,22 @@ class Rbyd: weight = weight_ # evaluate trunks - if (tag & 0x6000) != 0x2000 and ( + if (tag & 0xe000) != 0x2000 and ( not trunk or trunk >= j_-d or wastrunk): # new trunk? if not wastrunk: - trunk__ = j_-d - lower_, upper_ = 0, 0 wastrunk = True + trunk__ = j_-d + weight__ = 0 # keep track of weight - if tag & 0x4000: - if tag & 0x2000: - upper_ += w - else: - lower_ += w - else: - weight_ = lower_+upper_+w + weight__ += w + + # end of trunk? + if not tag & 0x4000 or tag == TAG_ALTA: wastrunk = False + # update weight + weight_ = weight__ # keep track of off for best matching trunk if trunk and j_ + size > trunk: trunkoff = j_ + size @@ -358,7 +359,7 @@ class Rbyd: tag_ = alt w_ = id_-lower - done = (id_, tag_) < (id, tag) or tag_ & 0x1000 + done = not tag_ or (id_, tag_) < (id, tag) return done, id_, tag_, w_, j, d, self.data[j+d:j+d+jump], path diff --git a/scripts/dbgrbyd.py b/scripts/dbgrbyd.py index c10341fa..65f0181b 100755 --- a/scripts/dbgrbyd.py +++ b/scripts/dbgrbyd.py @@ -17,7 +17,7 @@ COLORS = [ ] -TAG_UNR = 0x1000 +TAG_NULL = 0x0000 TAG_SUPERMAGIC = 0x0003 TAG_SUPERCONFIG = 0x0004 TAG_MROOT = 0x0304 @@ -32,9 +32,11 @@ TAG_BTREE = 0x0303 TAG_MDIR = 0x0305 TAG_UATTR = 0x0400 TAG_ALT = 0x4000 +TAG_ALTA = 0x6000 TAG_CRC = 0x2000 TAG_FCRC = 0x2100 + # parse some rbyd addr encodings # 0xa -> [0xa] # 0xa.b -> ([0xa], b) @@ -111,55 +113,56 @@ def xxd(data, width=16, crc=False): for b in map(chr, data[i:i+width]))) def tagrepr(tag, w, size, off=None): - if (tag & 0x7fff) == TAG_UNR: - return 'unr%s%s' % ( + if tag == TAG_NULL: + return 'null%s%s' % ( ' w%d' % w if w else '', ' %d' % size if size else '') - elif (tag & 0x6fff) == TAG_SUPERMAGIC: - return '%ssupermagic%s%s' % ( - 'rm' if tag & 0x1000 else '', + elif tag == TAG_SUPERMAGIC: + return 'supermagic%s %d' % ( ' w%d' % w if w else '', - ' %d' % size if not tag & 0x1000 or size else '') - elif (tag & 0x6fff) == TAG_SUPERCONFIG: - return '%ssuperconfig%s%s' % ( - 'rm' if tag & 0x1000 else '', + size) + elif tag == TAG_SUPERCONFIG: + return 'superconfig%s %d' % ( ' w%d' % w if w else '', - ' %d' % size if not tag & 0x1000 or size else '') - elif (tag & 0x6f00) == TAG_NAME: - return '%s%s%s%s' % ( - 'rm' if tag & 0x1000 else '', - 'branch' if (tag & 0xfffe) == TAG_BRANCH - else 'reg' if (tag & 0xfffe) == TAG_REG - else 'dir' if (tag & 0xfffe) == TAG_DIR - else 'name 0x%02x' % ((tag & 0x0ff0) >> 4), + size) + elif (tag & 0xff00) == TAG_NAME: + return '%s%s %d' % ( + 'branch' if tag == TAG_BRANCH + else 'reg' if tag == TAG_REG + else 'dir' if tag == TAG_DIR + else 'name 0x%02x' % (tag & 0xff), ' w%d' % w if w else '', - ' %d' % size if not tag & 0x1000 or size else '') - elif (tag & 0x6f00) == TAG_STRUCT: - return '%s%s%s%s' % ( - 'rm' if tag & 0x1000 else '', - 'inlined' if (tag & 0x6fff) == TAG_INLINED - else 'block' if (tag & 0x6fff) == TAG_BLOCK - else 'btree' if (tag & 0x6fff) == TAG_BTREE - else 'mdir' if (tag & 0x6fff) == TAG_MROOT - else 'mdir' if (tag & 0x6fff) == TAG_MDIR - else 'struct 0x%02x' % ((tag & 0x0ff0) >> 4), + size) + elif (tag & 0xff00) == TAG_STRUCT: + return '%s%s %d' % ( + 'inlined' if tag == TAG_INLINED + else 'block' if tag == TAG_BLOCK + else 'btree' if tag == TAG_BTREE + else 'mdir' if tag == TAG_MROOT + else 'mdir' if tag == TAG_MDIR + else 'struct 0x%02x' % (tag & 0xff), ' w%d' % w if w else '', - ' %d' % size if not tag & 0x1000 or size else '') - elif (tag & 0x6f00) == TAG_UATTR: - return '%suattr 0x%02x%s%s' % ( - 'rm' if tag & 0x1000 else '', + size) + elif (tag & 0xff00) == TAG_UATTR: + return 'uattr 0x%02x%s %d' % ( tag & 0xff, ' w%d' % w if w else '', - ' %d' % size if not tag & 0x1000 or size else '') - elif (tag & 0x7f00) == TAG_CRC: + size) + elif (tag & 0xff00) == TAG_CRC: return 'crc%x%s %d' % ( 1 if tag & 0x1 else 0, ' 0x%x' % w if w > 0 else '', size) - elif (tag & 0x7fff) == TAG_FCRC: + elif tag == TAG_FCRC: return 'fcrc%s %d' % ( ' 0x%x' % w if w > 0 else '', size) + elif tag == TAG_ALTA: + return 'alta w%d %s' % ( + w, + '0x%x' % (0xffffffff & (off-size)) + if off is not None + else '-%d' % off) elif tag & 0x4000: return 'alt%s%s 0x%x w%d %s' % ( 'r' if tag & 0x1000 else 'b', @@ -172,6 +175,7 @@ def tagrepr(tag, w, size, off=None): else: return '0x%04x w%d %d' % (tag, w, size) + def dbg_log(data, block_size, rev, off, weight, *, color=False, **args): @@ -285,22 +289,26 @@ def dbg_log(data, block_size, rev, off, weight, *, if not tag & 0x4000: j_ += size - # find trunk - if not wastrunk and (tag & 0x6000) != 0x2000: - lower_, upper_ = 0, 0 - wastrunk = not not tag & 0x4000 + # evaluate trunks + if (tag & 0xe000) != 0x2000: + if not wastrunk: + wastrunk = True + lower_, upper_ = 0, 0 - # keep track of weight - if tag & 0x4000: - if tag & 0x2000: - upper_ += w - else: + if (tag & 0xe000) == 0x4000: lower_ += w - elif (tag & 0x6000) == 0x2000: - delta = (lower_+upper_+w) - weight_ - weight_ = lower_+upper_+w - id = lower_+w-1 + else: + upper_ += w + if not tag & 0x4000 or tag == TAG_ALTA: + wastrunk = False + # derive the current tag's id from alt weights + delta = (lower_+upper_) - weight_ + weight_ = lower_+upper_ + id = lower_ + w-1 + + if ((tag & 0xe000) != 0x2000 + and (not tag & 0x4000 or tag == TAG_ALTA)): # note we ignore out-of-bounds here for debugging if delta > 0: # grow lifetimes @@ -339,7 +347,7 @@ def dbg_log(data, block_size, rev, off, weight, *, weights = weights_ lifetimes = lifetimes_ - if not tag & 0x1000 and id >= 0: + if id >= 0: # attach tag to lifetime i, id_ = index(weights, id) if i < len(weights): @@ -427,23 +435,9 @@ def dbg_log(data, block_size, rev, off, weight, *, crc = crc32c(data[j_:j_+d], crc) j_ += d - # find trunk - if not wastrunk and (tag & 0x6000) != 0x2000: - lower_, upper_ = 0, 0 - wastrunk = not not tag & 0x4000 - - # calculate id from alt weights - if tag & 0x4000: - if tag & 0x2000: - upper_ += w - else: - lower_ += w - elif (tag & 0x6000) == 0x0: - weight_ = lower_+upper_+w - id = lower_+w-1 - + # take care of crcs if not tag & 0x4000: - if (tag & 0x7f00) != TAG_CRC: + if (tag & 0xff00) != TAG_CRC: crc = crc32c(data[j_:j_+size], crc) # found a crc? else: @@ -452,6 +446,22 @@ def dbg_log(data, block_size, rev, off, weight, *, notes.append('crc!=%08x' % crc) j_ += size + # evaluate trunks + if (tag & 0xe000) != 0x2000: + if not wastrunk: + wastrunk = True + lower_, upper_ = 0, 0 + + if (tag & 0xe000) == 0x4000: + lower_ += w + else: + upper_ += w + + if not tag & 0x4000 or tag == TAG_ALTA: + wastrunk = False + # derive the current tag's id from alt weights + id = lower_ + w-1 + # show human-readable tag representation print('%s%08x:%s %*s%s%*s %-57s%s%s' % ( '\x1b[90m' if color and j >= off else '', @@ -459,7 +469,8 @@ def dbg_log(data, block_size, rev, off, weight, *, '\x1b[m' if color and j >= off else '', lifetime_width, lifetimerepr(j) if args.get('lifetimes') else '', '\x1b[90m' if color and j >= off else '', - w_width, '' if (tag & 0x6000) != 0x0 + w_width, '-' if tag == TAG_ALTA + else '' if (tag & 0xe000) != 0x0000 else '%d-%d' % (id-(w-1), id) if w > 1 else id, '%-22s%s' % ( @@ -576,7 +587,7 @@ def dbg_tree(data, block_size, rev, trunk, weight, *, tag_ = alt w_ = id_-lower - done = (id_, tag_) < (id, tag) or tag_ & 0x1000 + done = not tag_ or (id_, tag_) < (id, tag) return done, id_, tag_, w_, j, d, jump, path @@ -835,8 +846,8 @@ def main(disk, blocks=None, *, trunk_ = 0 trunk__ = 0 weight = 0 - lower_, upper_ = 0, 0 weight_ = 0 + weight__ = 0 wastrunk = False trunkoff = None while j_ < len(data) and (not trunk or off <= trunk): @@ -850,7 +861,7 @@ def main(disk, blocks=None, *, # take care of crcs if not tag & 0x4000: - if (tag & 0x7f00) != TAG_CRC: + if (tag & 0xff00) != TAG_CRC: crc_ = crc32c(data[j_:j_+size], crc_) # found a crc? else: @@ -864,23 +875,22 @@ def main(disk, blocks=None, *, weight = weight_ # evaluate trunks - if (tag & 0x6000) != 0x2000 and ( + if (tag & 0xe000) != 0x2000 and ( not trunk or trunk >= j_-d or wastrunk): # new trunk? if not wastrunk: - trunk__ = j_-d - lower_, upper_ = 0, 0 wastrunk = True + trunk__ = j_-d + weight__ = 0 # keep track of weight - if tag & 0x4000: - if tag & 0x2000: - upper_ += w - else: - lower_ += w - else: - weight_ = lower_+upper_+w + weight__ += w + + # end of trunk? + if not tag & 0x4000 or tag == TAG_ALTA: wastrunk = False + # update weight + weight_ = weight__ # keep track of off for best matching trunk if trunk and j_ + size > trunk: trunkoff = j_ + size