diff --git a/lfs.c b/lfs.c index 4ed4f6f7..66d9e782 100644 --- a/lfs.c +++ b/lfs.c @@ -449,32 +449,35 @@ static inline lfs_size_t lfs_tag_dsize(lfs_tag_t tag) { // 16-bit metadata tags enum lfsr_tag_type { - LFSR_TAG_NAME = 0x1000, - LFSR_TAG_BNAME = 0x1000, - LFSR_TAG_REG = 0x1010, - LFSR_TAG_DIR = 0x1020, + LFSR_TAG_UNREACHABLE = 0x0002, - LFSR_TAG_STRUCT = 0x3000, - LFSR_TAG_INLINED = 0x3000, - LFSR_TAG_BLOCK = 0x3100, - LFSR_TAG_BRANCH = 0x3200, - LFSR_TAG_BTREE = 0x3300, + LFSR_TAG_NAME = 0x1000, + LFSR_TAG_BNAME = 0x1000, + LFSR_TAG_REG = 0x1010, + LFSR_TAG_DIR = 0x1020, - LFSR_TAG_UATTR = 0x4000, - LFSR_TAG_RMUATTR = 0x4002, + LFSR_TAG_STRUCT = 0x3000, + LFSR_TAG_INLINED = 0x3000, + LFSR_TAG_BLOCK = 0x3100, + LFSR_TAG_BRANCH = 0x3200, + LFSR_TAG_BTREE = 0x3300, - LFSR_TAG_GROW = 0x0006, - LFSR_TAG_SHRINK = 0x0016, - LFSR_TAG_FROM = 0x0026, // in-device only + LFSR_TAG_UATTR = 0x4000, + LFSR_TAG_RMUATTR = 0x4002, - LFSR_TAG_ALT = 0x0008, - LFSR_TAG_ALTBLE = 0x0008, - LFSR_TAG_ALTRLE = 0x000a, - LFSR_TAG_ALTBGT = 0x000c, - LFSR_TAG_ALTRGT = 0x000e, + LFSR_TAG_ALT = 0x0008, + LFSR_TAG_ALTBLE = 0x0008, + LFSR_TAG_ALTRLE = 0x000a, + LFSR_TAG_ALTBGT = 0x000c, + LFSR_TAG_ALTRGT = 0x000e, - LFSR_TAG_CRC = 0x0004, - LFSR_TAG_FCRC = 0x1004, + LFSR_TAG_CRC = 0x0004, + LFSR_TAG_FCRC = 0x1004, + + // in-device only + LFSR_TAG_GROW = 0x0006, + LFSR_TAG_SHRINK = 0x0016, + LFSR_TAG_FROM = 0x0026, }; #define LFSR_TAG_ALT_(color, dir, key) \ @@ -512,12 +515,8 @@ static inline lfsr_tag_t lfsr_tag_mkrm(lfsr_tag_t tag) { return tag | 0x2; } -static inline bool lfsr_tag_hasdata(lfsr_tag_t tag) { - return (tag & 0xe) <= 0x4; -} - static inline bool lfsr_tag_istrunk(lfsr_tag_t tag) { - return (tag & 0xe) != 0x4; + return (tag & 0xc) != 0x4; } static inline bool lfsr_tag_isalt(lfsr_tag_t tag) { @@ -1282,6 +1281,8 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, lfs_off_t off = sizeof(uint32_t); lfs_off_t trunk = 0; bool wastrunk = false; + lfs_size_t lower = 0; + lfs_size_t upper = 0; lfs_size_t weight = 0; // assume unerased until proven otherwise @@ -1310,39 +1311,51 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd, // found trunk of tree? if (!wastrunk && lfsr_tag_istrunk(tag)) { trunk = off; + lower = 0; + upper = 0; } wastrunk = lfsr_tag_isalt(tag); off += delta; - // keep track of weight + // derive the new weight of the tree from alt pointers // // NOTE we can't check for overflow/underflow here because we // may be overeagerly parsing an invalid commit, it's ok for // this to overflow/underflow as long as we throw it out later // on a bad crc - if (tag == LFSR_TAG_GROW) { - weight += size; - - // adjust find - if (find && find->predicted_id >= id) { - find->predicted_id += size; + if (lfsr_tag_isalt(tag)) { + if (lfsr_tag_isgt(tag)) { + upper += id; + } else { + lower += id; } - } else if (tag == LFSR_TAG_SHRINK) { - weight -= size; - // adjust find + } else if (lfsr_tag_istrunk(tag)) { + // note we need to include our id's weight in our weight + // calculation, but only when our id is included in the tree + lfs_ssize_t delta = (lower+upper) - weight; + if (!lfsr_tag_isrm(tag) + && tag != LFSR_TAG_GROW + && tag != LFSR_TAG_SHRINK) { + delta += id+1-lower; + } + + // adjust any pending finds if (find && find->predicted_id >= id) { - if (find->predicted_id < id+(lfs_ssize_t)size) { + // pending find removed? + if (delta < 0 && find->predicted_id < id-delta) { find->predicted_tag = 0; find->predicted_id = id-1; } else { - find->predicted_id -= size; + find->predicted_id += delta; } } + + weight += delta; } // we mostly just skip non-data tags here - if (!lfsr_tag_hasdata(tag)) { + if (lfsr_tag_isalt(tag)) { continue; } @@ -2322,6 +2335,11 @@ leaf:; // // note we always need something after the alts! without something between // alts we may not be able to find the trunk of our tree + if (tag == LFSR_TAG_GROW || tag == LFSR_TAG_SHRINK) { + tag = LFSR_TAG_UNREACHABLE; + data = LFSR_DATA_NULL; + } + err = lfsr_rbyd_progtag(lfs, rbyd, tag, id, lfsr_data_len(data), &rbyd->crc); if (err) { @@ -2329,13 +2347,11 @@ leaf:; return err; } - if (lfsr_tag_hasdata(tag)) { - // don't forget the data! - err = lfsr_rbyd_progdata(lfs, rbyd, data, &rbyd->crc); - if (err) { - rbyd->erased = false; - return err; - } + // don't forget the data! + err = lfsr_rbyd_progdata(lfs, rbyd, data, &rbyd->crc); + if (err) { + rbyd->erased = false; + return err; } return 0; diff --git a/scripts/dbgbtree.py b/scripts/dbgbtree.py index d85951ba..bfcff549 100755 --- a/scripts/dbgbtree.py +++ b/scripts/dbgbtree.py @@ -7,21 +7,20 @@ import os import struct -TAG_NAME = 0x1000 -TAG_BNAME = 0x1000 -TAG_REG = 0x1010 -TAG_DIR = 0x1020 -TAG_STRUCT = 0x3000 -TAG_INLINED = 0x3000 -TAG_BLOCK = 0x3100 -TAG_BRANCH = 0x3200 -TAG_BTREE = 0x3300 -TAG_UATTR = 0x4000 -TAG_GROW = 0x0006 -TAG_SHRINK = 0x0016 -TAG_ALT = 0x0008 -TAG_CRC = 0x0004 -TAG_FCRC = 0x1004 +TAG_UNREACHABLE = 0x0002 +TAG_NAME = 0x1000 +TAG_BNAME = 0x1000 +TAG_REG = 0x1010 +TAG_DIR = 0x1020 +TAG_STRUCT = 0x3000 +TAG_INLINED = 0x3000 +TAG_BLOCK = 0x3100 +TAG_BRANCH = 0x3200 +TAG_BTREE = 0x3300 +TAG_UATTR = 0x4000 +TAG_ALT = 0x0008 +TAG_CRC = 0x0004 +TAG_FCRC = 0x1004 def blocklim(s): if '.' in s: @@ -84,7 +83,11 @@ def xxd(data, width=16, crc=False): for b in map(chr, data[i:i+width]))) def tagrepr(tag, id, size, off=None): - if (tag & 0xf00c) == TAG_NAME: + if (tag & 0xfffe) == TAG_UNREACHABLE: + return 'unreachable id%d%s' % ( + id, + ' %d' % size if size else '') + elif (tag & 0xf00c) == TAG_NAME: return '%s%s id%d %d' % ( 'rm' if tag & 0x2 else '', 'bname' if (tag & 0xfffe) == TAG_BNAME @@ -109,14 +112,6 @@ def tagrepr(tag, id, size, off=None): (tag & 0x0ff0) >> 4, ' id%d' % id if id != -1 else '', ' %d' % size if not tag & 0x2 or size else '') - elif (tag & 0xfffe) == TAG_GROW: - return 'grow id%d w%d' % ( - id, - size) - elif (tag & 0xfffe) == TAG_SHRINK: - return 'shrink id%d w%d' % ( - id, - size) elif (tag & 0xf00e) == TAG_CRC: return 'crc%x%s %d' % ( 1 if tag & 0x10 else 0, @@ -162,6 +157,7 @@ class Rbyd: trunk = None trunk_ = None weight = 0 + lower_, upper_ = 0, 0 weight_ = 0 wastrunk = False while j_ < limit: @@ -172,18 +168,24 @@ class Rbyd: j_ += delta # find trunk - if not wastrunk and (tag & 0xe) != 0x4: + if not wastrunk and (tag & 0xc) != 0x4: trunk_ = j_ - delta + lower_, upper_ = 0, 0 wastrunk = not not tag & 0x8 # keep track of weight - if tag == TAG_GROW: - weight_ += size - elif tag == TAG_SHRINK: - weight_ = max(weight_ - size, 0) + if tag & 0x8: + if tag & 0x4: + upper_ += id + else: + lower_ += id + elif (tag & 0xc) == 0x0: + weight_ = lower_+upper_ + if not tag & 0x2: + weight_ += id+1-lower_ # take care of crcs - if (tag & 0xe) <= 0x4: + if not tag & 0x8: if (tag & 0xf00f) != TAG_CRC: crc = crc32c(data[j_:j_+size], crc) # found a crc? diff --git a/scripts/dbgrbyd.py b/scripts/dbgrbyd.py index d855e2ff..1c8fa28c 100755 --- a/scripts/dbgrbyd.py +++ b/scripts/dbgrbyd.py @@ -16,21 +16,20 @@ COLORS = [ ] -TAG_NAME = 0x1000 -TAG_BNAME = 0x1000 -TAG_REG = 0x1010 -TAG_DIR = 0x1020 -TAG_STRUCT = 0x3000 -TAG_INLINED = 0x3000 -TAG_BLOCK = 0x3100 -TAG_BRANCH = 0x3200 -TAG_BTREE = 0x3300 -TAG_UATTR = 0x4000 -TAG_GROW = 0x0006 -TAG_SHRINK = 0x0016 -TAG_ALT = 0x0008 -TAG_CRC = 0x0004 -TAG_FCRC = 0x1004 +TAG_UNREACHABLE = 0x0002 +TAG_NAME = 0x1000 +TAG_BNAME = 0x1000 +TAG_REG = 0x1010 +TAG_DIR = 0x1020 +TAG_STRUCT = 0x3000 +TAG_INLINED = 0x3000 +TAG_BLOCK = 0x3100 +TAG_BRANCH = 0x3200 +TAG_BTREE = 0x3300 +TAG_UATTR = 0x4000 +TAG_ALT = 0x0008 +TAG_CRC = 0x0004 +TAG_FCRC = 0x1004 def blocklim(s): if '.' in s: @@ -93,7 +92,11 @@ def xxd(data, width=16, crc=False): for b in map(chr, data[i:i+width]))) def tagrepr(tag, id, size, off=None): - if (tag & 0xf00c) == TAG_NAME: + if (tag & 0xfffe) == TAG_UNREACHABLE: + return 'unreachable id%d%s' % ( + id, + ' %d' % size if size else '') + elif (tag & 0xf00c) == TAG_NAME: return '%s%s id%d %d' % ( 'rm' if tag & 0x2 else '', 'bname' if (tag & 0xfffe) == TAG_BNAME @@ -118,14 +121,6 @@ def tagrepr(tag, id, size, off=None): (tag & 0x0ff0) >> 4, ' id%d' % id if id != -1 else '', ' %d' % size if not tag & 0x2 or size else '') - elif (tag & 0xfffe) == TAG_GROW: - return 'grow id%d w%d' % ( - id, - size) - elif (tag & 0xfffe) == TAG_SHRINK: - return 'shrink id%d w%d' % ( - id, - size) elif (tag & 0xf00e) == TAG_CRC: return 'crc%x%s %d' % ( 1 if tag & 0x10 else 0, @@ -160,7 +155,7 @@ def show_log(block_size, data, rev, off, *, j = j_ v, tag, id, size, delta = fromtag(data[j_:]) j_ += delta - if (tag & 0xe) <= 0x4: + if not tag & 0x8: j_ += size if tag & 0x8: @@ -241,69 +236,91 @@ def show_log(block_size, data, rev, off, *, id -= w return len(weights), 0 - checkpoint_js = [] - checkpoints = [] + checkpoint_js = [0] + checkpoints = [([], [], set(), set(), set())] def checkpoint(j, weights, lifetimes, grows, shrinks, tags): checkpoint_js.append(j) checkpoints.append(( weights.copy(), lifetimes.copy(), grows, shrinks, tags)) + lower_, upper_ = 0, 0 + weight_ = 0 + wastrunk = False j_ = 4 while j_ < (block_size if args.get('all') else off): j = j_ v, tag, id, size, delta = fromtag(data[j_:]) j_ += delta - if (tag & 0xe) <= 0x4: + if not tag & 0x8: j_ += size - # note we ignore out-of-bounds here for debugging - if tag == TAG_GROW: - # grow lifetimes - i, id_ = index(weights, id) - if id_ > 0: - weights[i:i+1] = [id_, size, weights[i]-id_] - lifetimes[i:i+1] = [lifetimes[i], Lifetime(j), lifetimes[i]] + # find trunk + if not wastrunk and (tag & 0xc) != 0x4: + lower_, upper_ = 0, 0 + wastrunk = not not tag & 0x8 + + # keep track of weight + if tag & 0x8: + if tag & 0x4: + upper_ += id else: - weights[i:i] = [size] - lifetimes[i:i] = [Lifetime(j)] - - checkpoint(j, weights, lifetimes, {i}, set(), {i}) - - elif tag == TAG_SHRINK: - # shrink lifetimes - i, id_ = index(weights, id) - size_ = size - weights_ = weights.copy() - lifetimes_ = lifetimes.copy() - shrinks = set() - while size_ > 0 and i < len(weights_): - if id_ > 0: - diff = min(size_, weights_[i]-id_) - size_ -= diff - weights_[i] -= diff - i += 1 - id_ = 0 - elif weights_[i] > size_: - weights_[i] -= size_ - size_ = 0 - else: - size_ -= weights_[i] - weights_[i:i+1] = [] - lifetimes_[i:i+1] = [] - shrinks.add(i + len(shrinks)) - - checkpoint(j, weights, lifetimes, set(), shrinks, {i}) - weights = weights_ - lifetimes = lifetimes_ - + lower_ += id elif (tag & 0xc) == 0x0: - # attach tag to lifetime - i, id_ = index(weights, id) - if i < len(weights): - lifetimes[i].add(j) + delta = (lower_+upper_) - weight_ + if not tag & 0x2: + delta += id+1-lower_ + weight_ += delta - checkpoint(j, weights, lifetimes, set(), set(), {i}) + # note we ignore out-of-bounds here for debugging + if delta > 0: + # grow lifetimes + i, id_ = index(weights, lower_) + if id_ > 0: + weights[i:i+1] = [id_, delta, weights[i]-id_] + lifetimes[i:i+1] = [ + lifetimes[i], Lifetime(j), lifetimes[i]] + else: + weights[i:i] = [delta] + lifetimes[i:i] = [Lifetime(j)] + + checkpoint(j, weights, lifetimes, {i}, set(), {i}) + + elif delta < 0: + # shrink lifetimes + i, id_ = index(weights, lower_) + delta_ = -delta + weights_ = weights.copy() + lifetimes_ = lifetimes.copy() + shrinks = set() + while delta_ > 0 and i < len(weights_): + if id_ > 0: + delta__ = min(delta_, weights_[i]-id_) + delta_ -= delta__ + weights_[i] -= delta__ + i += 1 + id_ = 0 + elif weights_[i] > delta_: + weights_[i] -= delta_ + size_ = 0 + else: + delta_ -= weights_[i] + weights_[i:i+1] = [] + lifetimes_[i:i+1] = [] + shrinks.add(i + len(shrinks)) + + checkpoint(j, weights, lifetimes, set(), shrinks, {i}) + weights = weights_ + lifetimes = lifetimes_ + + if not tag & 0x2: + # attach tag to lifetime + i, id_ = index(weights, lower_) + if i < len(weights): + lifetimes[i].add(j) + + if delta == 0: + checkpoint(j, weights, lifetimes, set(), set(), {i}) lifetime_width = 2*max(( sum(1 for lifetime in lifetimes if lifetime) @@ -376,7 +393,7 @@ def show_log(block_size, data, rev, off, *, crc = crc32c(data[j_:j_+delta], crc) j_ += delta - if (tag & 0xe) <= 0x4: + if not tag & 0x8: if (tag & 0xf00f) != TAG_CRC: crc = crc32c(data[j_:j_+size], crc) # found a crc? @@ -398,7 +415,7 @@ def show_log(block_size, data, rev, off, *, ' %s' % next(xxd( data[j+delta:j+delta+min(size, 8)], 8), '') if not args.get('no_truncate') - and (tag & 0xe) <= 0x4 else ''), + and not tag & 0x8 else ''), '\x1b[m' if color and j >= off else '', ' (%s)' % ', '.join(notes) if notes else ' %s' % jumprepr(j) @@ -429,12 +446,12 @@ def show_log(block_size, data, rev, off, *, .ljust(4, b'\0')) for i in range(min(m.ceil(size/4), 3)))[:23] if not args.get('no_truncate') - and (tag & 0xe) <= 0x4 else ''), + and not tag & 0x8 else ''), crc, popc(crc) & 1, '\x1b[m' if color and j >= off else '')) - if (tag & 0xe) <= 0x4: + if not tag & 0x8: # show on-disk encoding of data if args.get('raw') or args.get('no_truncate'): for o, line in enumerate(xxd(data[j+delta:j+delta+size])): @@ -635,7 +652,7 @@ def show_tree(block_size, data, rev, trunk, weight, *, ' %s' % next(xxd( data[j+delta:j+delta+min(size, 8)], 8), '') if not args.get('no_truncate') - and (tag & 0xe) <= 0x4 else ''))) + and not tag & 0x8 else ''))) # show in-device representation if args.get('device'): @@ -651,7 +668,7 @@ def show_tree(block_size, data, rev, trunk, weight, *, .ljust(4, b'\0')) for i in range(min(m.ceil(size/4), 3)))[:23] if not args.get('no_truncate') - and (tag & 0xe) <= 0x4 else ''))) + and not tag & 0x8 else ''))) if args.get('raw'): # show on-disk encoding of tags @@ -660,7 +677,7 @@ def show_tree(block_size, data, rev, trunk, weight, *, '%04x' % (j + o*16), line)) - if (tag & 0xe) <= 0x4: + if not tag & 0x8: # show on-disk encoding of data if args.get('raw') or args.get('no_truncate'): for o, line in enumerate(xxd(data[j+delta:j+delta+size])): @@ -715,6 +732,7 @@ def main(disk, block_size=None, block1=0, block2=None, *, trunk = None trunk_ = None weight = 0 + lower_, upper_ = 0, 0 weight_ = 0 wastrunk = False while j_ < len(data): @@ -725,18 +743,24 @@ def main(disk, block_size=None, block1=0, block2=None, *, j_ += delta # find trunk - if not wastrunk and (tag & 0xe) != 0x4: + if not wastrunk and (tag & 0xc) != 0x4: trunk_ = j_ - delta + lower_, upper_ = 0, 0 wastrunk = not not tag & 0x8 # keep track of weight - if tag == TAG_GROW: - weight_ += size - elif tag == TAG_SHRINK: - weight_ = max(weight_ - size, 0) + if tag & 0x8: + if tag & 0x4: + upper_ += id + else: + lower_ += id + elif (tag & 0xc) == 0x0: + weight_ = lower_+upper_ + if not tag & 0x2: + weight_ += id+1-lower_ # take care of crcs - if (tag & 0xe) <= 0x4: + if not tag & 0x8: if (tag & 0xf00f) != TAG_CRC: crc = crc32c(data[j_:j_+size], crc) # found a crc?