Github/littlefs
1
Fork 0
Code Actions Activity

Derived grows/shrinks from rbyd trunk, no longer needing explicit tags

Browse Source 
I only recently noticed there is enough information in each rbyd trunk
to infer the effective grow/shrinks. This has a number of benefits:

- Cleans up the tag encoding a bit, no longer expecting tag size to
  sometimes contain a weight (though this could've been fixed other
  ways).

  0x6 in the lower nibble now reserved exclusively for in-device tags.

- grow/shrinks can be implicit to any tag. Will attempt to leverage this
  in the future.

- The weight of an rbyd can no longer go out-of-sync with itself. While
  this _shouldn't_ happen normally, if it does I imagine it'd be very
  hard to debug.

  Now, there is only one source of knowledge about the weight of the
  rbyd: The most recent set of alt-pointers.

Note that remove/unreachable tags now behave _very_ differently when it
comes to weight calculation, remove tags require the tree to make the
tag unreachable. This is a tradeoff for the above.
This commit is contained in:
Christopher Haster
2023-03-27 01:45:34 -05:00
parent 546fff77fb
commit 8f26b68af2
3 changed files with 203 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+62 -46
View File
@@ -449,32 +449,35 @@ static inline lfs_size_t lfs_tag_dsize(lfs_tag_t tag) {
// 16-bit metadata tags
enum lfsr_tag_type {
LFSR_TAG_NAME = 0x1000,
LFSR_TAG_BNAME = 0x1000,
LFSR_TAG_REG = 0x1010,
LFSR_TAG_DIR = 0x1020,
LFSR_TAG_UNREACHABLE = 0x0002,
LFSR_TAG_STRUCT = 0x3000,
LFSR_TAG_INLINED = 0x3000,
LFSR_TAG_BLOCK = 0x3100,
LFSR_TAG_BRANCH = 0x3200,
LFSR_TAG_BTREE = 0x3300,
LFSR_TAG_NAME = 0x1000,
LFSR_TAG_BNAME = 0x1000,
LFSR_TAG_REG = 0x1010,
LFSR_TAG_DIR = 0x1020,
LFSR_TAG_UATTR = 0x4000,
LFSR_TAG_RMUATTR = 0x4002,
LFSR_TAG_STRUCT = 0x3000,
LFSR_TAG_INLINED = 0x3000,
LFSR_TAG_BLOCK = 0x3100,
LFSR_TAG_BRANCH = 0x3200,
LFSR_TAG_BTREE = 0x3300,
LFSR_TAG_GROW = 0x0006,
LFSR_TAG_SHRINK = 0x0016,
LFSR_TAG_FROM = 0x0026, // in-device only
LFSR_TAG_UATTR = 0x4000,
LFSR_TAG_RMUATTR = 0x4002,
LFSR_TAG_ALT = 0x0008,
LFSR_TAG_ALTBLE = 0x0008,
LFSR_TAG_ALTRLE = 0x000a,
LFSR_TAG_ALTBGT = 0x000c,
LFSR_TAG_ALTRGT = 0x000e,
LFSR_TAG_ALT = 0x0008,
LFSR_TAG_ALTBLE = 0x0008,
LFSR_TAG_ALTRLE = 0x000a,
LFSR_TAG_ALTBGT = 0x000c,
LFSR_TAG_ALTRGT = 0x000e,
LFSR_TAG_CRC = 0x0004,
LFSR_TAG_FCRC = 0x1004,
LFSR_TAG_CRC = 0x0004,
LFSR_TAG_FCRC = 0x1004,
// in-device only
LFSR_TAG_GROW = 0x0006,
LFSR_TAG_SHRINK = 0x0016,
LFSR_TAG_FROM = 0x0026,
};
#define LFSR_TAG_ALT_(color, dir, key) \
@@ -512,12 +515,8 @@ static inline lfsr_tag_t lfsr_tag_mkrm(lfsr_tag_t tag) {
return tag | 0x2;
}
static inline bool lfsr_tag_hasdata(lfsr_tag_t tag) {
return (tag & 0xe) <= 0x4;
}
static inline bool lfsr_tag_istrunk(lfsr_tag_t tag) {
return (tag & 0xe) != 0x4;
return (tag & 0xc) != 0x4;
}
static inline bool lfsr_tag_isalt(lfsr_tag_t tag) {
@@ -1282,6 +1281,8 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd,
lfs_off_t off = sizeof(uint32_t);
lfs_off_t trunk = 0;
bool wastrunk = false;
lfs_size_t lower = 0;
lfs_size_t upper = 0;
lfs_size_t weight = 0;
// assume unerased until proven otherwise
@@ -1310,39 +1311,51 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd,
// found trunk of tree?
if (!wastrunk && lfsr_tag_istrunk(tag)) {
trunk = off;
lower = 0;
upper = 0;
}
wastrunk = lfsr_tag_isalt(tag);
off += delta;
// keep track of weight
// derive the new weight of the tree from alt pointers
//
// NOTE we can't check for overflow/underflow here because we
// may be overeagerly parsing an invalid commit, it's ok for
// this to overflow/underflow as long as we throw it out later
// on a bad crc
if (tag == LFSR_TAG_GROW) {
weight += size;
// adjust find
if (find && find->predicted_id >= id) {
find->predicted_id += size;
if (lfsr_tag_isalt(tag)) {
if (lfsr_tag_isgt(tag)) {
upper += id;
} else {
lower += id;
}
} else if (tag == LFSR_TAG_SHRINK) {
weight -= size;
// adjust find
} else if (lfsr_tag_istrunk(tag)) {
// note we need to include our id's weight in our weight
// calculation, but only when our id is included in the tree
lfs_ssize_t delta = (lower+upper) - weight;
if (!lfsr_tag_isrm(tag)
&& tag != LFSR_TAG_GROW
&& tag != LFSR_TAG_SHRINK) {
delta += id+1-lower;
}
// adjust any pending finds
if (find && find->predicted_id >= id) {
if (find->predicted_id < id+(lfs_ssize_t)size) {
// pending find removed?
if (delta < 0 && find->predicted_id < id-delta) {
find->predicted_tag = 0;
find->predicted_id = id-1;
} else {
find->predicted_id -= size;
find->predicted_id += delta;
}
}
weight += delta;
}
// we mostly just skip non-data tags here
if (!lfsr_tag_hasdata(tag)) {
if (lfsr_tag_isalt(tag)) {
continue;
}
@@ -2322,6 +2335,11 @@ leaf:;
//
// note we always need something after the alts! without something between
// alts we may not be able to find the trunk of our tree
if (tag == LFSR_TAG_GROW || tag == LFSR_TAG_SHRINK) {
tag = LFSR_TAG_UNREACHABLE;
data = LFSR_DATA_NULL;
}
err = lfsr_rbyd_progtag(lfs, rbyd,
tag, id, lfsr_data_len(data), &rbyd->crc);
if (err) {
@@ -2329,13 +2347,11 @@ leaf:;
return err;
}
if (lfsr_tag_hasdata(tag)) {
// don't forget the data!
err = lfsr_rbyd_progdata(lfs, rbyd, data, &rbyd->crc);
if (err) {
rbyd->erased = false;
return err;
}
// don't forget the data!
err = lfsr_rbyd_progdata(lfs, rbyd, data, &rbyd->crc);
if (err) {
rbyd->erased = false;
return err;
}
return 0;
scripts/dbgbtree.py
+32 -30
View File
@@ -7,21 +7,20 @@ import os
import struct
TAG_NAME = 0x1000
TAG_BNAME = 0x1000
TAG_REG = 0x1010
TAG_DIR = 0x1020
TAG_STRUCT = 0x3000
TAG_INLINED = 0x3000
TAG_BLOCK = 0x3100
TAG_BRANCH = 0x3200
TAG_BTREE = 0x3300
TAG_UATTR = 0x4000
TAG_GROW = 0x0006
TAG_SHRINK = 0x0016
TAG_ALT = 0x0008
TAG_CRC = 0x0004
TAG_FCRC = 0x1004
TAG_UNREACHABLE = 0x0002
TAG_NAME = 0x1000
TAG_BNAME = 0x1000
TAG_REG = 0x1010
TAG_DIR = 0x1020
TAG_STRUCT = 0x3000
TAG_INLINED = 0x3000
TAG_BLOCK = 0x3100
TAG_BRANCH = 0x3200
TAG_BTREE = 0x3300
TAG_UATTR = 0x4000
TAG_ALT = 0x0008
TAG_CRC = 0x0004
TAG_FCRC = 0x1004
def blocklim(s):
if '.' in s:
@@ -84,7 +83,11 @@ def xxd(data, width=16, crc=False):
for b in map(chr, data[i:i+width])))
def tagrepr(tag, id, size, off=None):
if (tag & 0xf00c) == TAG_NAME:
if (tag & 0xfffe) == TAG_UNREACHABLE:
return 'unreachable id%d%s' % (
id,
' %d' % size if size else '')
elif (tag & 0xf00c) == TAG_NAME:
return '%s%s id%d %d' % (
'rm' if tag & 0x2 else '',
'bname' if (tag & 0xfffe) == TAG_BNAME
@@ -109,14 +112,6 @@ def tagrepr(tag, id, size, off=None):
(tag & 0x0ff0) >> 4,
' id%d' % id if id != -1 else '',
' %d' % size if not tag & 0x2 or size else '')
elif (tag & 0xfffe) == TAG_GROW:
return 'grow id%d w%d' % (
id,
size)
elif (tag & 0xfffe) == TAG_SHRINK:
return 'shrink id%d w%d' % (
id,
size)
elif (tag & 0xf00e) == TAG_CRC:
return 'crc%x%s %d' % (
1 if tag & 0x10 else 0,
@@ -162,6 +157,7 @@ class Rbyd:
trunk = None
trunk_ = None
weight = 0
lower_, upper_ = 0, 0
weight_ = 0
wastrunk = False
while j_ < limit:
@@ -172,18 +168,24 @@ class Rbyd:
j_ += delta
# find trunk
if not wastrunk and (tag & 0xe) != 0x4:
if not wastrunk and (tag & 0xc) != 0x4:
trunk_ = j_ - delta
lower_, upper_ = 0, 0
wastrunk = not not tag & 0x8
# keep track of weight
if tag == TAG_GROW:
weight_ += size
elif tag == TAG_SHRINK:
weight_ = max(weight_ - size, 0)
if tag & 0x8:
if tag & 0x4:
upper_ += id
else:
lower_ += id
elif (tag & 0xc) == 0x0:
weight_ = lower_+upper_
if not tag & 0x2:
weight_ += id+1-lower_
# take care of crcs
if (tag & 0xe) <= 0x4:
if not tag & 0x8:
if (tag & 0xf00f) != TAG_CRC:
crc = crc32c(data[j_:j_+size], crc)
# found a crc?
scripts/dbgrbyd.py
+109 -85
View File
@@ -16,21 +16,20 @@ COLORS = [
]
TAG_NAME = 0x1000
TAG_BNAME = 0x1000
TAG_REG = 0x1010
TAG_DIR = 0x1020
TAG_STRUCT = 0x3000
TAG_INLINED = 0x3000
TAG_BLOCK = 0x3100
TAG_BRANCH = 0x3200
TAG_BTREE = 0x3300
TAG_UATTR = 0x4000
TAG_GROW = 0x0006
TAG_SHRINK = 0x0016
TAG_ALT = 0x0008
TAG_CRC = 0x0004
TAG_FCRC = 0x1004
TAG_UNREACHABLE = 0x0002
TAG_NAME = 0x1000
TAG_BNAME = 0x1000
TAG_REG = 0x1010
TAG_DIR = 0x1020
TAG_STRUCT = 0x3000
TAG_INLINED = 0x3000
TAG_BLOCK = 0x3100
TAG_BRANCH = 0x3200
TAG_BTREE = 0x3300
TAG_UATTR = 0x4000
TAG_ALT = 0x0008
TAG_CRC = 0x0004
TAG_FCRC = 0x1004
def blocklim(s):
if '.' in s:
@@ -93,7 +92,11 @@ def xxd(data, width=16, crc=False):
for b in map(chr, data[i:i+width])))
def tagrepr(tag, id, size, off=None):
if (tag & 0xf00c) == TAG_NAME:
if (tag & 0xfffe) == TAG_UNREACHABLE:
return 'unreachable id%d%s' % (
id,
' %d' % size if size else '')
elif (tag & 0xf00c) == TAG_NAME:
return '%s%s id%d %d' % (
'rm' if tag & 0x2 else '',
'bname' if (tag & 0xfffe) == TAG_BNAME
@@ -118,14 +121,6 @@ def tagrepr(tag, id, size, off=None):
(tag & 0x0ff0) >> 4,
' id%d' % id if id != -1 else '',
' %d' % size if not tag & 0x2 or size else '')
elif (tag & 0xfffe) == TAG_GROW:
return 'grow id%d w%d' % (
id,
size)
elif (tag & 0xfffe) == TAG_SHRINK:
return 'shrink id%d w%d' % (
id,
size)
elif (tag & 0xf00e) == TAG_CRC:
return 'crc%x%s %d' % (
1 if tag & 0x10 else 0,
@@ -160,7 +155,7 @@ def show_log(block_size, data, rev, off, *,
j = j_
v, tag, id, size, delta = fromtag(data[j_:])
j_ += delta
if (tag & 0xe) <= 0x4:
if not tag & 0x8:
j_ += size
if tag & 0x8:
@@ -241,69 +236,91 @@ def show_log(block_size, data, rev, off, *,
id -= w
return len(weights), 0
checkpoint_js = []
checkpoints = []
checkpoint_js = [0]
checkpoints = [([], [], set(), set(), set())]
def checkpoint(j, weights, lifetimes, grows, shrinks, tags):
checkpoint_js.append(j)
checkpoints.append((
weights.copy(), lifetimes.copy(),
grows, shrinks, tags))
lower_, upper_ = 0, 0
weight_ = 0
wastrunk = False
j_ = 4
while j_ < (block_size if args.get('all') else off):
j = j_
v, tag, id, size, delta = fromtag(data[j_:])
j_ += delta
if (tag & 0xe) <= 0x4:
if not tag & 0x8:
j_ += size
# note we ignore out-of-bounds here for debugging
if tag == TAG_GROW:
# grow lifetimes
i, id_ = index(weights, id)
if id_ > 0:
weights[i:i+1] = [id_, size, weights[i]-id_]
lifetimes[i:i+1] = [lifetimes[i], Lifetime(j), lifetimes[i]]
# find trunk
if not wastrunk and (tag & 0xc) != 0x4:
lower_, upper_ = 0, 0
wastrunk = not not tag & 0x8
# keep track of weight
if tag & 0x8:
if tag & 0x4:
upper_ += id
else:
weights[i:i] = [size]
lifetimes[i:i] = [Lifetime(j)]
checkpoint(j, weights, lifetimes, {i}, set(), {i})
elif tag == TAG_SHRINK:
# shrink lifetimes
i, id_ = index(weights, id)
size_ = size
weights_ = weights.copy()
lifetimes_ = lifetimes.copy()
shrinks = set()
while size_ > 0 and i < len(weights_):
if id_ > 0:
diff = min(size_, weights_[i]-id_)
size_ -= diff
weights_[i] -= diff
i += 1
id_ = 0
elif weights_[i] > size_:
weights_[i] -= size_
size_ = 0
else:
size_ -= weights_[i]
weights_[i:i+1] = []
lifetimes_[i:i+1] = []
shrinks.add(i + len(shrinks))
checkpoint(j, weights, lifetimes, set(), shrinks, {i})
weights = weights_
lifetimes = lifetimes_
lower_ += id
elif (tag & 0xc) == 0x0:
# attach tag to lifetime
i, id_ = index(weights, id)
if i < len(weights):
lifetimes[i].add(j)
delta = (lower_+upper_) - weight_
if not tag & 0x2:
delta += id+1-lower_
weight_ += delta
checkpoint(j, weights, lifetimes, set(), set(), {i})
# note we ignore out-of-bounds here for debugging
if delta > 0:
# grow lifetimes
i, id_ = index(weights, lower_)
if id_ > 0:
weights[i:i+1] = [id_, delta, weights[i]-id_]
lifetimes[i:i+1] = [
lifetimes[i], Lifetime(j), lifetimes[i]]
else:
weights[i:i] = [delta]
lifetimes[i:i] = [Lifetime(j)]
checkpoint(j, weights, lifetimes, {i}, set(), {i})
elif delta < 0:
# shrink lifetimes
i, id_ = index(weights, lower_)
delta_ = -delta
weights_ = weights.copy()
lifetimes_ = lifetimes.copy()
shrinks = set()
while delta_ > 0 and i < len(weights_):
if id_ > 0:
delta__ = min(delta_, weights_[i]-id_)
delta_ -= delta__
weights_[i] -= delta__
i += 1
id_ = 0
elif weights_[i] > delta_:
weights_[i] -= delta_
size_ = 0
else:
delta_ -= weights_[i]
weights_[i:i+1] = []
lifetimes_[i:i+1] = []
shrinks.add(i + len(shrinks))
checkpoint(j, weights, lifetimes, set(), shrinks, {i})
weights = weights_
lifetimes = lifetimes_
if not tag & 0x2:
# attach tag to lifetime
i, id_ = index(weights, lower_)
if i < len(weights):
lifetimes[i].add(j)
if delta == 0:
checkpoint(j, weights, lifetimes, set(), set(), {i})
lifetime_width = 2*max((
sum(1 for lifetime in lifetimes if lifetime)
@@ -376,7 +393,7 @@ def show_log(block_size, data, rev, off, *,
crc = crc32c(data[j_:j_+delta], crc)
j_ += delta
if (tag & 0xe) <= 0x4:
if not tag & 0x8:
if (tag & 0xf00f) != TAG_CRC:
crc = crc32c(data[j_:j_+size], crc)
# found a crc?
@@ -398,7 +415,7 @@ def show_log(block_size, data, rev, off, *,
' %s' % next(xxd(
data[j+delta:j+delta+min(size, 8)], 8), '')
if not args.get('no_truncate')
and (tag & 0xe) <= 0x4 else ''),
and not tag & 0x8 else ''),
'\x1b[m' if color and j >= off else '',
' (%s)' % ', '.join(notes) if notes
else ' %s' % jumprepr(j)
@@ -429,12 +446,12 @@ def show_log(block_size, data, rev, off, *,
.ljust(4, b'\0'))
for i in range(min(m.ceil(size/4), 3)))[:23]
if not args.get('no_truncate')
and (tag & 0xe) <= 0x4 else ''),
and not tag & 0x8 else ''),
crc,
popc(crc) & 1,
'\x1b[m' if color and j >= off else ''))
if (tag & 0xe) <= 0x4:
if not tag & 0x8:
# show on-disk encoding of data
if args.get('raw') or args.get('no_truncate'):
for o, line in enumerate(xxd(data[j+delta:j+delta+size])):
@@ -635,7 +652,7 @@ def show_tree(block_size, data, rev, trunk, weight, *,
' %s' % next(xxd(
data[j+delta:j+delta+min(size, 8)], 8), '')
if not args.get('no_truncate')
and (tag & 0xe) <= 0x4 else '')))
and not tag & 0x8 else '')))
# show in-device representation
if args.get('device'):
@@ -651,7 +668,7 @@ def show_tree(block_size, data, rev, trunk, weight, *,
.ljust(4, b'\0'))
for i in range(min(m.ceil(size/4), 3)))[:23]
if not args.get('no_truncate')
and (tag & 0xe) <= 0x4 else '')))
and not tag & 0x8 else '')))
if args.get('raw'):
# show on-disk encoding of tags
@@ -660,7 +677,7 @@ def show_tree(block_size, data, rev, trunk, weight, *,
'%04x' % (j + o*16),
line))
if (tag & 0xe) <= 0x4:
if not tag & 0x8:
# show on-disk encoding of data
if args.get('raw') or args.get('no_truncate'):
for o, line in enumerate(xxd(data[j+delta:j+delta+size])):
@@ -715,6 +732,7 @@ def main(disk, block_size=None, block1=0, block2=None, *,
trunk = None
trunk_ = None
weight = 0
lower_, upper_ = 0, 0
weight_ = 0
wastrunk = False
while j_ < len(data):
@@ -725,18 +743,24 @@ def main(disk, block_size=None, block1=0, block2=None, *,
j_ += delta
# find trunk
if not wastrunk and (tag & 0xe) != 0x4:
if not wastrunk and (tag & 0xc) != 0x4:
trunk_ = j_ - delta
lower_, upper_ = 0, 0
wastrunk = not not tag & 0x8
# keep track of weight
if tag == TAG_GROW:
weight_ += size
elif tag == TAG_SHRINK:
weight_ = max(weight_ - size, 0)
if tag & 0x8:
if tag & 0x4:
upper_ += id
else:
lower_ += id
elif (tag & 0xc) == 0x0:
weight_ = lower_+upper_
if not tag & 0x2:
weight_ += id+1-lower_
# take care of crcs
if (tag & 0xe) <= 0x4:
if not tag & 0x8:
if (tag & 0xf00f) != TAG_CRC:
crc = crc32c(data[j_:j_+size], crc)
# found a crc?