Implementation of lfsr_rbyd_pendinglookup in one pass

This ends up surprisingly tricky with sparse ids. I feel like I'm missing
a simpler solution, but this at least proves an implementation is possible.

The implementation here does a single pass through the attributes
backwards (which should probably be changed from a linked-list), keeping
track of the best matching tag/id while updating everything based on
grows/shrinks. Once we find the source of the best id we adjust things
back to the pending id space.

The implementation here only works with some significant caveats:

1. This solution might be able to find the id weights by keeping track
   of a lower bound, but it would be difficult and add complexity, so we
   don't do it. Really lfsr_rbyd_pendinglookup is only going to be used
   in full traversals as a part of compaction/splitting, so weight can
   be derived trivially from neighboring ids.

2. We don't know the difference between grows/shrinks used to change a
   branch's weight and used to create/delete ids. This is a bit of a
   problem here, but we can work around it by assuming that
   non-destructive grows/shrinks are always on the lower edge of a
   weighted id.

   Fortunately this assumption is only needed for in-flight attrs in
   lfsr_rbyd_pendinglookup, so this is not a requirement on-disk or in
   future implemenations.
This commit is contained in:
Christopher Haster
2023-03-01 10:01:42 -06:00
parent c0ee405cf2
commit 7f16c6e473
2 changed files with 159 additions and 132 deletions
+102 -88
View File
@@ -1615,23 +1615,30 @@ static lfs_ssize_t lfsr_rbyd_get(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
static int lfsr_rbyd_pendinglookup(lfs_t *lfs,
const lfsr_rbyd_t *rbyd, const struct lfsr_attr *attrs,
lfsr_tag_t tag, lfs_ssize_t id,
lfsr_tag_t *tag_, lfs_ssize_t *id_,
lfsr_data_t *data_, lfs_size_t *weight_) {
printf("pendinglookup(%x, %d)\n", tag, id);
lfsr_tag_t *tag_, lfs_ssize_t *id_, lfsr_data_t *data_) {
// For this lookup to work it requires a lot of caveats:
//
// 1. Finding the weight is much more difficult when attrs aren't on disk
// so we don't do this. Note that weight can still be derived during
// traversal by diffing ids.
//
// 2. Our grow/shrink checks expect the grow/shrink ids to always be on the
// lowest id. This is notably different from our rbyd bias. Fortunately
// this is only a requirement of in-flight attrs so this isn't a
// requirement on-disk.
again:;
// tag must be non-zero! zero tags may deceptively look like they work but
// fail when the tree contains a deleted id0
LFS_ASSERT(tag != 0);
lfsr_tag_t tag__ = 0xffff;
lfsr_data_t data__ = LFSR_DATA_NULL;
lfs_size_t weight = 0;
const struct lfsr_attr *attrs__ = attrs;
// keep track of best id/tag and upper/lower bounds to determine weight
lfs_ssize_t id__ = id;
lfs_ssize_t best_id = -2;
lfsr_tag_t best_tag = 0;
lfsr_data_t best_data = LFSR_DATA_NULL;
// first search backwards through tags to find the smallest, non-deleted,
// >= id, then search through tags in-order to find the most recent update,
// two passes are required to adjust for weight changes.
// search through our tags backwards to figure out the best tag/id
// TODO hmm, reverse iteration over a linked-list? this is a bad design
unsigned attr_count = 0;
for (const struct lfsr_attr *attr = attrs; attr; attr = attr->next) {
@@ -1644,112 +1651,119 @@ again:;
}
if (attr->tag == LFSR_TAG_GROW) {
printf("g %d %d\n", attr->id, attr->size);
if (id >= attr->id) {
if (id < attr->id + attr->size) {
// TODO
id = attr->id + attr->size-1;
weight = attr->size;
attrs__ = attr->next;
goto grown;
// found grow which includes both target and best ids? this
// must be the source of the best tag
if (attr->id <= id__
&& best_id != -2
&& attr->id+(lfs_ssize_t)attr->size > best_id) {
goto found;
// found grow which only includes target id? this must be a
// weight-changing grow so we can just adjust our target id to
// follow the upper edge of the grow
} else if (attr->id <= id__
&& attr->id+(lfs_ssize_t)attr->size > id__) {
id__ = attr->id;
tag = 0x10;
id += id__-attr->id+1;
// adjust ids
} else if (attr->id <= id__) {
id__ -= attr->size;
if (best_id != -2) {
best_id -= attr->size;
}
id -= attr->size;
// use grow as upper bound
} else if (best_id == -2 || attr->id <= best_id) {
best_id = attr->id-1;
best_tag = 0;
}
} else if (attr->tag == LFSR_TAG_SHRINK) {
if (id >= attr->id) {
id += attr->size;
// adjust ids
if (attr->id <= id__) {
id__ += attr->size;
if (best_id != -2) {
best_id += attr->size;
}
// use shrink as upper bound
} else if (best_id == -2 || attr->id <= best_id) {
best_id = attr->id-1;
best_tag = 0;
}
} else if (attr->tag == LFSR_TAG_FROM) {
// TODO
LFS_ASSERT(false);
} else {
// found better tag?
if ((attr->id > id__
|| (attr->id == id__
&& lfsr_tag_key(attr->tag)
>= lfsr_tag_key(tag)))
&& (best_id == -2
|| attr->id < best_id
|| (attr->id == best_id
&& (!best_tag
|| lfsr_tag_key(attr->tag)
< lfsr_tag_key(best_tag))))) {
best_id = attr->id;
best_tag = attr->tag;
best_data = LFSR_DATA_BUF(attr->buffer, attr->size);
}
}
}
// this id can't exist in our rbyd
if (id >= (lfs_ssize_t)rbyd->weight) {
printf("? %d >= %d\n", id, rbyd->weight);
return LFS_ERR_NOENT;
}
// if not created in attr list our id must have been created in the rbyd
lfs_off_t off__;
lfs_size_t size__;
int err = lfsr_rbyd_lookup(lfs, rbyd, tag, id,
&tag__, &id, &weight, &off__, &size__);
// try to found our id/tag on disk
lfsr_tag_t rbyd_tag;
lfs_ssize_t rbyd_id;
lfs_off_t rbyd_off;
lfs_size_t rbyd_size;
int err = lfsr_rbyd_lookup(lfs, rbyd, tag, id__,
&rbyd_tag, &rbyd_id, NULL, &rbyd_off, &rbyd_size);
if (err && err != LFS_ERR_NOENT) {
return err;
}
if (err != LFS_ERR_NOENT) {
data__ = LFSR_DATA_DISK(rbyd->block, off__, size__);
}
grown:;
// TODO different way to encode weight?
lfs_ssize_t lower = id-weight+1;
printf("raw %d %d (%d)\n", id, weight, lower);
// now replay the attr list, keeping track of changes to id, weight, tag
for (const struct lfsr_attr *attr = attrs__; attr; attr = attr->next) {
if (attr->tag == LFSR_TAG_GROW) {
if (lower >= attr->id) {
lower += attr->size;
}
if (id >= attr->id) {
id += attr->size;
}
} else if (attr->tag == LFSR_TAG_SHRINK) {
if (lower >= attr->id) {
lower -= attr->size;
}
if (id >= attr->id) {
id -= attr->size;
}
} else if (attr->tag == LFSR_TAG_FROM) {
// TODO
LFS_ASSERT(false);
} else if (attr->id == id
&& lfsr_tag_key(attr->tag) >= lfsr_tag_key(tag)
&& lfsr_tag_key(attr->tag) <= lfsr_tag_key(tag__)) {
tag__ = attr->tag;
data__ = LFSR_DATA_BUF(attr->buffer, attr->size);
// found a better tag?
if (best_id == -2
|| rbyd_id < best_id
|| (rbyd_id == best_id
&& (!best_tag
|| lfsr_tag_key(rbyd_tag)
< lfsr_tag_key(best_tag)))) {
best_id = rbyd_id;
best_tag = rbyd_tag;
best_data = LFSR_DATA_DISK(rbyd->block, rbyd_off, rbyd_size);
}
}
// TODO if we can't get rid of this we can at least move it into the
// below condition
weight = id-lower+1;
printf("fix %d %d (%d)\n", id, weight, lower);
found:;
// no better id found
if (best_id == -2) {
return LFS_ERR_NOENT;
}
// not found? increase id
if (tag__ == 0xffff) {
tag = 0x10;
id = id + 1;
// no tag found? increase id
if (!best_tag || lfsr_tag_isrm(best_tag)) {
tag = best_tag + 0x10;
id = best_id+(id-id__) + 1;
goto again;
}
// found rm? should continue
if (lfsr_tag_isrm(tag__)) {
tag = tag__ + 0x10;
goto again;
}
// found
// found an id/tag
// TODO how many of these should be conditional?
if (tag_) {
*tag_ = tag__;
*tag_ = best_tag;
}
if (id_) {
*id_ = id;
*id_ = best_id+(id-id__);
}
if (data_) {
*data_ = data__;
}
if (weight_) {
*weight_ = weight;
*data_ = best_data;
}
return 0;
@@ -1763,7 +1777,7 @@ static lfs_ssize_t lfsr_rbyd_pendingget(lfs_t *lfs,
lfs_ssize_t id_;
lfsr_data_t data_;
int err = lfsr_rbyd_pendinglookup(lfs, rbyd, attrs, tag, id,
&tag_, &id_, &data_, NULL);
&tag_, &id_, &data_);
if (err) {
return err;
}
+57 -44
View File
@@ -11416,7 +11416,6 @@ code = '''
lfsr_tag_t tag_ = 0;
lfs_ssize_t id_ = -1;
lfsr_data_t data_ = LFSR_DATA_NULL;
lfs_size_t weight_ = 0;
// test all permutations of a given size
uint16_t perm[N];
@@ -11448,8 +11447,8 @@ code = '''
UATTR(perm[j]+1), -1, "\xaa\xaa\xaa\xaa", 4,
(j+1 < N && j+1 != w) ? &attrs[j+1] : NULL);
}
struct lfsr_attr *written = w > 0 ? &attrs[0] : NULL;
struct lfsr_attr *unwritten = w < N ? &attrs[w] : NULL;
const struct lfsr_attr *written = w > 0 ? &attrs[0] : NULL;
const struct lfsr_attr *unwritten = w < N ? &attrs[w] : NULL;
// create rbyd with written attr
rbyd = init_rbyd;
@@ -11463,11 +11462,10 @@ code = '''
for (unsigned j = 0; j < N; j++) {
lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten,
LFSR_TAG_UATTR(j+1), -1,
&tag_, &id_, &data_, &weight_) => 0;
&tag_, &id_, &data_) => 0;
assert(tag_ == LFSR_TAG_UATTR(j+1));
assert(id_ == -1);
assert(lfsr_data_len(data_) == 4);
assert(weight_ == 0);
}
// test traverse both written/unwritten
@@ -11476,15 +11474,14 @@ code = '''
for (unsigned j = 0; j < N; j++) {
lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten,
lfsr_tag_next(tag_), id_,
&tag_, &id_, &data_, &weight_) => 0;
&tag_, &id_, &data_) => 0;
assert(tag_ == LFSR_TAG_UATTR(j+1));
assert(id_ == -1);
assert(lfsr_data_len(data_) == 4);
assert(weight_ == 0);
}
lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten,
lfsr_tag_next(tag_), id_,
&tag_, &id_, &data_, &weight_) => LFS_ERR_NOENT;
&tag_, &id_, &data_) => LFS_ERR_NOENT;
}
// next permutation using Heap's algorithm
@@ -11608,6 +11605,7 @@ code = '''
}
}
}
const struct lfsr_attr *unwritten = w < N ? attrs : NULL;
// compare rbyd vs simulation
printf("expd: [");
@@ -11625,7 +11623,7 @@ code = '''
printf("rbyd: [");
first = true;
for (unsigned attr = 0; attr < N; attr++) {
lfs_ssize_t size = lfsr_rbyd_pendingget(&lfs, &rbyd, attrs,
lfs_ssize_t size = lfsr_rbyd_pendingget(&lfs, &rbyd, unwritten,
LFSR_TAG_UATTR(attr), -1, buffer, 4);
if (size >= 0) {
if (!first) {
@@ -11638,7 +11636,7 @@ code = '''
printf("]\n");
for (unsigned attr = 0; attr < N; attr++) {
lfs_ssize_t size = lfsr_rbyd_pendingget(&lfs, &rbyd, attrs,
lfs_ssize_t size = lfsr_rbyd_pendingget(&lfs, &rbyd, unwritten,
LFSR_TAG_UATTR(attr), -1, buffer, 4);
if (sim[attr]) {
assert(size == 1);
@@ -11677,7 +11675,6 @@ code = '''
lfsr_tag_t tag_ = 0;
lfs_ssize_t id_ = -1;
lfsr_data_t data_ = LFSR_DATA_NULL;
lfs_size_t weight_ = 0;
const uint8_t names[6][4] = {
"\xaa\xaa\xaa\xaa",
"\xbb\xbb\xbb\xbb",
@@ -11729,8 +11726,8 @@ code = '''
MKREG, id, names[perm[j] % 6], 4,
(j+1 < N && j+1 != w) ? &attrs[2*j+2] : NULL);
}
struct lfsr_attr *written = w > 0 ? &attrs[0] : NULL;
struct lfsr_attr *unwritten = w < N ? &attrs[2*w] : NULL;
const struct lfsr_attr *written = w > 0 ? &attrs[0] : NULL;
const struct lfsr_attr *unwritten = w < N ? &attrs[2*w] : NULL;
// create rbyd with written attr
rbyd = init_rbyd;
@@ -11754,15 +11751,14 @@ code = '''
for (unsigned j = 0; j < N; j++) {
lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten,
lfsr_tag_next(tag_), id_,
&tag_, &id_, &data_, &weight_) => 0;
&tag_, &id_, &data_) => 0;
assert(tag_ == LFSR_TAG_MKREG);
assert(id_ == j);
assert(lfsr_data_len(data_) == 4);
assert(weight_ == 1);
}
lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten,
lfsr_tag_next(tag_), id_,
&tag_, &id_, &data_, NULL) => LFS_ERR_NOENT;
&tag_, &id_, &data_) => LFS_ERR_NOENT;
}
// next permutation using Heap's algorithm
@@ -11896,6 +11892,7 @@ code = '''
}
}
}
const struct lfsr_attr *unwritten = w < N ? attrs : NULL;
// compare rbyd vs simulation
printf("expd: [");
@@ -11908,7 +11905,7 @@ code = '''
printf("]\n");
printf("rbyd: [");
for (lfs_ssize_t id = 0; id < (lfs_ssize_t)count; id++) {
lfs_ssize_t size = lfsr_rbyd_pendingget(&lfs, &rbyd, attrs,
lfs_ssize_t size = lfsr_rbyd_pendingget(&lfs, &rbyd, unwritten,
LFSR_TAG_MKREG, id, buffer, 4);
if (size >= 0) {
printf("%.*s", size, buffer);
@@ -11922,7 +11919,7 @@ code = '''
printf("]\n");
for (lfs_ssize_t id = 0; id < (lfs_ssize_t)count; id++) {
lfsr_rbyd_pendingget(&lfs, &rbyd, attrs,
lfsr_rbyd_pendingget(&lfs, &rbyd, unwritten,
LFSR_TAG_MKREG, id, buffer, 4) => 1;
assert(memcmp(&sim[id], buffer, 1) == 0);
}
@@ -11957,7 +11954,6 @@ code = '''
lfsr_tag_t tag_ = 0;
lfs_ssize_t id_ = -1;
lfsr_data_t data_ = LFSR_DATA_NULL;
lfs_size_t weight_ = 0;
const uint8_t names[6][4] = {
"\xaa\xaa\xaa\xaa",
"\xbb\xbb\xbb\xbb",
@@ -11980,7 +11976,7 @@ code = '''
while (i < N) {
// test each number of written/unwritten tags, this gives us a quick
// way to test several unwritten situations
for (unsigned w = 0; w <= N; w++) {
for (signed w = -1; w <= N; w++) {
// print permutation to help debugging
printf("--- permutation: [");
for (unsigned j = 0; j < N; j++) {
@@ -11992,7 +11988,9 @@ code = '''
printf("], written: %d/%jd ---\n", w, N);
// build the attribute lists for the current permutation
struct lfsr_attr attrs[(2+M)*N];
struct lfsr_attr attrs[1+(2+M)*N];
attrs[0] = *LFSR_ATTR(UATTR(3), -1, "unrelated", 9, &attrs[1]);
for (unsigned j = 0; j < N; j++) {
// adjust id based on future insertions
uint16_t id = perm[j];
@@ -12002,24 +12000,29 @@ code = '''
}
}
attrs[(2+M)*j+0] = *LFSR_ATTR(
attrs[1+(2+M)*j+0] = *LFSR_ATTR(
GROW, id, NULL, 1,
&attrs[(2+M)*j+1]);
attrs[(2+M)*j+1] = *LFSR_ATTR(
&attrs[1+(2+M)*j+1]);
attrs[1+(2+M)*j+1] = *LFSR_ATTR(
MKREG, id, names[perm[j] % 6], 4,
&attrs[(2+M)*j+2]);
&attrs[1+(2+M)*j+2]);
for (unsigned u = 0; u < M; u++) {
attrs[(2+M)*j+2+u] = *LFSR_ATTR(
attrs[1+(2+M)*j+2+u] = *LFSR_ATTR(
UATTR(u+1), id, names[perm[j] % 6], 2,
&attrs[(2+M)*j+2+u+1]);
&attrs[1+(2+M)*j+2+u+1]);
}
}
if (w > 0) {
attrs[(2+M)*w-1].next = NULL;
if (w >= 0) {
attrs[1+(2+M)*w-1].next = NULL;
}
attrs[(2+M)*N-1].next = NULL;
struct lfsr_attr *written = w > 0 ? &attrs[0] : NULL;
struct lfsr_attr *unwritten = w < N ? &attrs[(2+M)*w] : NULL;
attrs[1+(2+M)*N-1].next = NULL;
const struct lfsr_attr *written
= w >= 0 ? &attrs[0]
: NULL;
const struct lfsr_attr *unwritten
= w < 0 ? &attrs[0]
: w < N ? &attrs[1+(2+M)*w]
: NULL;
// create rbyd with written attr
rbyd = init_rbyd;
@@ -12028,9 +12031,16 @@ code = '''
lfsr_rbyd_fetch(&lfs, &rbyd,
rbyd.block, cfg->block_size, NULL) => 0;
assert(rbyd.weight == w);
assert(rbyd.weight == (w >= 0 ? w : 0));
// test lookup both written/unwritten
lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten,
LFSR_TAG_UATTR(3), -1,
&tag_, &id_, &data_) => 0;
assert(tag_ == LFSR_TAG_UATTR(3));
assert(id_ == -1);
assert(lfsr_data_len(data_) == 9);
for (unsigned j = 0; j < N; j++) {
lfsr_rbyd_pendingget(&lfs, &rbyd, unwritten,
LFSR_TAG_MKREG, j, buffer, 4) => 4;
@@ -12046,28 +12056,33 @@ code = '''
// test traverse both written/unwritten
tag_ = 0;
id_ = -1;
lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten,
lfsr_tag_next(tag_), id_,
&tag_, &id_, &data_) => 0;
assert(tag_ == LFSR_TAG_UATTR(3));
assert(id_ == -1);
assert(lfsr_data_len(data_) == 9);
for (unsigned j = 0; j < N; j++) {
lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten,
lfsr_tag_next(tag_), id_,
&tag_, &id_, &data_, &weight_) => 0;
&tag_, &id_, &data_) => 0;
assert(tag_ == LFSR_TAG_MKREG);
assert(id_ == j);
assert(lfsr_data_len(data_) == 4);
assert(weight_ == 1);
for (unsigned u = 0; u < M; u++) {
lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten,
lfsr_tag_next(tag_), id_,
&tag_, &id_, &data_, &weight_) => 0;
&tag_, &id_, &data_) => 0;
assert(tag_ == LFSR_TAG_UATTR(u+1));
assert(id_ == j);
assert(lfsr_data_len(data_) == 2);
assert(weight_ == 1);
}
}
lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten,
lfsr_tag_next(tag_), id_,
&tag_, &id_, &data_, NULL) => LFS_ERR_NOENT;
&tag_, &id_, &data_) => LFS_ERR_NOENT;
}
// next permutation using Heap's algorithm
@@ -12113,7 +12128,6 @@ code = '''
lfsr_tag_t tag_ = 0;
lfs_ssize_t id_ = -1;
lfsr_data_t data_ = LFSR_DATA_NULL;
lfs_size_t weight_ = 0;
const uint8_t names[6][4] = {
"\xaa\xaa\xaa\xaa",
"\xbb\xbb\xbb\xbb",
@@ -12165,8 +12179,8 @@ code = '''
MKREG, id*W+W-1, names[perm[j] % 6], 4,
(j+1 < N && j+1 != w) ? &attrs[2*j+2] : NULL);
}
struct lfsr_attr *written = w > 0 ? &attrs[0] : NULL;
struct lfsr_attr *unwritten = w < N ? &attrs[2*w] : NULL;
const struct lfsr_attr *written = w > 0 ? &attrs[0] : NULL;
const struct lfsr_attr *unwritten = w < N ? &attrs[2*w] : NULL;
// create rbyd with written attr
rbyd = init_rbyd;
@@ -12190,15 +12204,14 @@ code = '''
for (unsigned j = 0; j < N; j++) {
lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten,
lfsr_tag_next(tag_), id_,
&tag_, &id_, &data_, &weight_) => 0;
&tag_, &id_, &data_) => 0;
assert(tag_ == LFSR_TAG_MKREG);
assert(id_ == j*W+W-1);
assert(lfsr_data_len(data_) == 4);
assert(weight_ == W);
}
lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten,
lfsr_tag_next(tag_), id_,
&tag_, &id_, &data_, NULL) => LFS_ERR_NOENT;
&tag_, &id_, &data_) => LFS_ERR_NOENT;
}
// next permutation using Heap's algorithm