From 7f16c6e473a65a653588b209448771f9ddc27fd9 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 1 Mar 2023 10:01:42 -0600 Subject: [PATCH] Implementation of lfsr_rbyd_pendinglookup in one pass This ends up surprisingly tricky with sparse ids. I feel like I'm missing a simpler solution, but this at least proves an implementation is possible. The implementation here does a single pass through the attributes backwards (which should probably be changed from a linked-list), keeping track of the best matching tag/id while updating everything based on grows/shrinks. Once we find the source of the best id we adjust things back to the pending id space. The implementation here only works with some significant caveats: 1. This solution might be able to find the id weights by keeping track of a lower bound, but it would be difficult and add complexity, so we don't do it. Really lfsr_rbyd_pendinglookup is only going to be used in full traversals as a part of compaction/splitting, so weight can be derived trivially from neighboring ids. 2. We don't know the difference between grows/shrinks used to change a branch's weight and used to create/delete ids. This is a bit of a problem here, but we can work around it by assuming that non-destructive grows/shrinks are always on the lower edge of a weighted id. Fortunately this assumption is only needed for in-flight attrs in lfsr_rbyd_pendinglookup, so this is not a requirement on-disk or in future implemenations. --- lfs.c | 190 +++++++++++++++++++++++-------------------- tests/test_rbyd.toml | 101 +++++++++++++---------- 2 files changed, 159 insertions(+), 132 deletions(-) diff --git a/lfs.c b/lfs.c index 5871d997..7bcf94b6 100644 --- a/lfs.c +++ b/lfs.c @@ -1615,23 +1615,30 @@ static lfs_ssize_t lfsr_rbyd_get(lfs_t *lfs, const lfsr_rbyd_t *rbyd, static int lfsr_rbyd_pendinglookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd, const struct lfsr_attr *attrs, lfsr_tag_t tag, lfs_ssize_t id, - lfsr_tag_t *tag_, lfs_ssize_t *id_, - lfsr_data_t *data_, lfs_size_t *weight_) { - printf("pendinglookup(%x, %d)\n", tag, id); + lfsr_tag_t *tag_, lfs_ssize_t *id_, lfsr_data_t *data_) { + // For this lookup to work it requires a lot of caveats: + // + // 1. Finding the weight is much more difficult when attrs aren't on disk + // so we don't do this. Note that weight can still be derived during + // traversal by diffing ids. + // + // 2. Our grow/shrink checks expect the grow/shrink ids to always be on the + // lowest id. This is notably different from our rbyd bias. Fortunately + // this is only a requirement of in-flight attrs so this isn't a + // requirement on-disk. again:; // tag must be non-zero! zero tags may deceptively look like they work but // fail when the tree contains a deleted id0 LFS_ASSERT(tag != 0); - lfsr_tag_t tag__ = 0xffff; - lfsr_data_t data__ = LFSR_DATA_NULL; - lfs_size_t weight = 0; - const struct lfsr_attr *attrs__ = attrs; + // keep track of best id/tag and upper/lower bounds to determine weight + lfs_ssize_t id__ = id; + lfs_ssize_t best_id = -2; + lfsr_tag_t best_tag = 0; + lfsr_data_t best_data = LFSR_DATA_NULL; - // first search backwards through tags to find the smallest, non-deleted, - // >= id, then search through tags in-order to find the most recent update, - // two passes are required to adjust for weight changes. + // search through our tags backwards to figure out the best tag/id // TODO hmm, reverse iteration over a linked-list? this is a bad design unsigned attr_count = 0; for (const struct lfsr_attr *attr = attrs; attr; attr = attr->next) { @@ -1644,112 +1651,119 @@ again:; } if (attr->tag == LFSR_TAG_GROW) { - printf("g %d %d\n", attr->id, attr->size); - if (id >= attr->id) { - if (id < attr->id + attr->size) { - // TODO - id = attr->id + attr->size-1; - weight = attr->size; - attrs__ = attr->next; - goto grown; + // found grow which includes both target and best ids? this + // must be the source of the best tag + if (attr->id <= id__ + && best_id != -2 + && attr->id+(lfs_ssize_t)attr->size > best_id) { + goto found; + + // found grow which only includes target id? this must be a + // weight-changing grow so we can just adjust our target id to + // follow the upper edge of the grow + } else if (attr->id <= id__ + && attr->id+(lfs_ssize_t)attr->size > id__) { + id__ = attr->id; + tag = 0x10; + id += id__-attr->id+1; + + // adjust ids + } else if (attr->id <= id__) { + id__ -= attr->size; + if (best_id != -2) { + best_id -= attr->size; } - id -= attr->size; + + // use grow as upper bound + } else if (best_id == -2 || attr->id <= best_id) { + best_id = attr->id-1; + best_tag = 0; } } else if (attr->tag == LFSR_TAG_SHRINK) { - if (id >= attr->id) { - id += attr->size; + // adjust ids + if (attr->id <= id__) { + id__ += attr->size; + if (best_id != -2) { + best_id += attr->size; + } + + // use shrink as upper bound + } else if (best_id == -2 || attr->id <= best_id) { + best_id = attr->id-1; + best_tag = 0; } + } else if (attr->tag == LFSR_TAG_FROM) { // TODO LFS_ASSERT(false); + } else { + // found better tag? + if ((attr->id > id__ + || (attr->id == id__ + && lfsr_tag_key(attr->tag) + >= lfsr_tag_key(tag))) + && (best_id == -2 + || attr->id < best_id + || (attr->id == best_id + && (!best_tag + || lfsr_tag_key(attr->tag) + < lfsr_tag_key(best_tag))))) { + best_id = attr->id; + best_tag = attr->tag; + best_data = LFSR_DATA_BUF(attr->buffer, attr->size); + } } } - // this id can't exist in our rbyd - if (id >= (lfs_ssize_t)rbyd->weight) { - printf("? %d >= %d\n", id, rbyd->weight); - return LFS_ERR_NOENT; - } - - // if not created in attr list our id must have been created in the rbyd - lfs_off_t off__; - lfs_size_t size__; - int err = lfsr_rbyd_lookup(lfs, rbyd, tag, id, - &tag__, &id, &weight, &off__, &size__); + // try to found our id/tag on disk + lfsr_tag_t rbyd_tag; + lfs_ssize_t rbyd_id; + lfs_off_t rbyd_off; + lfs_size_t rbyd_size; + int err = lfsr_rbyd_lookup(lfs, rbyd, tag, id__, + &rbyd_tag, &rbyd_id, NULL, &rbyd_off, &rbyd_size); if (err && err != LFS_ERR_NOENT) { return err; } if (err != LFS_ERR_NOENT) { - data__ = LFSR_DATA_DISK(rbyd->block, off__, size__); - } - -grown:; - // TODO different way to encode weight? - lfs_ssize_t lower = id-weight+1; - printf("raw %d %d (%d)\n", id, weight, lower); - - // now replay the attr list, keeping track of changes to id, weight, tag - for (const struct lfsr_attr *attr = attrs__; attr; attr = attr->next) { - if (attr->tag == LFSR_TAG_GROW) { - if (lower >= attr->id) { - lower += attr->size; - } - if (id >= attr->id) { - id += attr->size; - } - } else if (attr->tag == LFSR_TAG_SHRINK) { - if (lower >= attr->id) { - lower -= attr->size; - } - if (id >= attr->id) { - id -= attr->size; - } - } else if (attr->tag == LFSR_TAG_FROM) { - // TODO - LFS_ASSERT(false); - - } else if (attr->id == id - && lfsr_tag_key(attr->tag) >= lfsr_tag_key(tag) - && lfsr_tag_key(attr->tag) <= lfsr_tag_key(tag__)) { - tag__ = attr->tag; - data__ = LFSR_DATA_BUF(attr->buffer, attr->size); + // found a better tag? + if (best_id == -2 + || rbyd_id < best_id + || (rbyd_id == best_id + && (!best_tag + || lfsr_tag_key(rbyd_tag) + < lfsr_tag_key(best_tag)))) { + best_id = rbyd_id; + best_tag = rbyd_tag; + best_data = LFSR_DATA_DISK(rbyd->block, rbyd_off, rbyd_size); } } - // TODO if we can't get rid of this we can at least move it into the - // below condition - weight = id-lower+1; - printf("fix %d %d (%d)\n", id, weight, lower); +found:; + // no better id found + if (best_id == -2) { + return LFS_ERR_NOENT; + } - // not found? increase id - if (tag__ == 0xffff) { - tag = 0x10; - id = id + 1; + // no tag found? increase id + if (!best_tag || lfsr_tag_isrm(best_tag)) { + tag = best_tag + 0x10; + id = best_id+(id-id__) + 1; goto again; } - // found rm? should continue - if (lfsr_tag_isrm(tag__)) { - tag = tag__ + 0x10; - goto again; - } - - // found - + // found an id/tag // TODO how many of these should be conditional? if (tag_) { - *tag_ = tag__; + *tag_ = best_tag; } if (id_) { - *id_ = id; + *id_ = best_id+(id-id__); } if (data_) { - *data_ = data__; - } - if (weight_) { - *weight_ = weight; + *data_ = best_data; } return 0; @@ -1763,7 +1777,7 @@ static lfs_ssize_t lfsr_rbyd_pendingget(lfs_t *lfs, lfs_ssize_t id_; lfsr_data_t data_; int err = lfsr_rbyd_pendinglookup(lfs, rbyd, attrs, tag, id, - &tag_, &id_, &data_, NULL); + &tag_, &id_, &data_); if (err) { return err; } diff --git a/tests/test_rbyd.toml b/tests/test_rbyd.toml index f98bb85e..84b63524 100644 --- a/tests/test_rbyd.toml +++ b/tests/test_rbyd.toml @@ -11416,7 +11416,6 @@ code = ''' lfsr_tag_t tag_ = 0; lfs_ssize_t id_ = -1; lfsr_data_t data_ = LFSR_DATA_NULL; - lfs_size_t weight_ = 0; // test all permutations of a given size uint16_t perm[N]; @@ -11448,8 +11447,8 @@ code = ''' UATTR(perm[j]+1), -1, "\xaa\xaa\xaa\xaa", 4, (j+1 < N && j+1 != w) ? &attrs[j+1] : NULL); } - struct lfsr_attr *written = w > 0 ? &attrs[0] : NULL; - struct lfsr_attr *unwritten = w < N ? &attrs[w] : NULL; + const struct lfsr_attr *written = w > 0 ? &attrs[0] : NULL; + const struct lfsr_attr *unwritten = w < N ? &attrs[w] : NULL; // create rbyd with written attr rbyd = init_rbyd; @@ -11463,11 +11462,10 @@ code = ''' for (unsigned j = 0; j < N; j++) { lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten, LFSR_TAG_UATTR(j+1), -1, - &tag_, &id_, &data_, &weight_) => 0; + &tag_, &id_, &data_) => 0; assert(tag_ == LFSR_TAG_UATTR(j+1)); assert(id_ == -1); assert(lfsr_data_len(data_) == 4); - assert(weight_ == 0); } // test traverse both written/unwritten @@ -11476,15 +11474,14 @@ code = ''' for (unsigned j = 0; j < N; j++) { lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten, lfsr_tag_next(tag_), id_, - &tag_, &id_, &data_, &weight_) => 0; + &tag_, &id_, &data_) => 0; assert(tag_ == LFSR_TAG_UATTR(j+1)); assert(id_ == -1); assert(lfsr_data_len(data_) == 4); - assert(weight_ == 0); } lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten, lfsr_tag_next(tag_), id_, - &tag_, &id_, &data_, &weight_) => LFS_ERR_NOENT; + &tag_, &id_, &data_) => LFS_ERR_NOENT; } // next permutation using Heap's algorithm @@ -11608,6 +11605,7 @@ code = ''' } } } + const struct lfsr_attr *unwritten = w < N ? attrs : NULL; // compare rbyd vs simulation printf("expd: ["); @@ -11625,7 +11623,7 @@ code = ''' printf("rbyd: ["); first = true; for (unsigned attr = 0; attr < N; attr++) { - lfs_ssize_t size = lfsr_rbyd_pendingget(&lfs, &rbyd, attrs, + lfs_ssize_t size = lfsr_rbyd_pendingget(&lfs, &rbyd, unwritten, LFSR_TAG_UATTR(attr), -1, buffer, 4); if (size >= 0) { if (!first) { @@ -11638,7 +11636,7 @@ code = ''' printf("]\n"); for (unsigned attr = 0; attr < N; attr++) { - lfs_ssize_t size = lfsr_rbyd_pendingget(&lfs, &rbyd, attrs, + lfs_ssize_t size = lfsr_rbyd_pendingget(&lfs, &rbyd, unwritten, LFSR_TAG_UATTR(attr), -1, buffer, 4); if (sim[attr]) { assert(size == 1); @@ -11677,7 +11675,6 @@ code = ''' lfsr_tag_t tag_ = 0; lfs_ssize_t id_ = -1; lfsr_data_t data_ = LFSR_DATA_NULL; - lfs_size_t weight_ = 0; const uint8_t names[6][4] = { "\xaa\xaa\xaa\xaa", "\xbb\xbb\xbb\xbb", @@ -11729,8 +11726,8 @@ code = ''' MKREG, id, names[perm[j] % 6], 4, (j+1 < N && j+1 != w) ? &attrs[2*j+2] : NULL); } - struct lfsr_attr *written = w > 0 ? &attrs[0] : NULL; - struct lfsr_attr *unwritten = w < N ? &attrs[2*w] : NULL; + const struct lfsr_attr *written = w > 0 ? &attrs[0] : NULL; + const struct lfsr_attr *unwritten = w < N ? &attrs[2*w] : NULL; // create rbyd with written attr rbyd = init_rbyd; @@ -11754,15 +11751,14 @@ code = ''' for (unsigned j = 0; j < N; j++) { lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten, lfsr_tag_next(tag_), id_, - &tag_, &id_, &data_, &weight_) => 0; + &tag_, &id_, &data_) => 0; assert(tag_ == LFSR_TAG_MKREG); assert(id_ == j); assert(lfsr_data_len(data_) == 4); - assert(weight_ == 1); } lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten, lfsr_tag_next(tag_), id_, - &tag_, &id_, &data_, NULL) => LFS_ERR_NOENT; + &tag_, &id_, &data_) => LFS_ERR_NOENT; } // next permutation using Heap's algorithm @@ -11896,6 +11892,7 @@ code = ''' } } } + const struct lfsr_attr *unwritten = w < N ? attrs : NULL; // compare rbyd vs simulation printf("expd: ["); @@ -11908,7 +11905,7 @@ code = ''' printf("]\n"); printf("rbyd: ["); for (lfs_ssize_t id = 0; id < (lfs_ssize_t)count; id++) { - lfs_ssize_t size = lfsr_rbyd_pendingget(&lfs, &rbyd, attrs, + lfs_ssize_t size = lfsr_rbyd_pendingget(&lfs, &rbyd, unwritten, LFSR_TAG_MKREG, id, buffer, 4); if (size >= 0) { printf("%.*s", size, buffer); @@ -11922,7 +11919,7 @@ code = ''' printf("]\n"); for (lfs_ssize_t id = 0; id < (lfs_ssize_t)count; id++) { - lfsr_rbyd_pendingget(&lfs, &rbyd, attrs, + lfsr_rbyd_pendingget(&lfs, &rbyd, unwritten, LFSR_TAG_MKREG, id, buffer, 4) => 1; assert(memcmp(&sim[id], buffer, 1) == 0); } @@ -11957,7 +11954,6 @@ code = ''' lfsr_tag_t tag_ = 0; lfs_ssize_t id_ = -1; lfsr_data_t data_ = LFSR_DATA_NULL; - lfs_size_t weight_ = 0; const uint8_t names[6][4] = { "\xaa\xaa\xaa\xaa", "\xbb\xbb\xbb\xbb", @@ -11980,7 +11976,7 @@ code = ''' while (i < N) { // test each number of written/unwritten tags, this gives us a quick // way to test several unwritten situations - for (unsigned w = 0; w <= N; w++) { + for (signed w = -1; w <= N; w++) { // print permutation to help debugging printf("--- permutation: ["); for (unsigned j = 0; j < N; j++) { @@ -11992,7 +11988,9 @@ code = ''' printf("], written: %d/%jd ---\n", w, N); // build the attribute lists for the current permutation - struct lfsr_attr attrs[(2+M)*N]; + struct lfsr_attr attrs[1+(2+M)*N]; + attrs[0] = *LFSR_ATTR(UATTR(3), -1, "unrelated", 9, &attrs[1]); + for (unsigned j = 0; j < N; j++) { // adjust id based on future insertions uint16_t id = perm[j]; @@ -12002,24 +12000,29 @@ code = ''' } } - attrs[(2+M)*j+0] = *LFSR_ATTR( + attrs[1+(2+M)*j+0] = *LFSR_ATTR( GROW, id, NULL, 1, - &attrs[(2+M)*j+1]); - attrs[(2+M)*j+1] = *LFSR_ATTR( + &attrs[1+(2+M)*j+1]); + attrs[1+(2+M)*j+1] = *LFSR_ATTR( MKREG, id, names[perm[j] % 6], 4, - &attrs[(2+M)*j+2]); + &attrs[1+(2+M)*j+2]); for (unsigned u = 0; u < M; u++) { - attrs[(2+M)*j+2+u] = *LFSR_ATTR( + attrs[1+(2+M)*j+2+u] = *LFSR_ATTR( UATTR(u+1), id, names[perm[j] % 6], 2, - &attrs[(2+M)*j+2+u+1]); + &attrs[1+(2+M)*j+2+u+1]); } } - if (w > 0) { - attrs[(2+M)*w-1].next = NULL; + if (w >= 0) { + attrs[1+(2+M)*w-1].next = NULL; } - attrs[(2+M)*N-1].next = NULL; - struct lfsr_attr *written = w > 0 ? &attrs[0] : NULL; - struct lfsr_attr *unwritten = w < N ? &attrs[(2+M)*w] : NULL; + attrs[1+(2+M)*N-1].next = NULL; + const struct lfsr_attr *written + = w >= 0 ? &attrs[0] + : NULL; + const struct lfsr_attr *unwritten + = w < 0 ? &attrs[0] + : w < N ? &attrs[1+(2+M)*w] + : NULL; // create rbyd with written attr rbyd = init_rbyd; @@ -12028,9 +12031,16 @@ code = ''' lfsr_rbyd_fetch(&lfs, &rbyd, rbyd.block, cfg->block_size, NULL) => 0; - assert(rbyd.weight == w); + assert(rbyd.weight == (w >= 0 ? w : 0)); // test lookup both written/unwritten + lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten, + LFSR_TAG_UATTR(3), -1, + &tag_, &id_, &data_) => 0; + assert(tag_ == LFSR_TAG_UATTR(3)); + assert(id_ == -1); + assert(lfsr_data_len(data_) == 9); + for (unsigned j = 0; j < N; j++) { lfsr_rbyd_pendingget(&lfs, &rbyd, unwritten, LFSR_TAG_MKREG, j, buffer, 4) => 4; @@ -12046,28 +12056,33 @@ code = ''' // test traverse both written/unwritten tag_ = 0; id_ = -1; + lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten, + lfsr_tag_next(tag_), id_, + &tag_, &id_, &data_) => 0; + assert(tag_ == LFSR_TAG_UATTR(3)); + assert(id_ == -1); + assert(lfsr_data_len(data_) == 9); + for (unsigned j = 0; j < N; j++) { lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten, lfsr_tag_next(tag_), id_, - &tag_, &id_, &data_, &weight_) => 0; + &tag_, &id_, &data_) => 0; assert(tag_ == LFSR_TAG_MKREG); assert(id_ == j); assert(lfsr_data_len(data_) == 4); - assert(weight_ == 1); for (unsigned u = 0; u < M; u++) { lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten, lfsr_tag_next(tag_), id_, - &tag_, &id_, &data_, &weight_) => 0; + &tag_, &id_, &data_) => 0; assert(tag_ == LFSR_TAG_UATTR(u+1)); assert(id_ == j); assert(lfsr_data_len(data_) == 2); - assert(weight_ == 1); } } lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten, lfsr_tag_next(tag_), id_, - &tag_, &id_, &data_, NULL) => LFS_ERR_NOENT; + &tag_, &id_, &data_) => LFS_ERR_NOENT; } // next permutation using Heap's algorithm @@ -12113,7 +12128,6 @@ code = ''' lfsr_tag_t tag_ = 0; lfs_ssize_t id_ = -1; lfsr_data_t data_ = LFSR_DATA_NULL; - lfs_size_t weight_ = 0; const uint8_t names[6][4] = { "\xaa\xaa\xaa\xaa", "\xbb\xbb\xbb\xbb", @@ -12165,8 +12179,8 @@ code = ''' MKREG, id*W+W-1, names[perm[j] % 6], 4, (j+1 < N && j+1 != w) ? &attrs[2*j+2] : NULL); } - struct lfsr_attr *written = w > 0 ? &attrs[0] : NULL; - struct lfsr_attr *unwritten = w < N ? &attrs[2*w] : NULL; + const struct lfsr_attr *written = w > 0 ? &attrs[0] : NULL; + const struct lfsr_attr *unwritten = w < N ? &attrs[2*w] : NULL; // create rbyd with written attr rbyd = init_rbyd; @@ -12190,15 +12204,14 @@ code = ''' for (unsigned j = 0; j < N; j++) { lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten, lfsr_tag_next(tag_), id_, - &tag_, &id_, &data_, &weight_) => 0; + &tag_, &id_, &data_) => 0; assert(tag_ == LFSR_TAG_MKREG); assert(id_ == j*W+W-1); assert(lfsr_data_len(data_) == 4); - assert(weight_ == W); } lfsr_rbyd_pendinglookup(&lfs, &rbyd, unwritten, lfsr_tag_next(tag_), id_, - &tag_, &id_, &data_, NULL) => LFS_ERR_NOENT; + &tag_, &id_, &data_) => LFS_ERR_NOENT; } // next permutation using Heap's algorithm