Partial implementation of B-tree name split/lookup
Name lookup brings back the O(m') scan-during-fetch approach of the previous metadata layout. Since our rbyd trees map id+attr pairs and not actual names, this beats the alternative O(m log(m)) scan of the tree. Though tree searching does only include the current attributes, where as scanning during fetch needs to also look at outdated attributes. Which may make the winner less obvious depending on how we find the rbyd. But being able to do the search in the same pass as fetch is an extra plus. --- What turned out to be surprisingly complicated was the propagation of names during B-tree splits and merges. The on-disk reference, lfsr_data_t, does most of the heavy lifting here, but there's just a lot of corner cases to consider. At the moment this isn't working due to outdated names on the leading entries of the rbyds, but to fix this bigger changes to the B-tree layout may be needed.
This commit is contained in:
Christopher Haster
+950
-5
@@ -308,7 +308,7 @@ code = '''
|
||||
# try larger trees, when exactly a tree splits depends on the disk geometry, so
|
||||
# we don't really have a better way of testing multi-rbyd trees
|
||||
[cases.test_btree_push]
|
||||
defines.N = [4, 8, 16, 32, 64, 128, 256, 512, 1024]
|
||||
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
|
||||
in = 'lfs.c'
|
||||
code = '''
|
||||
lfs_t lfs;
|
||||
@@ -2106,7 +2106,7 @@ code = '''
|
||||
|
||||
# test btree splits
|
||||
[cases.test_btree_split]
|
||||
defines.N = [4, 8, 16, 32, 64, 128, 256, 512, 1024]
|
||||
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
|
||||
in = 'lfs.c'
|
||||
code = '''
|
||||
lfs_t lfs;
|
||||
@@ -2125,7 +2125,7 @@ code = '''
|
||||
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1,
|
||||
&alphas[0 % 26], 1) => 0;
|
||||
for (lfs_size_t i = 1; i < N; i++) {
|
||||
lfsr_btree_split(&lfs, &btree, i-1,
|
||||
lfsr_btree_split(&lfs, &btree, i-1, NULL, 0,
|
||||
LFSR_TAG_INLINED, 1, &alphas[(i-1) % 26], 1,
|
||||
LFSR_TAG_INLINED, 1, &alphas[(i-0) % 26], 1) => 0;
|
||||
}
|
||||
@@ -2206,7 +2206,7 @@ code = '''
|
||||
lfs_size_t id = TEST_PRNG(&prng) % sim_size;
|
||||
|
||||
// split btree
|
||||
lfsr_btree_split(&lfs, &btree, id,
|
||||
lfsr_btree_split(&lfs, &btree, id, NULL, 0,
|
||||
LFSR_TAG_INLINED, 1, &alphas[i % 26], 1,
|
||||
LFSR_TAG_INLINED, 1, &uppers[i % 26], 1) => 0;
|
||||
|
||||
@@ -2281,7 +2281,7 @@ code = '''
|
||||
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, W,
|
||||
&alphas[0 % 26], 1) => 0;
|
||||
for (lfs_size_t i = 1; i < N; i++) {
|
||||
lfsr_btree_split(&lfs, &btree, (i-1)*W+W-1,
|
||||
lfsr_btree_split(&lfs, &btree, (i-1)*W+W-1, NULL, 0,
|
||||
LFSR_TAG_INLINED, W, &alphas[(i-1) % 26], 1,
|
||||
LFSR_TAG_INLINED, W, &alphas[(i-0) % 26], 1) => 0;
|
||||
}
|
||||
@@ -2376,6 +2376,7 @@ code = '''
|
||||
|
||||
// split btree
|
||||
lfsr_btree_split(&lfs, &btree, weighted_id+sim_weights[id]-1,
|
||||
NULL, 0,
|
||||
LFSR_TAG_INLINED, weight1, &alphas[i % 26], 1,
|
||||
LFSR_TAG_INLINED, weight2, &uppers[i % 26], 1) => 0;
|
||||
|
||||
@@ -2758,3 +2759,947 @@ code = '''
|
||||
}
|
||||
'''
|
||||
|
||||
|
||||
# test key-value btrees
|
||||
[cases.test_btree_find_one]
|
||||
in = 'lfs.c'
|
||||
code = '''
|
||||
lfs_t lfs;
|
||||
lfs_init(&lfs, cfg) => 0;
|
||||
// create free lookahead
|
||||
memset(lfs.free.buffer, 0, lfs.cfg->lookahead_size);
|
||||
lfs.free.off = 0;
|
||||
lfs.free.size = lfs_min(8*lfs.cfg->lookahead_size,
|
||||
lfs.cfg->block_count);
|
||||
lfs.free.i = 0;
|
||||
lfs_alloc_ack(&lfs);
|
||||
|
||||
// create a single-entry tree
|
||||
lfsr_btree_t btree = LFSR_BTREE_NULL;
|
||||
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, "0", 1) => 0;
|
||||
printf("btree: 0x%x.%x 0x%x w%d\n",
|
||||
btree.u.trunk.block,
|
||||
btree.u.trunk.limit,
|
||||
btree.tag,
|
||||
btree.weight);
|
||||
assert(btree.weight == 1);
|
||||
|
||||
// try to find tags
|
||||
uint8_t buffer[4];
|
||||
lfsr_tag_t tag_;
|
||||
lfs_size_t id_;
|
||||
lfs_size_t weight_;
|
||||
|
||||
lfsr_btree_find(&lfs, &btree, "aaa", 3,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == 0);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, "0", 1) == 0);
|
||||
|
||||
lfsr_btree_find(&lfs, &btree, "aab", 3,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == 0);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, "0", 1) == 0);
|
||||
'''
|
||||
|
||||
[cases.test_btree_find_two]
|
||||
in = 'lfs.c'
|
||||
code = '''
|
||||
lfs_t lfs;
|
||||
lfs_init(&lfs, cfg) => 0;
|
||||
// create free lookahead
|
||||
memset(lfs.free.buffer, 0, lfs.cfg->lookahead_size);
|
||||
lfs.free.off = 0;
|
||||
lfs.free.size = lfs_min(8*lfs.cfg->lookahead_size,
|
||||
lfs.cfg->block_count);
|
||||
lfs.free.i = 0;
|
||||
lfs_alloc_ack(&lfs);
|
||||
|
||||
// create a two-entry tree
|
||||
lfsr_btree_t btree = LFSR_BTREE_NULL;
|
||||
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, "0", 1) => 0;
|
||||
lfsr_btree_split(&lfs, &btree, 0, "aab", 3,
|
||||
LFSR_TAG_INLINED, 1, "0", 1,
|
||||
LFSR_TAG_INLINED, 1, "1", 1) => 0;
|
||||
printf("btree: 0x%x.%x 0x%x w%d\n",
|
||||
btree.u.trunk.block,
|
||||
btree.u.trunk.limit,
|
||||
btree.tag,
|
||||
btree.weight);
|
||||
assert(btree.weight == 2);
|
||||
|
||||
// try to find tags
|
||||
uint8_t buffer[4];
|
||||
lfsr_tag_t tag_;
|
||||
lfs_size_t id_;
|
||||
lfs_size_t weight_;
|
||||
|
||||
lfsr_btree_find(&lfs, &btree, "aaa", 3,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == 0);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, "0", 1) == 0);
|
||||
|
||||
lfsr_btree_find(&lfs, &btree, "aab", 3,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == 1);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, "1", 1) == 0);
|
||||
|
||||
lfsr_btree_find(&lfs, &btree, "aac", 3,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == 1);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, "1", 1) == 0);
|
||||
'''
|
||||
|
||||
[cases.test_btree_find_three]
|
||||
in = 'lfs.c'
|
||||
code = '''
|
||||
lfs_t lfs;
|
||||
lfs_init(&lfs, cfg) => 0;
|
||||
// create free lookahead
|
||||
memset(lfs.free.buffer, 0, lfs.cfg->lookahead_size);
|
||||
lfs.free.off = 0;
|
||||
lfs.free.size = lfs_min(8*lfs.cfg->lookahead_size,
|
||||
lfs.cfg->block_count);
|
||||
lfs.free.i = 0;
|
||||
lfs_alloc_ack(&lfs);
|
||||
|
||||
// create a two-entry tree
|
||||
lfsr_btree_t btree = LFSR_BTREE_NULL;
|
||||
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, "0", 1) => 0;
|
||||
lfsr_btree_split(&lfs, &btree, 0, "aab", 3,
|
||||
LFSR_TAG_INLINED, 1, "0", 1,
|
||||
LFSR_TAG_INLINED, 1, "1", 1) => 0;
|
||||
lfsr_btree_split(&lfs, &btree, 1, "aac", 3,
|
||||
LFSR_TAG_INLINED, 1, "1", 1,
|
||||
LFSR_TAG_INLINED, 1, "2", 1) => 0;
|
||||
printf("btree: 0x%x.%x 0x%x w%d\n",
|
||||
btree.u.trunk.block,
|
||||
btree.u.trunk.limit,
|
||||
btree.tag,
|
||||
btree.weight);
|
||||
assert(btree.weight == 3);
|
||||
|
||||
// try to find tags
|
||||
uint8_t buffer[4];
|
||||
lfsr_tag_t tag_;
|
||||
lfs_size_t id_;
|
||||
lfs_size_t weight_;
|
||||
|
||||
lfsr_btree_find(&lfs, &btree, "aaa", 3,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == 0);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, "0", 1) == 0);
|
||||
|
||||
lfsr_btree_find(&lfs, &btree, "aab", 3,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == 1);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, "1", 1) == 0);
|
||||
|
||||
lfsr_btree_find(&lfs, &btree, "aac", 3,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == 2);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, "2", 1) == 0);
|
||||
|
||||
lfsr_btree_find(&lfs, &btree, "aad", 3,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == 2);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, "2", 1) == 0);
|
||||
'''
|
||||
|
||||
[cases.test_btree_find_three_backwards]
|
||||
in = 'lfs.c'
|
||||
code = '''
|
||||
lfs_t lfs;
|
||||
lfs_init(&lfs, cfg) => 0;
|
||||
// create free lookahead
|
||||
memset(lfs.free.buffer, 0, lfs.cfg->lookahead_size);
|
||||
lfs.free.off = 0;
|
||||
lfs.free.size = lfs_min(8*lfs.cfg->lookahead_size,
|
||||
lfs.cfg->block_count);
|
||||
lfs.free.i = 0;
|
||||
lfs_alloc_ack(&lfs);
|
||||
|
||||
// create a two-entry tree
|
||||
lfsr_btree_t btree = LFSR_BTREE_NULL;
|
||||
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, "0", 1) => 0;
|
||||
lfsr_btree_split(&lfs, &btree, 0, "aac", 3,
|
||||
LFSR_TAG_INLINED, 1, "1", 1,
|
||||
LFSR_TAG_INLINED, 1, "2", 1) => 0;
|
||||
lfsr_btree_split(&lfs, &btree, 0, "aab", 3,
|
||||
LFSR_TAG_INLINED, 1, "0", 1,
|
||||
LFSR_TAG_INLINED, 1, "1", 1) => 0;
|
||||
printf("btree: 0x%x.%x 0x%x w%d\n",
|
||||
btree.u.trunk.block,
|
||||
btree.u.trunk.limit,
|
||||
btree.tag,
|
||||
btree.weight);
|
||||
assert(btree.weight == 3);
|
||||
|
||||
// try to find tags
|
||||
uint8_t buffer[4];
|
||||
lfsr_tag_t tag_;
|
||||
lfs_size_t id_;
|
||||
lfs_size_t weight_;
|
||||
|
||||
lfsr_btree_find(&lfs, &btree, "aaa", 3,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == 0);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, "0", 1) == 0);
|
||||
|
||||
lfsr_btree_find(&lfs, &btree, "aab", 3,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == 1);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, "1", 1) == 0);
|
||||
|
||||
lfsr_btree_find(&lfs, &btree, "aac", 3,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == 2);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, "2", 1) == 0);
|
||||
|
||||
lfsr_btree_find(&lfs, &btree, "aad", 3,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == 2);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, "2", 1) == 0);
|
||||
'''
|
||||
|
||||
[cases.test_btree_find]
|
||||
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
|
||||
in = 'lfs.c'
|
||||
code = '''
|
||||
lfs_t lfs;
|
||||
lfs_init(&lfs, cfg) => 0;
|
||||
// create free lookahead
|
||||
memset(lfs.free.buffer, 0, lfs.cfg->lookahead_size);
|
||||
lfs.free.off = 0;
|
||||
lfs.free.size = lfs_min(8*lfs.cfg->lookahead_size,
|
||||
lfs.cfg->block_count);
|
||||
lfs.free.i = 0;
|
||||
lfs_alloc_ack(&lfs);
|
||||
|
||||
// create a tree with N elements
|
||||
lfsr_btree_t btree = LFSR_BTREE_NULL;
|
||||
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
|
||||
const char *nums = "0123456789";
|
||||
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1,
|
||||
&nums[0 % 10], 1) => 0;
|
||||
for (lfs_size_t i = 1; i < N; i++) {
|
||||
char name[3] = {
|
||||
alphas[(i/26/26) % 26], alphas[(i/26) % 26], alphas[i % 26]
|
||||
};
|
||||
lfsr_btree_split(&lfs, &btree, i-1, name, 3,
|
||||
LFSR_TAG_INLINED, 1, &nums[(i-1) % 10], 1,
|
||||
LFSR_TAG_INLINED, 1, &nums[(i-0) % 10], 1) => 0;
|
||||
}
|
||||
printf("btree: 0x%x.%x 0x%x w%d\n",
|
||||
btree.u.trunk.block,
|
||||
btree.u.trunk.limit,
|
||||
btree.tag,
|
||||
btree.weight);
|
||||
assert(btree.weight == N);
|
||||
|
||||
// try to find tags
|
||||
uint8_t buffer[4];
|
||||
lfsr_tag_t tag_;
|
||||
lfs_size_t id_;
|
||||
lfs_size_t weight_;
|
||||
|
||||
for (lfs_size_t i = 0; i < N; i++) {
|
||||
char name[3] = {
|
||||
alphas[(i/26/26) % 26], alphas[(i/26) % 26], alphas[i % 26]
|
||||
};
|
||||
|
||||
lfsr_btree_find(&lfs, &btree, name, 3,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == i);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, &nums[i % 10], 1) == 0);
|
||||
}
|
||||
'''
|
||||
|
||||
[cases.test_btree_find_fuzz]
|
||||
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
|
||||
defines.SAMPLES = 10
|
||||
# -1 => all pseudo-random seeds
|
||||
# n => reproduce a specific seed
|
||||
defines.SEED = -1
|
||||
in = 'lfs.c'
|
||||
code = '''
|
||||
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
|
||||
const char *nums = "0123456789";
|
||||
|
||||
// iterate through severals seeds that we can reproduce easily
|
||||
for (uint32_t seed = (SEED == -1 ? 1 : SEED);
|
||||
(SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
|
||||
seed++) {
|
||||
printf("--- seed: %d ---\n", seed);
|
||||
// create lfs here since we need to reset each iteration, we're
|
||||
// space constrained and we can't expect gc to work at this point
|
||||
lfs_t lfs;
|
||||
lfs_init(&lfs, cfg) => 0;
|
||||
// create free lookahead
|
||||
memset(lfs.free.buffer, 0, lfs.cfg->lookahead_size);
|
||||
lfs.free.off = 0;
|
||||
lfs.free.size = lfs_min(8*lfs.cfg->lookahead_size,
|
||||
lfs.cfg->block_count);
|
||||
lfs.free.i = 0;
|
||||
lfs_alloc_ack(&lfs);
|
||||
|
||||
// create a btree
|
||||
lfsr_btree_t btree = LFSR_BTREE_NULL;
|
||||
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1,
|
||||
&alphas[0 % 26], 1) => 0;
|
||||
|
||||
// set up a simulation to compare against
|
||||
//
|
||||
// fun fact this is slower than our actual tree! unfun fact this is
|
||||
// starting to be a problem...
|
||||
char *sim = malloc(N);
|
||||
char (*sim_names)[3] = malloc(N*3);
|
||||
lfs_size_t sim_size = 1;
|
||||
memset(sim, 0, N);
|
||||
memset(sim_names, 0, N*3);
|
||||
sim[0] = alphas[0 % 26];
|
||||
memcpy(&sim_names[0], "___", 3);
|
||||
|
||||
uint32_t prng = seed;
|
||||
for (lfs_size_t i = 1; i < N; i++) {
|
||||
// choose a pseudo-random name
|
||||
lfs_size_t x = TEST_PRNG(&prng) % (26*26*26);
|
||||
char name[3] = {
|
||||
alphas[(x/26/26) % 26], alphas[(x/26) % 26], alphas[x % 26]
|
||||
};
|
||||
|
||||
// find where to split
|
||||
lfs_size_t id = 0;
|
||||
while (id+1 < sim_size && memcmp(sim_names[id+1], name, 3) <= 0) {
|
||||
id += 1;
|
||||
}
|
||||
// just skip exact matches for now
|
||||
if (memcmp(sim_names[id], name, 3) == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// split btree
|
||||
lfsr_btree_split(&lfs, &btree, id, name, 3,
|
||||
LFSR_TAG_INLINED, 1, &nums[i % 10], 1,
|
||||
LFSR_TAG_INLINED, 1, &nums[i % 10], 1) => 0;
|
||||
|
||||
// split sim
|
||||
memmove(&sim[id+1], &sim[id], sim_size-id);
|
||||
memmove(&sim_names[id+1], &sim_names[id], (sim_size-id)*3);
|
||||
sim[id+0] = nums[i % 10];
|
||||
sim[id+1] = nums[i % 10];
|
||||
memcpy(&sim_names[id+1], name, 3);
|
||||
sim_size += 1;
|
||||
}
|
||||
|
||||
// check that btree matches sim
|
||||
printf("expd: [");
|
||||
bool first = true;
|
||||
for (lfs_size_t i = 0; i < sim_size; i++) {
|
||||
if (!first) {
|
||||
printf(", ");
|
||||
}
|
||||
first = false;
|
||||
printf("%.3s=%c", sim_names[i], sim[i]);
|
||||
}
|
||||
printf("]\n");
|
||||
printf("btree: 0x%x.%x 0x%x w%d\n",
|
||||
btree.u.trunk.block,
|
||||
btree.u.trunk.limit,
|
||||
btree.tag,
|
||||
btree.weight);
|
||||
assert(btree.weight == sim_size);
|
||||
|
||||
uint8_t buffer[4];
|
||||
lfsr_tag_t tag_;
|
||||
lfs_size_t id_;
|
||||
lfs_size_t weight_;
|
||||
for (lfs_size_t i = 0; i < sim_size; i++) {
|
||||
lfsr_btree_find(&lfs, &btree, sim_names[i], 3,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == i);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, &sim[i], 1) == 0);
|
||||
}
|
||||
|
||||
// clean up sim
|
||||
free(sim);
|
||||
free(sim_names);
|
||||
lfs_deinit(&lfs) => 0;
|
||||
}
|
||||
'''
|
||||
|
||||
[cases.test_btree_find_sparse]
|
||||
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
|
||||
defines.W = 5
|
||||
in = 'lfs.c'
|
||||
code = '''
|
||||
lfs_t lfs;
|
||||
lfs_init(&lfs, cfg) => 0;
|
||||
// create free lookahead
|
||||
memset(lfs.free.buffer, 0, lfs.cfg->lookahead_size);
|
||||
lfs.free.off = 0;
|
||||
lfs.free.size = lfs_min(8*lfs.cfg->lookahead_size,
|
||||
lfs.cfg->block_count);
|
||||
lfs.free.i = 0;
|
||||
lfs_alloc_ack(&lfs);
|
||||
|
||||
// create a tree with N elements
|
||||
lfsr_btree_t btree = LFSR_BTREE_NULL;
|
||||
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
|
||||
const char *nums = "0123456789";
|
||||
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, W,
|
||||
&nums[0 % 10], 1) => 0;
|
||||
for (lfs_size_t i = 1; i < N; i++) {
|
||||
char name[3] = {
|
||||
alphas[(i/26/26) % 26], alphas[(i/26) % 26], alphas[i % 26]
|
||||
};
|
||||
lfsr_btree_split(&lfs, &btree, (i-1)*W+W-1, name, 3,
|
||||
LFSR_TAG_INLINED, W, &nums[(i-1) % 10], 1,
|
||||
LFSR_TAG_INLINED, W, &nums[(i-0) % 10], 1) => 0;
|
||||
}
|
||||
printf("btree: 0x%x.%x 0x%x w%d\n",
|
||||
btree.u.trunk.block,
|
||||
btree.u.trunk.limit,
|
||||
btree.tag,
|
||||
btree.weight);
|
||||
assert(btree.weight == N*W);
|
||||
|
||||
// try to find tags
|
||||
uint8_t buffer[4];
|
||||
lfsr_tag_t tag_;
|
||||
lfs_size_t id_;
|
||||
lfs_size_t weight_;
|
||||
|
||||
for (lfs_size_t i = 0; i < N; i++) {
|
||||
char name[3] = {
|
||||
alphas[(i/26/26) % 26], alphas[(i/26) % 26], alphas[i % 26]
|
||||
};
|
||||
|
||||
lfsr_btree_find(&lfs, &btree, name, 3,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == i*W+W-1);
|
||||
assert(weight_ == W);
|
||||
assert(memcmp(buffer, &nums[i % 10], 1) == 0);
|
||||
}
|
||||
'''
|
||||
|
||||
[cases.test_btree_find_sparse_fuzz]
|
||||
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
|
||||
defines.W = 5
|
||||
defines.SAMPLES = 10
|
||||
# -1 => all pseudo-random seeds
|
||||
# n => reproduce a specific seed
|
||||
defines.SEED = -1
|
||||
in = 'lfs.c'
|
||||
code = '''
|
||||
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
|
||||
const char *nums = "0123456789";
|
||||
|
||||
// iterate through severals seeds that we can reproduce easily
|
||||
for (uint32_t seed = (SEED == -1 ? 1 : SEED);
|
||||
(SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
|
||||
seed++) {
|
||||
printf("--- seed: %d ---\n", seed);
|
||||
// create lfs here since we need to reset each iteration, we're
|
||||
// space constrained and we can't expect gc to work at this point
|
||||
lfs_t lfs;
|
||||
lfs_init(&lfs, cfg) => 0;
|
||||
// create free lookahead
|
||||
memset(lfs.free.buffer, 0, lfs.cfg->lookahead_size);
|
||||
lfs.free.off = 0;
|
||||
lfs.free.size = lfs_min(8*lfs.cfg->lookahead_size,
|
||||
lfs.cfg->block_count);
|
||||
lfs.free.i = 0;
|
||||
lfs_alloc_ack(&lfs);
|
||||
|
||||
// create a btree
|
||||
lfsr_btree_t btree = LFSR_BTREE_NULL;
|
||||
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, W,
|
||||
&alphas[0 % 26], 1) => 0;
|
||||
|
||||
// set up a simulation to compare against
|
||||
//
|
||||
// fun fact this is slower than our actual tree! unfun fact this is
|
||||
// starting to be a problem...
|
||||
char *sim = malloc(N);
|
||||
char (*sim_names)[3] = malloc(N*3);
|
||||
lfs_size_t *sim_weights = malloc(N*sizeof(lfs_size_t));
|
||||
lfs_size_t sim_size = 1;
|
||||
memset(sim, 0, N);
|
||||
memset(sim_names, 0, N*3);
|
||||
memset(sim_weights, 0, N*sizeof(lfs_size_t));
|
||||
sim[0] = alphas[0 % 26];
|
||||
memcpy(&sim_names[0], "___", 3);
|
||||
sim_weights[0] = W;
|
||||
|
||||
uint32_t prng = seed;
|
||||
for (lfs_size_t i = 1; i < N; i++) {
|
||||
// choose a pseudo-random name
|
||||
lfs_size_t x = TEST_PRNG(&prng) % (26*26*26);
|
||||
char name[3] = {
|
||||
alphas[(x/26/26) % 26], alphas[(x/26) % 26], alphas[x % 26]
|
||||
};
|
||||
// choose pseudo-random weights
|
||||
lfs_size_t weight1 = 1 + (TEST_PRNG(&prng) % W);
|
||||
lfs_size_t weight2 = 1 + (TEST_PRNG(&prng) % W);
|
||||
|
||||
// find where to split
|
||||
lfs_size_t id = 0;
|
||||
while (id+1 < sim_size && memcmp(sim_names[id+1], name, 3) <= 0) {
|
||||
id += 1;
|
||||
}
|
||||
// just skip exact matches for now
|
||||
if (memcmp(sim_names[id], name, 3) == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// calculate actual id in btree space
|
||||
lfs_size_t weighted_id = 0;
|
||||
for (lfs_size_t j = 0; j < id; j++) {
|
||||
weighted_id += sim_weights[j];
|
||||
}
|
||||
|
||||
// split btree
|
||||
lfsr_btree_split(&lfs, &btree, weighted_id+sim_weights[id]-1,
|
||||
name, 3,
|
||||
LFSR_TAG_INLINED, weight1, &nums[i % 10], 1,
|
||||
LFSR_TAG_INLINED, weight2, &nums[i % 10], 1) => 0;
|
||||
|
||||
// split sim
|
||||
memmove(&sim[id+1], &sim[id], sim_size-id);
|
||||
memmove(&sim_names[id+1], &sim_names[id], (sim_size-id)*3);
|
||||
memmove(&sim_weights[id+1], &sim_weights[id],
|
||||
(sim_size-id)*sizeof(lfs_size_t));
|
||||
sim[id+0] = nums[i % 10];
|
||||
sim[id+1] = nums[i % 10];
|
||||
memcpy(&sim_names[id+1], name, 3);
|
||||
sim_weights[id+0] = weight1;
|
||||
sim_weights[id+1] = weight2;
|
||||
sim_size += 1;
|
||||
}
|
||||
|
||||
// check that btree matches sim
|
||||
printf("expd: [");
|
||||
bool first = true;
|
||||
for (lfs_size_t i = 0; i < sim_size; i++) {
|
||||
// calculate actual id in btree space
|
||||
lfs_size_t weighted_id = 0;
|
||||
for (lfs_size_t j = 0; j < i; j++) {
|
||||
weighted_id += sim_weights[j];
|
||||
}
|
||||
|
||||
if (!first) {
|
||||
printf(", ");
|
||||
}
|
||||
first = false;
|
||||
printf("%.3sid%dw%d=%c",
|
||||
sim_names[i],
|
||||
weighted_id+sim_weights[i]-1,
|
||||
sim_weights[i],
|
||||
sim[i]);
|
||||
}
|
||||
printf("]\n");
|
||||
printf("btree: 0x%x.%x 0x%x w%d\n",
|
||||
btree.u.trunk.block,
|
||||
btree.u.trunk.limit,
|
||||
btree.tag,
|
||||
btree.weight);
|
||||
|
||||
lfs_size_t total_weight = 0;
|
||||
for (lfs_size_t j = 0; j < N; j++) {
|
||||
total_weight += sim_weights[j];
|
||||
}
|
||||
assert(btree.weight == total_weight);
|
||||
|
||||
uint8_t buffer[4];
|
||||
lfsr_tag_t tag_;
|
||||
lfs_size_t id_;
|
||||
lfs_size_t weight_;
|
||||
for (lfs_size_t i = 0; i < sim_size; i++) {
|
||||
// calculate actual id in btree space
|
||||
lfs_size_t weighted_id = 0;
|
||||
for (lfs_size_t j = 0; j < i; j++) {
|
||||
weighted_id += sim_weights[j];
|
||||
}
|
||||
|
||||
lfsr_btree_find(&lfs, &btree, sim_names[i], 3,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == weighted_id+sim_weights[i]-1);
|
||||
assert(weight_ == sim_weights[i]);
|
||||
assert(memcmp(buffer, &sim[i], 1) == 0);
|
||||
}
|
||||
|
||||
// clean up sim
|
||||
free(sim);
|
||||
free(sim_names);
|
||||
free(sim_weights);
|
||||
lfs_deinit(&lfs) => 0;
|
||||
}
|
||||
'''
|
||||
|
||||
# make sure we test finds with other operations
|
||||
[cases.test_btree_find_general_fuzz]
|
||||
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
|
||||
defines.SAMPLES = 100
|
||||
# -1 => all pseudo-random seeds
|
||||
# n => reproduce a specific seed
|
||||
defines.SEED = -1
|
||||
in = 'lfs.c'
|
||||
code = '''
|
||||
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
|
||||
const char *nums = "0123456789";
|
||||
|
||||
// iterate through severals seeds that we can reproduce easily
|
||||
for (uint32_t seed = (SEED == -1 ? 1 : SEED);
|
||||
(SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
|
||||
seed++) {
|
||||
printf("--- seed: %d ---\n", seed);
|
||||
// create lfs here since we need to reset each iteration, we're
|
||||
// space constrained and we can't expect gc to work at this point
|
||||
lfs_t lfs;
|
||||
lfs_init(&lfs, cfg) => 0;
|
||||
// create free lookahead
|
||||
memset(lfs.free.buffer, 0, lfs.cfg->lookahead_size);
|
||||
lfs.free.off = 0;
|
||||
lfs.free.size = lfs_min(8*lfs.cfg->lookahead_size,
|
||||
lfs.cfg->block_count);
|
||||
lfs.free.i = 0;
|
||||
lfs_alloc_ack(&lfs);
|
||||
|
||||
// create a btree
|
||||
lfsr_btree_t btree = LFSR_BTREE_NULL;
|
||||
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1,
|
||||
&alphas[0 % 26], 1) => 0;
|
||||
|
||||
// set up a simulation to compare against
|
||||
//
|
||||
// fun fact this is slower than our actual tree! unfun fact this is
|
||||
// starting to be a problem...
|
||||
char *sim = malloc(N);
|
||||
char (*sim_names)[3] = malloc(N*3);
|
||||
lfs_size_t sim_size = 1;
|
||||
memset(sim, 0, N);
|
||||
memset(sim_names, 0, N*3);
|
||||
sim[0] = alphas[0 % 26];
|
||||
memcpy(&sim_names[0], "___", 3);
|
||||
|
||||
uint32_t prng = seed;
|
||||
for (lfs_size_t i = 0; i < N; i++) {
|
||||
// choose a pseudo-random op
|
||||
uint8_t op = TEST_PRNG(&prng) % 3;
|
||||
// choose a pseudo-random id
|
||||
lfs_size_t id = TEST_PRNG(&prng) % (sim_size == 0 ? 1 : sim_size);
|
||||
// choose a pseudo-random name
|
||||
lfs_size_t x = TEST_PRNG(&prng) % (26*26*26);
|
||||
char name[3] = {
|
||||
alphas[(x/26/26) % 26], alphas[(x/26) % 26], alphas[x % 26]
|
||||
};
|
||||
|
||||
// don't let sim drop below one element
|
||||
if (op == 0 || sim_size <= 1) {
|
||||
// find where to split
|
||||
printf("- split(\"%.3s\", \"%c\")\n", name, nums[i % 10]);
|
||||
lfs_size_t id = 0;
|
||||
while (id+1 < sim_size
|
||||
&& memcmp(sim_names[id+1], name, 3) <= 0) {
|
||||
id += 1;
|
||||
}
|
||||
// just skip exact matches for now
|
||||
if (memcmp(sim_names[id], name, 3) == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// split btree
|
||||
lfsr_btree_split(&lfs, &btree, id, name, 3,
|
||||
LFSR_TAG_INLINED, 1, &nums[i % 10], 1,
|
||||
LFSR_TAG_INLINED, 1, &nums[i % 10], 1) => 0;
|
||||
|
||||
// split sim
|
||||
memmove(&sim[id+1], &sim[id], sim_size-id);
|
||||
memmove(&sim_names[id+1], &sim_names[id], (sim_size-id)*3);
|
||||
sim[id+0] = nums[i % 10];
|
||||
sim[id+1] = nums[i % 10];
|
||||
memcpy(&sim_names[id+1], name, 3);
|
||||
sim_size += 1;
|
||||
|
||||
} else if (op == 1) {
|
||||
// update btree
|
||||
printf("- update(%d, \"%c\")\n", id, nums[i % 10]);
|
||||
lfsr_btree_update(&lfs, &btree, id,
|
||||
LFSR_TAG_INLINED, 1,
|
||||
&nums[i % 10], 1) => 0;
|
||||
|
||||
// update sim
|
||||
sim[id] = nums[i % 10];
|
||||
|
||||
} else {
|
||||
// pop from btree
|
||||
printf("- pop(%d)\n", id);
|
||||
lfsr_btree_pop(&lfs, &btree, id) => 0;
|
||||
|
||||
// pop from sim
|
||||
memmove(&sim[id], &sim[id+1], sim_size-(id+1));
|
||||
memmove(&sim_names[id], &sim_names[id+1], (sim_size-(id+1))*3);
|
||||
sim_size -= 1;
|
||||
|
||||
// our B-tree doesn't actually track the name of id0, so we need
|
||||
// mirror this in our sim
|
||||
if (id == 0) {
|
||||
memcpy(&sim_names[0], "___", 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// check that btree matches sim
|
||||
printf("expd: [");
|
||||
bool first = true;
|
||||
for (lfs_size_t i = 0; i < sim_size; i++) {
|
||||
if (!first) {
|
||||
printf(", ");
|
||||
}
|
||||
first = false;
|
||||
printf("%.3s=%c", sim_names[i], sim[i]);
|
||||
}
|
||||
printf("]\n");
|
||||
printf("btree: 0x%x.%x 0x%x w%d\n",
|
||||
btree.u.trunk.block,
|
||||
btree.u.trunk.limit,
|
||||
btree.tag,
|
||||
btree.weight);
|
||||
assert(btree.weight == sim_size);
|
||||
|
||||
uint8_t buffer[4];
|
||||
lfsr_tag_t tag_;
|
||||
lfs_size_t id_;
|
||||
lfs_size_t weight_;
|
||||
for (lfs_size_t i = 0; i < sim_size; i++) {
|
||||
lfsr_btree_find(&lfs, &btree, sim_names[i], 3,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == i);
|
||||
assert(weight_ == 1);
|
||||
assert(memcmp(buffer, &sim[i], 1) == 0);
|
||||
}
|
||||
|
||||
// clean up sim
|
||||
free(sim);
|
||||
free(sim_names);
|
||||
}
|
||||
'''
|
||||
|
||||
[cases.test_btree_find_general_sparse_fuzz]
|
||||
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
|
||||
defines.W = 5
|
||||
defines.SAMPLES = 100
|
||||
# -1 => all pseudo-random seeds
|
||||
# n => reproduce a specific seed
|
||||
defines.SEED = -1
|
||||
in = 'lfs.c'
|
||||
code = '''
|
||||
const char *alphas = "abcdefghijklmnopqrstuvwxyz";
|
||||
|
||||
// iterate through severals seeds that we can reproduce easily
|
||||
for (uint32_t seed = (SEED == -1 ? 1 : SEED);
|
||||
(SEED == -1 ? seed < SAMPLES+1 : seed == SEED);
|
||||
seed++) {
|
||||
printf("--- seed: %d ---\n", seed);
|
||||
// create lfs here since we need to reset each iteration, we're
|
||||
// space constrained and we can't expect gc to work at this point
|
||||
lfs_t lfs;
|
||||
lfs_init(&lfs, cfg) => 0;
|
||||
// create free lookahead
|
||||
memset(lfs.free.buffer, 0, lfs.cfg->lookahead_size);
|
||||
lfs.free.off = 0;
|
||||
lfs.free.size = lfs_min(8*lfs.cfg->lookahead_size,
|
||||
lfs.cfg->block_count);
|
||||
lfs.free.i = 0;
|
||||
lfs_alloc_ack(&lfs);
|
||||
|
||||
// create a btree
|
||||
lfsr_btree_t btree = LFSR_BTREE_NULL;
|
||||
|
||||
// set up a simulation to compare against
|
||||
//
|
||||
// fun fact this is slower than our actual tree! unfun fact this is
|
||||
// starting to be a problem...
|
||||
char *sim = malloc(N);
|
||||
lfs_size_t *sim_weights = malloc(N*sizeof(lfs_size_t));
|
||||
lfs_size_t sim_size = 0;
|
||||
memset(sim, 0, N);
|
||||
memset(sim_weights, 0, N*sizeof(lfs_size_t));
|
||||
|
||||
uint32_t prng = seed;
|
||||
for (lfs_size_t i = 0; i < N; i++) {
|
||||
// choose a pseudo-random op
|
||||
uint8_t op = TEST_PRNG(&prng) % 3;
|
||||
// choose a pseudo-random id
|
||||
lfs_size_t id = TEST_PRNG(&prng) % (sim_size+1);
|
||||
// choose a pseudo-random weight
|
||||
lfs_size_t weight = 1 + (TEST_PRNG(&prng) % W);
|
||||
|
||||
// calculate actual id in btree space
|
||||
lfs_size_t weighted_id = 0;
|
||||
for (lfs_size_t j = 0; j < id; j++) {
|
||||
weighted_id += sim_weights[j];
|
||||
}
|
||||
|
||||
if (op == 0 || id == sim_size) {
|
||||
// push to btree
|
||||
lfsr_btree_push(&lfs, &btree, weighted_id,
|
||||
LFSR_TAG_INLINED, weight,
|
||||
&alphas[i % 26], 1) => 0;
|
||||
|
||||
// push to sim
|
||||
memmove(&sim[id+1], &sim[id], sim_size-id);
|
||||
memmove(&sim_weights[id+1], &sim_weights[id],
|
||||
(sim_size-id)*sizeof(lfs_size_t));
|
||||
sim[id] = alphas[i % 26];
|
||||
sim_weights[id] = weight;
|
||||
sim_size += 1;
|
||||
|
||||
} else if (op == 1) {
|
||||
// update btree
|
||||
lfsr_btree_update(&lfs, &btree,
|
||||
weighted_id+sim_weights[id]-1, LFSR_TAG_INLINED, weight,
|
||||
&alphas[i % 26], 1) => 0;
|
||||
|
||||
// update sim
|
||||
sim[id] = alphas[i % 26];
|
||||
sim_weights[id] = weight;
|
||||
|
||||
} else {
|
||||
// remove from btree
|
||||
lfsr_btree_pop(&lfs, &btree,
|
||||
weighted_id+sim_weights[id]-1) => 0;
|
||||
|
||||
// remove from sim
|
||||
memmove(&sim[id], &sim[id+1], sim_size-(id+1));
|
||||
memmove(&sim_weights[id], &sim_weights[id+1],
|
||||
(sim_size-(id+1))*sizeof(lfs_size_t));
|
||||
sim_size -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
// check that btree matches sim
|
||||
printf("expd: [");
|
||||
bool first = true;
|
||||
for (lfs_size_t i = 0; i < sim_size; i++) {
|
||||
if (!first) {
|
||||
printf(", ");
|
||||
}
|
||||
first = false;
|
||||
printf("%c", sim[i]);
|
||||
}
|
||||
printf("]\n");
|
||||
printf("btree: 0x%x.%x 0x%x w%d\n",
|
||||
btree.u.trunk.block,
|
||||
btree.u.trunk.limit,
|
||||
btree.tag,
|
||||
btree.weight);
|
||||
|
||||
lfs_size_t total_weight = 0;
|
||||
for (lfs_size_t j = 0; j < sim_size; j++) {
|
||||
total_weight += sim_weights[j];
|
||||
}
|
||||
assert(btree.weight == total_weight);
|
||||
|
||||
uint8_t buffer[4];
|
||||
lfsr_tag_t tag_;
|
||||
lfs_size_t id_;
|
||||
lfs_size_t weight_;
|
||||
for (lfs_size_t i = 0; i < sim_size; i++) {
|
||||
// calculate actual id in btree space
|
||||
lfs_size_t weighted_id = 0;
|
||||
for (lfs_size_t j = 0; j < i; j++) {
|
||||
weighted_id += sim_weights[j];
|
||||
}
|
||||
|
||||
lfsr_btree_get(&lfs, &btree, weighted_id+sim_weights[i]-1,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == weighted_id+sim_weights[i]-1);
|
||||
assert(weight_ == sim_weights[i]);
|
||||
assert(memcmp(buffer, &sim[i], 1) == 0);
|
||||
}
|
||||
|
||||
// and no extra elements
|
||||
lfsr_btree_get(&lfs, &btree, total_weight,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => LFS_ERR_NOENT;
|
||||
|
||||
// also test that we can traverse the tree without prior knowledge
|
||||
id_ = -1;
|
||||
for (lfs_size_t i = 0; i < sim_size; i++) {
|
||||
// calculate actual id in btree space
|
||||
lfs_size_t weighted_id = 0;
|
||||
for (lfs_size_t j = 0; j < i; j++) {
|
||||
weighted_id += sim_weights[j];
|
||||
}
|
||||
|
||||
lfsr_btree_get(&lfs, &btree, id_+1,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => 1;
|
||||
assert(tag_ == LFSR_TAG_INLINED);
|
||||
assert(id_ == weighted_id+sim_weights[i]-1);
|
||||
assert(weight_ == sim_weights[i]);
|
||||
assert(memcmp(buffer, &sim[i], 1) == 0);
|
||||
}
|
||||
lfsr_btree_get(&lfs, &btree, id_+1,
|
||||
&tag_, &id_, &weight_,
|
||||
buffer, 4) => LFS_ERR_NOENT;
|
||||
|
||||
// clean up sim
|
||||
free(sim);
|
||||
}
|
||||
'''
|
||||
|
||||
|
||||
Reference in New Issue
Block a user