Christopher Haster
316ca1cc05
At least at a proof-of-concept level, there's still a lot of cleanup
needed.
To make things work, lfs3_alloc_ckpoint now takes an mdir, which
provides the target for gbmap gstate updates.
When the bmap is close to empty (configurable via bmap_scan_thresh), we
opportunistically rebuild it during lfs3_alloc_ckpoints. The nice thing
about lfs3_alloc_ckpoint is we know the state of all in-flight blocks,
so rebuilding the bmap just requires traversing the filesystem + in-RAM
state.
We might still fall back to the lookahead buffer, but in theory a well
tuned bmap_scan_thresh can prevent this from becoming a bottleneck (at
the cost of more frequent bmap rebuilds).
---
This is also probably a good time to resume measuring code/ram costs,
though it's worth repeating the above note about the bmap work still
needing cleanup:
code stack ctx
before: 36840 2368 684
after: 36920 (+0.2%) 2368 (+0.0%) 684 (+0.0%)
Haha, no, the bmap isn't basically free, it's just an opt-in features.
With -DLFS3_YES_BMAP=1:
code stack ctx
no bmap: 36920 2368 684
yes bmap: 38552 (+4.4%) 2472 (+4.4%) 812 (+18.7%)
378 lines
12 KiB
TOML
378 lines
12 KiB
TOML
# Test some low-level block-map operations
|
|
after = ['test_btree', 'test_mtree']
|
|
ifdef = 'LFS3_BMAP'
|
|
|
|
# test simple set operations
|
|
[cases.test_bmap_set_split]
|
|
in = 'lfs3.c'
|
|
code = '''
|
|
lfs3_t lfs3;
|
|
lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
|
|
lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
|
|
lfs3_alloc_ckpoint(&lfs3, NULL);
|
|
|
|
// create an initial bmap
|
|
lfs3_btree_t bmap;
|
|
lfs3_btree_init(&bmap);
|
|
lfs3_bmap_commit(&lfs3, &bmap, 0, LFS3_RATTRS(
|
|
LFS3_RATTR(LFS3_TAG_BMFREE, +BLOCK_COUNT))) => 0;
|
|
|
|
// set some blocks as in-use, avoid any weird merges for now
|
|
lfs3_bmap_set(&lfs3, &bmap, 6, LFS3_TAG_BMINUSE) => 0;
|
|
lfs3_bmap_set(&lfs3, &bmap, 8, LFS3_TAG_BMINUSE) => 0;
|
|
lfs3_bmap_set(&lfs3, &bmap, 10, LFS3_TAG_BMINUSE) => 0;
|
|
printf("bmap: w%d 0x%x.%x\n",
|
|
bmap.r.weight,
|
|
bmap.r.blocks[0],
|
|
bmap.r.trunk);
|
|
|
|
// weight should stay the same
|
|
assert(bmap.r.weight == BLOCK_COUNT);
|
|
|
|
// check ranges
|
|
lfs3_bid_t bid_;
|
|
lfs3_bid_t weight_;
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 0,
|
|
&bid_, &weight_) => LFS3_TAG_BMFREE;
|
|
assert(bid_ == 5);
|
|
assert(weight_ == 6);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 6,
|
|
&bid_, &weight_) => LFS3_TAG_BMINUSE;
|
|
assert(bid_ == 6);
|
|
assert(weight_ == 1);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 7,
|
|
&bid_, &weight_) => LFS3_TAG_BMFREE;
|
|
assert(bid_ == 7);
|
|
assert(weight_ == 1);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 8,
|
|
&bid_, &weight_) => LFS3_TAG_BMINUSE;
|
|
assert(bid_ == 8);
|
|
assert(weight_ == 1);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 9,
|
|
&bid_, &weight_) => LFS3_TAG_BMFREE;
|
|
assert(bid_ == 9);
|
|
assert(weight_ == 1);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 10,
|
|
&bid_, &weight_) => LFS3_TAG_BMINUSE;
|
|
assert(bid_ == 10);
|
|
assert(weight_ == 1);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 11,
|
|
&bid_, &weight_) => LFS3_TAG_BMFREE;
|
|
assert(bid_ == BLOCK_COUNT-1);
|
|
assert(weight_ == BLOCK_COUNT-11);
|
|
|
|
lfs3_unmount(&lfs3) => 0;
|
|
'''
|
|
|
|
[cases.test_bmap_set_replace]
|
|
in = 'lfs3.c'
|
|
code = '''
|
|
lfs3_t lfs3;
|
|
lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
|
|
lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
|
|
lfs3_alloc_ckpoint(&lfs3, NULL);
|
|
|
|
// create an initial bmap
|
|
lfs3_btree_t bmap;
|
|
lfs3_btree_init(&bmap);
|
|
lfs3_bmap_commit(&lfs3, &bmap, 0, LFS3_RATTRS(
|
|
LFS3_RATTR(LFS3_TAG_BMFREE, +BLOCK_COUNT))) => 0;
|
|
|
|
// set some blocks as in-use, avoid any weird merges for now
|
|
lfs3_bmap_set(&lfs3, &bmap, 6, LFS3_TAG_BMINUSE) => 0;
|
|
lfs3_bmap_set(&lfs3, &bmap, 8, LFS3_TAG_BMINUSE) => 0;
|
|
lfs3_bmap_set(&lfs3, &bmap, 10, LFS3_TAG_BMINUSE) => 0;
|
|
// replace those blocks as bad, this tests deleting ranges
|
|
lfs3_bmap_set(&lfs3, &bmap, 6, LFS3_TAG_BMINFLIGHT) => 0;
|
|
lfs3_bmap_set(&lfs3, &bmap, 8, LFS3_TAG_BMINFLIGHT) => 0;
|
|
lfs3_bmap_set(&lfs3, &bmap, 10, LFS3_TAG_BMINFLIGHT) => 0;
|
|
printf("bmap: w%d 0x%x.%x\n",
|
|
bmap.r.weight,
|
|
bmap.r.blocks[0],
|
|
bmap.r.trunk);
|
|
|
|
// weight should stay the same
|
|
assert(bmap.r.weight == BLOCK_COUNT);
|
|
|
|
// check ranges
|
|
lfs3_bid_t bid_;
|
|
lfs3_bid_t weight_;
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 0,
|
|
&bid_, &weight_) => LFS3_TAG_BMFREE;
|
|
assert(bid_ == 5);
|
|
assert(weight_ == 6);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 6,
|
|
&bid_, &weight_) => LFS3_TAG_BMINFLIGHT;
|
|
assert(bid_ == 6);
|
|
assert(weight_ == 1);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 7,
|
|
&bid_, &weight_) => LFS3_TAG_BMFREE;
|
|
assert(bid_ == 7);
|
|
assert(weight_ == 1);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 8,
|
|
&bid_, &weight_) => LFS3_TAG_BMINFLIGHT;
|
|
assert(bid_ == 8);
|
|
assert(weight_ == 1);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 9,
|
|
&bid_, &weight_) => LFS3_TAG_BMFREE;
|
|
assert(bid_ == 9);
|
|
assert(weight_ == 1);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 10,
|
|
&bid_, &weight_) => LFS3_TAG_BMINFLIGHT;
|
|
assert(bid_ == 10);
|
|
assert(weight_ == 1);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 11,
|
|
&bid_, &weight_) => LFS3_TAG_BMFREE;
|
|
assert(bid_ == BLOCK_COUNT-1);
|
|
assert(weight_ == BLOCK_COUNT-11);
|
|
|
|
lfs3_unmount(&lfs3) => 0;
|
|
'''
|
|
|
|
[cases.test_bmap_set_merge]
|
|
in = 'lfs3.c'
|
|
code = '''
|
|
lfs3_t lfs3;
|
|
lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
|
|
lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
|
|
lfs3_alloc_ckpoint(&lfs3, NULL);
|
|
|
|
// create an initial bmap
|
|
lfs3_btree_t bmap;
|
|
lfs3_btree_init(&bmap);
|
|
lfs3_bmap_commit(&lfs3, &bmap, 0, LFS3_RATTRS(
|
|
LFS3_RATTR(LFS3_TAG_BMFREE, +BLOCK_COUNT))) => 0;
|
|
|
|
// set some blocks as in-use, avoid any weird merges for now
|
|
lfs3_bmap_set(&lfs3, &bmap, 6, LFS3_TAG_BMINUSE) => 0;
|
|
lfs3_bmap_set(&lfs3, &bmap, 8, LFS3_TAG_BMINUSE) => 0;
|
|
lfs3_bmap_set(&lfs3, &bmap, 10, LFS3_TAG_BMINUSE) => 0;
|
|
lfs3_bmap_set(&lfs3, &bmap, 12, LFS3_TAG_BMINUSE) => 0;
|
|
// set neighboring blocks as in-use, triggering merges
|
|
lfs3_bmap_set(&lfs3, &bmap, 5, LFS3_TAG_BMINUSE) => 0;
|
|
lfs3_bmap_set(&lfs3, &bmap, 9, LFS3_TAG_BMINUSE) => 0;
|
|
lfs3_bmap_set(&lfs3, &bmap, 13, LFS3_TAG_BMINUSE) => 0;
|
|
printf("bmap: w%d 0x%x.%x\n",
|
|
bmap.r.weight,
|
|
bmap.r.blocks[0],
|
|
bmap.r.trunk);
|
|
|
|
// weight should stay the same
|
|
assert(bmap.r.weight == BLOCK_COUNT);
|
|
|
|
// check ranges
|
|
lfs3_bid_t bid_;
|
|
lfs3_bid_t weight_;
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 0,
|
|
&bid_, &weight_) => LFS3_TAG_BMFREE;
|
|
assert(bid_ == 4);
|
|
assert(weight_ == 5);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 5,
|
|
&bid_, &weight_) => LFS3_TAG_BMINUSE;
|
|
assert(bid_ == 6);
|
|
assert(weight_ == 2);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 7,
|
|
&bid_, &weight_) => LFS3_TAG_BMFREE;
|
|
assert(bid_ == 7);
|
|
assert(weight_ == 1);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 8,
|
|
&bid_, &weight_) => LFS3_TAG_BMINUSE;
|
|
assert(bid_ == 10);
|
|
assert(weight_ == 3);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 11,
|
|
&bid_, &weight_) => LFS3_TAG_BMFREE;
|
|
assert(bid_ == 11);
|
|
assert(weight_ == 1);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 12,
|
|
&bid_, &weight_) => LFS3_TAG_BMINUSE;
|
|
assert(bid_ == 13);
|
|
assert(weight_ == 2);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 14,
|
|
&bid_, &weight_) => LFS3_TAG_BMFREE;
|
|
assert(bid_ == BLOCK_COUNT-1);
|
|
assert(weight_ == BLOCK_COUNT-14);
|
|
|
|
lfs3_unmount(&lfs3) => 0;
|
|
'''
|
|
|
|
[cases.test_bmap_set_noop]
|
|
in = 'lfs3.c'
|
|
code = '''
|
|
lfs3_t lfs3;
|
|
lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
|
|
lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
|
|
lfs3_alloc_ckpoint(&lfs3, NULL);
|
|
|
|
// create an initial bmap
|
|
lfs3_btree_t bmap;
|
|
lfs3_btree_init(&bmap);
|
|
lfs3_bmap_commit(&lfs3, &bmap, 0, LFS3_RATTRS(
|
|
LFS3_RATTR(LFS3_TAG_BMFREE, +BLOCK_COUNT))) => 0;
|
|
|
|
// set some blocks as in-use, avoid any weird merges for now
|
|
lfs3_bmap_set(&lfs3, &bmap, 6, LFS3_TAG_BMINUSE) => 0;
|
|
lfs3_bmap_set(&lfs3, &bmap, 8, LFS3_TAG_BMINUSE) => 0;
|
|
lfs3_bmap_set(&lfs3, &bmap, 10, LFS3_TAG_BMINUSE) => 0;
|
|
// test a bunch of noops
|
|
lfs3_bmap_set(&lfs3, &bmap, 6, LFS3_TAG_BMINUSE) => 0;
|
|
lfs3_bmap_set(&lfs3, &bmap, 7, LFS3_TAG_BMFREE) => 0;
|
|
lfs3_bmap_set(&lfs3, &bmap, 8, LFS3_TAG_BMINUSE) => 0;
|
|
lfs3_bmap_set(&lfs3, &bmap, 9, LFS3_TAG_BMFREE) => 0;
|
|
lfs3_bmap_set(&lfs3, &bmap, 10, LFS3_TAG_BMINUSE) => 0;
|
|
printf("bmap: w%d 0x%x.%x\n",
|
|
bmap.r.weight,
|
|
bmap.r.blocks[0],
|
|
bmap.r.trunk);
|
|
|
|
// weight should stay the same
|
|
assert(bmap.r.weight == BLOCK_COUNT);
|
|
|
|
// check ranges
|
|
lfs3_bid_t bid_;
|
|
lfs3_bid_t weight_;
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 0,
|
|
&bid_, &weight_) => LFS3_TAG_BMFREE;
|
|
assert(bid_ == 5);
|
|
assert(weight_ == 6);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 6,
|
|
&bid_, &weight_) => LFS3_TAG_BMINUSE;
|
|
assert(bid_ == 6);
|
|
assert(weight_ == 1);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 7,
|
|
&bid_, &weight_) => LFS3_TAG_BMFREE;
|
|
assert(bid_ == 7);
|
|
assert(weight_ == 1);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 8,
|
|
&bid_, &weight_) => LFS3_TAG_BMINUSE;
|
|
assert(bid_ == 8);
|
|
assert(weight_ == 1);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 9,
|
|
&bid_, &weight_) => LFS3_TAG_BMFREE;
|
|
assert(bid_ == 9);
|
|
assert(weight_ == 1);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 10,
|
|
&bid_, &weight_) => LFS3_TAG_BMINUSE;
|
|
assert(bid_ == 10);
|
|
assert(weight_ == 1);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 11,
|
|
&bid_, &weight_) => LFS3_TAG_BMFREE;
|
|
assert(bid_ == BLOCK_COUNT-1);
|
|
assert(weight_ == BLOCK_COUNT-11);
|
|
|
|
lfs3_unmount(&lfs3) => 0;
|
|
'''
|
|
|
|
[cases.test_bmap_set_bounds]
|
|
in = 'lfs3.c'
|
|
code = '''
|
|
lfs3_t lfs3;
|
|
lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
|
|
lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
|
|
lfs3_alloc_ckpoint(&lfs3, NULL);
|
|
|
|
// create an initial bmap
|
|
lfs3_btree_t bmap;
|
|
lfs3_btree_init(&bmap);
|
|
lfs3_bmap_commit(&lfs3, &bmap, 0, LFS3_RATTRS(
|
|
LFS3_RATTR(LFS3_TAG_BMFREE, +BLOCK_COUNT))) => 0;
|
|
|
|
// test that setting the first and last blocks don't break anything
|
|
//
|
|
// though in theory blocks 0x{0,1} are immutable...
|
|
lfs3_bmap_set(&lfs3, &bmap, 0, LFS3_TAG_BMINUSE) => 0;
|
|
lfs3_bmap_set(&lfs3, &bmap, BLOCK_COUNT-1, LFS3_TAG_BMINUSE) => 0;
|
|
printf("bmap: w%d 0x%x.%x\n",
|
|
bmap.r.weight,
|
|
bmap.r.blocks[0],
|
|
bmap.r.trunk);
|
|
|
|
// weight should stay the same
|
|
assert(bmap.r.weight == BLOCK_COUNT);
|
|
|
|
// check ranges
|
|
lfs3_bid_t bid_;
|
|
lfs3_bid_t weight_;
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 0,
|
|
&bid_, &weight_) => LFS3_TAG_BMINUSE;
|
|
assert(bid_ == 0);
|
|
assert(weight_ == 1);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, 1,
|
|
&bid_, &weight_) => LFS3_TAG_BMFREE;
|
|
assert(bid_ == BLOCK_COUNT-2);
|
|
assert(weight_ == BLOCK_COUNT-2);
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, BLOCK_COUNT-1,
|
|
&bid_, &weight_) => LFS3_TAG_BMINUSE;
|
|
assert(bid_ == BLOCK_COUNT-1);
|
|
assert(weight_ == 1);
|
|
|
|
lfs3_unmount(&lfs3) => 0;
|
|
'''
|
|
|
|
[cases.test_bmap_set_fuzz]
|
|
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
|
|
defines.TYPES = [2, 3, 4]
|
|
defines.SEED = 'range(20)'
|
|
fuzz = 'SEED'
|
|
in = 'lfs3.c'
|
|
code = '''
|
|
lfs3_t lfs3;
|
|
lfs3_format(&lfs3, LFS3_F_RDWR, CFG) => 0;
|
|
lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
|
|
lfs3_alloc_ckpoint(&lfs3, NULL);
|
|
|
|
// create an initial bmap
|
|
lfs3_btree_t bmap;
|
|
lfs3_btree_init(&bmap);
|
|
lfs3_bmap_commit(&lfs3, &bmap, 0, LFS3_RATTRS(
|
|
LFS3_RATTR(LFS3_TAG_BMFREE, +BLOCK_COUNT))) => 0;
|
|
|
|
// create a simulation to compare against
|
|
lfs3_tag_t *sim = malloc(BLOCK_COUNT*sizeof(lfs3_tag_t));
|
|
for (lfs3_size_t i = 0; i < BLOCK_COUNT; i++) {
|
|
sim[i] = LFS3_TAG_BMFREE;
|
|
}
|
|
|
|
uint32_t prng = SEED;
|
|
for (lfs3_size_t i = 0; i < N; i++) {
|
|
// choose a pseudo-random block
|
|
lfs3_block_t block = TEST_PRNG(&prng) % BLOCK_COUNT;
|
|
// and pseudo-random bmap type
|
|
lfs3_tag_t tag = LFS3_TAG_BMRANGE + (TEST_PRNG(&prng) % TYPES);
|
|
|
|
// set in bmap
|
|
lfs3_bmap_set(&lfs3, &bmap, block, tag) => 0;
|
|
|
|
// and set in sim
|
|
sim[block] = tag;
|
|
}
|
|
printf("bmap: w%d 0x%x.%x\n",
|
|
bmap.r.weight,
|
|
bmap.r.blocks[0],
|
|
bmap.r.trunk);
|
|
|
|
// check if bmap matches sim
|
|
lfs3_size_t i = 0;
|
|
while (i < BLOCK_COUNT) {
|
|
// we need to convert our sim's raw blocks to compressed ranges,
|
|
// in theory our bmap is optimal
|
|
lfs3_tag_t tag = sim[i];
|
|
lfs3_size_t d = 1;
|
|
while (i+d < BLOCK_COUNT && sim[i+d] == tag) {
|
|
d += 1;
|
|
}
|
|
|
|
// does our bmap contain the optimal range?
|
|
lfs3_bid_t bid_;
|
|
lfs3_bid_t weight_;
|
|
lfs3_bmap_lookupnext(&lfs3, &bmap, i,
|
|
&bid_, &weight_) => tag;
|
|
assert(bid_ == i+(weight_-1));
|
|
assert(weight_ == d);
|
|
|
|
i += d;
|
|
}
|
|
|
|
free(sim);
|
|
lfs3_unmount(&lfs3) => 0;
|
|
'''
|