Github/littlefs
1
Fork 0
Code Actions Activity

Attempted to implement slice dereferencing

Browse Source 
But already there are some pretty fundamental problems.

The main issue is that, while we correctly dereference slices during
compaction, pending commits that get delayed after compaction still
point to the old block. I'm not sure there's an easy way around this
aside from aborting compaction commits or fully simulating commits,
both of which seem too costly to implement...

Also coalescing during compaction is flawed as well, since our
attributes will be outdated by the time they are committed if there is a
compaction...

Looks like it's back to the drawing board. Either our approach to
compaction needs to change, or this slice/coalescing work needs to be
reverted/redesigned...
This commit is contained in:
Christopher Haster
2023-10-19 01:05:22 -05:00
parent 865477d7e1
commit 2940555caa
2 changed files with 489 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_files.toml
+420 -69
View File
@@ -647,17 +647,15 @@ code = '''
}
// write first chunk?
if (ORDER == 0) {
if (MASK & 0x1) {
if (MASK & 0x1) {
if (ORDER == 0) {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[i] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_seek(&lfs, &file, 0, LFS_SEEK_SET) => 0;
lfsr_file_write(&lfs, &file, &sim[0], CHUNK) => CHUNK;
}
} else {
if (MASK & 0x4) {
} else {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[SIZE-CHUNK+i] = 'a' + (TEST_PRNG(&prng) % 26);
}
@@ -665,23 +663,23 @@ code = '''
lfsr_file_seek(&lfs, &file, SIZE-CHUNK, LFS_SEEK_SET) => SIZE-CHUNK;
lfsr_file_write(&lfs, &file, &sim[SIZE-CHUNK], CHUNK) => CHUNK;
}
}
// sync?
if (SYNC) {
lfsr_file_sync(&lfs, &file) => 0;
}
// sync?
if (SYNC) {
lfsr_file_sync(&lfs, &file) => 0;
}
// remount?
if (REMOUNT) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0;
// remount?
if (REMOUNT) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0;
}
}
// write second chunk?
if (MASK & 2) {
if (MASK & 0x2) {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[SIZE/2-CHUNK/2+i] = 'a' + (TEST_PRNG(&prng) % 26);
}
@@ -689,33 +687,31 @@ code = '''
lfsr_file_seek(&lfs, &file, SIZE/2 - CHUNK/2, LFS_SEEK_SET)
=> SIZE/2 - CHUNK/2;
lfsr_file_write(&lfs, &file, &sim[SIZE/2-CHUNK/2], CHUNK) => CHUNK;
}
// sync?
if (SYNC) {
lfsr_file_sync(&lfs, &file) => 0;
}
// sync?
if (SYNC) {
lfsr_file_sync(&lfs, &file) => 0;
}
// remount?
if (REMOUNT) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0;
// remount?
if (REMOUNT) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0;
}
}
// write third chunk?
if (ORDER == 0) {
if (MASK & 0x4) {
if (MASK & 0x4) {
if (ORDER == 0) {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[SIZE-CHUNK+i] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_seek(&lfs, &file, SIZE-CHUNK, LFS_SEEK_SET) => SIZE-CHUNK;
lfsr_file_write(&lfs, &file, &sim[SIZE-CHUNK], CHUNK) => CHUNK;
}
} else {
if (MASK & 0x1) {
} else {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[i] = 'a' + (TEST_PRNG(&prng) % 26);
}
@@ -804,9 +800,9 @@ code = '''
memset(sim, 0, SIZE);
// we may not write the entire file
lfs_off_t size
= (MASK & 0x4) ? SIZE
: (MASK & 0x2) ? SIZE/2 + (CHUNK+2-1)/2
: (MASK & 0x1) ? CHUNK
= (MASK & ((ORDER == 0) ? 0x4 : 0x1)) ? SIZE
: (MASK & ((ORDER == 0) ? 0x2 : 0x2)) ? SIZE/2 + (CHUNK+2-1)/2
: (MASK & ((ORDER == 0) ? 0x1 : 0x4)) ? CHUNK
: 0;
// sync?
@@ -823,17 +819,15 @@ code = '''
}
// write first chunk?
if (ORDER == 0) {
if (MASK & 0x1) {
if (MASK & 0x1) {
if (ORDER == 0) {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[i] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_seek(&lfs, &file, 0, LFS_SEEK_SET) => 0;
lfsr_file_write(&lfs, &file, &sim[0], CHUNK) => CHUNK;
}
} else {
if (MASK & 0x4) {
} else {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[SIZE-CHUNK+i] = 'a' + (TEST_PRNG(&prng) % 26);
}
@@ -841,23 +835,23 @@ code = '''
lfsr_file_seek(&lfs, &file, SIZE-CHUNK, LFS_SEEK_SET) => SIZE-CHUNK;
lfsr_file_write(&lfs, &file, &sim[SIZE-CHUNK], CHUNK) => CHUNK;
}
}
// sync?
if (SYNC) {
lfsr_file_sync(&lfs, &file) => 0;
}
// sync?
if (SYNC) {
lfsr_file_sync(&lfs, &file) => 0;
}
// remount?
if (REMOUNT) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0;
// remount?
if (REMOUNT) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0;
}
}
// write second chunk?
if (MASK & 2) {
if (MASK & 0x2) {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[SIZE/2-CHUNK/2+i] = 'a' + (TEST_PRNG(&prng) % 26);
}
@@ -865,33 +859,31 @@ code = '''
lfsr_file_seek(&lfs, &file, SIZE/2 - CHUNK/2, LFS_SEEK_SET)
=> SIZE/2 - CHUNK/2;
lfsr_file_write(&lfs, &file, &sim[SIZE/2-CHUNK/2], CHUNK) => CHUNK;
}
// sync?
if (SYNC) {
lfsr_file_sync(&lfs, &file) => 0;
}
// sync?
if (SYNC) {
lfsr_file_sync(&lfs, &file) => 0;
}
// remount?
if (REMOUNT) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0;
// remount?
if (REMOUNT) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0;
}
}
// write third chunk?
if (ORDER == 0) {
if (MASK & 0x4) {
if (MASK & 0x4) {
if (ORDER == 0) {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[SIZE-CHUNK+i] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_seek(&lfs, &file, SIZE-CHUNK, LFS_SEEK_SET) => SIZE-CHUNK;
lfsr_file_write(&lfs, &file, &sim[SIZE-CHUNK], CHUNK) => CHUNK;
}
} else {
if (MASK & 0x1) {
} else {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[i] = 'a' + (TEST_PRNG(&prng) % 26);
}
@@ -1496,6 +1488,365 @@ code = '''
lfsr_unmount(&lfs) => 0;
'''
# these are like the overwrite/hole tests, but with enough rewrites to
# trigger compaction
[cases.test_files_overwrite_compaction]
defines.SIZE = ['CACHE_SIZE/2', '2*CACHE_SIZE']
defines.CHUNK = ['CACHE_SIZE/2', '4', '1']
# bit 0 => first chunk
# bit 1 => middle chunk
# bit 2 => last chunk
defines.MASK = [0, 1, 2, 3, 4, 5, 6, 7]
# 0 => in-order
# 1 => reversed
defines.ORDER = [0, 1]
# writing this many times guarantees a compaction
defines.WRITES = '2*(BLOCK_SIZE/PROG_SIZE)'
# TODO is setting PROG_SIZE here reasonable?
defines.PROG_SIZE = 64
defines.SYNC = [false, true]
defines.REMOUNT = [false, true]
code = '''
// format once per test
lfs_t lfs;
int err = lfsr_mount(&lfs, CFG);
if (err) {
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
}
// create a file, truncating in case of powerloss
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "hello",
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0;
// simulate our file in ram
uint8_t sim[SIZE];
uint32_t prng = 42;
for (lfs_size_t i = 0; i < SIZE; i++) {
sim[i] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, sim, SIZE) => SIZE;
// sync?
if (SYNC) {
lfsr_file_sync(&lfs, &file) => 0;
}
// remount?
if (REMOUNT) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0;
}
// write first chunk?
if (MASK & 0x1) {
for (lfs_size_t w = 0; w < WRITES; w++) {
if (ORDER == 0) {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[i] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_seek(&lfs, &file, 0, LFS_SEEK_SET) => 0;
lfsr_file_write(&lfs, &file, &sim[0], CHUNK) => CHUNK;
} else {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[SIZE-CHUNK+i] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_seek(&lfs, &file, SIZE-CHUNK, LFS_SEEK_SET) => SIZE-CHUNK;
lfsr_file_write(&lfs, &file, &sim[SIZE-CHUNK], CHUNK) => CHUNK;
}
// sync?
if (SYNC) {
lfsr_file_sync(&lfs, &file) => 0;
}
// remount?
if (REMOUNT) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0;
}
}
}
// write second chunk?
if (MASK & 0x2) {
for (lfs_size_t w = 0; w < WRITES; w++) {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[SIZE/2-CHUNK/2+i] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_seek(&lfs, &file, SIZE/2 - CHUNK/2, LFS_SEEK_SET)
=> SIZE/2 - CHUNK/2;
lfsr_file_write(&lfs, &file, &sim[SIZE/2-CHUNK/2], CHUNK) => CHUNK;
// sync?
if (SYNC) {
lfsr_file_sync(&lfs, &file) => 0;
}
// remount?
if (REMOUNT) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0;
}
}
}
// write third chunk?
if (MASK & 0x4) {
for (lfs_size_t w = 0; w < WRITES; w++) {
if (ORDER == 0) {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[SIZE-CHUNK+i] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_seek(&lfs, &file, SIZE-CHUNK, LFS_SEEK_SET) => SIZE-CHUNK;
lfsr_file_write(&lfs, &file, &sim[SIZE-CHUNK], CHUNK) => CHUNK;
} else {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[i] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_seek(&lfs, &file, 0, LFS_SEEK_SET) => 0;
lfsr_file_write(&lfs, &file, &sim[0], CHUNK) => CHUNK;
}
}
}
lfsr_file_close(&lfs, &file) => 0;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
}
// check our file with stat
struct lfs_info info;
lfsr_stat(&lfs, "hello", &info) => 0;
assert(strcmp(info.name, "hello") == 0);
assert(info.type == LFS_TYPE_REG);
assert(info.size == SIZE);
// and with dir read
lfsr_dir_t dir;
lfsr_dir_open(&lfs, &dir, "/") => 0;
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, ".") == 0);
assert(info.type == LFS_TYPE_DIR);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, "..") == 0);
assert(info.type == LFS_TYPE_DIR);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, "hello") == 0);
assert(info.type == LFS_TYPE_REG);
assert(info.size == SIZE);
lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT;
lfsr_dir_close(&lfs, &dir) => 0;
// try reading our file
lfsr_file_open(&lfs, &file, "hello", LFS_O_RDONLY) => 0;
// is size correct?
lfsr_file_size(&lfs, &file) => SIZE;
// try reading
uint8_t rbuf[2*SIZE];
memset(rbuf, 0xaa, 2*SIZE);
lfsr_file_read(&lfs, &file, rbuf, 2*SIZE) => SIZE;
// does our file match our simulation?
assert(memcmp(rbuf, sim, SIZE) == 0);
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
'''
[cases.test_files_hole_compaction]
defines.SIZE = ['CACHE_SIZE/2', '2*CACHE_SIZE']
defines.CHUNK = ['CACHE_SIZE/2', '4', '1']
# bit 0 => first chunk
# bit 1 => middle chunk
# bit 2 => last chunk
defines.MASK = [0, 1, 2, 3, 4, 5, 6, 7]
# 0 => in-order
# 1 => reversed
defines.ORDER = [0, 1]
# writing this many times guarantees a compaction
defines.WRITES = '2*(BLOCK_SIZE/PROG_SIZE)'
# TODO is setting PROG_SIZE here reasonable?
defines.PROG_SIZE = 64
defines.SYNC = [false, true]
defines.REMOUNT = [false, true]
code = '''
// format once per test
lfs_t lfs;
int err = lfsr_mount(&lfs, CFG);
if (err) {
lfsr_format(&lfs, CFG) => 0;
lfsr_mount(&lfs, CFG) => 0;
}
// create a file, truncating in case of powerloss
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "hello",
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0;
// simulate our file in ram
uint8_t sim[SIZE];
uint32_t prng = 42;
memset(sim, 0, SIZE);
// we may not write the entire file
lfs_off_t size
= (MASK & ((ORDER == 0) ? 0x4 : 0x1)) ? SIZE
: (MASK & ((ORDER == 0) ? 0x2 : 0x2)) ? SIZE/2 + (CHUNK+2-1)/2
: (MASK & ((ORDER == 0) ? 0x1 : 0x4)) ? CHUNK
: 0;
// sync?
if (SYNC) {
lfsr_file_sync(&lfs, &file) => 0;
}
// remount?
if (REMOUNT) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0;
}
// write first chunk?
if (MASK & 0x1) {
for (lfs_size_t w = 0; w < WRITES; w++) {
if (ORDER == 0) {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[i] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_seek(&lfs, &file, 0, LFS_SEEK_SET) => 0;
lfsr_file_write(&lfs, &file, &sim[0], CHUNK) => CHUNK;
} else {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[SIZE-CHUNK+i] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_seek(&lfs, &file, SIZE-CHUNK, LFS_SEEK_SET) => SIZE-CHUNK;
lfsr_file_write(&lfs, &file, &sim[SIZE-CHUNK], CHUNK) => CHUNK;
}
// sync?
if (SYNC) {
lfsr_file_sync(&lfs, &file) => 0;
}
// remount?
if (REMOUNT) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0;
}
}
}
// write second chunk?
if (MASK & 0x2) {
for (lfs_size_t w = 0; w < WRITES; w++) {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[SIZE/2-CHUNK/2+i] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_seek(&lfs, &file, SIZE/2 - CHUNK/2, LFS_SEEK_SET)
=> SIZE/2 - CHUNK/2;
lfsr_file_write(&lfs, &file, &sim[SIZE/2-CHUNK/2], CHUNK) => CHUNK;
// sync?
if (SYNC) {
lfsr_file_sync(&lfs, &file) => 0;
}
// remount?
if (REMOUNT) {
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0;
}
}
}
// write third chunk?
if (MASK & 0x4) {
for (lfs_size_t w = 0; w < WRITES; w++) {
if (ORDER == 0) {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[SIZE-CHUNK+i] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_seek(&lfs, &file, SIZE-CHUNK, LFS_SEEK_SET) => SIZE-CHUNK;
lfsr_file_write(&lfs, &file, &sim[SIZE-CHUNK], CHUNK) => CHUNK;
} else {
for (lfs_size_t i = 0; i < CHUNK; i++) {
sim[i] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_seek(&lfs, &file, 0, LFS_SEEK_SET) => 0;
lfsr_file_write(&lfs, &file, &sim[0], CHUNK) => CHUNK;
}
}
}
lfsr_file_close(&lfs, &file) => 0;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, CFG) => 0;
}
// check our file with stat
struct lfs_info info;
lfsr_stat(&lfs, "hello", &info) => 0;
assert(strcmp(info.name, "hello") == 0);
assert(info.type == LFS_TYPE_REG);
assert(info.size == size);
// and with dir read
lfsr_dir_t dir;
lfsr_dir_open(&lfs, &dir, "/") => 0;
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, ".") == 0);
assert(info.type == LFS_TYPE_DIR);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, "..") == 0);
assert(info.type == LFS_TYPE_DIR);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, "hello") == 0);
assert(info.type == LFS_TYPE_REG);
assert(info.size == size);
lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT;
lfsr_dir_close(&lfs, &dir) => 0;
// try reading our file
lfsr_file_open(&lfs, &file, "hello", LFS_O_RDONLY) => 0;
// is size correct?
lfsr_file_size(&lfs, &file) => size;
// try reading
uint8_t rbuf[2*SIZE];
memset(rbuf, 0xaa, 2*SIZE);
lfsr_file_read(&lfs, &file, rbuf, 2*SIZE) => size;
// does our file match our simulation?
assert(memcmp(rbuf, sim, size) == 0);
lfsr_file_close(&lfs, &file) => 0;
lfsr_unmount(&lfs) => 0;
'''
# fuzz testing
[cases.test_files_fuzz_aligned]
defines.N = 100