From 8f2a6a3095cf39904ca5d7c994c40b357fc5e685 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 1 Jan 2024 13:50:24 -0600 Subject: [PATCH] Implemented file sync broadcasting Now, when files are synced, they broadcast their disk changes to any other opened file handles. In effect, all open files match disk after a sync call to any opened file handle pointing to that file. This was a much requested feature, as the previous behavior (multiple opened file handles maintain independent snapshots) is pretty different from other filesystems. It's also quite difficult to implement outside of the filesystem, since you need to track all opened files, requiring either unbounded RAM or a known upper limit. --- A bit unrelated, but this commit also changes bshrub estimate calculation to include all opened file handles. This adds some annoying complexity, but is necessary to prevent sporadic ERANGE errors when the same file is opened multiple times. The current implementation just refetches on-disk metadata. This adds some maybe unnecessary metadata lookups, but simplifies things by avoiding the tracking of on-disk sprout/shrub size, which risks falling out of date. Keep in mind we only recalculate the estimate every ~inline_size/2 bytes written. Just like lfsr_mdir_estimate, this scales O(n^2) with the number of opened files (this are basically the same function... hmmm... can they be deduplicated?). This is unlikely to be a problem for littlefs's use case, but just something to be aware of. Code changes: code stack before: 32920 3032 after: 33192 (+0.8%) 3048 (+0.5%) --- lfs.c | 88 ++++- runners/bench_runner.h | 2 +- runners/test_runner.h | 2 +- tests/test_fmulti.toml | 819 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 902 insertions(+), 9 deletions(-) create mode 100644 tests/test_fmulti.toml diff --git a/lfs.c b/lfs.c index f588baaa..b011183d 100644 --- a/lfs.c +++ b/lfs.c @@ -7080,23 +7080,81 @@ static int lfsr_bshrub_commit(lfs_t *lfs, // block_size and rbyds interact, and amortizes the estimate cost. // figure out how much data this commit progs + lfs_size_t progged = 0; for (lfs_size_t i = 0; i < attr_count; i++) { // only include tag overhead if tag is not a grow tag if (!lfsr_tag_isgrow(attrs[i].tag)) { - bshrub->progged += LFSR_ATTR_ESTIMATE; + progged += LFSR_ATTR_ESTIMATE; } - bshrub->progged += lfsr_data_size(&attrs[i].data); + progged += lfsr_data_size(&attrs[i].data); } // does progged exceed our shrub_size? need to recalculate an // accurate our estimate? + bshrub->progged += progged; if (bshrub->progged > lfs->cfg->shrub_size) { - lfs_ssize_t estimate = lfsr_rbyd_estimate(lfs, - &bshrub->rbyd, -1, -1, NULL); - if (estimate < 0) { - return estimate; + // include all unique sprouts/shrubs related to our file, + // including the on-disk sprout/shrub + lfs_size_t estimate = 0; + lfsr_tag_t tag; + lfsr_data_t data; + err = lfsr_mdir_lookupnext(lfs, mdir, mdir->mid, LFSR_TAG_DATA, + &tag, &data); + if (err && err != LFS_ERR_NOENT) { + return err; } - bshrub->progged = estimate; + + if (err != LFS_ERR_NOENT && tag == LFSR_TAG_DATA) { + lfs_ssize_t dsize = lfsr_bsprout_estimate__(lfs, + (const lfsr_bsprout_t*)&data); + if (dsize < 0) { + return dsize; + } + estimate += lfsr_data_size(&data); + + } else if (err != LFS_ERR_NOENT && tag == LFSR_TAG_BSHRUB) { + lfsr_rbyd_t shrub = mdir->rbyd; + err = lfsr_data_readtrunk(lfs, &data, + &shrub.trunk, (lfsr_rid_t*)&shrub.weight); + if (err) { + return err; + } + + lfs_ssize_t dsize = lfsr_bshrub_estimate__(lfs, + (const lfsr_bshrub_t*)&shrub); + if (dsize < 0) { + return dsize; + } + estimate += dsize; + } + + // this includes our current shrub + for (lfsr_openedmdir_t *opened = lfs->opened[ + LFS_TYPE_REG-LFS_TYPE_REG]; + opened; + opened = opened->next) { + lfsr_ftree_t *ftree = (lfsr_ftree_t*)opened; + if (ftree->mdir.mid == mdir->mid) { + if (lfsr_ftree_isbsprout(ftree)) { + lfs_ssize_t dsize = lfsr_bsprout_estimate__(lfs, + &ftree->u.bsprout); + if (dsize < 0) { + return dsize; + } + estimate += dsize; + + } else if (lfsr_ftree_isbshrub(ftree)) { + lfs_ssize_t dsize = lfsr_bshrub_estimate__(lfs, + &ftree->u.bshrub); + if (dsize < 0) { + return dsize; + } + estimate += dsize; + } + } + } + + bshrub->progged = estimate + progged; // do we overflow shrub_size/2? the 1/2 here prevents runaway // performance when the shrub is near full @@ -10745,7 +10803,23 @@ int lfsr_file_sync(lfs_t *lfs, lfsr_file_t *file) { goto failed; } + // mark as synced file->flags &= ~LFS_F_UNSYNCED; + // update other file handles + for (lfsr_openedmdir_t *opened = lfs->opened[ + LFS_TYPE_REG-LFS_TYPE_REG]; + opened; + opened = opened->next) { + lfsr_file_t *file_ = (lfsr_file_t*)opened; + if (file_->ftree.mdir.mid == file->ftree.mdir.mid) { + file_->size = file->size; + file_->ftree.u = file->ftree.u; + file_->buffer_pos = file->buffer_pos; + memcpy(file_->buffer, file->buffer, file->buffer_size); + file_->buffer_size = file->buffer_size; + } + } + return 0; failed:; diff --git a/runners/bench_runner.h b/runners/bench_runner.h index 0d055715..c6a9efd0 100644 --- a/runners/bench_runner.h +++ b/runners/bench_runner.h @@ -156,7 +156,7 @@ intmax_t bench_define(size_t define); BENCH_DEF(BLOCK_COUNT, DISK_SIZE/BLOCK_SIZE ) \ BENCH_DEF(DISK_SIZE, 1024*1024 ) \ BENCH_DEF(CACHE_SIZE, lfs_max(16, lfs_max(READ_SIZE, PROG_SIZE))) \ - BENCH_DEF(INLINE_SIZE, BLOCK_SIZE/8 ) \ + BENCH_DEF(INLINE_SIZE, BLOCK_SIZE/4 ) \ BENCH_DEF(SHRUB_SIZE, INLINE_SIZE ) \ BENCH_DEF(FRAGMENT_SIZE, CACHE_SIZE ) \ BENCH_DEF(CRYSTAL_THRESH, BLOCK_SIZE/8 ) \ diff --git a/runners/test_runner.h b/runners/test_runner.h index da38fd43..0274fda7 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -142,7 +142,7 @@ intmax_t test_define(size_t define); TEST_DEF(BLOCK_COUNT, DISK_SIZE/BLOCK_SIZE ) \ TEST_DEF(DISK_SIZE, 1024*1024 ) \ TEST_DEF(CACHE_SIZE, lfs_max(16, lfs_max(READ_SIZE, PROG_SIZE)) ) \ - TEST_DEF(INLINE_SIZE, BLOCK_SIZE/8 ) \ + TEST_DEF(INLINE_SIZE, BLOCK_SIZE/4 ) \ TEST_DEF(SHRUB_SIZE, INLINE_SIZE ) \ TEST_DEF(FRAGMENT_SIZE, CACHE_SIZE ) \ TEST_DEF(CRYSTAL_THRESH, BLOCK_SIZE/8 ) \ diff --git a/tests/test_fmulti.toml b/tests/test_fmulti.toml new file mode 100644 index 00000000..1cbca13a --- /dev/null +++ b/tests/test_fmulti.toml @@ -0,0 +1,819 @@ +# Test multiple open file handles in different r/w configurations +after = 'test_fwrite' + +# Test multiple readers, this shouldn't really have any issues +[cases.test_fmulti_rrrr] +defines.R = 4 +defines.SIZE = [ + '0', + 'CACHE_SIZE/2', + '2*CACHE_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '4*BLOCK_SIZE', +] +defines.CHUNK = [32, 8, 1] +if = 'CHUNK <= SIZE' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + // create a file + uint32_t prng = 42; + uint8_t before[SIZE]; + for (lfs_size_t i = 0; i < SIZE; i++) { + before[i] = 'a' + (TEST_PRNG(&prng) % 26); + } + + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jello", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + lfsr_file_write(&lfs, &file, before, SIZE) => SIZE; + lfsr_file_close(&lfs, &file) => 0; + + // read R handles in parallel + lfsr_file_t readers[R]; + for (lfs_size_t r = 0; r < R; r++) { + lfsr_file_open(&lfs, &readers[r], "jello", LFS_O_RDONLY) => 0; + } + for (lfs_size_t i = 0; i < SIZE; i += CHUNK) { + for (lfs_size_t r = 0; r < R; r++) { + uint8_t rbuf[CHUNK]; + lfsr_file_read(&lfs, &readers[r], rbuf, CHUNK) => CHUNK; + assert(memcmp(rbuf, &before[i], CHUNK) == 0); + } + } + for (lfs_size_t r = 0; r < R; r++) { + lfsr_file_close(&lfs, &readers[r]) => 0; + } + + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_fmulti_rrrr_fuzz] +defines.R = 4 +defines.SEED = 'range(10)' +defines.N = 20 +defines.SIZE = [ + '0', + 'CACHE_SIZE/2', + '2*CACHE_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '4*BLOCK_SIZE', +] +defines.CHUNK = [32, 8, 1] +if = 'CHUNK <= SIZE' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + // create a file + uint32_t prng = SEED; + uint8_t before[SIZE]; + for (lfs_size_t i = 0; i < SIZE; i++) { + before[i] = 'a' + (TEST_PRNG(&prng) % 26); + } + + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jello", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + lfsr_file_write(&lfs, &file, before, SIZE) => SIZE; + lfsr_file_close(&lfs, &file) => 0; + + // read R handles in parallel + lfsr_file_t readers[R]; + for (lfs_size_t r = 0; r < R; r++) { + lfsr_file_open(&lfs, &readers[r], "jello", LFS_O_RDONLY) => 0; + } + for (lfs_size_t i = 0; i < N; i++) { + for (lfs_size_t r = 0; r < R; r++) { + // choose a random offset + lfs_off_t off = TEST_PRNG(&prng) % SIZE; + lfs_size_t size = lfs_min32(CHUNK, SIZE - off); + lfsr_file_seek(&lfs, &readers[r], off, LFS_SEEK_SET) => off; + + // read + uint8_t rbuf[CHUNK]; + lfsr_file_read(&lfs, &readers[r], rbuf, CHUNK) => size; + assert(memcmp(rbuf, &before[off], size) == 0); + } + } + for (lfs_size_t r = 0; r < R; r++) { + lfsr_file_close(&lfs, &readers[r]) => 0; + } + + lfsr_unmount(&lfs) => 0; +''' + +# Test one writer, multiple readers +[cases.test_fmulti_wrrr] +defines.R = 4 +# 0 => no sync, readers not updated +# 1 => sync via lfsr_file_sync +defines.SYNC = [0, 1] +defines.SIZE = [ + '0', + 'CACHE_SIZE/2', + '2*CACHE_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '4*BLOCK_SIZE', +] +defines.CHUNK = [32, 8, 1] +if = 'CHUNK <= SIZE' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + // create a file + uint32_t prng = 42; + uint8_t before[SIZE]; + for (lfs_size_t i = 0; i < SIZE; i++) { + before[i] = 'a' + (TEST_PRNG(&prng) % 26); + } + uint8_t after[SIZE]; + memcpy(after, before, SIZE); + + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jello", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + lfsr_file_write(&lfs, &file, before, SIZE) => SIZE; + lfsr_file_close(&lfs, &file) => 0; + + // write 1 handle, read R handles in parallel + lfsr_file_t writer; + lfsr_file_t readers[R]; + lfsr_file_open(&lfs, &writer, "jello", LFS_O_WRONLY) => 0; + for (lfs_size_t r = 0; r < R; r++) { + lfsr_file_open(&lfs, &readers[r], "jello", LFS_O_RDONLY) => 0; + } + for (lfs_size_t i = 0; i < SIZE; i += CHUNK) { + uint8_t wbuf[CHUNK]; + for (lfs_size_t j = 0; j < CHUNK; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &writer, wbuf, CHUNK) => CHUNK; + memcpy(&after[i], wbuf, CHUNK); + if (SYNC == 1) { + lfsr_file_sync(&lfs, &writer) => 0; + } + + for (lfs_size_t r = 0; r < R; r++) { + uint8_t rbuf[CHUNK]; + lfsr_file_read(&lfs, &readers[r], rbuf, CHUNK) => CHUNK; + if (SYNC == 0) { + assert(memcmp(rbuf, &before[i], CHUNK) == 0); + } else { + assert(memcmp(rbuf, &after[i], CHUNK) == 0); + } + } + } + lfsr_file_close(&lfs, &writer) => 0; + for (lfs_size_t r = 0; r < R; r++) { + lfsr_file_close(&lfs, &readers[r]) => 0; + } + + // check that file was written as expected + lfsr_file_open(&lfs, &file, "jello", LFS_O_RDONLY) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, &after, SIZE) == 0); + lfsr_file_close(&lfs, &file) => 0; + + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_fmulti_wrrr_fuzz] +defines.R = 4 +# 0 => no sync, readers not updated +# 1 => sync via lfsr_file_sync +defines.SYNC = [0, 1] +defines.SEED = 'range(10)' +defines.N = 20 +defines.SIZE = [ + '0', + 'CACHE_SIZE/2', + '2*CACHE_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '4*BLOCK_SIZE', +] +defines.CHUNK = [32, 8, 1] +if = 'CHUNK <= SIZE' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + // create a file + uint32_t prng = 42; + uint8_t before[SIZE]; + for (lfs_size_t i = 0; i < SIZE; i++) { + before[i] = 'a' + (TEST_PRNG(&prng) % 26); + } + uint8_t after[SIZE]; + memcpy(after, before, SIZE); + + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jello", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + lfsr_file_write(&lfs, &file, before, SIZE) => SIZE; + lfsr_file_close(&lfs, &file) => 0; + + // write 1 handle, read R handles in parallel + lfsr_file_t writer; + lfsr_file_t readers[R]; + lfsr_file_open(&lfs, &writer, "jello", LFS_O_WRONLY) => 0; + for (lfs_size_t r = 0; r < R; r++) { + lfsr_file_open(&lfs, &readers[r], "jello", LFS_O_RDONLY) => 0; + } + for (lfs_size_t i = 0; i < N; i++) { + // choose a random offset + lfs_off_t off = TEST_PRNG(&prng) % SIZE; + lfs_size_t size = lfs_min32(CHUNK, SIZE - off); + lfsr_file_seek(&lfs, &writer, off, LFS_SEEK_SET) => off; + + // write + uint8_t wbuf[CHUNK]; + for (lfs_size_t j = 0; j < size; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &writer, wbuf, size) => size; + memcpy(&after[off], wbuf, size); + if (SYNC == 1) { + lfsr_file_sync(&lfs, &writer) => 0; + } + + for (lfs_size_t r = 0; r < R; r++) { + // choose a random offset + lfs_off_t off = TEST_PRNG(&prng) % SIZE; + lfs_size_t size = lfs_min32(CHUNK, SIZE - off); + lfsr_file_seek(&lfs, &readers[r], off, LFS_SEEK_SET) => off; + + // read + uint8_t rbuf[CHUNK]; + lfsr_file_read(&lfs, &readers[r], rbuf, CHUNK) => size; + if (SYNC == 0) { + assert(memcmp(rbuf, &before[off], size) == 0); + } else { + assert(memcmp(rbuf, &after[off], size) == 0); + } + } + } + lfsr_file_close(&lfs, &writer) => 0; + for (lfs_size_t r = 0; r < R; r++) { + lfsr_file_close(&lfs, &readers[r]) => 0; + } + + // check that file was written as expected + lfsr_file_open(&lfs, &file, "jello", LFS_O_RDONLY) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, &after, SIZE) == 0); + lfsr_file_close(&lfs, &file) => 0; + + lfsr_unmount(&lfs) => 0; +''' + +# Test multiple writers +[cases.test_fmulti_wwww] +defines.W = 4 +# 0 => no sync, readers not updated +# 1 => sync via lfsr_file_sync +defines.SYNC = [0, 1] +defines.SIZE = [ + '0', + 'CACHE_SIZE/2', + '2*CACHE_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '4*BLOCK_SIZE', +] +defines.CHUNK = [32, 8, 1] +if = 'CHUNK <= SIZE' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + // create a file + uint32_t prng = 42; + uint8_t before[SIZE]; + for (lfs_size_t i = 0; i < SIZE; i++) { + before[i] = 'a' + (TEST_PRNG(&prng) % 26); + } + uint8_t after[SIZE]; + memcpy(after, before, SIZE); + + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jello", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + lfsr_file_write(&lfs, &file, before, SIZE) => SIZE; + lfsr_file_close(&lfs, &file) => 0; + + // write W handles in parallel + lfsr_file_t writers[W]; + for (lfs_size_t w = 0; w < W; w++) { + lfsr_file_open(&lfs, &writers[w], "jello", LFS_O_WRONLY) => 0; + } + for (lfs_size_t i = 0; i < SIZE; i += CHUNK) { + for (lfs_size_t w = 0; w < W; w++) { + uint8_t wbuf[CHUNK]; + for (lfs_size_t j = 0; j < CHUNK; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &writers[w], wbuf, CHUNK) => CHUNK; + if (SYNC == 1) { + lfsr_file_sync(&lfs, &writers[w]) => 0; + memcpy(&after[i], wbuf, CHUNK); + } else { + if (w == 0) { + memcpy(&after[i], wbuf, CHUNK); + } + } + } + } + for (lfs_size_t w = 0; w < W; w++) { + lfsr_file_close(&lfs, &writers[w]) => 0; + } + + // check that file was written as expected + lfsr_file_open(&lfs, &file, "jello", LFS_O_RDONLY) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, &after, SIZE) == 0); + lfsr_file_close(&lfs, &file) => 0; + + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_fmulti_wwww_fuzz] +defines.W = 4 +# 0 => no sync, readers not updated +# 1 => sync via lfsr_file_sync +defines.SYNC = [0, 1] +defines.SEED = 'range(10)' +defines.N = 20 +defines.SIZE = [ + '0', + 'CACHE_SIZE/2', + '2*CACHE_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '4*BLOCK_SIZE', +] +defines.CHUNK = 32 +if = 'CHUNK <= SIZE' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + // create a file + uint32_t prng = 42; + uint8_t before[SIZE]; + for (lfs_size_t i = 0; i < SIZE; i++) { + before[i] = 'a' + (TEST_PRNG(&prng) % 26); + } + uint8_t after[SIZE]; + memcpy(after, before, SIZE); + + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jello", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + lfsr_file_write(&lfs, &file, before, SIZE) => SIZE; + lfsr_file_close(&lfs, &file) => 0; + + // write W files in parallel + lfsr_file_t writers[W]; + for (lfs_size_t w = 0; w < W; w++) { + lfsr_file_open(&lfs, &writers[w], "jello", LFS_O_WRONLY) => 0; + } + for (lfs_size_t i = 0; i < SIZE; i += CHUNK) { + for (lfs_size_t w = 0; w < W; w++) { + // choose a random offset + lfs_off_t off = TEST_PRNG(&prng) % SIZE; + lfs_size_t size = lfs_min32(CHUNK, SIZE - off); + lfsr_file_seek(&lfs, &writers[w], off, LFS_SEEK_SET) => off; + + // write + uint8_t wbuf[CHUNK]; + for (lfs_size_t j = 0; j < size; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &writers[w], wbuf, size) => size; + if (SYNC == 1) { + lfsr_file_sync(&lfs, &writers[w]) => 0; + memcpy(&after[off], wbuf, size); + } else { + if (w == 0) { + memcpy(&after[off], wbuf, size); + } + } + } + } + for (lfs_size_t w = 0; w < W; w++) { + lfsr_file_close(&lfs, &writers[w]) => 0; + } + + // check that file was written as expected + lfsr_file_open(&lfs, &file, "jello", LFS_O_RDONLY) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, &after, SIZE) == 0); + lfsr_file_close(&lfs, &file) => 0; + + lfsr_unmount(&lfs) => 0; +''' + +# Test multiple writers and multiple readers +[cases.test_fmulti_wwrr] +defines.W = 4 +defines.R = 4 +# 0 => no sync, readers not updated +# 1 => sync via lfsr_file_sync +defines.SYNC = [0, 1] +defines.SIZE = [ + '0', + 'CACHE_SIZE/2', + '2*CACHE_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '4*BLOCK_SIZE', +] +defines.CHUNK = [32, 8, 1] +if = 'CHUNK <= SIZE' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + // create a file + uint32_t prng = 42; + uint8_t before[SIZE]; + for (lfs_size_t i = 0; i < SIZE; i++) { + before[i] = 'a' + (TEST_PRNG(&prng) % 26); + } + uint8_t after[SIZE]; + memcpy(after, before, SIZE); + + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jello", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + lfsr_file_write(&lfs, &file, before, SIZE) => SIZE; + lfsr_file_close(&lfs, &file) => 0; + + // write W handles, R handles in parallel + lfsr_file_t writers[W]; + lfsr_file_t readers[R]; + for (lfs_size_t w = 0; w < W; w++) { + lfsr_file_open(&lfs, &writers[w], "jello", LFS_O_WRONLY) => 0; + } + for (lfs_size_t r = 0; r < R; r++) { + lfsr_file_open(&lfs, &readers[r], "jello", LFS_O_RDONLY) => 0; + } + for (lfs_size_t i = 0; i < SIZE; i += CHUNK) { + for (lfs_size_t w = 0; w < W; w++) { + uint8_t wbuf[CHUNK]; + for (lfs_size_t j = 0; j < CHUNK; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &writers[w], wbuf, CHUNK) => CHUNK; + if (SYNC == 1) { + lfsr_file_sync(&lfs, &writers[w]) => 0; + memcpy(&after[i], wbuf, CHUNK); + } else { + if (w == 0) { + memcpy(&after[i], wbuf, CHUNK); + } + } + } + for (lfs_size_t r = 0; r < R; r++) { + uint8_t rbuf[CHUNK]; + lfsr_file_read(&lfs, &readers[r], rbuf, CHUNK) => CHUNK; + if (SYNC == 0) { + assert(memcmp(rbuf, &before[i], CHUNK) == 0); + } else { + assert(memcmp(rbuf, &after[i], CHUNK) == 0); + } + } + } + for (lfs_size_t w = 0; w < W; w++) { + lfsr_file_close(&lfs, &writers[w]) => 0; + } + for (lfs_size_t r = 0; r < R; r++) { + lfsr_file_close(&lfs, &readers[r]) => 0; + } + + // check that file was written as expected + lfsr_file_open(&lfs, &file, "jello", LFS_O_RDONLY) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, &after, SIZE) == 0); + lfsr_file_close(&lfs, &file) => 0; + + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_fmulti_wwrr_fuzz] +defines.W = 4 +defines.R = 4 +# 0 => no sync, readers not updated +# 1 => sync via lfsr_file_sync +defines.SYNC = [0, 1] +defines.SEED = 'range(10)' +defines.N = 20 +defines.SIZE = [ + '0', + 'CACHE_SIZE/2', + '2*CACHE_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '4*BLOCK_SIZE', +] +defines.CHUNK = 32 +if = 'CHUNK <= SIZE' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + // create a file + uint32_t prng = 42; + uint8_t before[SIZE]; + for (lfs_size_t i = 0; i < SIZE; i++) { + before[i] = 'a' + (TEST_PRNG(&prng) % 26); + } + uint8_t after[SIZE]; + memcpy(after, before, SIZE); + + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jello", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + lfsr_file_write(&lfs, &file, before, SIZE) => SIZE; + lfsr_file_close(&lfs, &file) => 0; + + // write W files in parallel + lfsr_file_t writers[W]; + lfsr_file_t readers[R]; + for (lfs_size_t w = 0; w < W; w++) { + lfsr_file_open(&lfs, &writers[w], "jello", LFS_O_WRONLY) => 0; + } + for (lfs_size_t r = 0; r < R; r++) { + lfsr_file_open(&lfs, &readers[r], "jello", LFS_O_RDONLY) => 0; + } + for (lfs_size_t i = 0; i < SIZE; i += CHUNK) { + for (lfs_size_t w = 0; w < W; w++) { + // choose a random offset + lfs_off_t off = TEST_PRNG(&prng) % SIZE; + lfs_size_t size = lfs_min32(CHUNK, SIZE - off); + lfsr_file_seek(&lfs, &writers[w], off, LFS_SEEK_SET) => off; + + // write + uint8_t wbuf[CHUNK]; + for (lfs_size_t j = 0; j < size; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &writers[w], wbuf, size) => size; + if (SYNC == 1) { + lfsr_file_sync(&lfs, &writers[w]) => 0; + memcpy(&after[off], wbuf, size); + } else { + if (w == 0) { + memcpy(&after[off], wbuf, size); + } + } + } + for (lfs_size_t r = 0; r < R; r++) { + // choose a random offset + lfs_off_t off = TEST_PRNG(&prng) % SIZE; + lfs_size_t size = lfs_min32(CHUNK, SIZE - off); + lfsr_file_seek(&lfs, &readers[r], off, LFS_SEEK_SET) => off; + + // read + uint8_t rbuf[CHUNK]; + lfsr_file_read(&lfs, &readers[r], rbuf, CHUNK) => size; + if (SYNC == 0) { + assert(memcmp(rbuf, &before[off], size) == 0); + } else { + assert(memcmp(rbuf, &after[off], size) == 0); + } + } + } + for (lfs_size_t w = 0; w < W; w++) { + lfsr_file_close(&lfs, &writers[w]) => 0; + } + for (lfs_size_t r = 0; r < R; r++) { + lfsr_file_close(&lfs, &readers[r]) => 0; + } + + // check that file was written as expected + lfsr_file_open(&lfs, &file, "jello", LFS_O_RDONLY) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, &after, SIZE) == 0); + lfsr_file_close(&lfs, &file) => 0; + + lfsr_unmount(&lfs) => 0; +''' + +# Test multiple rd/wrers +[cases.test_fmulti_mmmm] +defines.M = 4 +# 0 => no sync, readers not updated +# 1 => sync via lfsr_file_sync +defines.SYNC = [0, 1] +defines.SIZE = [ + '0', + 'CACHE_SIZE/2', + '2*CACHE_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '4*BLOCK_SIZE', +] +defines.CHUNK = [32, 8, 1] +if = 'CHUNK <= SIZE' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + // create a file + uint32_t prng = 42; + uint8_t before[SIZE]; + for (lfs_size_t i = 0; i < SIZE; i++) { + before[i] = 'a' + (TEST_PRNG(&prng) % 26); + } + uint8_t between[M][SIZE]; + for (lfs_size_t m = 0; m < M; m++) { + memcpy(between[M], before, SIZE); + } + uint8_t after[SIZE]; + memcpy(after, before, SIZE); + + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jello", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + lfsr_file_write(&lfs, &file, before, SIZE) => SIZE; + lfsr_file_close(&lfs, &file) => 0; + + // write M rdwrs in parallel + lfsr_file_t rdwrs[M]; + for (lfs_size_t m = 0; m < M; m++) { + lfsr_file_open(&lfs, &rdwrs[m], "jello", LFS_O_RDWR) => 0; + } + for (lfs_size_t i = 0; i < SIZE; i += CHUNK) { + for (lfs_size_t m = 0; m < M; m++) { + uint8_t wbuf[CHUNK]; + for (lfs_size_t j = 0; j < CHUNK; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_seek(&lfs, &rdwrs[m], i, LFS_SEEK_SET) => i; + lfsr_file_write(&lfs, &rdwrs[m], wbuf, CHUNK) => CHUNK; + memcpy(&between[m][i], wbuf, CHUNK); + if (SYNC == 1) { + lfsr_file_sync(&lfs, &rdwrs[m]) => 0; + memcpy(&after[i], wbuf, CHUNK); + } else { + if (m == 0) { + memcpy(&after[i], wbuf, CHUNK); + } + } + } + for (lfs_size_t m = 0; m < M; m++) { + lfsr_file_seek(&lfs, &rdwrs[m], i, LFS_SEEK_SET) => i; + uint8_t rbuf[CHUNK]; + lfsr_file_read(&lfs, &rdwrs[m], rbuf, CHUNK) => CHUNK; + if (SYNC == 0) { + assert(memcmp(rbuf, &between[m][i], CHUNK) == 0); + } else { + assert(memcmp(rbuf, &after[i], CHUNK) == 0); + } + } + } + for (lfs_size_t m = 0; m < M; m++) { + lfsr_file_close(&lfs, &rdwrs[m]) => 0; + } + + // check that file was written as expected + lfsr_file_open(&lfs, &file, "jello", LFS_O_RDONLY) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, &after, SIZE) == 0); + lfsr_file_close(&lfs, &file) => 0; + + lfsr_unmount(&lfs) => 0; +''' + +[cases.test_fmulti_mmmm_fuzz] +defines.M = 4 +# 0 => no sync, readers not updated +# 1 => sync via lfsr_file_sync +defines.SYNC = [0, 1] +defines.SEED = 'range(10)' +defines.N = 20 +defines.SIZE = [ + '0', + 'CACHE_SIZE/2', + '2*CACHE_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '4*BLOCK_SIZE', +] +defines.CHUNK = 32 +if = 'CHUNK <= SIZE' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + // create a file + uint32_t prng = 42; + uint8_t before[SIZE]; + for (lfs_size_t i = 0; i < SIZE; i++) { + before[i] = 'a' + (TEST_PRNG(&prng) % 26); + } + uint8_t between[M][SIZE]; + for (lfs_size_t m = 0; m < M; m++) { + memcpy(between[m], before, SIZE); + } + uint8_t after[SIZE]; + memcpy(after, before, SIZE); + + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jello", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + lfsr_file_write(&lfs, &file, before, SIZE) => SIZE; + lfsr_file_close(&lfs, &file) => 0; + + // write M rdwrs in parallel + lfsr_file_t rdwrs[M]; + for (lfs_size_t m = 0; m < M; m++) { + lfsr_file_open(&lfs, &rdwrs[m], "jello", LFS_O_RDWR) => 0; + } + for (lfs_size_t i = 0; i < SIZE; i += CHUNK) { + for (lfs_size_t m = 0; m < M; m++) { + // choose a random offset + lfs_off_t off = TEST_PRNG(&prng) % SIZE; + lfs_size_t size = lfs_min32(CHUNK, SIZE - off); + lfsr_file_seek(&lfs, &rdwrs[m], off, LFS_SEEK_SET) => off; + + // write + uint8_t wbuf[CHUNK]; + for (lfs_size_t j = 0; j < size; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &rdwrs[m], wbuf, size) => size; + memcpy(&between[m][off], wbuf, size); + if (SYNC == 1) { + lfsr_file_sync(&lfs, &rdwrs[m]) => 0; + memcpy(&after[off], wbuf, size); + } else { + if (m == 0) { + memcpy(&after[off], wbuf, size); + } + } + } + for (lfs_size_t m = 0; m < M; m++) { + // choose a random offset + lfs_off_t off = TEST_PRNG(&prng) % SIZE; + lfs_size_t size = lfs_min32(CHUNK, SIZE - off); + lfsr_file_seek(&lfs, &rdwrs[m], off, LFS_SEEK_SET) => off; + + // read + uint8_t rbuf[CHUNK]; + lfsr_file_read(&lfs, &rdwrs[m], rbuf, CHUNK) => size; + if (SYNC == 0) { + assert(memcmp(rbuf, &between[m][off], size) == 0); + } else { + assert(memcmp(rbuf, &after[off], size) == 0); + } + } + } + for (lfs_size_t m = 0; m < M; m++) { + lfsr_file_close(&lfs, &rdwrs[m]) => 0; + } + + // check that file was written as expected + lfsr_file_open(&lfs, &file, "jello", LFS_O_RDONLY) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, &after, SIZE) == 0); + lfsr_file_close(&lfs, &file) => 0; + + lfsr_unmount(&lfs) => 0; +'''