From 61c03cf2757def9cdd00d7a7b429954d63aae845 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 27 May 2024 23:07:04 -0500 Subject: [PATCH] Added wear-leveling litmus tests, test_relocations_wl_* These tests provide a litmus test for if wear-leveling is working: - test_relocations_wl_dir_fuzz - test_relocations_wl_file_fuzz - test_relocations_wl_orphanzombie_fuzz - test_relocations_wl_orphanzombiedir_fuzz We can't test the uniformity of wear, because we only implement static wear-leveling, but what we can test is that doubling the size of storage results in roughly doubling the lifetime of the storage. I did try to implement some wear-leveling tests under powerloss, this has some promise storing the current run/state on disk, but gave up after realizing the way our linear powerloss heuristic works would interfere with the assumption that both runs run in identical environments... --- Suprisingly enough, all of this fuzz testing did find another bug! We were returning LFS_ERR_CORRUPT instead of LFS_ERR_NOSPC if overcompaction failed to erase/prog the revision count. This is very hard to hit, only being reachable if a block goes bad on the same erase cycle an mdir's recycle counter overflows, and if there are no more blocks in our filesystem, triggering overcompaction. Difficult to hit bug, but easy fix. Just a tiny bit of extra code: code stack before: 33550 2624 after: 33566 2624 I guess these wear-leveling tests are also doubling as aggressive LFS_ERR_NOSPC exhaustion tests... --- lfs.c | 4 + tests/test_relocations.toml | 1357 ++++++++++++++++++++++++++++++++++- 2 files changed, 1355 insertions(+), 6 deletions(-) diff --git a/lfs.c b/lfs.c index 155ea19a..f2569291 100644 --- a/lfs.c +++ b/lfs.c @@ -6604,6 +6604,10 @@ relocate:; err = lfsr_mdir_swap__(lfs, &mdir_, mdir, true); if (err) { + // bad prog? try another block + if (err == LFS_ERR_CORRUPT) { + goto relocate; + } return err; } } diff --git a/tests/test_relocations.toml b/tests/test_relocations.toml index 83a797a8..498c2b23 100644 --- a/tests/test_relocations.toml +++ b/tests/test_relocations.toml @@ -4,7 +4,8 @@ after = [ 'test_dirs', 'test_files', 'test_forphans', - 'test_alloc' + 'test_alloc', + 'test_badblocks', ] # Note that most of the delicate relocation operations are already tested @@ -1339,11 +1340,1355 @@ code = ''' lfsr_unmount(&lfs) => 0; ''' -# TODO, but this needs bad blocks -## test that wear-leveling converges to a more-or-less even distribution -#[cases.test_relocations_distribution_dirs] -#[cases.test_relocations_distribution_files] -#[cases.test_relocations_distribution_mixed] + +# High-level wear-leveling litmus tests +# +# littlefs implements the weaker form of wear-leveling: dynamic +# wear-leveling. This means we can't guarantee evenly distributed wear, +# but we can at least guarantee the lifetime of storage scales with the +# size of storage. +# +# This gives us something concrete we can test, that doubling the size of +# storage roughly doubles the lifetime of the storage. + +# test dir wear-leveling +[cases.test_relocations_wl_dir_fuzz] +defines.ERASE_CYCLES = 10 +defines.BLOCK_RECYCLES = 4 +defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256] +defines.SEED = 42 +code = ''' + // run our test twice, once with 1/2 the storage, once with 2/2 the + // storage, and compare how many operations we were able to perform + // before filesystem death + uint32_t run_bc[2] = {BLOCK_COUNT/2, BLOCK_COUNT}; + uint32_t run_ops[2] = {0, 0}; + + for (int run = 0; run < 2; run++) { + // clear any wear from the previous run + for (lfs_block_t i = 0; i < BLOCK_COUNT; i++) { + lfs_emubd_setwear(CFG, i, 0) => 0; + } + + // configure the filesystem size + struct lfs_config cfg = *CFG; + cfg.block_count = run_bc[run]; + + // run the test + lfs_t lfs; + lfsr_format(&lfs, &cfg) => 0; + lfsr_mount(&lfs, &cfg) => 0; + + // set up a simulation to compare against + lfs_size_t *sim = malloc(N*sizeof(lfs_size_t)); + lfs_size_t sim_size = 0; + + uint32_t prng = SEED; + for (;; run_ops[run]++) { + // choose a pseudo-random op, either mkdir, remove, or rename + uint8_t op = TEST_PRNG(&prng) % 3; + + if (op == 0 || sim_size == 0) { + // choose a pseudo-random number, truncate to 3 hexadecimals + lfs_size_t x = TEST_PRNG(&prng) % N; + + // insert into our sim + for (lfs_size_t j = 0;; j++) { + if (j >= sim_size || sim[j] >= x) { + // already seen? + if (j < sim_size && sim[j] == x) { + // do nothing + } else { + // insert + memmove(&sim[j+1], &sim[j], + (sim_size-j)*sizeof(lfs_size_t)); + sim_size += 1; + sim[j] = x; + } + break; + } + } + + // create a directory here + char name[256]; + sprintf(name, "dir%03x", x); + int err = lfsr_mkdir(&lfs, name); + assert(!err || err == LFS_ERR_EXIST || err == LFS_ERR_NOSPC); + if (err == LFS_ERR_NOSPC) { + goto dead; + } + + } else if (op == 1) { + // choose a pseudo-random entry to delete + lfs_size_t j = TEST_PRNG(&prng) % sim_size; + lfs_size_t x = sim[j]; + + // delete from our sim + memmove(&sim[j], &sim[j+1], + (sim_size-(j+1))*sizeof(lfs_size_t)); + sim_size -= 1; + + // remove this directory + char name[256]; + sprintf(name, "dir%03x", x); + int err = lfsr_remove(&lfs, name); + assert(!err || err == LFS_ERR_NOSPC); + if (err == LFS_ERR_NOSPC) { + goto dead; + } + + } else { + // choose a pseudo-random entry to rename, and a pseudo-random + // number to rename to + lfs_size_t j = TEST_PRNG(&prng) % sim_size; + lfs_size_t x = sim[j]; + lfs_size_t y = TEST_PRNG(&prng) % N; + for (lfs_size_t k = 0;; k++) { + if (k >= sim_size || sim[k] >= y) { + // already seen and not a noop? + if (k < sim_size && sim[k] == y && x != y) { + // just delete the original entry + memmove(&sim[j], &sim[j+1], + (sim_size-(j+1))*sizeof(lfs_size_t)); + sim_size -= 1; + } else { + // first delete + memmove(&sim[j], &sim[j+1], + (sim_size-(j+1))*sizeof(lfs_size_t)); + if (k > j) { + k -= 1; + } + // then insert + memmove(&sim[k+1], &sim[k], + (sim_size-k)*sizeof(lfs_size_t)); + sim[k] = y; + } + break; + } + } + + // rename this directory + char old_name[256]; + sprintf(old_name, "dir%03x", x); + char new_name[256]; + sprintf(new_name, "dir%03x", y); + int err = lfsr_rename(&lfs, old_name, new_name); + assert(!err || err == LFS_ERR_NOSPC); + if (err == LFS_ERR_NOSPC) { + goto dead; + } + } + + // check our simulation every power-of-2 ops + if (lfs_popc(run_ops[run]) == 1) { + // test that our directories match our simulation + for (lfs_size_t j = 0; j < sim_size; j++) { + char name[256]; + sprintf(name, "dir%03x", sim[j]); + struct lfs_info info; + lfsr_stat(&lfs, name, &info) => 0; + char name2[256]; + sprintf(name2, "dir%03x", sim[j]); + assert(strcmp(info.name, name2) == 0); + assert(info.type == LFS_TYPE_DIR); + assert(info.size == 0); + } + + lfsr_dir_t dir; + lfsr_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, ".") == 0); + assert(info.type == LFS_TYPE_DIR); + assert(info.size == 0); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "..") == 0); + assert(info.type == LFS_TYPE_DIR); + assert(info.size == 0); + for (lfs_size_t j = 0; j < sim_size; j++) { + char name[256]; + sprintf(name, "dir%03x", sim[j]); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, name) == 0); + assert(info.type == LFS_TYPE_DIR); + assert(info.size == 0); + } + lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT; + lfsr_dir_close(&lfs, &dir) => 0; + } + } + + dead:; + // clean up sim/lfs + free(sim); + lfsr_unmount(&lfs) => 0; + + // print how many ops + printf("run %d, %dx%d, %d ec: %d ops\n", + run, + (int)BLOCK_SIZE, + run_bc[run], + (int)ERASE_CYCLES, + run_ops[run]); + } + + // check that we increased the liftime by ~2x, with ~10% error + printf("lifetime: %d -> %d (x%.2f)\n", + run_ops[0], + run_ops[1], + (double)run_ops[1] / (double)run_ops[0]); + assert(run_ops[1]*110/100 > 2*run_ops[0]); +''' + +# test file wear-leveling +[cases.test_relocations_wl_file_fuzz] +defines.ERASE_CYCLES = 10 +defines.BLOCK_RECYCLES = 4 +defines.N = [1, 2, 4, 8, 16, 32, 64] +defines.SIZE = [ + '0', + 'FBUFFER_SIZE/2', + '2*FBUFFER_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '4*BLOCK_SIZE', +] +defines.SEED = 42 +if = '(SIZE*N)/BLOCK_SIZE <= 16' +code = ''' + // run our test twice, once with 1/2 the storage, once with 2/2 the + // storage, and compare how many operations we were able to perform + // before filesystem death + uint32_t run_bc[2] = {BLOCK_COUNT/2, BLOCK_COUNT}; + uint32_t run_ops[2] = {0, 0}; + + for (int run = 0; run < 2; run++) { + // clear any wear from the previous run + for (lfs_block_t i = 0; i < BLOCK_COUNT; i++) { + lfs_emubd_setwear(CFG, i, 0) => 0; + } + + // configure the filesystem size + struct lfs_config cfg = *CFG; + cfg.block_count = run_bc[run]; + + // run the test + lfs_t lfs; + lfsr_format(&lfs, &cfg) => 0; + lfsr_mount(&lfs, &cfg) => 0; + + // set up a simulation to compare against + lfs_size_t *sim = malloc(N*sizeof(lfs_size_t)); + uint32_t *sim_prngs = malloc(N*sizeof(uint32_t)); + lfs_size_t sim_size = 0; + + uint32_t prng = SEED; + for (;; run_ops[run]++) { + // choose which operation to do + uint8_t op = TEST_PRNG(&prng) % 3; + + // creating a new file? + if (op == 0 || sim_size == 0) { + // choose a pseudo-random number + lfs_size_t x = TEST_PRNG(&prng) % N; + // associate each file with a prng that generates its contents + uint32_t wprng = TEST_PRNG(&prng); + + // insert into our sim + for (lfs_size_t j = 0;; j++) { + if (j >= sim_size || sim[j] >= x) { + // already seen? + if (j < sim_size && sim[j] == x) { + // new prng + sim_prngs[j] = wprng; + } else { + // insert + memmove(&sim[j+1], &sim[j], + (sim_size-j)*sizeof(lfs_size_t)); + memmove(&sim_prngs[j+1], &sim_prngs[j], + (sim_size-j)*sizeof(uint32_t)); + sim_size += 1; + sim[j] = x; + sim_prngs[j] = wprng; + } + break; + } + } + + // create a file here + char name[256]; + sprintf(name, "amethyst%03x", x); + uint8_t wbuf[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&wprng) % 26); + } + + lfsr_file_t file; + int err = lfsr_file_open(&lfs, &file, name, + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC); + assert(!err || err == LFS_ERR_NOSPC); + if (err) { + goto dead; + } + lfs_ssize_t d = lfsr_file_write(&lfs, &file, wbuf, SIZE); + assert(d == SIZE || d == LFS_ERR_NOSPC); + if (d == LFS_ERR_NOSPC) { + goto dead; + } + err = lfsr_file_close(&lfs, &file); + assert(!err || err == LFS_ERR_NOSPC); + if (err == LFS_ERR_NOSPC) { + goto dead; + } + + // deleting a file? + } else if (op == 1) { + // choose a random file to delete + lfs_size_t j = TEST_PRNG(&prng) % sim_size; + lfs_size_t x = sim[j]; + // delete from our sim + memmove(&sim[j], &sim[j+1], + (sim_size-(j+1))*sizeof(lfs_size_t)); + memmove(&sim_prngs[j], &sim_prngs[j+1], + (sim_size-(j+1))*sizeof(uint32_t)); + sim_size -= 1; + + // delete this file + char name[256]; + sprintf(name, "amethyst%03x", x); + int err = lfsr_remove(&lfs, name); + assert(!err || err == LFS_ERR_NOSPC); + if (err == LFS_ERR_NOSPC) { + goto dead; + } + + // renaming a file? + } else { + // choose a random file to rename, and a random number to + // rename to + lfs_size_t j = TEST_PRNG(&prng) % sim_size; + lfs_size_t x = sim[j]; + lfs_size_t y = TEST_PRNG(&prng) % N; + uint32_t wprng = sim_prngs[j]; + + // update our sim + for (lfs_size_t k = 0;; k++) { + if (k >= sim_size || sim[k] >= y) { + // renaming and replacing + if (k < sim_size && sim[k] == y && x != y) { + // delete the original entry + memmove(&sim[j], &sim[j+1], + (sim_size-(j+1))*sizeof(lfs_size_t)); + memmove(&sim_prngs[j], &sim_prngs[j+1], + (sim_size-(j+1))*sizeof(uint32_t)); + sim_size -= 1; + if (k > j) { + k -= 1; + } + // update the prng + sim_prngs[k] = wprng; + // just renaming + } else { + // first delete + memmove(&sim[j], &sim[j+1], + (sim_size-(j+1))*sizeof(lfs_size_t)); + memmove(&sim_prngs[j], &sim_prngs[j+1], + (sim_size-(j+1))*sizeof(uint32_t)); + if (k > j) { + k -= 1; + } + // then insert + memmove(&sim[k+1], &sim[k], + (sim_size-k)*sizeof(lfs_size_t)); + memmove(&sim_prngs[k+1], &sim_prngs[k], + (sim_size-k)*sizeof(uint32_t)); + sim[k] = y; + sim_prngs[k] = wprng; + } + break; + } + } + + // rename this file + char old_name[256]; + sprintf(old_name, "amethyst%03x", x); + char new_name[256]; + sprintf(new_name, "amethyst%03x", y); + int err = lfsr_rename(&lfs, old_name, new_name); + assert(!err || err == LFS_ERR_NOSPC); + if (err == LFS_ERR_NOSPC) { + goto dead; + } + } + + // check our simulation every power-of-2 ops + if (lfs_popc(run_ops[run]) == 1) { + // check that our files match our simulation + for (lfs_size_t j = 0; j < sim_size; j++) { + char name[256]; + sprintf(name, "amethyst%03x", sim[j]); + struct lfs_info info; + lfsr_stat(&lfs, name, &info) => 0; + assert(strcmp(info.name, name) == 0); + assert(info.type == LFS_TYPE_REG); + assert(info.size == SIZE); + } + + lfsr_dir_t dir; + lfsr_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, ".") == 0); + assert(info.type == LFS_TYPE_DIR); + assert(info.size == 0); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "..") == 0); + assert(info.type == LFS_TYPE_DIR); + assert(info.size == 0); + for (lfs_size_t j = 0; j < sim_size; j++) { + char name[256]; + sprintf(name, "amethyst%03x", sim[j]); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, name) == 0); + assert(info.type == LFS_TYPE_REG); + assert(info.size == SIZE); + } + lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT; + lfsr_dir_close(&lfs, &dir) => 0; + + // check the file contents + for (lfs_size_t j = 0; j < sim_size; j++) { + char name[256]; + sprintf(name, "amethyst%03x", sim[j]); + lfsr_file_t file; + lfsr_file_open(&lfs, &file, name, LFS_O_RDONLY) => 0; + + uint32_t wprng = sim_prngs[j]; + uint8_t wbuf[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&wprng) % 26); + } + + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf, SIZE) == 0); + lfsr_file_close(&lfs, &file) => 0; + } + } + } + + dead:; + // clean up sim/lfs + free(sim); + free(sim_prngs); + lfsr_unmount(&lfs) => 0; + + // print how many ops + printf("run %d, %dx%d, %d ec: %d ops\n", + run, + (int)BLOCK_SIZE, + run_bc[run], + (int)ERASE_CYCLES, + run_ops[run]); + } + + // check that we increased the liftime by ~2x, with ~10% error + printf("lifetime: %d -> %d (x%.2f)\n", + run_ops[0], + run_ops[1], + (double)run_ops[1] / (double)run_ops[0]); + assert(run_ops[1]*110/100 > 2*run_ops[0]); +''' + +# just more things that could go wrong +[cases.test_relocations_wl_orphanzombie_fuzz] +defines.ERASE_CYCLES = 10 +defines.BLOCK_RECYCLES = 4 +defines.N = [1, 2, 4, 8, 16, 32, 64] +defines.SIZE = [ + '0', + 'FBUFFER_SIZE/2', + '2*FBUFFER_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '4*BLOCK_SIZE', +] +defines.SEED = 42 +if = '(SIZE*N)/BLOCK_SIZE <= 16' +code = ''' + // run our test twice, once with 1/2 the storage, once with 2/2 the + // storage, and compare how many operations we were able to perform + // before filesystem death + uint32_t run_bc[2] = {BLOCK_COUNT/2, BLOCK_COUNT}; + uint32_t run_ops[2] = {0, 0}; + + for (int run = 0; run < 2; run++) { + // clear any wear from the previous run + for (lfs_block_t i = 0; i < BLOCK_COUNT; i++) { + lfs_emubd_setwear(CFG, i, 0) => 0; + } + + // configure the filesystem size + struct lfs_config cfg = *CFG; + cfg.block_count = run_bc[run]; + + // run the test + lfs_t lfs; + lfsr_format(&lfs, &cfg) => 0; + lfsr_mount(&lfs, &cfg) => 0; + + // set up a simulation to compare against + lfs_size_t *sim = malloc(N*sizeof(lfs_size_t)); + uint32_t *sim_prngs = malloc(N*sizeof(uint32_t)); + lfs_size_t sim_size = 0; + + typedef struct sim_file { + lfs_size_t x; + bool orphan; + bool zombie; + uint32_t prng; + lfsr_file_t file; + } sim_file_t; + sim_file_t **sim_files = malloc(N*sizeof(sim_file_t*)); + lfs_size_t sim_file_count = 0; + + uint32_t prng = SEED; + for (;; run_ops[run]++) { + nonsense:; + // choose which operation to do + uint8_t op = TEST_PRNG(&prng) % 5; + + // open a new file? + if (op == 0) { + if (sim_file_count >= N) { + goto nonsense; + } + // choose a pseudo-random number + lfs_size_t x = TEST_PRNG(&prng) % N; + + // already exists? + bool orphan = true; + uint32_t wprng = 0; + for (lfs_size_t j = 0; j < sim_size; j++) { + if (sim[j] == x) { + orphan = false; + wprng = sim_prngs[j]; + break; + } + } + // choose a random seed if we don't exist + if (orphan) { + wprng = TEST_PRNG(&prng); + } + + // open in our sim + lfs_size_t j = sim_file_count; + sim_files[j] = malloc(sizeof(sim_file_t)); + sim_files[j]->x = x; + sim_files[j]->orphan = orphan; + sim_files[j]->zombie = false; + sim_files[j]->prng = wprng; + + // open the actual file + char name[256]; + sprintf(name, "batman%03x", x); + int err = lfsr_file_open(&lfs, &sim_files[j]->file, name, + LFS_O_RDWR | LFS_O_CREAT); + assert(!err || err == LFS_ERR_NOSPC); + if (err == LFS_ERR_NOSPC) { + goto dead; + } + sim_file_count++; + + // write some initial data if we don't exist + if (orphan) { + uint8_t wbuf[SIZE]; + for (lfs_size_t k = 0; k < SIZE; k++) { + wbuf[k] = 'a' + (TEST_PRNG(&wprng) % 26); + } + lfs_ssize_t d = lfsr_file_write(&lfs, &sim_files[j]->file, + wbuf, SIZE); + assert(d == SIZE || d == LFS_ERR_NOSPC); + if (err == LFS_ERR_NOSPC) { + goto dead; + } + } + + // write/rewrite a file? + } else if (op == 1) { + if (sim_file_count == 0) { + goto nonsense; + } + // choose a random file handle + lfs_size_t j = TEST_PRNG(&prng) % sim_file_count; + lfs_size_t x = sim_files[j]->x; + // choose a random seed + uint32_t wprng = TEST_PRNG(&prng); + + // update sim + sim_files[j]->prng = wprng; + if (!sim_files[j]->zombie) { + // insert into our sim + for (lfs_size_t k = 0;; k++) { + if (k >= sim_size || sim[k] >= x) { + // already seen? + if (k < sim_size && sim[k] == x) { + // new prng + sim_prngs[k] = wprng; + } else { + // insert + memmove(&sim[k+1], &sim[k], + (sim_size-k)*sizeof(lfs_size_t)); + memmove(&sim_prngs[k+1], &sim_prngs[k], + (sim_size-k)*sizeof(uint32_t)); + sim_size += 1; + sim[k] = x; + sim_prngs[k] = wprng; + } + break; + } + } + + // update related sim files + for (lfs_size_t k = 0; k < sim_file_count; k++) { + if (sim_files[k]->x == x && !sim_files[k]->zombie) { + sim_files[k]->orphan = false; + sim_files[k]->prng = wprng; + } + } + } + + // write to the file + lfsr_file_rewind(&lfs, &sim_files[j]->file) => 0; + uint8_t wbuf[SIZE]; + for (lfs_size_t k = 0; k < SIZE; k++) { + wbuf[k] = 'a' + (TEST_PRNG(&wprng) % 26); + } + lfs_ssize_t d = lfsr_file_write(&lfs, &sim_files[j]->file, + wbuf, SIZE); + assert(d == SIZE || d == LFS_ERR_NOSPC); + if (d == LFS_ERR_NOSPC) { + goto dead; + } + int err = lfsr_file_sync(&lfs, &sim_files[j]->file); + assert(err == ((!sim_files[j]->zombie) ? 0 : LFS_ERR_NOENT) + || err == LFS_ERR_NOSPC); + if (err == LFS_ERR_NOSPC) { + goto dead; + } + + // close a file? + } else if (op == 2) { + if (sim_file_count == 0) { + goto nonsense; + } + // choose a random file handle + lfs_size_t j = TEST_PRNG(&prng) % sim_file_count; + + // this doesn't really test anything, but if we don't close + // files eventually everything will end up zombies + + // close the file without affected disk + lfsr_file_desync(&lfs, &sim_files[j]->file) => 0; + lfsr_file_close(&lfs, &sim_files[j]->file) => 0; + + // remove from list + free(sim_files[j]); + sim_files[j] = sim_files[sim_file_count-1]; + sim_file_count -= 1; + + // remove a file? + } else if (op == 3) { + if (sim_size == 0) { + goto nonsense; + } + // choose a random file to delete + lfs_size_t j = TEST_PRNG(&prng) % sim_size; + lfs_size_t x = sim[j]; + + // delete from our sim + memmove(&sim[j], &sim[j+1], + (sim_size-(j+1))*sizeof(lfs_size_t)); + memmove(&sim_prngs[j], &sim_prngs[j+1], + (sim_size-(j+1))*sizeof(uint32_t)); + sim_size -= 1; + + // mark any related sim files as zombied + for (lfs_size_t k = 0; k < sim_file_count; k++) { + if (sim_files[k]->x == x) { + sim_files[k]->zombie = true; + } + } + + // delete this file + char name[256]; + sprintf(name, "batman%03x", x); + int err = lfsr_remove(&lfs, name); + assert(!err || err == LFS_ERR_NOSPC); + if (err == LFS_ERR_NOSPC) { + goto dead; + } + + // rename a file? + } else if (op == 4) { + if (sim_size == 0) { + goto nonsense; + } + // choose a random file to rename, and a random number to + // rename to + lfs_size_t j = TEST_PRNG(&prng) % sim_size; + lfs_size_t x = sim[j]; + lfs_size_t y = TEST_PRNG(&prng) % N; + uint32_t wprng = sim_prngs[j]; + + // update our sim + for (lfs_size_t k = 0;; k++) { + if (k >= sim_size || sim[k] >= y) { + // renaming and replacing + if (k < sim_size && sim[k] == y && x != y) { + // delete the original entry + memmove(&sim[j], &sim[j+1], + (sim_size-(j+1))*sizeof(lfs_size_t)); + memmove(&sim_prngs[j], &sim_prngs[j+1], + (sim_size-(j+1))*sizeof(uint32_t)); + sim_size -= 1; + if (k > j) { + k -= 1; + } + // update the prng + sim_prngs[k] = wprng; + // just renaming + } else { + // first delete + memmove(&sim[j], &sim[j+1], + (sim_size-(j+1))*sizeof(lfs_size_t)); + memmove(&sim_prngs[j], &sim_prngs[j+1], + (sim_size-(j+1))*sizeof(uint32_t)); + if (k > j) { + k -= 1; + } + // then insert + memmove(&sim[k+1], &sim[k], + (sim_size-k)*sizeof(lfs_size_t)); + memmove(&sim_prngs[k+1], &sim_prngs[k], + (sim_size-k)*sizeof(uint32_t)); + sim[k] = y; + sim_prngs[k] = wprng; + } + break; + } + } + + // update any related sim files + for (lfs_size_t k = 0; k < sim_file_count; k++) { + // move source files + if (sim_files[k]->x == x) { + sim_files[k]->x = y; + + // mark target files as zombied + } else if (sim_files[k]->x == y) { + sim_files[k]->zombie = true; + } + } + + // rename this file + char old_name[256]; + sprintf(old_name, "batman%03x", x); + char new_name[256]; + sprintf(new_name, "batman%03x", y); + int err = lfsr_rename(&lfs, old_name, new_name); + assert(!err || err == LFS_ERR_NOSPC); + if (err == LFS_ERR_NOSPC) { + goto dead; + } + } + + // check our simulation every power-of-2 ops + if (lfs_popc(run_ops[run]) == 1) { + // check that disk matches our simulation + for (lfs_size_t j = 0; j < sim_size; j++) { + char name[256]; + sprintf(name, "batman%03x", sim[j]); + struct lfs_info info; + lfsr_stat(&lfs, name, &info) => 0; + assert(strcmp(info.name, name) == 0); + assert(info.type == LFS_TYPE_REG); + assert(info.size == SIZE); + } + + lfsr_dir_t dir; + lfsr_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, ".") == 0); + assert(info.type == LFS_TYPE_DIR); + assert(info.size == 0); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "..") == 0); + assert(info.type == LFS_TYPE_DIR); + assert(info.size == 0); + for (lfs_size_t j = 0; j < sim_size; j++) { + char name[256]; + sprintf(name, "batman%03x", sim[j]); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, name) == 0); + assert(info.type == LFS_TYPE_REG); + assert(info.size == SIZE); + } + lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT; + lfsr_dir_close(&lfs, &dir) => 0; + + for (lfs_size_t j = 0; j < sim_size; j++) { + char name[256]; + sprintf(name, "batman%03x", sim[j]); + lfsr_file_t file; + lfsr_file_open(&lfs, &file, name, LFS_O_RDONLY) => 0; + + uint32_t wprng = sim_prngs[j]; + uint8_t wbuf[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&wprng) % 26); + } + + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf, SIZE) == 0); + lfsr_file_close(&lfs, &file) => 0; + } + + // check that our file handles match our simulation + for (lfs_size_t j = 0; j < sim_file_count; j++) { + uint32_t wprng = sim_files[j]->prng; + uint8_t wbuf[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&wprng) % 26); + } + + lfsr_file_rewind(&lfs, &sim_files[j]->file) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &sim_files[j]->file, + rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf, SIZE) == 0); + } + } + } + + dead:; + // clean up sim/lfs + free(sim); + free(sim_prngs); + for (lfs_size_t j = 0; j < sim_file_count; j++) { + lfsr_file_desync(&lfs, &sim_files[j]->file) => 0; + lfsr_file_close(&lfs, &sim_files[j]->file) => 0; + free(sim_files[j]); + } + free(sim_files); + lfsr_unmount(&lfs) => 0; + + // print how many ops + printf("run %d, %dx%d, %d ec: %d ops\n", + run, + (int)BLOCK_SIZE, + run_bc[run], + (int)ERASE_CYCLES, + run_ops[run]); + } + + // check that we increased the liftime by ~2x, with ~10% error + printf("lifetime: %d -> %d (x%.2f)\n", + run_ops[0], + run_ops[1], + (double)run_ops[1] / (double)run_ops[0]); + assert(run_ops[1]*110/100 > 2*run_ops[0]); +''' + +# just more things that could go wrong +[cases.test_relocations_wl_orphanzombiedir_fuzz] +defines.ERASE_CYCLES = 10 +defines.BLOCK_RECYCLES = 4 +defines.N = [1, 2, 4, 8, 16, 32, 64] +defines.SIZE = [ + '0', + 'FBUFFER_SIZE/2', + '2*FBUFFER_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '4*BLOCK_SIZE', +] +defines.SEED = 42 +if = '(SIZE*N)/BLOCK_SIZE <= 16' +code = ''' + // run our test twice, once with 1/2 the storage, once with 2/2 the + // storage, and compare how many operations we were able to perform + // before filesystem death + uint32_t run_bc[2] = {BLOCK_COUNT/2, BLOCK_COUNT}; + uint32_t run_ops[2] = {0, 0}; + + for (int run = 0; run < 2; run++) { + // clear any wear from the previous run + for (lfs_block_t i = 0; i < BLOCK_COUNT; i++) { + lfs_emubd_setwear(CFG, i, 0) => 0; + } + + // configure the filesystem size + struct lfs_config cfg = *CFG; + cfg.block_count = run_bc[run]; + + // run the test + lfs_t lfs; + lfsr_format(&lfs, &cfg) => 0; + lfsr_mount(&lfs, &cfg) => 0; + + // set up a simulation to compare against + lfs_size_t *sim = malloc(N*sizeof(lfs_size_t)); + uint32_t *sim_prngs = malloc(N*sizeof(uint32_t)); + bool *sim_isdirs = malloc(N*sizeof(bool)); + lfs_size_t sim_size = 0; + + typedef struct sim_file { + lfs_size_t x; + bool orphan; + bool zombie; + uint32_t prng; + lfsr_file_t file; + } sim_file_t; + sim_file_t **sim_files = malloc(N*sizeof(sim_file_t*)); + lfs_size_t sim_file_count = 0; + + uint32_t prng = SEED; + for (;; run_ops[run]++) { + nonsense:; + // choose which operation to do + uint8_t op = TEST_PRNG(&prng) % 8; + + // open a new file? + if (op == 0) { + if (sim_file_count >= N) { + goto nonsense; + } + // choose a pseudo-random number + lfs_size_t x = TEST_PRNG(&prng) % N; + + // already exists? + bool orphan = true; + uint32_t wprng = 0; + for (lfs_size_t j = 0; j < sim_size; j++) { + if (sim[j] == x) { + if (sim_isdirs[j]) { + goto nonsense; + } + orphan = false; + wprng = sim_prngs[j]; + break; + } + } + // choose a random seed if we don't exist + if (orphan) { + wprng = TEST_PRNG(&prng); + } + + // open in our sim + lfs_size_t j = sim_file_count; + sim_files[j] = malloc(sizeof(sim_file_t)); + sim_files[j]->x = x; + sim_files[j]->orphan = orphan; + sim_files[j]->zombie = false; + sim_files[j]->prng = wprng; + + // open the actual file + char name[256]; + sprintf(name, "batman%03x", x); + int err = lfsr_file_open(&lfs, &sim_files[j]->file, name, + LFS_O_RDWR | LFS_O_CREAT); + assert(!err || err == LFS_ERR_NOSPC); + if (err == LFS_ERR_NOSPC) { + goto dead; + } + sim_file_count++; + + // write some initial data if we don't exist + if (orphan) { + uint8_t wbuf[SIZE]; + for (lfs_size_t k = 0; k < SIZE; k++) { + wbuf[k] = 'a' + (TEST_PRNG(&wprng) % 26); + } + lfs_ssize_t d = lfsr_file_write(&lfs, &sim_files[j]->file, + wbuf, SIZE); + assert(d == SIZE || d == LFS_ERR_NOSPC); + if (d == LFS_ERR_NOSPC) { + goto dead; + } + } + + // write/rewrite a file? + } else if (op == 1) { + if (sim_file_count == 0) { + goto nonsense; + } + // choose a random file handle + lfs_size_t j = TEST_PRNG(&prng) % sim_file_count; + lfs_size_t x = sim_files[j]->x; + // choose a random seed + uint32_t wprng = TEST_PRNG(&prng); + + // update sim + sim_files[j]->prng = wprng; + if (!sim_files[j]->zombie) { + // insert into our sim + for (lfs_size_t k = 0;; k++) { + if (k >= sim_size || sim[k] >= x) { + // already seen? + if (k < sim_size && sim[k] == x) { + // new prng + sim_prngs[k] = wprng; + } else { + // insert + memmove(&sim[k+1], &sim[k], + (sim_size-k)*sizeof(lfs_size_t)); + memmove(&sim_prngs[k+1], &sim_prngs[k], + (sim_size-k)*sizeof(uint32_t)); + memmove(&sim_isdirs[k+1], &sim_isdirs[k], + (sim_size-k)*sizeof(bool)); + sim_size += 1; + sim[k] = x; + sim_prngs[k] = wprng; + sim_isdirs[k] = false; + } + break; + } + } + + // update related sim files + for (lfs_size_t k = 0; k < sim_file_count; k++) { + if (sim_files[k]->x == x && !sim_files[k]->zombie) { + sim_files[k]->orphan = false; + sim_files[k]->prng = wprng; + } + } + } + + // write to the file + lfsr_file_rewind(&lfs, &sim_files[j]->file) => 0; + uint8_t wbuf[SIZE]; + for (lfs_size_t k = 0; k < SIZE; k++) { + wbuf[k] = 'a' + (TEST_PRNG(&wprng) % 26); + } + lfs_ssize_t d = lfsr_file_write(&lfs, &sim_files[j]->file, + wbuf, SIZE); + assert(d == SIZE || d == LFS_ERR_NOSPC); + if (d == LFS_ERR_NOSPC) { + goto dead; + } + int err = lfsr_file_sync(&lfs, &sim_files[j]->file); + assert(err == ((!sim_files[j]->zombie) ? 0 : LFS_ERR_NOENT) + || err == LFS_ERR_NOSPC); + if (err == LFS_ERR_NOSPC) { + goto dead; + } + + // close a file? + } else if (op == 2) { + if (sim_file_count == 0) { + goto nonsense; + } + // choose a random file handle + lfs_size_t j = TEST_PRNG(&prng) % sim_file_count; + + // this doesn't really test anything, but if we don't close + // files eventually everything will end up zombies + + // close the file without affected disk + lfsr_file_desync(&lfs, &sim_files[j]->file) => 0; + lfsr_file_close(&lfs, &sim_files[j]->file) => 0; + + // remove from list + free(sim_files[j]); + sim_files[j] = sim_files[sim_file_count-1]; + sim_file_count -= 1; + + // remove a file? + } else if (op == 3) { + if (sim_size == 0) { + goto nonsense; + } + // choose a random file to delete + lfs_size_t j = TEST_PRNG(&prng) % sim_size; + lfs_size_t x = sim[j]; + + // delete from our sim + memmove(&sim[j], &sim[j+1], + (sim_size-(j+1))*sizeof(lfs_size_t)); + memmove(&sim_prngs[j], &sim_prngs[j+1], + (sim_size-(j+1))*sizeof(uint32_t)); + memmove(&sim_isdirs[j], &sim_isdirs[j+1], + (sim_size-(j+1))*sizeof(bool)); + sim_size -= 1; + + // mark any related sim files as zombied + for (lfs_size_t k = 0; k < sim_file_count; k++) { + if (sim_files[k]->x == x) { + sim_files[k]->zombie = true; + } + } + + // delete this file + char name[256]; + sprintf(name, "batman%03x", x); + int err = lfsr_remove(&lfs, name); + assert(!err || err == LFS_ERR_NOSPC); + if (err == LFS_ERR_NOSPC) { + goto dead; + } + + // rename a file? + } else if (op == 4) { + if (sim_size == 0) { + goto nonsense; + } + // choose a random file to rename, and a random number to + // rename to + lfs_size_t j = TEST_PRNG(&prng) % sim_size; + lfs_size_t x = sim[j]; + lfs_size_t y = TEST_PRNG(&prng) % N; + uint32_t wprng = sim_prngs[j]; + bool isdir = sim_isdirs[j]; + + // update our sim + for (lfs_size_t k = 0;; k++) { + if (k >= sim_size || sim[k] >= y) { + // renaming and replacing + if (k < sim_size && sim[k] == y && x != y) { + // type mismatch? + if (sim_isdirs[k] != isdir) { + goto nonsense; + } + + // delete the original entry + memmove(&sim[j], &sim[j+1], + (sim_size-(j+1))*sizeof(lfs_size_t)); + memmove(&sim_prngs[j], &sim_prngs[j+1], + (sim_size-(j+1))*sizeof(uint32_t)); + memmove(&sim_isdirs[j], &sim_isdirs[j+1], + (sim_size-(j+1))*sizeof(bool)); + sim_size -= 1; + if (k > j) { + k -= 1; + } + // update the prng + sim_prngs[k] = wprng; + // just renaming + } else { + // first delete + memmove(&sim[j], &sim[j+1], + (sim_size-(j+1))*sizeof(lfs_size_t)); + memmove(&sim_prngs[j], &sim_prngs[j+1], + (sim_size-(j+1))*sizeof(uint32_t)); + memmove(&sim_isdirs[j], &sim_isdirs[j+1], + (sim_size-(j+1))*sizeof(bool)); + if (k > j) { + k -= 1; + } + // then insert + memmove(&sim[k+1], &sim[k], + (sim_size-k)*sizeof(lfs_size_t)); + memmove(&sim_prngs[k+1], &sim_prngs[k], + (sim_size-k)*sizeof(uint32_t)); + memmove(&sim_isdirs[k+1], &sim_isdirs[k], + (sim_size-k)*sizeof(bool)); + sim[k] = y; + sim_prngs[k] = wprng; + sim_isdirs[k] = isdir; + } + break; + } + } + + // update any related sim files + for (lfs_size_t k = 0; k < sim_file_count; k++) { + // move source files + if (sim_files[k]->x == x) { + sim_files[k]->x = y; + + // mark target files as zombied + } else if (sim_files[k]->x == y) { + sim_files[k]->zombie = true; + } + } + + // rename this file + char old_name[256]; + sprintf(old_name, "batman%03x", x); + char new_name[256]; + sprintf(new_name, "batman%03x", y); + int err = lfsr_rename(&lfs, old_name, new_name); + assert(!err || err == LFS_ERR_NOSPC); + if (err == LFS_ERR_NOSPC) { + goto dead; + } + + // toss a directory into the mix + } else if (op == 5) { + // choose a pseudo-random number + lfs_size_t x = TEST_PRNG(&prng) % N; + + // insert into our sim, use negative numbers for dirs + for (lfs_size_t k = 0;; k++) { + if (k >= sim_size || sim[k] >= x) { + // already seen? + if (k < sim_size && sim[k] == x) { + goto nonsense; + } else { + // insert + memmove(&sim[k+1], &sim[k], + (sim_size-k)*sizeof(lfs_size_t)); + memmove(&sim_prngs[k+1], &sim_prngs[k], + (sim_size-k)*sizeof(uint32_t)); + memmove(&sim_isdirs[k+1], &sim_isdirs[k], + (sim_size-k)*sizeof(bool)); + sim_size += 1; + sim[k] = x; + sim_prngs[k] = 0; + sim_isdirs[k] = true; + } + break; + } + } + + // mark any related sim files as zombied + for (lfs_size_t k = 0; k < sim_file_count; k++) { + if (sim_files[k]->x == x) { + sim_files[k]->zombie = true; + } + } + + // make the directory + char name[256]; + sprintf(name, "batman%03x", x); + int err = lfsr_mkdir(&lfs, name); + assert(!err || err == LFS_ERR_NOSPC); + if (err == LFS_ERR_NOSPC) { + goto dead; + } + } + + // check our simulation every power-of-2 ops + if (lfs_popc(run_ops[run]) == 1) { + // check that disk matches our simulation + for (lfs_size_t j = 0; j < sim_size; j++) { + char name[256]; + sprintf(name, "batman%03x", sim[j]); + struct lfs_info info; + lfsr_stat(&lfs, name, &info) => 0; + assert(strcmp(info.name, name) == 0); + if (sim_isdirs[j]) { + assert(info.type == LFS_TYPE_DIR); + } else { + assert(info.type == LFS_TYPE_REG); + assert(info.size == SIZE); + } + } + + lfsr_dir_t dir; + lfsr_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, ".") == 0); + assert(info.type == LFS_TYPE_DIR); + assert(info.size == 0); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "..") == 0); + assert(info.type == LFS_TYPE_DIR); + assert(info.size == 0); + for (lfs_size_t j = 0; j < sim_size; j++) { + char name[256]; + sprintf(name, "batman%03x", sim[j]); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, name) == 0); + if (sim_isdirs[j]) { + assert(info.type == LFS_TYPE_DIR); + } else { + assert(info.type == LFS_TYPE_REG); + assert(info.size == SIZE); + } + } + lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT; + lfsr_dir_close(&lfs, &dir) => 0; + + for (lfs_size_t j = 0; j < sim_size; j++) { + if (sim_isdirs[j]) { + char name[256]; + sprintf(name, "batman%03x", sim[j]); + lfsr_file_t file; + lfsr_file_open(&lfs, &file, name, LFS_O_RDONLY) + => LFS_ERR_ISDIR; + + } else { + char name[256]; + sprintf(name, "batman%03x", sim[j]); + lfsr_file_t file; + lfsr_file_open(&lfs, &file, name, LFS_O_RDONLY) => 0; + + uint32_t wprng = sim_prngs[j]; + uint8_t wbuf[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&wprng) % 26); + } + + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => SIZE; + assert(memcmp(rbuf, wbuf, SIZE) == 0); + lfsr_file_close(&lfs, &file) => 0; + } + } + + // check that our file handles match our simulation + for (lfs_size_t j = 0; j < sim_file_count; j++) { + uint32_t wprng = sim_files[j]->prng; + uint8_t wbuf[SIZE]; + for (lfs_size_t j = 0; j < SIZE; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&wprng) % 26); + } + + lfsr_file_rewind(&lfs, &sim_files[j]->file) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &sim_files[j]->file, rbuf, SIZE) + => SIZE; + assert(memcmp(rbuf, wbuf, SIZE) == 0); + } + } + } + + dead:; + // clean up sim/lfs + free(sim); + free(sim_prngs); + for (lfs_size_t j = 0; j < sim_file_count; j++) { + lfsr_file_desync(&lfs, &sim_files[j]->file) => 0; + lfsr_file_close(&lfs, &sim_files[j]->file) => 0; + free(sim_files[j]); + } + free(sim_files); + lfsr_unmount(&lfs) => 0; + + // print how many ops + printf("run %d, %dx%d, %d ec: %d ops\n", + run, + (int)BLOCK_SIZE, + run_bc[run], + (int)ERASE_CYCLES, + run_ops[run]); + } + + // check that we increased the liftime by ~2x, with ~10% error + printf("lifetime: %d -> %d (x%.2f)\n", + run_ops[0], + run_ops[1], + (double)run_ops[1] / (double)run_ops[0]); + assert(run_ops[1]*110/100 > 2*run_ops[0]); +'''