diff --git a/lfs.c b/lfs.c index 02a44406..c62646d5 100644 --- a/lfs.c +++ b/lfs.c @@ -1561,11 +1561,6 @@ static int lfsr_gdelta_xor(lfs_t *lfs, // GRM (global remove) things -typedef struct lfsr_grm { - lfs_ssize_t mid; - lfs_size_t rid; -} lfsr_grm_t; - static lfs_ssize_t lfsr_grm_todisk(lfs_t *lfs, const lfsr_grm_t *grm, uint8_t buffer[static LFSR_GRM_DSIZE]) { (void)lfs; @@ -1781,6 +1776,9 @@ static int lfsr_grm_split(lfs_t *lfs, static int lfs_alloc(lfs_t *lfs, lfs_block_t *block); static void lfs_alloc_ack(lfs_t *lfs); +// and our main "fix everything before writing" function +static int lfsr_fs_preparemutation(lfs_t *lfs); + /// Red-black-yellow Dhara tree operations /// @@ -5260,6 +5258,10 @@ static lfs_ssize_t lfsr_mdir_get(lfs_t *lfs, const lfsr_mdir_t *mdir, // some mdir-related gstate things we need +static void lfsr_fs_flushgdelta(lfs_t *lfs) { + memset(lfs->grmd, 0, LFSR_GRM_DSIZE); +} + static int lfsr_fs_consumegdelta(lfs_t *lfs, const lfsr_mdir_t *mdir) { lfsr_data_t data; int err = lfsr_mdir_lookup(lfs, mdir, -1, LFSR_TAG_GRM, NULL, &data); @@ -5277,10 +5279,6 @@ static int lfsr_fs_consumegdelta(lfs_t *lfs, const lfsr_mdir_t *mdir) { return 0; } -static void lfsr_fs_flushgdelta(lfs_t *lfs) { - memset(lfs->grmd, 0, LFSR_GRM_DSIZE); -} - // mtree is the core tree of mdirs in littlefs @@ -5377,9 +5375,18 @@ static int lfsr_mdir_compact_(lfs_t *lfs, lfsr_mdir_t *mdir, // - mid = wl => only alloc if mdir is tired (wear-leveling) // - otherwise => always alloc, use this mid (new mdir) + // consume gstate on original rbyd, we need this even if we drop + // our mdir to avoid losing info + // + // if succesful, this should get immediately appended to our new commit + int err = lfsr_fs_consumegdelta(lfs, msource); + if (err) { + return err; + } + // first thing we need to do is read our current revision count uint32_t rev; - int err = lfsr_bd_read(lfs, msource->rbyd.block, 0, sizeof(uint32_t), + err = lfsr_bd_read(lfs, msource->rbyd.block, 0, sizeof(uint32_t), &rev, sizeof(uint32_t)); if (err && err != LFS_ERR_CORRUPT) { return err; @@ -5460,14 +5467,9 @@ static int lfsr_mdir_compact_(lfs_t *lfs, lfsr_mdir_t *mdir, return err; } + // drop commit if weight goes to zero if (mdir->mid >= 0 && mdir->rbyd.weight == 0) { - // consume gstate so we don't lose any info - int err = lfsr_fs_consumegdelta(lfs, mdir); - if (err) { - return err; - } - // TODO should we just make our pcache not assert? // drop our pcache, we're not going to complete this commit lfs_cache_zero(lfs, &lfs->pcache); @@ -6697,6 +6699,9 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg); static int lfs_deinit(lfs_t *lfs); static int lfsr_mountinited(lfs_t *lfs) { + // zero gdeltas, we'll read these from our mdirs + lfsr_fs_flushgdelta(lfs); + // traverse the mtree rooted at mroot 0x{1,0} // // note that lfsr_mtree_traversal_next will update our mroot/mtree @@ -6963,6 +6968,12 @@ static int lfsr_mountinited(lfs_t *lfs) { } } } + + // collect any gdeltas from this mdir + err = lfsr_fs_consumegdelta(lfs, mdir); + if (err) { + return err; + } } // once we've mounted and derived a pseudo-random seed, initialize our @@ -6973,6 +6984,24 @@ static int lfsr_mountinited(lfs_t *lfs) { // lfs->lookahead.start = lfs->seed % lfs->cfg->block_count; + // TODO should the consumegdelta above take gstate/gdelta as a parameter? + // keep track of the current gstate on disk + memcpy(lfs->grm, lfs->grmd, LFSR_GRM_DSIZE); + + // decode grm so we can report any removed files as missing + // TODO wait, should mdir_commit update lfs->grm_ as well? + lfs_ssize_t d = lfsr_grm_fromdisk(lfs, &lfs->grm_, + LFSR_DATA_BUF(lfs->grm, LFSR_GRM_DSIZE)); + if (d < 0) { + return d; + } + + if (lfs->grm_.mid >= 0) { + LFS_DEBUG("Found pending grm (0x%"PRIx32".%"PRIx32")\n", + lfs->grm_.mid, + lfs->grm_.rid); + } + return 0; } @@ -7167,16 +7196,18 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { /// Directory operations /// int lfsr_mkdir(lfs_t *lfs, const char *path) { - // checkpoint block allocations - // TODO we should just name this lfsr_alloc_checkpoint - lfs_alloc_ack(lfs); + // prepare our filesystem for writing + int err= lfsr_fs_preparemutation(lfs); + if (err) { + return err; + } // lookup our parent lfsr_openedmdir_t parent; lfs_size_t parent_did; const char *name; lfs_size_t name_size; - int err = lfsr_mtree_pathlookup(lfs, path, + err = lfsr_mtree_pathlookup(lfs, path, &parent.mdir, &parent.rid, NULL, &parent_did, &name, &name_size); if (err && (err != LFS_ERR_NOENT || parent.rid == -1)) { @@ -7426,6 +7457,45 @@ int lfsr_dir_read(lfs_t *lfs, lfsr_dir_t *dir, struct lfs_info *info) { } +/// Prepare the filesystem for mutation /// +static int lfsr_fs_fixgrm(lfs_t *lfs) { + // find our mdir + lfsr_mdir_t mdir; + int err = lfsr_mtree_lookup(lfs, lfs->grm_.mid, &mdir); + if (err) { + return err; + } + + // remove the rid while also zeroing our grm + err = lfsr_mdir_commit(lfs, &mdir, (lfs_ssize_t*)&lfs->grm_.rid, LFSR_ATTRS( + LFSR_ATTR(lfs->grm_.rid, UNR, -1, NULL, 0), + LFSR_ATTR(-1, GRM, 0, NULL, 0))); + + // mark grm as taken care of + lfs->grm_.mid = 0; + return 0; +} + +static int lfsr_fs_preparemutation(lfs_t *lfs) { + // fix pending grms + if (lfs->grm_.mid >= 0) { + LFS_DEBUG("Fixing grm (0x%"PRIx32".%"PRIx32")", + lfs->grm_.mid, + lfs->grm_.rid); + int err = lfsr_fs_fixgrm(lfs); + if (err) { + return err; + } + } + + // checkpoint the allocator + lfs_alloc_ack(lfs); + + return 0; +} + + + ///// Metadata pair and directory operations /// diff --git a/lfs.h b/lfs.h index 8f92d8d8..b21608c6 100644 --- a/lfs.h +++ b/lfs.h @@ -368,12 +368,6 @@ typedef union lfsr_btree { } inlined; } lfsr_btree_t; -// space for: -// - type - 1 leb128 - 1 byte (worst case) -// - mid - 1 leb128 - 5 bytes (worst case) -// - rid - 1 leb128 - 5 bytes (worst case) -#define LFSR_GRM_DSIZE (5+5) - typedef struct lfsr_mdir { // -2 => deleted // -1 => mroot @@ -390,6 +384,17 @@ typedef struct lfsr_openedmdir { lfsr_mdir_t mdir; } lfsr_openedmdir_t; +// space for: +// - type - 1 leb128 - 1 byte (worst case) +// - mid - 1 leb128 - 5 bytes (worst case) +// - rid - 1 leb128 - 5 bytes (worst case) +#define LFSR_GRM_DSIZE (5+5) + +typedef struct lfsr_grm { + lfs_ssize_t mid; + lfs_size_t rid; +} lfsr_grm_t; + typedef struct lfs_mdir { lfs_block_t pair[2]; @@ -488,6 +493,9 @@ typedef struct lfs { lfsr_mdir_t mroot; lfsr_btree_t mtree; + // TODO do we really need separate decoded/encoded grms? + lfsr_grm_t grm_; + uint8_t grm[LFSR_GRM_DSIZE]; uint8_t grmd[LFSR_GRM_DSIZE]; diff --git a/scripts/dbgbtree.py b/scripts/dbgbtree.py index 55c37b21..ba55752a 100755 --- a/scripts/dbgbtree.py +++ b/scripts/dbgbtree.py @@ -225,7 +225,8 @@ class Rbyd: for i_, rbyd in enumerate(rbyds): # compare with sequence arithmetic if rbyd and ( - not ((rbyd.rev - rbyds[i].rev) & 0x80000000) + not rbyds[i] + or not ((rbyd.rev - rbyds[i].rev) & 0x80000000) or (rbyd.rev == rbyds[i].rev and rbyd.trunk > rbyds[i].trunk)): i = i_ diff --git a/scripts/dbgmtree.py b/scripts/dbgmtree.py index 4c17227d..eb05e8ab 100755 --- a/scripts/dbgmtree.py +++ b/scripts/dbgmtree.py @@ -233,7 +233,8 @@ class Rbyd: for i_, rbyd in enumerate(rbyds): # compare with sequence arithmetic if rbyd and ( - not ((rbyd.rev - rbyds[i].rev) & 0x80000000) + not rbyds[i] + or not ((rbyd.rev - rbyds[i].rev) & 0x80000000) or (rbyd.rev == rbyds[i].rev and rbyd.trunk > rbyds[i].trunk)): i = i_ diff --git a/scripts/dbgrbyd.py b/scripts/dbgrbyd.py index e63baeed..7a07fd8c 100755 --- a/scripts/dbgrbyd.py +++ b/scripts/dbgrbyd.py @@ -924,7 +924,8 @@ def main(disk, blocks=None, *, # compare with sequence arithmetic if trunk_ and ( - not ((rev - revs[i]) & 0x80000000) + not trunks_[i] + or not ((rev - revs[i]) & 0x80000000) or (rev == revs[i] and trunk_ > trunks_[i])): i = i_ diff --git a/tests/t5_dirs.toml b/tests/t5_dirs.toml index 22d91e64..060c8424 100644 --- a/tests/t5_dirs.toml +++ b/tests/t5_dirs.toml @@ -1,6 +1,7 @@ # Directory tests [cases.t5_dirs_mkdir] +defines.REMOUNT = [false, true] code = ''' lfs_t lfs; lfsr_format(&lfs, cfg) => 0; @@ -9,6 +10,14 @@ code = ''' // make a directory lfsr_mkdir(&lfs, "ardvark") => 0; + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, cfg) => 0; + // grm should be zero here + assert(lfs.grm[0] == 0); + } + // check that our mkdir worked with stat struct lfs_info info; lfsr_stat(&lfs, "ardvark", &info) => 0; @@ -34,6 +43,7 @@ code = ''' # test that noent errors work [cases.t5_dirs_noent] +defines.REMOUNT = [false, true] code = ''' lfs_t lfs; lfsr_format(&lfs, cfg) => 0; @@ -49,6 +59,14 @@ code = ''' lfsr_dir_t dir; lfsr_dir_open(&lfs, &dir, "no") => LFS_ERR_NOENT; + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, cfg) => 0; + // grm should be zero here + assert(lfs.grm[0] == 0); + } + // and check that this didn't interfere with our original directory lfsr_stat(&lfs, "ardvark", &info) => 0; assert(strcmp(info.name, "ardvark") == 0); @@ -71,6 +89,7 @@ code = ''' # test that creating the same directory twice errors [cases.t5_dirs_mkdir_exists] +defines.REMOUNT = [false, true] code = ''' lfs_t lfs; lfsr_format(&lfs, cfg) => 0; @@ -82,6 +101,14 @@ code = ''' // make the same directory, should error lfsr_mkdir(&lfs, "ardvark") => LFS_ERR_EXIST; + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, cfg) => 0; + // grm should be zero here + assert(lfs.grm[0] == 0); + } + // and check that this didn't interfere with our original directory struct lfs_info info; lfsr_stat(&lfs, "ardvark", &info) => 0; @@ -106,6 +133,7 @@ code = ''' # test that creating a directory with an invalid path errors [cases.t5_dirs_mkdir_noent] +defines.REMOUNT = [false, true] code = ''' lfs_t lfs; lfsr_format(&lfs, cfg) => 0; @@ -120,6 +148,14 @@ code = ''' // make a nonsense child directory, should error lfsr_mkdir(&lfs, "ardvark/no/hmm") => LFS_ERR_NOENT; + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, cfg) => 0; + // grm should be zero here + assert(lfs.grm[0] == 0); + } + // and check that this didn't interfere with our original directory struct lfs_info info; lfsr_stat(&lfs, "ardvark", &info) => 0; @@ -143,6 +179,7 @@ code = ''' ''' [cases.t5_dirs_mkdir_siblings] +defines.REMOUNT = [false, true] code = ''' lfs_t lfs; lfsr_format(&lfs, cfg) => 0; @@ -153,6 +190,14 @@ code = ''' lfsr_mkdir(&lfs, "batman") => 0; lfsr_mkdir(&lfs, "cantaloupe") => 0; + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, cfg) => 0; + // grm should be zero here + assert(lfs.grm[0] == 0); + } + // check that our mkdir worked struct lfs_info info; lfsr_stat(&lfs, "ardvark", &info) => 0; @@ -188,6 +233,7 @@ code = ''' ''' [cases.t5_dirs_mkdir_children] +defines.REMOUNT = [false, true] code = ''' lfs_t lfs; lfsr_format(&lfs, cfg) => 0; @@ -198,6 +244,14 @@ code = ''' lfsr_mkdir(&lfs, "ardvark/batman") => 0; lfsr_mkdir(&lfs, "ardvark/batman/cantaloupe") => 0; + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, cfg) => 0; + // grm should be zero here + assert(lfs.grm[0] == 0); + } + // check that our mkdirs worked struct lfs_info info; lfsr_stat(&lfs, "ardvark", &info) => 0; @@ -252,6 +306,7 @@ code = ''' [cases.t5_dirs_mkdir_many] defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] +defines.REMOUNT = [false, true] code = ''' lfs_t lfs; lfsr_format(&lfs, cfg) => 0; @@ -262,6 +317,14 @@ code = ''' char name[256]; sprintf(name, "dir%04d", i); lfsr_mkdir(&lfs, name) => 0; + + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, cfg) => 0; + // grm should be zero here + assert(lfs.grm[0] == 0); + } } // check that our mkdir worked @@ -297,6 +360,7 @@ code = ''' [cases.t5_dirs_mkdir_many_2layers] defines.N = [1, 2, 4, 8, 16, 32] +defines.REMOUNT = [false, true] code = ''' lfs_t lfs; lfsr_format(&lfs, cfg) => 0; @@ -312,6 +376,14 @@ code = ''' for (lfs_size_t j = 0; j < N; j++) { sprintf(name, "dir%04d/child%04d", i, j); lfsr_mkdir(&lfs, name) => 0; + + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, cfg) => 0; + // grm should be zero here + assert(lfs.grm[0] == 0); + } } } @@ -378,6 +450,7 @@ code = ''' [cases.t5_dirs_mkdir_many_3layers] defines.N = [1, 2, 4, 8] +defines.REMOUNT = [false, true] code = ''' lfs_t lfs; lfsr_format(&lfs, cfg) => 0; @@ -398,6 +471,14 @@ code = ''' for (lfs_size_t k = 0; k < N; k++) { sprintf(name, "dir%04d/child%04d/grandchild%04d", i, j, k); lfsr_mkdir(&lfs, name) => 0; + + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, cfg) => 0; + // grm should be zero here + assert(lfs.grm[0] == 0); + } } } } @@ -495,6 +576,7 @@ code = ''' [cases.t5_dirs_mkdir_many_linkedlist] defines.N = [1, 2, 4, 8, 16, 32, 64] +defines.REMOUNT = [false, true] code = ''' lfs_t lfs; lfsr_format(&lfs, cfg) => 0; @@ -506,6 +588,14 @@ code = ''' for (lfs_size_t i = 0; i < N; i++) { sprintf(&name[strlen(name)], "/dir%04d", i); lfsr_mkdir(&lfs, name) => 0; + + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, cfg) => 0; + // grm should be zero here + assert(lfs.grm[0] == 0); + } } // check that our mkdir worked @@ -547,9 +637,8 @@ code = ''' [cases.t5_dirs_mkdir_fuzz] defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] -# 0 => do this test in the root dir -# 1 => do this test in a directory named "parent" -defines.PARENT = [0, 1] +defines.PARENT = [false, true] +defines.REMOUNT = [false, true] defines.SAMPLES = 10 # -1 => all pseudo-random seeds # n => reproduce a specific seed @@ -590,6 +679,14 @@ code = ''' (sim_size-j)*sizeof(lfs_size_t)); sim_size += 1; sim[j] = x; + + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, cfg) => 0; + // grm should be zero here + assert(lfs.grm[0] == 0); + } break; } } @@ -640,6 +737,7 @@ code = ''' # test that did collisions don't cause issues [cases.t5_dirs_did_collisions] +defines.REMOUNT = [false, true] code = ''' lfs_t lfs; lfsr_format(&lfs, cfg) => 0; @@ -660,6 +758,14 @@ code = ''' lfsr_mkdir(&lfs, "e_LptKHkHH") => 0; lfsr_mkdir(&lfs, "f_lUoVuhJH") => 0; + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, cfg) => 0; + // grm should be zero here + assert(lfs.grm[0] == 0); + } + // check that our mkdirs worked struct lfs_info info; lfsr_stat(&lfs, "a_SNmwMTHH", &info) => 0; @@ -714,6 +820,7 @@ code = ''' # these will also collide with the root [cases.t5_dirs_did_zero] +defines.REMOUNT = [false, true] code = ''' lfs_t lfs; lfsr_format(&lfs, cfg) => 0; @@ -726,6 +833,14 @@ code = ''' assert(lfs_crc32c(0, "e_VNunKMPH", 10) == 0x00000000); assert(lfs_crc32c(0, "f_vknsvNRH", 10) == 0x00000000); + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, cfg) => 0; + // grm should be zero here + assert(lfs.grm[0] == 0); + } + // make directories lfsr_mkdir(&lfs, "a_IplRNrPH") => 0; lfsr_mkdir(&lfs, "b_iUwOsqRH") => 0; @@ -734,6 +849,14 @@ code = ''' lfsr_mkdir(&lfs, "e_VNunKMPH") => 0; lfsr_mkdir(&lfs, "f_vknsvNRH") => 0; + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, cfg) => 0; + // grm should be zero here + assert(lfs.grm[0] == 0); + } + // check that our mkdirs worked struct lfs_info info; lfsr_stat(&lfs, "a_IplRNrPH", &info) => 0; @@ -790,6 +913,7 @@ code = ''' # # note this is true even if you truncate [cases.t5_dirs_did_ones] +defines.REMOUNT = [false, true] code = ''' lfs_t lfs; lfsr_format(&lfs, cfg) => 0; @@ -810,6 +934,14 @@ code = ''' lfsr_mkdir(&lfs, "e_vQtPStPH") => 0; lfsr_mkdir(&lfs, "f_VtoMnwRH") => 0; + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, cfg) => 0; + // grm should be zero here + assert(lfs.grm[0] == 0); + } + // check that our mkdirs worked struct lfs_info info; lfsr_stat(&lfs, "a_iomlVKPH", &info) => 0; @@ -866,6 +998,7 @@ code = ''' # if the leb128 disk-size is not calculated correctly these can cause # issues [cases.t5_dirs_did_leb128_boundaries] +defines.REMOUNT = [false, true] code = ''' lfs_t lfs; lfsr_format(&lfs, cfg) => 0; @@ -886,6 +1019,14 @@ code = ''' lfsr_mkdir(&lfs, "e_thrRIsRH") => 0; lfsr_mkdir(&lfs, "f_pNtQTPJH") => 0; + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, cfg) => 0; + // grm should be zero here + assert(lfs.grm[0] == 0); + } + // check that our mkdirs worked struct lfs_info info; lfsr_stat(&lfs, "a_IOtUptRH", &info) => 0;