Github/littlefs
1
Fork 0
Code Actions Activity

Implemented, but untested, global-removes

Browse Source 
This implementation is in theory correct, but of course, being untested,
who knows?

Though this does come with remounting added to all of the directory
tests. This effectively tests that all of the directory creation tests
we have so far maintain grm=0 after each unmount-mount cycle. Which is
valuable.
This commit is contained in:
Christopher Haster
2023-07-10 01:35:54 -05:00
parent cc0ac25b5e
commit c2d9f1b047
6 changed files with 254 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+90 -20
View File
@@ -1561,11 +1561,6 @@ static int lfsr_gdelta_xor(lfs_t *lfs,
// GRM (global remove) things
typedef struct lfsr_grm {
lfs_ssize_t mid;
lfs_size_t rid;
} lfsr_grm_t;
static lfs_ssize_t lfsr_grm_todisk(lfs_t *lfs, const lfsr_grm_t *grm,
uint8_t buffer[static LFSR_GRM_DSIZE]) {
(void)lfs;
@@ -1781,6 +1776,9 @@ static int lfsr_grm_split(lfs_t *lfs,
static int lfs_alloc(lfs_t *lfs, lfs_block_t *block);
static void lfs_alloc_ack(lfs_t *lfs);
// and our main "fix everything before writing" function
static int lfsr_fs_preparemutation(lfs_t *lfs);
/// Red-black-yellow Dhara tree operations ///
@@ -5260,6 +5258,10 @@ static lfs_ssize_t lfsr_mdir_get(lfs_t *lfs, const lfsr_mdir_t *mdir,
// some mdir-related gstate things we need
static void lfsr_fs_flushgdelta(lfs_t *lfs) {
memset(lfs->grmd, 0, LFSR_GRM_DSIZE);
}
static int lfsr_fs_consumegdelta(lfs_t *lfs, const lfsr_mdir_t *mdir) {
lfsr_data_t data;
int err = lfsr_mdir_lookup(lfs, mdir, -1, LFSR_TAG_GRM, NULL, &data);
@@ -5277,10 +5279,6 @@ static int lfsr_fs_consumegdelta(lfs_t *lfs, const lfsr_mdir_t *mdir) {
return 0;
}
static void lfsr_fs_flushgdelta(lfs_t *lfs) {
memset(lfs->grmd, 0, LFSR_GRM_DSIZE);
}
// mtree is the core tree of mdirs in littlefs
@@ -5377,9 +5375,18 @@ static int lfsr_mdir_compact_(lfs_t *lfs, lfsr_mdir_t *mdir,
// - mid = wl => only alloc if mdir is tired (wear-leveling)
// - otherwise => always alloc, use this mid (new mdir)
// consume gstate on original rbyd, we need this even if we drop
// our mdir to avoid losing info
//
// if succesful, this should get immediately appended to our new commit
int err = lfsr_fs_consumegdelta(lfs, msource);
if (err) {
return err;
}
// first thing we need to do is read our current revision count
uint32_t rev;
int err = lfsr_bd_read(lfs, msource->rbyd.block, 0, sizeof(uint32_t),
err = lfsr_bd_read(lfs, msource->rbyd.block, 0, sizeof(uint32_t),
&rev, sizeof(uint32_t));
if (err && err != LFS_ERR_CORRUPT) {
return err;
@@ -5460,14 +5467,9 @@ static int lfsr_mdir_compact_(lfs_t *lfs, lfsr_mdir_t *mdir,
return err;
}
// drop commit if weight goes to zero
if (mdir->mid >= 0 && mdir->rbyd.weight == 0) {
// consume gstate so we don't lose any info
int err = lfsr_fs_consumegdelta(lfs, mdir);
if (err) {
return err;
}
// TODO should we just make our pcache not assert?
// drop our pcache, we're not going to complete this commit
lfs_cache_zero(lfs, &lfs->pcache);
@@ -6697,6 +6699,9 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg);
static int lfs_deinit(lfs_t *lfs);
static int lfsr_mountinited(lfs_t *lfs) {
// zero gdeltas, we'll read these from our mdirs
lfsr_fs_flushgdelta(lfs);
// traverse the mtree rooted at mroot 0x{1,0}
//
// note that lfsr_mtree_traversal_next will update our mroot/mtree
@@ -6963,6 +6968,12 @@ static int lfsr_mountinited(lfs_t *lfs) {
}
}
}
// collect any gdeltas from this mdir
err = lfsr_fs_consumegdelta(lfs, mdir);
if (err) {
return err;
}
}
// once we've mounted and derived a pseudo-random seed, initialize our
@@ -6973,6 +6984,24 @@ static int lfsr_mountinited(lfs_t *lfs) {
//
lfs->lookahead.start = lfs->seed % lfs->cfg->block_count;
// TODO should the consumegdelta above take gstate/gdelta as a parameter?
// keep track of the current gstate on disk
memcpy(lfs->grm, lfs->grmd, LFSR_GRM_DSIZE);
// decode grm so we can report any removed files as missing
// TODO wait, should mdir_commit update lfs->grm_ as well?
lfs_ssize_t d = lfsr_grm_fromdisk(lfs, &lfs->grm_,
LFSR_DATA_BUF(lfs->grm, LFSR_GRM_DSIZE));
if (d < 0) {
return d;
}
if (lfs->grm_.mid >= 0) {
LFS_DEBUG("Found pending grm (0x%"PRIx32".%"PRIx32")\n",
lfs->grm_.mid,
lfs->grm_.rid);
}
return 0;
}
@@ -7167,16 +7196,18 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) {
/// Directory operations ///
int lfsr_mkdir(lfs_t *lfs, const char *path) {
// checkpoint block allocations
// TODO we should just name this lfsr_alloc_checkpoint
lfs_alloc_ack(lfs);
// prepare our filesystem for writing
int err= lfsr_fs_preparemutation(lfs);
if (err) {
return err;
}
// lookup our parent
lfsr_openedmdir_t parent;
lfs_size_t parent_did;
const char *name;
lfs_size_t name_size;
int err = lfsr_mtree_pathlookup(lfs, path,
err = lfsr_mtree_pathlookup(lfs, path,
&parent.mdir, &parent.rid, NULL,
&parent_did, &name, &name_size);
if (err && (err != LFS_ERR_NOENT || parent.rid == -1)) {
@@ -7426,6 +7457,45 @@ int lfsr_dir_read(lfs_t *lfs, lfsr_dir_t *dir, struct lfs_info *info) {
}
/// Prepare the filesystem for mutation ///
static int lfsr_fs_fixgrm(lfs_t *lfs) {
// find our mdir
lfsr_mdir_t mdir;
int err = lfsr_mtree_lookup(lfs, lfs->grm_.mid, &mdir);
if (err) {
return err;
}
// remove the rid while also zeroing our grm
err = lfsr_mdir_commit(lfs, &mdir, (lfs_ssize_t*)&lfs->grm_.rid, LFSR_ATTRS(
LFSR_ATTR(lfs->grm_.rid, UNR, -1, NULL, 0),
LFSR_ATTR(-1, GRM, 0, NULL, 0)));
// mark grm as taken care of
lfs->grm_.mid = 0;
return 0;
}
static int lfsr_fs_preparemutation(lfs_t *lfs) {
// fix pending grms
if (lfs->grm_.mid >= 0) {
LFS_DEBUG("Fixing grm (0x%"PRIx32".%"PRIx32")",
lfs->grm_.mid,
lfs->grm_.rid);
int err = lfsr_fs_fixgrm(lfs);
if (err) {
return err;
}
}
// checkpoint the allocator
lfs_alloc_ack(lfs);
return 0;
}
///// Metadata pair and directory operations ///
lfs.h
+14 -6
View File
@@ -368,12 +368,6 @@ typedef union lfsr_btree {
} inlined;
} lfsr_btree_t;
// space for:
// - type - 1 leb128 - 1 byte (worst case)
// - mid - 1 leb128 - 5 bytes (worst case)
// - rid - 1 leb128 - 5 bytes (worst case)
#define LFSR_GRM_DSIZE (5+5)
typedef struct lfsr_mdir {
// -2 => deleted
// -1 => mroot
@@ -390,6 +384,17 @@ typedef struct lfsr_openedmdir {
lfsr_mdir_t mdir;
} lfsr_openedmdir_t;
// space for:
// - type - 1 leb128 - 1 byte (worst case)
// - mid - 1 leb128 - 5 bytes (worst case)
// - rid - 1 leb128 - 5 bytes (worst case)
#define LFSR_GRM_DSIZE (5+5)
typedef struct lfsr_grm {
lfs_ssize_t mid;
lfs_size_t rid;
} lfsr_grm_t;
typedef struct lfs_mdir {
lfs_block_t pair[2];
@@ -488,6 +493,9 @@ typedef struct lfs {
lfsr_mdir_t mroot;
lfsr_btree_t mtree;
// TODO do we really need separate decoded/encoded grms?
lfsr_grm_t grm_;
uint8_t grm[LFSR_GRM_DSIZE];
uint8_t grmd[LFSR_GRM_DSIZE];
scripts/dbgbtree.py
+2 -1
View File
@@ -225,7 +225,8 @@ class Rbyd:
for i_, rbyd in enumerate(rbyds):
# compare with sequence arithmetic
if rbyd and (
not ((rbyd.rev - rbyds[i].rev) & 0x80000000)
not rbyds[i]
or not ((rbyd.rev - rbyds[i].rev) & 0x80000000)
or (rbyd.rev == rbyds[i].rev
and rbyd.trunk > rbyds[i].trunk)):
i = i_
scripts/dbgmtree.py
+2 -1
View File
@@ -233,7 +233,8 @@ class Rbyd:
for i_, rbyd in enumerate(rbyds):
# compare with sequence arithmetic
if rbyd and (
not ((rbyd.rev - rbyds[i].rev) & 0x80000000)
not rbyds[i]
or not ((rbyd.rev - rbyds[i].rev) & 0x80000000)
or (rbyd.rev == rbyds[i].rev
and rbyd.trunk > rbyds[i].trunk)):
i = i_
scripts/dbgrbyd.py
+2 -1
View File
@@ -924,7 +924,8 @@ def main(disk, blocks=None, *,
# compare with sequence arithmetic
if trunk_ and (
not ((rev - revs[i]) & 0x80000000)
not trunks_[i]
or not ((rev - revs[i]) & 0x80000000)
or (rev == revs[i] and trunk_ > trunks_[i])):
i = i_
tests/t5_dirs.toml
+144 -3
View File
@@ -1,6 +1,7 @@
# Directory tests
[cases.t5_dirs_mkdir]
defines.REMOUNT = [false, true]
code = '''
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
@@ -9,6 +10,14 @@ code = '''
// make a directory
lfsr_mkdir(&lfs, "ardvark") => 0;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// grm should be zero here
assert(lfs.grm[0] == 0);
}
// check that our mkdir worked with stat
struct lfs_info info;
lfsr_stat(&lfs, "ardvark", &info) => 0;
@@ -34,6 +43,7 @@ code = '''
# test that noent errors work
[cases.t5_dirs_noent]
defines.REMOUNT = [false, true]
code = '''
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
@@ -49,6 +59,14 @@ code = '''
lfsr_dir_t dir;
lfsr_dir_open(&lfs, &dir, "no") => LFS_ERR_NOENT;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// grm should be zero here
assert(lfs.grm[0] == 0);
}
// and check that this didn't interfere with our original directory
lfsr_stat(&lfs, "ardvark", &info) => 0;
assert(strcmp(info.name, "ardvark") == 0);
@@ -71,6 +89,7 @@ code = '''
# test that creating the same directory twice errors
[cases.t5_dirs_mkdir_exists]
defines.REMOUNT = [false, true]
code = '''
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
@@ -82,6 +101,14 @@ code = '''
// make the same directory, should error
lfsr_mkdir(&lfs, "ardvark") => LFS_ERR_EXIST;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// grm should be zero here
assert(lfs.grm[0] == 0);
}
// and check that this didn't interfere with our original directory
struct lfs_info info;
lfsr_stat(&lfs, "ardvark", &info) => 0;
@@ -106,6 +133,7 @@ code = '''
# test that creating a directory with an invalid path errors
[cases.t5_dirs_mkdir_noent]
defines.REMOUNT = [false, true]
code = '''
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
@@ -120,6 +148,14 @@ code = '''
// make a nonsense child directory, should error
lfsr_mkdir(&lfs, "ardvark/no/hmm") => LFS_ERR_NOENT;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// grm should be zero here
assert(lfs.grm[0] == 0);
}
// and check that this didn't interfere with our original directory
struct lfs_info info;
lfsr_stat(&lfs, "ardvark", &info) => 0;
@@ -143,6 +179,7 @@ code = '''
'''
[cases.t5_dirs_mkdir_siblings]
defines.REMOUNT = [false, true]
code = '''
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
@@ -153,6 +190,14 @@ code = '''
lfsr_mkdir(&lfs, "batman") => 0;
lfsr_mkdir(&lfs, "cantaloupe") => 0;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// grm should be zero here
assert(lfs.grm[0] == 0);
}
// check that our mkdir worked
struct lfs_info info;
lfsr_stat(&lfs, "ardvark", &info) => 0;
@@ -188,6 +233,7 @@ code = '''
'''
[cases.t5_dirs_mkdir_children]
defines.REMOUNT = [false, true]
code = '''
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
@@ -198,6 +244,14 @@ code = '''
lfsr_mkdir(&lfs, "ardvark/batman") => 0;
lfsr_mkdir(&lfs, "ardvark/batman/cantaloupe") => 0;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// grm should be zero here
assert(lfs.grm[0] == 0);
}
// check that our mkdirs worked
struct lfs_info info;
lfsr_stat(&lfs, "ardvark", &info) => 0;
@@ -252,6 +306,7 @@ code = '''
[cases.t5_dirs_mkdir_many]
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
defines.REMOUNT = [false, true]
code = '''
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
@@ -262,6 +317,14 @@ code = '''
char name[256];
sprintf(name, "dir%04d", i);
lfsr_mkdir(&lfs, name) => 0;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// grm should be zero here
assert(lfs.grm[0] == 0);
}
}
// check that our mkdir worked
@@ -297,6 +360,7 @@ code = '''
[cases.t5_dirs_mkdir_many_2layers]
defines.N = [1, 2, 4, 8, 16, 32]
defines.REMOUNT = [false, true]
code = '''
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
@@ -312,6 +376,14 @@ code = '''
for (lfs_size_t j = 0; j < N; j++) {
sprintf(name, "dir%04d/child%04d", i, j);
lfsr_mkdir(&lfs, name) => 0;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// grm should be zero here
assert(lfs.grm[0] == 0);
}
}
}
@@ -378,6 +450,7 @@ code = '''
[cases.t5_dirs_mkdir_many_3layers]
defines.N = [1, 2, 4, 8]
defines.REMOUNT = [false, true]
code = '''
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
@@ -398,6 +471,14 @@ code = '''
for (lfs_size_t k = 0; k < N; k++) {
sprintf(name, "dir%04d/child%04d/grandchild%04d", i, j, k);
lfsr_mkdir(&lfs, name) => 0;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// grm should be zero here
assert(lfs.grm[0] == 0);
}
}
}
}
@@ -495,6 +576,7 @@ code = '''
[cases.t5_dirs_mkdir_many_linkedlist]
defines.N = [1, 2, 4, 8, 16, 32, 64]
defines.REMOUNT = [false, true]
code = '''
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
@@ -506,6 +588,14 @@ code = '''
for (lfs_size_t i = 0; i < N; i++) {
sprintf(&name[strlen(name)], "/dir%04d", i);
lfsr_mkdir(&lfs, name) => 0;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// grm should be zero here
assert(lfs.grm[0] == 0);
}
}
// check that our mkdir worked
@@ -547,9 +637,8 @@ code = '''
[cases.t5_dirs_mkdir_fuzz]
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
# 0 => do this test in the root dir
# 1 => do this test in a directory named "parent"
defines.PARENT = [0, 1]
defines.PARENT = [false, true]
defines.REMOUNT = [false, true]
defines.SAMPLES = 10
# -1 => all pseudo-random seeds
# n => reproduce a specific seed
@@ -590,6 +679,14 @@ code = '''
(sim_size-j)*sizeof(lfs_size_t));
sim_size += 1;
sim[j] = x;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// grm should be zero here
assert(lfs.grm[0] == 0);
}
break;
}
}
@@ -640,6 +737,7 @@ code = '''
# test that did collisions don't cause issues
[cases.t5_dirs_did_collisions]
defines.REMOUNT = [false, true]
code = '''
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
@@ -660,6 +758,14 @@ code = '''
lfsr_mkdir(&lfs, "e_LptKHkHH") => 0;
lfsr_mkdir(&lfs, "f_lUoVuhJH") => 0;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// grm should be zero here
assert(lfs.grm[0] == 0);
}
// check that our mkdirs worked
struct lfs_info info;
lfsr_stat(&lfs, "a_SNmwMTHH", &info) => 0;
@@ -714,6 +820,7 @@ code = '''
# these will also collide with the root
[cases.t5_dirs_did_zero]
defines.REMOUNT = [false, true]
code = '''
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
@@ -726,6 +833,14 @@ code = '''
assert(lfs_crc32c(0, "e_VNunKMPH", 10) == 0x00000000);
assert(lfs_crc32c(0, "f_vknsvNRH", 10) == 0x00000000);
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// grm should be zero here
assert(lfs.grm[0] == 0);
}
// make directories
lfsr_mkdir(&lfs, "a_IplRNrPH") => 0;
lfsr_mkdir(&lfs, "b_iUwOsqRH") => 0;
@@ -734,6 +849,14 @@ code = '''
lfsr_mkdir(&lfs, "e_VNunKMPH") => 0;
lfsr_mkdir(&lfs, "f_vknsvNRH") => 0;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// grm should be zero here
assert(lfs.grm[0] == 0);
}
// check that our mkdirs worked
struct lfs_info info;
lfsr_stat(&lfs, "a_IplRNrPH", &info) => 0;
@@ -790,6 +913,7 @@ code = '''
#
# note this is true even if you truncate
[cases.t5_dirs_did_ones]
defines.REMOUNT = [false, true]
code = '''
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
@@ -810,6 +934,14 @@ code = '''
lfsr_mkdir(&lfs, "e_vQtPStPH") => 0;
lfsr_mkdir(&lfs, "f_VtoMnwRH") => 0;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// grm should be zero here
assert(lfs.grm[0] == 0);
}
// check that our mkdirs worked
struct lfs_info info;
lfsr_stat(&lfs, "a_iomlVKPH", &info) => 0;
@@ -866,6 +998,7 @@ code = '''
# if the leb128 disk-size is not calculated correctly these can cause
# issues
[cases.t5_dirs_did_leb128_boundaries]
defines.REMOUNT = [false, true]
code = '''
lfs_t lfs;
lfsr_format(&lfs, cfg) => 0;
@@ -886,6 +1019,14 @@ code = '''
lfsr_mkdir(&lfs, "e_thrRIsRH") => 0;
lfsr_mkdir(&lfs, "f_pNtQTPJH") => 0;
// remount?
if (REMOUNT) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, cfg) => 0;
// grm should be zero here
assert(lfs.grm[0] == 0);
}
// check that our mkdirs worked
struct lfs_info info;
lfsr_stat(&lfs, "a_IOtUptRH", &info) => 0;