Github/littlefs
1
Fork 0
Code Actions Activity

Added support for recursive removes in directories

Browse Source 
"Recursion" here just refers to the ability to remove entries in a
directory while iterating over it. This is very useful when you just
want a directory gone, and can be extended to a "true" recursive remove
straightforwardly. This mainly tests that mid/rid updates in opened
mdirs are correct.

To make this work, we need to update opened dirs differently than files,
since opened dirs do not get marked as removed when its rid is removed
and contain an additional position in the dir that needs to be updated.

To keep track of the different types, littlefs now contains 2
linked-lists for opened mdirs. Maybe these should be correctly typed,
but by hiding the specific types behind an array of mdir linked-lists,
we can more efficiently iterate over both lists when necessary.

We should probably compare this approach to the type-tagged approach in
the previous littlefs implementation, but I think the idea of an array
of type-hidden linked-lists just didn't come to me then. There was also
a bit more room in the mdir structs to hide a 1-bit type field. The mdir
structs here are getting pretty squeezed since they are used everywhere.
This commit is contained in:
Christopher Haster
2023-07-20 15:22:43 -05:00
parent 0ddd851f6f
commit b1187595d6
4 changed files with 221 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/t3_mtree.toml
+42 -44
View File
@@ -2680,8 +2680,8 @@ code = '''
lfsr_openedmdir_t left_neighbor = {.rid=0, .mdir=lfs.mroot};
lfsr_openedmdir_t right_neighbor = {.rid=1, .mdir=lfs.mroot};
lfsr_mdir_addopened(&lfs, &left_neighbor);
lfsr_mdir_addopened(&lfs, &right_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &right_neighbor);
// insert a new entry, this should update our neighbors
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS(
@@ -2703,8 +2703,8 @@ code = '''
assert(right_neighbor.mdir.mid == -1);
assert(memcmp(&right_neighbor.mdir, &lfs.mroot, sizeof(lfsr_mdir_t)) == 0);
lfsr_mdir_removeopened(&lfs, &left_neighbor);
lfsr_mdir_removeopened(&lfs, &right_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &right_neighbor);
lfsr_unmount(&lfs) => 0;
'''
@@ -2729,8 +2729,8 @@ code = '''
lfsr_openedmdir_t left_neighbor = {.rid=0, .mdir=lfs.mroot};
lfsr_openedmdir_t right_neighbor = {.rid=1, .mdir=lfs.mroot};
lfsr_mdir_addopened(&lfs, &left_neighbor);
lfsr_mdir_addopened(&lfs, &right_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &right_neighbor);
// try removing our left entry
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS(
@@ -2740,14 +2740,13 @@ code = '''
assert(lfs.mroot.rbyd.weight == 1);
// assert that our neighbors were updated correctly
assert(left_neighbor.rid == -2);
assert(left_neighbor.mdir.mid == -2);
assert(left_neighbor.mdir.mid == LFSR_MID_RM);
assert(right_neighbor.rid == 0);
assert(right_neighbor.mdir.mid == -1);
assert(memcmp(&right_neighbor.mdir, &lfs.mroot, sizeof(lfsr_mdir_t)) == 0);
lfsr_mdir_removeopened(&lfs, &left_neighbor);
lfsr_mdir_removeopened(&lfs, &right_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &right_neighbor);
lfsr_unmount(&lfs) => 0;
'''
@@ -2772,8 +2771,8 @@ code = '''
lfsr_openedmdir_t left_neighbor = {.rid=0, .mdir=lfs.mroot};
lfsr_openedmdir_t right_neighbor = {.rid=1, .mdir=lfs.mroot};
lfsr_mdir_addopened(&lfs, &left_neighbor);
lfsr_mdir_addopened(&lfs, &right_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &right_neighbor);
// try removing our left entry
lfsr_mdir_commit(&lfs, &lfs.mroot, &(lfs_ssize_t){-1}, LFSR_ATTRS(
@@ -2786,11 +2785,10 @@ code = '''
assert(left_neighbor.rid == 0);
assert(left_neighbor.mdir.mid == -1);
assert(memcmp(&left_neighbor.mdir, &lfs.mroot, sizeof(lfsr_mdir_t)) == 0);
assert(right_neighbor.rid == -2);
assert(right_neighbor.mdir.mid == -2);
assert(right_neighbor.mdir.mid == LFSR_MID_RM);
lfsr_mdir_removeopened(&lfs, &left_neighbor);
lfsr_mdir_removeopened(&lfs, &right_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &right_neighbor);
lfsr_unmount(&lfs) => 0;
'''
@@ -2817,8 +2815,8 @@ code = '''
lfsr_openedmdir_t left_neighbor = {.rid=0, .mdir=lfs.mroot};
lfsr_openedmdir_t right_neighbor = {.rid=1, .mdir=lfs.mroot};
lfsr_mdir_addopened(&lfs, &left_neighbor);
lfsr_mdir_addopened(&lfs, &right_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &right_neighbor);
// prepare mroot with a large attr so the next entry can not fit
uint8_t buffer[SIZE];
@@ -2862,8 +2860,8 @@ code = '''
assert(right_neighbor.mdir.mid == 0);
assert(memcmp(&right_neighbor.mdir, &mdir, sizeof(lfsr_mdir_t)) == 0);
lfsr_mdir_removeopened(&lfs, &left_neighbor);
lfsr_mdir_removeopened(&lfs, &right_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &right_neighbor);
lfsr_unmount(&lfs) => 0;
'''
@@ -2890,8 +2888,8 @@ code = '''
lfsr_openedmdir_t left_neighbor = {.rid=0, .mdir=lfs.mroot};
lfsr_openedmdir_t right_neighbor = {.rid=1, .mdir=lfs.mroot};
lfsr_mdir_addopened(&lfs, &left_neighbor);
lfsr_mdir_addopened(&lfs, &right_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &right_neighbor);
// create 2 large entries that needs to be uninlined and split
uint8_t buffer[SIZE];
@@ -2935,8 +2933,8 @@ code = '''
assert(right_neighbor.mdir.mid == 1);
assert(memcmp(&right_neighbor.mdir, &msibling, sizeof(lfsr_mdir_t)) == 0);
lfsr_mdir_removeopened(&lfs, &left_neighbor);
lfsr_mdir_removeopened(&lfs, &right_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &right_neighbor);
lfsr_unmount(&lfs) => 0;
'''
@@ -2963,8 +2961,8 @@ code = '''
lfsr_openedmdir_t left_neighbor = {.rid=0, .mdir=lfs.mroot};
lfsr_openedmdir_t right_neighbor = {.rid=1, .mdir=lfs.mroot};
lfsr_mdir_addopened(&lfs, &left_neighbor);
lfsr_mdir_addopened(&lfs, &right_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &right_neighbor);
// create an uninlined mdir
uint8_t buffer[SIZE];
@@ -3030,8 +3028,8 @@ code = '''
assert(right_neighbor.mdir.mid == 1);
assert(memcmp(&right_neighbor.mdir, &msibling, sizeof(lfsr_mdir_t)) == 0);
lfsr_mdir_removeopened(&lfs, &left_neighbor);
lfsr_mdir_removeopened(&lfs, &right_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &right_neighbor);
lfsr_unmount(&lfs) => 0;
'''
@@ -3060,8 +3058,8 @@ code = '''
lfsr_openedmdir_t left_neighbor = {.rid=0, .mdir=lfs.mroot};
lfsr_openedmdir_t right_neighbor = {.rid=1, .mdir=lfs.mroot};
lfsr_mdir_addopened(&lfs, &left_neighbor);
lfsr_mdir_addopened(&lfs, &right_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &right_neighbor);
// prepare mroot with an attr
uint8_t buffer[SIZE];
@@ -3095,8 +3093,8 @@ code = '''
assert(right_neighbor.mdir.mid == -1);
assert(memcmp(&right_neighbor.mdir, &lfs.mroot, sizeof(lfsr_mdir_t)) == 0);
lfsr_mdir_removeopened(&lfs, &left_neighbor);
lfsr_mdir_removeopened(&lfs, &right_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &right_neighbor);
lfsr_unmount(&lfs) => 0;
'''
@@ -3125,8 +3123,8 @@ code = '''
lfsr_openedmdir_t left_neighbor = {.rid=0, .mdir=lfs.mroot};
lfsr_openedmdir_t right_neighbor = {.rid=1, .mdir=lfs.mroot};
lfsr_mdir_addopened(&lfs, &left_neighbor);
lfsr_mdir_addopened(&lfs, &right_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &right_neighbor);
// prepare mroot with a large attr so the next entry can not fit
uint8_t buffer[SIZE];
@@ -3186,8 +3184,8 @@ code = '''
assert(right_neighbor.mdir.mid == 0);
assert(memcmp(&right_neighbor.mdir, &mdir, sizeof(lfsr_mdir_t)) == 0);
lfsr_mdir_removeopened(&lfs, &left_neighbor);
lfsr_mdir_removeopened(&lfs, &right_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &right_neighbor);
lfsr_unmount(&lfs) => 0;
'''
@@ -3254,8 +3252,8 @@ code = '''
assert(right_neighbor.mdir.rbyd.weight == 1);
right_neighbor.rid = 0;
lfsr_mdir_addopened(&lfs, &left_neighbor);
lfsr_mdir_addopened(&lfs, &right_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &right_neighbor);
// cause middle mdir to split
lfsr_mtree_lookup(&lfs, 1, &mdir) => 0;
@@ -3282,8 +3280,8 @@ code = '''
lfsr_mtree_lookup(&lfs, 3, &mdir) => 0;
assert(memcmp(&right_neighbor.mdir, &mdir, sizeof(lfsr_mdir_t)) == 0);
lfsr_mdir_removeopened(&lfs, &left_neighbor);
lfsr_mdir_removeopened(&lfs, &right_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &right_neighbor);
lfsr_unmount(&lfs) => 0;
'''
@@ -3350,8 +3348,8 @@ code = '''
assert(right_neighbor.mdir.rbyd.weight == 1);
right_neighbor.rid = 0;
lfsr_mdir_addopened(&lfs, &left_neighbor);
lfsr_mdir_addopened(&lfs, &right_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_addopened(&lfs, LFS_TYPE_REG, &right_neighbor);
// cause middle mdir to drop
lfsr_mtree_lookup(&lfs, 1, &mdir) => 0;
@@ -3373,8 +3371,8 @@ code = '''
lfsr_mtree_lookup(&lfs, 1, &mdir) => 0;
assert(memcmp(&right_neighbor.mdir, &mdir, sizeof(lfsr_mdir_t)) == 0);
lfsr_mdir_removeopened(&lfs, &left_neighbor);
lfsr_mdir_removeopened(&lfs, &right_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &left_neighbor);
lfsr_mdir_removeopened(&lfs, LFS_TYPE_REG, &right_neighbor);
lfsr_unmount(&lfs) => 0;
'''
tests/t5_dirs.toml
+106 -108
View File
@@ -2803,115 +2803,113 @@ code = '''
lfsr_unmount(&lfs) => 0;
'''
# TODO this eventually
# Recursive here just refers to deleting entries in a directory while
# iterating over the directory
#
## Recursive here just refers to deleting entries in a directory while
## iterating over the directory
##
## This is a useful feature, but it's unintuitive that this should have
## well-defined behavior, so make sure to test for it
#[cases.t5_dirs_rm_many_recursive]
#defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
#defines.PARENT = [false, true]
## this test sort of fights against itself when powerloss testing,
## limit it to a _very_ small number of entries for this reason
#if = '!TEST_PL || N <= 32'
#reentrant = true
#code = '''
# // format once per test
# lfs_t lfs;
# int err = lfsr_mount(&lfs, cfg);
# if (err) {
# lfsr_format(&lfs, cfg) => 0;
# lfsr_mount(&lfs, cfg) => 0;
# }
#
# if (PARENT) {
# err = lfsr_mkdir(&lfs, "pricklypear");
# assert(!err || (TEST_PL && err == LFS_ERR_EXIST));
# }
#
# // make this many directories
# for (lfs_size_t i = 0; i < N; i++) {
# char name[256];
# sprintf(name, "%s/dir%04d", (PARENT ? "pricklypear" : ""), i);
# err = lfsr_mkdir(&lfs, name);
# assert(!err || (TEST_PL && err == LFS_ERR_EXIST));
# }
#
# // check that our mkdir worked
# for (lfs_size_t i = 0; i < N; i++) {
# char name[256];
# sprintf(name, "%s/dir%04d", (PARENT ? "pricklypear" : ""), i);
# struct lfs_info info;
# lfsr_stat(&lfs, name, &info) => 0;
# char name2[256];
# sprintf(name2, "dir%04d", i);
# assert(strcmp(info.name, name2) == 0);
# assert(info.type == LFS_TYPE_DIR);
# }
#
# lfsr_dir_t dir;
# lfsr_dir_open(&lfs, &dir, (PARENT ? "pricklypear" : "/")) => 0;
# struct lfs_info info;
# lfsr_dir_read(&lfs, &dir, &info) => 0;
# assert(strcmp(info.name, ".") == 0);
# assert(info.type == LFS_TYPE_DIR);
# lfsr_dir_read(&lfs, &dir, &info) => 0;
# assert(strcmp(info.name, "..") == 0);
# assert(info.type == LFS_TYPE_DIR);
# for (lfs_size_t i = 0; i < N; i++) {
# char name[256];
# sprintf(name, "dir%04d", i);
# lfsr_dir_read(&lfs, &dir, &info) => 0;
# assert(strcmp(info.name, name) == 0);
# assert(info.type == LFS_TYPE_DIR);
# }
# lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT;
# lfsr_dir_close(&lfs, &dir) => 0;
#
# // now remove directories recursively
# lfsr_dir_open(&lfs, &dir, (PARENT ? "pricklypear" : "/")) => 0;
# lfsr_dir_read(&lfs, &dir, &info) => 0;
# assert(strcmp(info.name, ".") == 0);
# assert(info.type == LFS_TYPE_DIR);
# lfsr_dir_read(&lfs, &dir, &info) => 0;
# assert(strcmp(info.name, "..") == 0);
# assert(info.type == LFS_TYPE_DIR);
# for (lfs_size_t i = 0; i < N; i++) {
# char name[256];
# sprintf(name, "dir%04d", i);
# lfsr_dir_read(&lfs, &dir, &info) => 0;
# assert(strcmp(info.name, name) == 0);
# assert(info.type == LFS_TYPE_DIR);
#
# char path[1024];
# sprintf(path, "%s/%s", (PARENT ? "pricklypear" : ""), info.name);
# lfsr_remove(&lfs, path) => 0;
# }
# lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT;
# lfsr_dir_close(&lfs, &dir) => 0;
#
# // check that our removes worked
# for (lfs_size_t i = 0; i < N; i++) {
# char name[256];
# sprintf(name, "%s/dir%04d", (PARENT ? "pricklypear" : ""), i);
# struct lfs_info info;
# lfsr_stat(&lfs, name, &info) => LFS_ERR_NOENT;
# }
#
# lfsr_dir_open(&lfs, &dir, "/") => 0;
# lfsr_dir_read(&lfs, &dir, &info) => 0;
# assert(strcmp(info.name, ".") == 0);
# assert(info.type == LFS_TYPE_DIR);
# lfsr_dir_read(&lfs, &dir, &info) => 0;
# assert(strcmp(info.name, "..") == 0);
# assert(info.type == LFS_TYPE_DIR);
# lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT;
# lfsr_dir_close(&lfs, &dir) => 0;
#
# lfsr_unmount(&lfs) => 0;
#'''
# This is a useful feature, but it's unintuitive that this should have
# well-defined behavior, so make sure to test for it
[cases.t5_dirs_rm_many_recursive]
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]
defines.PARENT = [false, true]
# this test sort of fights against itself when powerloss testing,
# limit it to a _very_ small number of entries for this reason
if = '!TEST_PL || N <= 32'
reentrant = true
code = '''
// format once per test
lfs_t lfs;
int err = lfsr_mount(&lfs, cfg);
if (err) {
lfsr_format(&lfs, cfg) => 0;
lfsr_mount(&lfs, cfg) => 0;
}
if (PARENT) {
err = lfsr_mkdir(&lfs, "pricklypear");
assert(!err || (TEST_PL && err == LFS_ERR_EXIST));
}
// make this many directories
for (lfs_size_t i = 0; i < N; i++) {
char name[256];
sprintf(name, "%s/dir%04d", (PARENT ? "pricklypear" : ""), i);
err = lfsr_mkdir(&lfs, name);
assert(!err || (TEST_PL && err == LFS_ERR_EXIST));
}
// check that our mkdir worked
for (lfs_size_t i = 0; i < N; i++) {
char name[256];
sprintf(name, "%s/dir%04d", (PARENT ? "pricklypear" : ""), i);
struct lfs_info info;
lfsr_stat(&lfs, name, &info) => 0;
char name2[256];
sprintf(name2, "dir%04d", i);
assert(strcmp(info.name, name2) == 0);
assert(info.type == LFS_TYPE_DIR);
}
lfsr_dir_t dir;
lfsr_dir_open(&lfs, &dir, (PARENT ? "pricklypear" : "/")) => 0;
struct lfs_info info;
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, ".") == 0);
assert(info.type == LFS_TYPE_DIR);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, "..") == 0);
assert(info.type == LFS_TYPE_DIR);
for (lfs_size_t i = 0; i < N; i++) {
char name[256];
sprintf(name, "dir%04d", i);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, name) == 0);
assert(info.type == LFS_TYPE_DIR);
}
lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT;
lfsr_dir_close(&lfs, &dir) => 0;
// now remove directories recursively
lfsr_dir_open(&lfs, &dir, (PARENT ? "pricklypear" : "/")) => 0;
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, ".") == 0);
assert(info.type == LFS_TYPE_DIR);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, "..") == 0);
assert(info.type == LFS_TYPE_DIR);
for (lfs_size_t i = 0; i < N; i++) {
char name[256];
sprintf(name, "dir%04d", i);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, name) == 0);
assert(info.type == LFS_TYPE_DIR);
char path[1024];
sprintf(path, "%s/%s", (PARENT ? "pricklypear" : ""), info.name);
lfsr_remove(&lfs, path) => 0;
}
lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT;
lfsr_dir_close(&lfs, &dir) => 0;
// check that our removes worked
for (lfs_size_t i = 0; i < N; i++) {
char name[256];
sprintf(name, "%s/dir%04d", (PARENT ? "pricklypear" : ""), i);
struct lfs_info info;
lfsr_stat(&lfs, name, &info) => LFS_ERR_NOENT;
}
lfsr_dir_open(&lfs, &dir, (PARENT ? "pricklypear" : "/")) => 0;
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, ".") == 0);
assert(info.type == LFS_TYPE_DIR);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, "..") == 0);
assert(info.type == LFS_TYPE_DIR);
lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT;
lfsr_dir_close(&lfs, &dir) => 0;
lfsr_unmount(&lfs) => 0;
'''
[cases.t5_dirs_rm_fuzz]
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512]