Github/littlefs
1
Fork 0
Code Actions Activity

Reverted most of dir offset changes, dirs to follow dstart when open

Browse Source 
Unfortunately the previous attempt to fix the dir seek system didn't
really work. Using a packed mid/rid integer for the offset is tempting,
but since mid/rid can change with any metadata id change in the
filesystem, dir tell offsets would become invalidated if you modified
files in unrelated directories, which isn't great and likely to catch
users by surprise.

This solution builds on the previous dir offset design, which tracks the
dstart-relative position independently from the current mid/rid in our
directory. To update this correctly when there are unrelated changes to
the filesystem, we need to know if metadata id changes are in the range
between our directories dstart and current mid/rid. This in turn means
we need to track our dstart. So our opened directories need three
separate pointers we need to update on every mdir commit:

             dir->pos
                |
        .-------+-------.
  a b c d e f g h i j k l m n o p
        ^               ^
        |               |
    dir->dstart     dir->mdir

This has quite a few moving parts, which I was hoping to avoid.
Fortunately we don't need a second mdir, so the RAM cost is pretty
small.

We can also drop dir->did, since the dstart mid/rid render it redundant,
which is interesting.
This commit is contained in:
Christopher Haster
2023-07-28 12:58:16 -05:00
parent edd12e1f93
commit 4cf5509c91
3 changed files with 276 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+99 -116
View File
@@ -5056,54 +5056,6 @@ static int lfsr_btree_traversal_next(lfs_t *lfs,
/// Metadata pair operations ///
// mids are a tuple of bid (index in the mtree) and rid (index in the mdir)
// shoved into an integer
//
// We want to maximize our encoding space, so we conservatively carve out
// only enough space to fit the maximum possible number of rids in an mdir.
// In theory, as the mdirs get larger, we need fewer mdirs for a filesystem of
// a given word size.
//
// - Each file neads 1 tag minimum
// - Each tag needs ~2 alts with our current compaction strategy
// - Each tag/alt encodes to a minimum of 4 bytes
//
// This gives us ~1*3*4 or ~12 bytes per rid minimum. Rounding down to the
// nearest power of 2 gives us ~block_size/8 rids per block at most.
//
#define LFSR_MID(_lfs, _bid, _rid) \
(((_bid) << lfs_nlog2((_lfs)->cfg->block_size/8)) \
| ((_rid) & ((1 << lfs_nlog2((_lfs)->cfg->block_size/8))-1)))
// reserved values to represent "." and ".." entries
//
// note these need to be positive values to fit in the lfsr_soff_t returned
// by lfsr_dir_tell
//
// TODO is this really the best way to carve out space for these entries?
// TODO do we need an mdir_limit in the superconfig? since this makes
// mid=max problematic
enum {
LFSR_MID_DOT = 0x7fffffff & -2,
LFSR_MID_DOTDOT = 0x7fffffff & -1,
};
static inline lfs_ssize_t lfsr_mid_bid(lfs_t *lfs, lfsr_mid_t mid) {
// TODO can we get rid of this?
// adjust mid if mtree is inlined
if (lfsr_mtree_isinlined(lfs)) {
return LFSR_MID_MROOT;
}
// TODO should this be stored in lfs_t?
return mid >> lfs_nlog2(lfs->cfg->block_size/8);
}
static inline lfs_size_t lfsr_mid_rid(lfs_t *lfs, lfsr_mid_t mid) {
// TODO should this be stored in lfs_t?
return mid & ((1 << lfs_nlog2(lfs->cfg->block_size/8))-1);
}
// mptr things
typedef struct lfsr_mptr {
lfs_block_t blocks[2];
@@ -6288,6 +6240,8 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
// first play out any attrs that change our rid
for (lfs_size_t i = 0; i < attr_count; i++) {
// TODO clean this up a bit
// adjust opened mdirs?
if (opened->mdir.mid == mdir->mid
&& opened->rid >= attrs[i].id) {
// removed?
@@ -6303,6 +6257,41 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
}
} else {
opened->rid += attrs[i].delta;
// adjust dir position?
if (type == LFS_TYPE_DIR
&& ((((lfsr_dir_t*)opened)->dstart_mid
== mdir->mid
&& ((lfsr_dir_t*)opened)->dstart_rid
< attrs[i].id)
|| ((lfsr_dir_t*)opened)->dstart_mid
< mdir->mid)) {
((lfsr_dir_t*)opened)->pos += attrs[i].delta;
}
}
} else if (opened->mdir.mid > mdir->mid) {
// adjust dir position?
if (type == LFS_TYPE_DIR
&& ((((lfsr_dir_t*)opened)->dstart_mid
== mdir->mid
&& ((lfsr_dir_t*)opened)->dstart_rid
< attrs[i].id)
|| ((lfsr_dir_t*)opened)->dstart_mid
< mdir->mid)) {
((lfsr_dir_t*)opened)->pos += attrs[i].delta;
}
}
// TODO combine with above?
// adjust opened dstarts?
if (type == LFS_TYPE_DIR
&& ((lfsr_dir_t*)opened)->dstart_mid == mdir->mid
&& ((lfsr_dir_t*)opened)->dstart_rid >= attrs[i].id) {
// removed?
if (((lfsr_dir_t*)opened)->dstart_rid
+ attrs[i].delta < attrs[i].id) {
((lfsr_dir_t*)opened)->dstart_mid = LFSR_MID_RM;
} else {
((lfsr_dir_t*)opened)->dstart_rid += attrs[i].delta;
}
}
}
@@ -7830,9 +7819,8 @@ int lfsr_dir_open(lfs_t *lfs, lfsr_dir_t *dir, const char *path) {
}
// read our did from the mdir, unless we're root
if (err == LFS_ERR_INVAL) {
dir->did = 0;
} else {
lfs_size_t did = 0;
if (err != LFS_ERR_INVAL) {
lfsr_data_t data;
int err = lfsr_mdir_lookup(lfs, &mdir, rid, LFSR_TAG_DID,
NULL, &data);
@@ -7840,15 +7828,32 @@ int lfsr_dir_open(lfs_t *lfs, lfsr_dir_t *dir, const char *path) {
return err;
}
lfs_ssize_t d = lfsr_data_readleb128(lfs, data, 0, &dir->did);
lfs_ssize_t d = lfsr_data_readleb128(lfs, data, 0, &did);
if (d < 0) {
return d;
}
}
// leave it up to rewind to initialize mid/dots
err = lfsr_dir_rewind(lfs, dir);
// reset pos
dir->pos = 0;
// lookup our dstart in the mtree
err = lfsr_mtree_dnamelookup(lfs, did, NULL, 0,
&dir->mdir.mdir, &dir->mdir.rid, NULL, NULL);
if (err) {
LFS_ASSERT(err != LFS_ERR_NOENT);
return err;
}
// keep track of the mid/rid of our dstart
dir->dstart_mid = dir->mdir.mdir.mid;
dir->dstart_rid = dir->mdir.rid;
// eagerly look up the next entry
//
// this makes handling of corner cases with mixed removes/dir reads easier
err = lfsr_mtree_seek(lfs, &dir->mdir.mdir, &dir->mdir.rid, 1);
if (err && err != LFS_ERR_NOENT) {
return err;
}
@@ -7867,15 +7872,15 @@ int lfsr_dir_read(lfs_t *lfs, lfsr_dir_t *dir, struct lfs_info *info) {
memset(info, 0, sizeof(struct lfs_info));
// handle dots specially
if (dir->dots == 0) {
if (dir->pos == 0) {
info->type = LFS_TYPE_DIR;
strcpy(info->name, ".");
dir->dots += 1;
dir->pos += 1;
return 0;
} else if (dir->dots == 1) {
} else if (dir->pos == 1) {
info->type = LFS_TYPE_DIR;
strcpy(info->name, "..");
dir->dots += 1;
dir->pos += 1;
return 0;
}
@@ -7895,18 +7900,19 @@ int lfsr_dir_read(lfs_t *lfs, lfsr_dir_t *dir, struct lfs_info *info) {
return err;
}
lfs_size_t did;
lfs_ssize_t d = lfsr_data_readleb128(lfs, data, 0, &did);
if (d < 0) {
return d;
}
// did mismatch? we must be done
if (did != dir->did) {
// found another directory's dstart? we must be done
if (tag == LFSR_TAG_DSTART) {
return LFS_ERR_NOENT;
}
// get file type from the tag
info->type = lfsr_tag_filetype(tag);
// get file name from the name entry
lfs_ssize_t d = lfsr_data_readleb128(lfs, data, 0, &(uint32_t){0});
if (d < 0) {
return d;
}
LFS_ASSERT(lfsr_data_size(data)-d <= LFS_NAME_MAX);
d = lfsr_data_read(lfs, data, d, info->name, LFS_NAME_MAX);
if (d < 0) {
@@ -7914,84 +7920,61 @@ int lfsr_dir_read(lfs_t *lfs, lfsr_dir_t *dir, struct lfs_info *info) {
}
info->name[d] = '\0';
// get file type from the tag
info->type = lfsr_tag_filetype(tag);
// TODO get size once we actually have regular files
// TODO get size once we actually have regular files
// eagerly look up the next entry
err = lfsr_mtree_seek(lfs, &dir->mdir.mdir, &dir->mdir.rid, 1);
if (err && err != LFS_ERR_NOENT) {
return err;
}
dir->pos += 1;
return 0;
}
int lfsr_dir_seek(lfs_t *lfs, lfsr_dir_t *dir, lfs_off_t off) {
// off should never be negative, most likely an error went unchecked
//
// TODO is this the right approach? should we just assert if we detect
// a modified directory during seek?
// Note that aside from negative offsets, all other offset are accepted.
// This makes it so seeking to an outdated offset in the case of concurrent
// modification is at least not an assert, though it may repeat files in
// the directory.
LFS_ASSERT((lfs_soff_t)off > 0);
// handle dots specially
if (off == LFSR_MID_DOT || off == LFSR_MID_DOTDOT) {
int err = lfsr_dir_rewind(lfs, dir);
if (err) {
return err;
}
dir->dots = off - LFSR_MID_DOT;
return 0;
}
dir->dots = 2;
// find our mdir
lfs_ssize_t mid = lfsr_mid_bid(lfs, off);
int err = lfsr_mtree_lookup(lfs,
lfs_smin32(mid, (lfs_ssize_t)lfsr_mtree_weight(lfs)-1),
&dir->mdir.mdir);
// first rewind
int err = lfsr_dir_rewind(lfs, dir);
if (err) {
return err;
}
// and update our rid, clamping to rbyd weight
lfs_ssize_t rid = lfsr_mid_rid(lfs, off);
if (mid > dir->mdir.mdir.mid
|| rid >= (lfs_ssize_t)dir->mdir.mdir.rbyd.weight) {
dir->mdir.rid = dir->mdir.mdir.rbyd.weight;
} else {
dir->mdir.rid = rid;
// then seek to the requested offset, we leave it up to lfsr_mtree_seek
// to make this efficient
//
// note the -2 to adjust for dot entries
if (off > 2) {
err = lfsr_mtree_seek(lfs, &dir->mdir.mdir, &dir->mdir.rid, off - 2);
if (err && err != LFS_ERR_NOENT) {
return err;
}
}
dir->pos = off;
return 0;
}
lfs_soff_t lfsr_dir_tell(lfs_t *lfs, lfsr_dir_t *dir) {
if (dir->dots == 0) {
return LFSR_MID_DOT;
} else if (dir->dots == 1) {
return LFSR_MID_DOTDOT;
} else {
return LFSR_MID(lfs, lfs_smax32(dir->mdir.mdir.mid, 0), dir->mdir.rid);
}
(void)lfs;
return dir->pos;
}
int lfsr_dir_rewind(lfs_t *lfs, lfsr_dir_t *dir) {
// reset dots
dir->dots = 0;
// do nothing if removed
if (dir->dstart_mid == LFSR_MID_RM) {
return 0;
}
// lookup our dstart in the mtree
int err = lfsr_mtree_dnamelookup(lfs, dir->did, NULL, 0,
&dir->mdir.mdir, &dir->mdir.rid, NULL, NULL);
// reset pos
dir->pos = 0;
// lookup our dstart in the mtree again
int err = lfsr_mtree_lookup(lfs, dir->dstart_mid, &dir->mdir.mdir);
if (err) {
LFS_ASSERT(err != LFS_ERR_NOENT);
return err;
}
dir->mdir.rid = dir->dstart_rid;
// eagerly look up the next entry
//