From da810aca26607a287e0a73ab740dc128950f22b1 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 5 Jul 2023 13:34:50 -0500 Subject: [PATCH] Implemented mtree path/dname lookup, rudimentary lfsr_mkdir/lfsr_dir_read This makes it now possible to create directories in the new system. The new system now uses a single global "mtree" to store all metadata entries in the filesystem. In this system, a directory is simply a range of metadata entries. This has a number of benefits, but does come with its own problems: 1. We need to indicate which directory each file belongs to. To do this the file's name entry has been changed to a tuple of leb128-encoded directory-id + actual file name: 01 66 69 6c 65 2e 74 78 74 .file.txt ^ '----------+----------' '------------|------------ leb128 directory-id '------------ ascii/utf8 name If we include the directory-id as part of filename comparison, files should naturally be next to other files in the same directory. 2. We need a way allocate directory-ids for new directories. This turns out to be a bit more tricky than I expected. We can't use any mid/bid/rid inherent to the mtree, because these change on any file creation/deletion. And since we commit the did into the tree, that's not acceptable. Initially I though you could just find the largest did and increment, but this gives you no way to reclaim deleted dids. And sure, deleted dids have no storage consumption, but eventually you will overflow the did integer. Since this can suddenly happen in a filesystem that's been in a steady-state for years, that's pretty unnacceptable. One solution is to do a simple linear search over the mtree for an unused did. But with a runtime of O(n^2 log(n)), this raises performance concerns. Sidenote: It's interesting to note that the Linux kernel's allocation of process-ids, a very similar problem, is surprisingly complex and relies on a radix-tree of bitmaps (struct idr). This suggests I'm not missing an obvious solution somewhere. The solution I settled on here is to instead treat the set of dids as a sort of hash table: 1. Hash the full directory path into a did. 2. Perform a linear search until we have no collision. leb128(truncate28(crc32c("dir"))) .--------' v 9e cd c8 30 66 69 6c 65 2e 74 78 74 ...0file.txt '----+----' '----------+----------' '-----------------|------------ leb128 directory-id '------------ ascii/utf8 name Worst case, this can still exhibit the worst case O(n^2 log(n)) performance when we are close to full dids. However that seems unlikely to happen in practice, since we don't truncate our hashes, unlike normal hash tables. An additional 32-bit word for each file is a small price to pay for a low-chance of collisions. In the current implementation, I do truncate the hash to 28-bits. Since we encode the hash with leb128, and hashes are statistically random, this gives us better usage of the leb128 encoding. However it does limit a 32-bit littlefs to 256 Mi directories. Maybe this should be a configurable limit in the future. But that highlights another benefit of this scheme. It's easy to change in the future without disk changes. 3. We need a way to know if a directory-id is allocated, even if the directory is empty. For this we just introduce a new tag: LFSR_TAG_DSTART, which is an empty file entry that indicates the directory at the given did in the mtree is allocated. To create/delete these atomically with the reference in our parent directory, we can use the GRM system for atomic renames. Note this isn't implemented yet. This is also the first time we finally get around to testing all of the dname lookup functions, so this did find a few bugs, mostly around reporting the root correctly. --- lfs.c | 692 ++++++++++++++++++++----- lfs.h | 67 +-- scripts/dbgbtree.py | 13 +- scripts/dbgmtree.py | 12 +- scripts/dbgrbyd.py | 13 +- tests/{test_dirs.toml => t5_dirs.toml} | 120 +++++ 6 files changed, 751 insertions(+), 166 deletions(-) rename tests/{test_dirs.toml => t5_dirs.toml} (89%) diff --git a/lfs.c b/lfs.c index 622e47c9..497551ef 100644 --- a/lfs.c +++ b/lfs.c @@ -586,32 +586,35 @@ enum lfsr_tag_type { LFSR_TAG_SUPERMAGIC = 0x0003, LFSR_TAG_SUPERCONFIG = 0x0004, - LFSR_TAG_NAME = 0x0100, - LFSR_TAG_BRANCH = 0x0100, - LFSR_TAG_REG = 0x0101, - LFSR_TAG_GROWREG = 0x2101, // test only? TODO - LFSR_TAG_DIR = 0x0102, + LFSR_TAG_NAME = 0x0200, + LFSR_TAG_WIDENAME = 0x4200, // in-device only + LFSR_TAG_BRANCH = 0x0200, + LFSR_TAG_DSTART = 0x0201, + LFSR_TAG_REG = 0x0202, + LFSR_TAG_GROWREG = 0x2202, // test only? TODO + LFSR_TAG_DIR = 0x0203, LFSR_TAG_STRUCT = 0x0300, - LFSR_TAG_WIDESTRUCT = 0x4300, - LFSR_TAG_RMWIDESTRUCT = 0x5300, + LFSR_TAG_WIDESTRUCT = 0x4300, // in-device only + LFSR_TAG_RMWIDESTRUCT = 0x5300, // in-device only LFSR_TAG_INLINED = 0x0300, LFSR_TAG_BLOCK = 0x0302, LFSR_TAG_BTREE = 0x0303, - LFSR_TAG_RMBTREE = 0x1303, + LFSR_TAG_RMBTREE = 0x1303, // in-device only LFSR_TAG_MROOT = 0x0304, LFSR_TAG_MDIR = 0x0305, - LFSR_TAG_RMMDIR = 0x1305, + LFSR_TAG_RMMDIR = 0x1305, // in-device only LFSR_TAG_MTREE = 0x0306, - LFSR_TAG_RMMTREE = 0x1306, + LFSR_TAG_RMMTREE = 0x1306, // in-device only + LFSR_TAG_DID = 0x0307, LFSR_TAG_UATTR = 0x0400, LFSR_TAG_WIDEUATTR = 0x4400, // test only? TODO LFSR_TAG_GROWUATTR = 0x2400, // test only? TODO - LFSR_TAG_RMUATTR = 0x1400, + LFSR_TAG_RMUATTR = 0x1400, // in-device only LFSR_TAG_RMWIDEUATTR = 0x5400, // test only? TODO LFSR_TAG_SATTR = 0x0500, // test only? TODO - LFSR_TAG_RMWIDESATTR = 0x5500, + LFSR_TAG_RMWIDESATTR = 0x5500, // in-device only LFSR_TAG_ALT = 0x4000, LFSR_TAG_ALTLE = 0x4000, @@ -1021,10 +1024,17 @@ typedef union lfsr_data { #define LFSR_DATA_DNAME(_did, _buffer, _size) \ ((lfsr_data_t){.buf={ \ /* note this find the effective leb128 size */ \ - .size=_size + lfs_min32(lfs_nlog2(_did)/7, 1), \ + .size=_size + (lfs_nlog2(lfs_max32(_did, 1))+7-1)/7, \ .buffer=(const void*)(_buffer), \ .did=_did}}) +#define LFSR_DATA_LEB128(_did) \ + ((lfsr_data_t){.buf={ \ + /* note this find the effective leb128 size */ \ + .size=(lfs_nlog2(lfs_max32(_did, 1))+7-1)/7, \ + .buffer=NULL, \ + .did=_did}}) + #define LFSR_DATA_DISK(_block, _off, _size) \ ((lfsr_data_t){.disk={ \ .size=(0x80000000 | (_size)), \ @@ -1100,26 +1110,30 @@ static lfs_scmp_t lfsr_data_cmp(lfs_t *lfs, lfsr_data_t data, lfs_off_t off_ = lfs_min32(off, lfsr_data_size(data)); lfs_size_t hint_ = lfsr_data_size(data)-off_; - // return early if our size doesn't match - if (hint_ < size) { - return LFS_CMP_LT; - } else if (hint_ > size) { - return LFS_CMP_GT; - } - + // compare our data if (lfsr_data_ondisk(data)) { - return lfsr_bd_cmp(lfs, data.disk.block, data.disk.off+off_, 0, - buffer, size); + int cmp = lfsr_bd_cmp(lfs, data.disk.block, data.disk.off+off_, 0, + buffer, lfs_min32(hint_, size)); + if (cmp != LFS_CMP_EQ) { + return cmp; + } } else { int cmp = memcmp(data.buf.buffer+off_, buffer, size); if (cmp < 0) { return LFS_CMP_LT; - } else if (cmp == 0) { - return LFS_CMP_EQ; - } else { + } else if (cmp > 0) { return LFS_CMP_GT; } } + + // if data is equal, check for size mismatch + if (hint_ < size) { + return LFS_CMP_LT; + } else if (hint_ > size) { + return LFS_CMP_GT; + } else { + return LFS_CMP_EQ; + } } static lfs_scmp_t lfsr_data_dnamecmp(lfs_t *lfs, lfsr_data_t data, @@ -1227,6 +1241,18 @@ typedef struct lfsr_attr { #define LFSR_ATTR_DATA(_id, _type, _delta, _data) \ LFSR_ATTR_DATA_(_id, LFSR_TAG_##_type, _delta, _data) +#define LFSR_ATTR_DNAME_(_id, _tag, _delta, _did, _buffer, _size) \ + LFSR_ATTR_DATA_(_id, _tag, _delta, LFSR_DATA_DNAME(_did, _buffer, _size)) + +#define LFSR_ATTR_DNAME(_id, _type, _delta, _did, _buffer, _size) \ + LFSR_ATTR_DNAME_(_id, LFSR_TAG_##_type, _delta, _did, _buffer, _size) + +#define LFSR_ATTR_LEB128_(_id, _tag, _delta, _did) \ + LFSR_ATTR_DATA_(_id, _tag, _delta, LFSR_DATA_LEB128(_did)) + +#define LFSR_ATTR_LEB128(_id, _type, _delta, _did) \ + LFSR_ATTR_LEB128_(_id, LFSR_TAG_##_type, _delta, _did) + #define LFSR_ATTR_(_id, _tag, _delta, _buffer, _size) \ LFSR_ATTR_DATA_(_id, _tag, _delta, LFSR_DATA_BUF(_buffer, _size)) @@ -3129,6 +3155,10 @@ static int lfsr_rbyd_dnamelookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd, // binary search for our name lfs_ssize_t lower = 0; lfs_ssize_t upper = rbyd->weight; + // if we have an empty mdir, default to id = 0 + if (id_) { + *id_ = 0; + } while (lower < upper) { lfsr_tag_t tag__; @@ -3147,7 +3177,7 @@ static int lfsr_rbyd_dnamelookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd, // if we have no name or a vestigial name, treat this id as always lt lfs_scmp_t cmp; - if (id__-(weight__-1) == 0 + if ((tag__ == LFSR_TAG_BRANCH && id__-(weight__-1) == 0) || lfsr_tag_suptype(tag__) != LFSR_TAG_NAME) { cmp = LFS_CMP_LT; @@ -3166,18 +3196,9 @@ static int lfsr_rbyd_dnamelookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd, } else if (lfs_cmp(cmp) < 0) { lower = id__ + 1; - // keep track of best-matching name so far + // keep track of best-matching id >= our target if (id_) { - *id_ = id__; - } - if (tag_) { - *tag_ = tag__; - } - if (weight_) { - *weight_ = weight__; - } - if (data_) { - *data_ = data__; + *id_ = id__ + weight__; } } else { @@ -3557,88 +3578,6 @@ static int lfsr_btree_parent(lfs_t *lfs, } } -static lfs_ssize_t lfsr_btree_dnamelookup(lfs_t *lfs, const lfsr_btree_t *btree, - lfs_size_t did, const char *name, lfs_size_t name_size, - lfs_size_t *bid_, lfsr_tag_t *tag_, lfs_size_t *weight_, - lfsr_data_t *data_) { - // an empty tree? - if (lfsr_btree_weight(btree) == 0) { - return LFS_ERR_NOENT; - } - - // inlined? - if (lfsr_btree_isinlined(btree)) { - // TODO how many of these should be conditional? - if (bid_) { - *bid_ = lfsr_btree_weight(btree)-1; - } - if (tag_) { - *tag_ = btree->inlined.tag; - } - if (weight_) { - *weight_ = lfsr_btree_weight(btree); - } - if (data_) { - *data_ = LFSR_DATA_BUF(btree->inlined.buffer, btree->inlined.size); - } - return 0; - } - - // descend down the btree looking for our name - lfsr_rbyd_t branch = btree->root; - lfs_ssize_t bid = 0; - while (true) { - // lookup our name in the rbyd via binary search - lfs_ssize_t rid__; - lfs_size_t weight__; - int err = lfsr_rbyd_dnamelookup(lfs, &branch, did, name, name_size, - &rid__, NULL, &weight__, NULL); - if (err && err != LFS_ERR_NOENT) { - return err; - } - - // the name may not match exactly, but indicates which branch to follow - lfsr_tag_t tag__; - lfsr_data_t data__; - err = lfsr_rbyd_lookup(lfs, &branch, rid__, LFSR_TAG_WIDESTRUCT, - &tag__, &data__); - if (err) { - LFS_ASSERT(err != LFS_ERR_NOENT); - return err; - } - - // found another branch - if (tag__ == LFSR_TAG_BTREE) { - // update our bid - bid += rid__ - (weight__-1); - - // fetch the next branch - lfs_ssize_t d = lfsr_bptr_fromdisk(lfs, &branch, data__); - if (d < 0) { - return d; - } - LFS_ASSERT(branch.weight == weight__); - - // found our id - } else { - // TODO how many of these should be conditional? - if (bid_) { - *bid_ = bid + rid__; - } - if (tag_) { - *tag_ = tag__; - } - if (weight_) { - *weight_ = weight__; - } - if (data_) { - *data_ = data__; - } - return 0; - } - } -} - // we need some scratch space for tail-recursive attr in lfsr_btree_commit // @@ -4530,6 +4469,88 @@ static int lfsr_btree_split(lfs_t *lfs, lfsr_btree_t *btree, } } +// lookup in a btree by dname +static int lfsr_btree_dnamelookup(lfs_t *lfs, const lfsr_btree_t *btree, + lfs_size_t did, const char *name, lfs_size_t name_size, + lfs_size_t *bid_, lfsr_tag_t *tag_, lfs_size_t *weight_, + lfsr_data_t *data_) { + // an empty tree? + if (lfsr_btree_weight(btree) == 0) { + return LFS_ERR_NOENT; + } + + // inlined? + if (lfsr_btree_isinlined(btree)) { + // TODO how many of these should be conditional? + if (bid_) { + *bid_ = lfsr_btree_weight(btree)-1; + } + if (tag_) { + *tag_ = btree->inlined.tag; + } + if (weight_) { + *weight_ = lfsr_btree_weight(btree); + } + if (data_) { + *data_ = LFSR_DATA_BUF(btree->inlined.buffer, btree->inlined.size); + } + return 0; + } + + // descend down the btree looking for our name + lfsr_rbyd_t branch = btree->root; + lfs_ssize_t bid = 0; + while (true) { + // lookup our name in the rbyd via binary search + lfs_ssize_t rid__; + lfs_size_t weight__; + int err = lfsr_rbyd_dnamelookup(lfs, &branch, did, name, name_size, + &rid__, NULL, &weight__, NULL); + if (err && err != LFS_ERR_NOENT) { + return err; + } + + // the name may not match exactly, but indicates which branch to follow + lfsr_tag_t tag__; + lfsr_data_t data__; + err = lfsr_rbyd_lookup(lfs, &branch, rid__, LFSR_TAG_WIDESTRUCT, + &tag__, &data__); + if (err) { + LFS_ASSERT(err != LFS_ERR_NOENT); + return err; + } + + // found another branch + if (tag__ == LFSR_TAG_BTREE) { + // update our bid + bid += rid__ - (weight__-1); + + // fetch the next branch + lfs_ssize_t d = lfsr_bptr_fromdisk(lfs, &branch, data__); + if (d < 0) { + return d; + } + LFS_ASSERT(branch.weight == weight__); + + // found our id + } else { + // TODO how many of these should be conditional? + if (bid_) { + *bid_ = bid + rid__; + } + if (tag_) { + *tag_ = tag__; + } + if (weight_) { + *weight_ = weight__; + } + if (data_) { + *data_ = data__; + } + return 0; + } + } +} // incremental btree traversal // @@ -4898,10 +4919,9 @@ static int lfsr_mdir_fetch(lfs_t *lfs, lfsr_mdir_t *mdir, static int lfsr_mdir_lookupnext(lfs_t *lfs, const lfsr_mdir_t *mdir, lfs_ssize_t id, lfsr_tag_t tag, - lfs_ssize_t *id_, lfsr_tag_t *tag_, lfs_size_t *weight_, - lfsr_data_t *data_) { + lfs_ssize_t *id_, lfsr_tag_t *tag_, lfsr_data_t *data_) { return lfsr_rbyd_lookupnext(lfs, &mdir->rbyd, id, tag, - id_, tag_, weight_, data_); + id_, tag_, NULL, data_); } static int lfsr_mdir_lookup(lfs_t *lfs, const lfsr_mdir_t *mdir, @@ -5241,7 +5261,7 @@ static int lfsr_mtree_split_(lfs_t *lfs, lfsr_btree_t *mtree, lfsr_tag_t stag; lfsr_data_t sdata; err = lfsr_mdir_lookupnext(lfs, msibling, 0, LFSR_TAG_NAME, - NULL, &stag, NULL, &sdata); + NULL, &stag, &sdata); if (err) { LFS_ASSERT(err != LFS_ERR_NOENT); return err; @@ -5617,7 +5637,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid, if (opened->mdir.mid == mdir->mid // avoid double-updating our current mdir && &opened->mdir != mdir) { - LFS_ASSERT(opened->rid < (lfs_ssize_t)opened->mdir.rbyd.weight); + LFS_ASSERT(opened->rid <= (lfs_ssize_t)opened->mdir.rbyd.weight); LFS_ASSERT(opened->rid != -1); // first play out any attrs that change our rid @@ -5662,6 +5682,203 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid, } +// lookup dnames in our mtree +static int lfsr_mdir_dnamelookup(lfs_t *lfs, const lfsr_mdir_t *mdir, + lfs_size_t did, const char *name, lfs_size_t name_size, + lfs_ssize_t *id_, lfsr_tag_t *tag_, lfsr_data_t *data_) { + return lfsr_rbyd_dnamelookup(lfs, &mdir->rbyd, + did, name, name_size, + id_, tag_, NULL, data_); +} + +// note if we fail, we at least leave mdir_/rid_ with the best place to insert +static int lfsr_mtree_dnamelookup(lfs_t *lfs, + lfs_size_t did, const char *name, lfs_size_t name_size, + lfsr_mdir_t *mdir_, lfs_ssize_t *rid_, lfsr_tag_t *tag_, + lfsr_data_t *data_) { + // do we only have mroot? + lfsr_mdir_t mdir; + if (lfsr_mtree_isinlined(lfs)) { + mdir = lfs->mroot; + + // lookup dname in actual mtree + } else { + lfs_size_t mid; + lfsr_tag_t tag; + lfsr_data_t data; + int err = lfsr_btree_dnamelookup(lfs, &lfs->mtree, + did, name, name_size, + &mid, &tag, NULL, &data); + if (err) { + return err; + } + LFS_ASSERT(tag == LFSR_TAG_MDIR); + + // decode mptr + lfsr_mptr_t mptr; + lfs_ssize_t d = lfsr_mptr_fromdisk(lfs, &mptr, data); + if (d < 0) { + return d; + } + + // fetch mdir + err = lfsr_mdir_fetch(lfs, &mdir, mid, mptr); + if (err) { + return err; + } + } + + if (mdir_) { + *mdir_ = mdir; + } + + // and finally lookup dname in our mdir + return lfsr_mdir_dnamelookup(lfs, &mdir, + did, name, name_size, + rid_, tag_, data_); +} + + +// special directory-ids +enum { + LFSR_DID_ROOT = 0, +}; + +// lookup full paths in our mtree +// +// if not found, mdir_/rid_/did_/name_ will at least be set up +// with what should be the parent +static int lfsr_mtree_pathlookup(lfs_t *lfs, const char *path, + // TODO originally path itself was a double pointer, is that a + // better design? + lfsr_mdir_t *mdir_, lfs_ssize_t *rid_, lfsr_tag_t *tag_, + lfs_size_t *did_, const char **name_, lfs_size_t *name_size_) { + // setup root + lfsr_mdir_t mdir = {.mid = LFSR_MID_RM}; + lfs_ssize_t rid = -1; + lfsr_tag_t tag = LFSR_TAG_DIR; + lfs_size_t did = LFSR_DID_ROOT; + + if (mdir_) { + *mdir_ = mdir; + } + if (rid_) { + *rid_ = rid; + } + if (tag_) { + *tag_ = tag; + } + + // we reduce path to a single name if we can find it + const char *name = path; + + while (true) { + // skip slashes + name += strspn(name, "/"); + lfs_size_t name_size = strcspn(name, "/"); + + // skip '.' and root '..' + if ((name_size == 1 && memcmp(name, ".", 1) == 0) + || (name_size == 2 && memcmp(name, "..", 2) == 0)) { + name += name_size; + goto next; + } + + // skip if matched by '..' in name + const char *suffix = name + name_size; + lfs_size_t suffix_size; + int depth = 1; + while (true) { + suffix += strspn(suffix, "/"); + suffix_size = strcspn(suffix, "/"); + if (suffix_size == 0) { + break; + } + + if (suffix_size == 2 && memcmp(suffix, "..", 2) == 0) { + depth -= 1; + if (depth == 0) { + name = suffix + suffix_size; + goto next; + } + } else { + depth += 1; + } + + suffix += suffix_size; + } + + // found end of path, we must be done parsing our path now + if (name[0] == '\0') { + return 0; + } + + // only continue if we hit a directory + if (tag != LFSR_TAG_DIR) { + return LFS_ERR_NOTDIR; + } + + // read the next did from the mdir if this is not the root + if (rid != -1) { + lfsr_data_t data; + int err = lfsr_mdir_lookup(lfs, &mdir, rid, LFSR_TAG_DID, + NULL, &data); + if (err) { + return err; + } + + lfs_ssize_t d = lfsr_data_readleb128(lfs, data, 0, &did); + if (d < 0) { + return d; + } + + // TODO should we put this in lfsr_data_readleb128? + // TODO should readtag then call lfsr_data_readleb128? + if (did > 0x7fffffff) { + return LFS_ERR_CORRUPT; + } + } + + // lookup up this dname in the mtree + int err = lfsr_mtree_dnamelookup(lfs, did, name, name_size, + &mdir, &rid, &tag, NULL); + if (err && err != LFS_ERR_NOENT) { + return err; + } + + // keep track of what we've seen so far + if (mdir_) { + *mdir_ = mdir; + } + if (rid_) { + *rid_ = rid; + } + if (tag_) { + *tag_ = tag; + } + if (did_) { + *did_ = did; + } + if (name_) { + *name_ = name; + } + if (name_size_) { + *name_size_ = name_size; + } + + // error if not found, note we update things first so mdir/rid + // get updated with where to insert correctly + if (err == LFS_ERR_NOENT) { + return LFS_ERR_NOENT; + } + + // go on to next name + name += name_size; +next:; + } +} + + // incremental mtree traversal typedef struct lfsr_mtree_traversal { // core traversal state @@ -6286,9 +6503,14 @@ static int lfsr_formatinited(lfs_t *lfs) { return err; } + // our initial superblock contains a couple things: + // - our magic string, "littlefs" + // - the superconfig, format-time configuration + // - the root's dstart tag, which reserves did = 0 for the root err = lfsr_rbyd_commit(lfs, &rbyd, LFSR_ATTRS( LFSR_ATTR(-1, SUPERMAGIC, 0, "littlefs", 8), - LFSR_ATTR(-1, SUPERCONFIG, 0, buf, d))); + LFSR_ATTR(-1, SUPERCONFIG, 0, buf, d), + LFSR_ATTR_DNAME(0, DSTART, +1, 0, NULL, 0))); if (err) { return err; } @@ -6447,6 +6669,226 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { } +/// Directory operations /// + +int lfsr_mkdir(lfs_t *lfs, const char *path) { + // checkpoint block allocations + // TODO we should just name this lfsr_alloc_checkpoint + lfs_alloc_ack(lfs); + + // lookup our parent + lfsr_openedmdir_t parent; + lfs_size_t parent_did; + const char *name; + lfs_size_t name_size; + int err = lfsr_mtree_pathlookup(lfs, path, + &parent.mdir, &parent.rid, NULL, + &parent_did, &name, &name_size); + if (err && err != LFS_ERR_NOENT) { + return err; + } + + // woah, already exists? + if (err != LFS_ERR_NOENT) { + return LFS_ERR_EXIST; + } + + // check that name fits + if (name_size > lfs->name_max) { + return LFS_ERR_NAMETOOLONG; + } + + // Our directory needs an arbitrary directory-id. To find one with + // hopefully few collisions, we use a hash of the full path. Since + // we have a CRC handy, we can use that. + // + // We truncate to 28-bits to more optimally use our leb128 encoding. + // TODO should we have a configurable limit for this? dir_limit? or + // just loose ~3 bits from the configured file limit? + // + lfs_size_t did = lfs_crc32c(0, path, strlen(path)) & 0xfffffff; + + // Check if we have a collision. If we do, search for the next + // available did + lfsr_mdir_t mdir; + lfs_ssize_t rid; + while (true) { + int err = lfsr_mtree_dnamelookup(lfs, did, NULL, 0, + &mdir, &rid, NULL, NULL); + if (err && err != LFS_ERR_NOENT) { + return err; + } + if (err == LFS_ERR_NOENT) { + break; + } + + // try the next did + did = (did + 1) & 0xfffffff; + } + + // Note when we write to the mtree, it's possible it changes our + // parent's mdir/rid. We can catch this by tracking our parent + // as "opened" temporarily + // TODO is this the best workaround for rid update issues? + parent.rid -= 1; + lfsr_mdir_addopened(lfs, &parent); + + // TODO GRM, make this power-safe + // Conveniently, we just found where our dstart should go. The dstart + // tag is an empty entry that marks our directory as being allocated. + err = lfsr_mdir_commit(lfs, &mdir, &rid, LFSR_ATTRS( + LFSR_ATTR_DNAME(rid, DSTART, +1, did, NULL, 0))); + if (err) { + goto failed_with_parent; + } + + lfsr_mdir_removeopened(lfs, &parent); + parent.rid += 1; + + // commit our new directory into our parent + err = lfsr_mdir_commit(lfs, &parent.mdir, &parent.rid, LFSR_ATTRS( + LFSR_ATTR_DNAME(parent.rid, DIR, +1, parent_did, name, name_size), + LFSR_ATTR_LEB128(parent.rid, DID, 0, did))); + if (err) { + return err; + } + + return 0; + +failed_with_parent: + lfsr_mdir_removeopened(lfs, &parent); + return err; +} + +int lfsr_dir_open(lfs_t *lfs, lfsr_dir_t *dir, const char *path) { + // lookup our directory + lfsr_mdir_t mdir; + lfs_ssize_t rid; + lfsr_tag_t tag; + int err = lfsr_mtree_pathlookup(lfs, path, + &mdir, &rid, &tag, + NULL, NULL, NULL); + if (err) { + return err; + } + + // are we a directory? + if (tag != LFSR_TAG_DIR) { + return LFS_ERR_NOENT; + } + + // read our did from the mdir, unless we're root + lfs_size_t did = 0; + if (rid != -1) { + lfsr_data_t data; + int err = lfsr_mdir_lookup(lfs, &mdir, rid, LFSR_TAG_DID, + NULL, &data); + if (err) { + return err; + } + + lfs_ssize_t d = lfsr_data_readleb128(lfs, data, 0, &did); + if (d < 0) { + return d; + } + + // TODO should we put this in lfsr_data_readleb128? + // TODO should readtag then call lfsr_data_readleb128? + if (did > 0x7fffffff) { + return LFS_ERR_CORRUPT; + } + } + + // now lookup our dstart in the mtree + err = lfsr_mtree_dnamelookup(lfs, did, NULL, 0, + &dir->mdir.mdir, &dir->mdir.rid, NULL, NULL); + if (err) { + LFS_ASSERT(err != LFS_ERR_NOENT); + return err; + } + + // add to tracked mdirs + lfsr_mdir_addopened(lfs, &dir->mdir); + dir->off = 0; + return 0; +} + +int lfsr_dir_close(lfs_t *lfs, lfsr_dir_t *dir) { + // remove from tracked mdirs + lfsr_mdir_removeopened(lfs, &dir->mdir); + return 0; +} + +int lfsr_dir_read(lfs_t *lfs, lfsr_dir_t *dir, struct lfs_info *info) { + memset(info, 0, sizeof(struct lfs_info)); + + // handle "." and ".." specially + if (dir->off == 0) { + info->type = LFS_TYPE_DIR; + strcpy(info->name, "."); + dir->off += 1; + return 0; + } else if (dir->off == 1) { + info->type = LFS_TYPE_DIR; + strcpy(info->name, ".."); + dir->off += 1; + return 0; + } + + // lookup the next entry in our dir + lfs_ssize_t rid = dir->mdir.rid + 1; + // need to lookup the next mdir? + if (rid >= (lfs_ssize_t)dir->mdir.mdir.rbyd.weight) { + // out of mdirs? + lfs_ssize_t mid = dir->mdir.mdir.mid + 1; + if (mid >= lfsr_mtree_weight(lfs)) { + return LFS_ERR_NOENT; + } + + int err = lfsr_mtree_lookup(lfs, mid, &dir->mdir.mdir); + if (err) { + return err; + } + rid = 0; + } + dir->mdir.rid = rid; + + // lookup our name tag + lfsr_tag_t tag; + lfsr_data_t data; + int err = lfsr_mdir_lookup(lfs, &dir->mdir.mdir, rid, LFSR_TAG_WIDENAME, + &tag, &data); + if (err) { + return err; + } + + // found another directory's dstart? we must be done + if (tag == LFSR_TAG_DSTART) { + return LFS_ERR_NOENT; + } + + // fill in our info struct + info->type = tag - LFSR_TAG_REG; + + LFS_ASSERT(lfsr_data_size(data) <= lfs->name_max); + lfs_ssize_t d = lfsr_data_readleb128(lfs, data, 0, &(uint32_t){0}); + if (d < 0) { + return d; + } + d = lfsr_data_read(lfs, data, d, info->name, LFS_NAME_MAX); + if (d < 0) { + return d; + } + info->name[d] = '\0'; + + // TODO size once we actually have regular files + + return 0; +} + + + + ///// Metadata pair and directory operations /// //static lfs_stag_t lfs_dir_getslice(lfs_t *lfs, const lfs_mdir_t *dir, // lfs_tag_t gmask, lfs_tag_t gtag, diff --git a/lfs.h b/lfs.h index 78fdb2cf..20b38559 100644 --- a/lfs.h +++ b/lfs.h @@ -93,36 +93,36 @@ enum lfs_error { // File types enum lfs_type { // file types - LFS_TYPE_REG = 0x001, - LFS_TYPE_DIR = 0x002, + LFS_TYPE_REG = 0, + LFS_TYPE_DIR = 1, - // internally used types - LFS_TYPE_SPLICE = 0x400, - LFS_TYPE_NAME = 0x000, - LFS_TYPE_STRUCT = 0x200, - LFS_TYPE_USERATTR = 0x300, - LFS_TYPE_FROM = 0x100, - LFS_TYPE_TAIL = 0x600, - LFS_TYPE_GLOBALS = 0x700, - LFS_TYPE_CRC = 0x500, - - // internally used type specializations - LFS_TYPE_CREATE = 0x401, - LFS_TYPE_DELETE = 0x4ff, - LFS_TYPE_SUPERBLOCK = 0x0ff, - LFS_TYPE_DIRSTRUCT = 0x200, - LFS_TYPE_CTZSTRUCT = 0x202, - LFS_TYPE_INLINESTRUCT = 0x201, - LFS_TYPE_SOFTTAIL = 0x600, - LFS_TYPE_HARDTAIL = 0x601, - LFS_TYPE_MOVESTATE = 0x7ff, - LFS_TYPE_CCRC = 0x500, - LFS_TYPE_FCRC = 0x5ff, - - // internal chip sources - LFS_FROM_NOOP = 0x000, - LFS_FROM_MOVE = 0x101, - LFS_FROM_USERATTRS = 0x102, +// // internally used types +// LFS_TYPE_SPLICE = 0x400, +// LFS_TYPE_NAME = 0x000, +// LFS_TYPE_STRUCT = 0x200, +// LFS_TYPE_USERATTR = 0x300, +// LFS_TYPE_FROM = 0x100, +// LFS_TYPE_TAIL = 0x600, +// LFS_TYPE_GLOBALS = 0x700, +// LFS_TYPE_CRC = 0x500, +// +// // internally used type specializations +// LFS_TYPE_CREATE = 0x401, +// LFS_TYPE_DELETE = 0x4ff, +// LFS_TYPE_SUPERBLOCK = 0x0ff, +// LFS_TYPE_DIRSTRUCT = 0x200, +// LFS_TYPE_CTZSTRUCT = 0x202, +// LFS_TYPE_INLINESTRUCT = 0x201, +// LFS_TYPE_SOFTTAIL = 0x600, +// LFS_TYPE_HARDTAIL = 0x601, +// LFS_TYPE_MOVESTATE = 0x7ff, +// LFS_TYPE_CCRC = 0x500, +// LFS_TYPE_FCRC = 0x5ff, +// +// // internal chip sources +// LFS_FROM_NOOP = 0x000, +// LFS_FROM_MOVE = 0x101, +// LFS_FROM_USERATTRS = 0x102, }; // File open flags @@ -407,6 +407,11 @@ typedef struct lfs_dir { lfs_block_t head[2]; } lfs_dir_t; +typedef struct lfsr_dir { + lfsr_openedmdir_t mdir; + lfs_off_t off; +} lfsr_dir_t; + // littlefs file type typedef struct lfs_file { struct lfs_file *next; @@ -683,6 +688,7 @@ lfs_soff_t lfs_file_size(lfs_t *lfs, lfs_file_t *file); // // Returns a negative error code on failure. int lfs_mkdir(lfs_t *lfs, const char *path); +int lfsr_mkdir(lfs_t *lfs, const char *path); #endif // Open a directory @@ -690,12 +696,14 @@ int lfs_mkdir(lfs_t *lfs, const char *path); // Once open a directory can be used with read to iterate over files. // Returns a negative error code on failure. int lfs_dir_open(lfs_t *lfs, lfs_dir_t *dir, const char *path); +int lfsr_dir_open(lfs_t *lfs, lfsr_dir_t *dir, const char *path); // Close a directory // // Releases any allocated resources. // Returns a negative error code on failure. int lfs_dir_close(lfs_t *lfs, lfs_dir_t *dir); +int lfsr_dir_close(lfs_t *lfs, lfsr_dir_t *dir); // Read an entry in the directory // @@ -703,6 +711,7 @@ int lfs_dir_close(lfs_t *lfs, lfs_dir_t *dir); // Returns a positive value on success, 0 at the end of directory, // or a negative error code on failure. int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info); +int lfsr_dir_read(lfs_t *lfs, lfsr_dir_t *dir, struct lfs_info *info); // Change the position of the directory // diff --git a/scripts/dbgbtree.py b/scripts/dbgbtree.py index 4098abbf..1def0ced 100755 --- a/scripts/dbgbtree.py +++ b/scripts/dbgbtree.py @@ -11,10 +11,11 @@ import struct TAG_NULL = 0x0000 TAG_SUPERMAGIC = 0x0003 TAG_SUPERCONFIG = 0x0004 -TAG_NAME = 0x0100 -TAG_BRANCH = 0x0100 -TAG_REG = 0x0101 -TAG_DIR = 0x0102 +TAG_NAME = 0x0200 +TAG_BRANCH = 0x0200 +TAG_DSTART = 0x0201 +TAG_REG = 0x0202 +TAG_DIR = 0x0203 TAG_STRUCT = 0x0300 TAG_INLINED = 0x0300 TAG_BLOCK = 0x0302 @@ -22,6 +23,7 @@ TAG_BTREE = 0x0303 TAG_MROOT = 0x0304 TAG_MDIR = 0x0305 TAG_MTREE = 0x0306 +TAG_DID = 0x0307 TAG_UATTR = 0x0400 TAG_SATTR = 0x0500 TAG_ALT = 0x4000 @@ -29,6 +31,7 @@ TAG_CRC = 0x2000 TAG_FCRC = 0x2100 + # parse some rbyd addr encodings # 0xa -> [0xa] # 0xa.b -> ([0xa], b) @@ -128,6 +131,7 @@ def tagrepr(tag, w, size, off=None): elif (tag & 0xff00) == TAG_NAME: return '%s%s %d' % ( 'branch' if tag == TAG_BRANCH + else 'dstart' if tag == TAG_DSTART else 'reg' if tag == TAG_REG else 'dir' if tag == TAG_DIR else 'name 0x%02x' % (tag & 0xff), @@ -141,6 +145,7 @@ def tagrepr(tag, w, size, off=None): else 'mroot' if tag == TAG_MROOT else 'mdir' if tag == TAG_MDIR else 'mtree' if tag == TAG_MTREE + else 'did' if tag == TAG_DID else 'struct 0x%02x' % (tag & 0xff), ' w%d' % w if w else '', size) diff --git a/scripts/dbgmtree.py b/scripts/dbgmtree.py index 5c452fcc..b9b0c199 100755 --- a/scripts/dbgmtree.py +++ b/scripts/dbgmtree.py @@ -11,10 +11,11 @@ import struct TAG_NULL = 0x0000 TAG_SUPERMAGIC = 0x0003 TAG_SUPERCONFIG = 0x0004 -TAG_NAME = 0x0100 -TAG_BRANCH = 0x0100 -TAG_REG = 0x0101 -TAG_DIR = 0x0102 +TAG_NAME = 0x0200 +TAG_BRANCH = 0x0200 +TAG_DSTART = 0x0201 +TAG_REG = 0x0202 +TAG_DIR = 0x0203 TAG_STRUCT = 0x0300 TAG_INLINED = 0x0300 TAG_BLOCK = 0x0302 @@ -22,6 +23,7 @@ TAG_BTREE = 0x0303 TAG_MROOT = 0x0304 TAG_MDIR = 0x0305 TAG_MTREE = 0x0306 +TAG_DID = 0x0307 TAG_UATTR = 0x0400 TAG_SATTR = 0x0500 TAG_ALT = 0x4000 @@ -137,6 +139,7 @@ def tagrepr(tag, w, size, off=None): elif (tag & 0xff00) == TAG_NAME: return '%s%s %d' % ( 'branch' if tag == TAG_BRANCH + else 'dstart' if tag == TAG_DSTART else 'reg' if tag == TAG_REG else 'dir' if tag == TAG_DIR else 'name 0x%02x' % (tag & 0xff), @@ -150,6 +153,7 @@ def tagrepr(tag, w, size, off=None): else 'mroot' if tag == TAG_MROOT else 'mdir' if tag == TAG_MDIR else 'mtree' if tag == TAG_MTREE + else 'did' if tag == TAG_DID else 'struct 0x%02x' % (tag & 0xff), ' w%d' % w if w else '', size) diff --git a/scripts/dbgrbyd.py b/scripts/dbgrbyd.py index 101893ec..a9edf406 100755 --- a/scripts/dbgrbyd.py +++ b/scripts/dbgrbyd.py @@ -20,10 +20,11 @@ COLORS = [ TAG_NULL = 0x0000 TAG_SUPERMAGIC = 0x0003 TAG_SUPERCONFIG = 0x0004 -TAG_NAME = 0x0100 -TAG_BRANCH = 0x0100 -TAG_REG = 0x0101 -TAG_DIR = 0x0102 +TAG_NAME = 0x0200 +TAG_BRANCH = 0x0200 +TAG_DSTART = 0x0201 +TAG_REG = 0x0202 +TAG_DIR = 0x0203 TAG_STRUCT = 0x0300 TAG_INLINED = 0x0300 TAG_BLOCK = 0x0302 @@ -31,6 +32,7 @@ TAG_BTREE = 0x0303 TAG_MROOT = 0x0304 TAG_MDIR = 0x0305 TAG_MTREE = 0x0306 +TAG_DID = 0x0307 TAG_UATTR = 0x0400 TAG_SATTR = 0x0500 TAG_ALT = 0x4000 @@ -38,6 +40,7 @@ TAG_CRC = 0x2000 TAG_FCRC = 0x2100 + # parse some rbyd addr encodings # 0xa -> [0xa] # 0xa.b -> ([0xa], b) @@ -130,6 +133,7 @@ def tagrepr(tag, w, size, off=None): elif (tag & 0xff00) == TAG_NAME: return '%s%s %d' % ( 'branch' if tag == TAG_BRANCH + else 'dstart' if tag == TAG_DSTART else 'reg' if tag == TAG_REG else 'dir' if tag == TAG_DIR else 'name 0x%02x' % (tag & 0xff), @@ -143,6 +147,7 @@ def tagrepr(tag, w, size, off=None): else 'mroot' if tag == TAG_MROOT else 'mdir' if tag == TAG_MDIR else 'mtree' if tag == TAG_MTREE + else 'did' if tag == TAG_DID else 'struct 0x%02x' % (tag & 0xff), ' w%d' % w if w else '', size) diff --git a/tests/test_dirs.toml b/tests/t5_dirs.toml similarity index 89% rename from tests/test_dirs.toml rename to tests/t5_dirs.toml index bda62e1b..9db61ac9 100644 --- a/tests/test_dirs.toml +++ b/tests/t5_dirs.toml @@ -1,3 +1,123 @@ +# Directory tests + +[cases.t5_dirs_mkdir] +code = ''' + lfs_t lfs; + lfsr_format(&lfs, cfg) => 0; + lfsr_mount(&lfs, cfg) => 0; + + // make a directory + lfsr_mkdir(&lfs, "ardvark") => 0; + + // check that our mkdir worked + lfsr_dir_t dir; + lfsr_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, ".") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "..") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "ardvark") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT; + + lfsr_unmount(&lfs) => 0; +''' + +[cases.t5_dirs_mkdir_siblings] +code = ''' + lfs_t lfs; + lfsr_format(&lfs, cfg) => 0; + lfsr_mount(&lfs, cfg) => 0; + + // make some directories + lfsr_mkdir(&lfs, "ardvark") => 0; + lfsr_mkdir(&lfs, "batman") => 0; + lfsr_mkdir(&lfs, "cantaloupe") => 0; + + // check that our mkdir worked + lfsr_dir_t dir; + lfsr_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, ".") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "..") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "ardvark") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "batman") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "cantaloupe") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT; + + lfsr_unmount(&lfs) => 0; +''' + +[cases.t5_dirs_mkdir_children] +code = ''' + lfs_t lfs; + lfsr_format(&lfs, cfg) => 0; + lfsr_mount(&lfs, cfg) => 0; + + // make some directories + lfsr_mkdir(&lfs, "ardvark") => 0; + lfsr_mkdir(&lfs, "ardvark/batman") => 0; + lfsr_mkdir(&lfs, "ardvark/batman/cantaloupe") => 0; + + // check that our mkdir worked + lfsr_dir_t dir; + lfsr_dir_open(&lfs, &dir, "/") => 0; + struct lfs_info info; + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, ".") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "..") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "ardvark") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT; + + lfsr_dir_open(&lfs, &dir, "/ardvark") => 0; + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, ".") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "..") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "batman") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT; + + lfsr_dir_open(&lfs, &dir, "/ardvark/batman") => 0; + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, ".") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "..") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "cantaloupe") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT; + + lfsr_unmount(&lfs) => 0; +''' + + + + #[cases.test_dirs_root] #code = ''' # lfs_t lfs;