From a60a986c9c7064f4b1a393879338505146f357ce Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 18 Mar 2024 23:36:55 -0500 Subject: [PATCH 1/6] Duplicate the superblock entry during superblock expansion The documentation does not match the implementation here. The intended behavior of superblock expansion was to duplicate the current superblock entry into the new superblock: .--------. .--------. .|littlefs|->|littlefs| ||bs=4096 | ||bs=4096 | ||bc=256 | ||bc=256 | ||crc32 | ||root dir| || | ||crc32 | |'--------' |'--------' '--------' '--------' The main benefit is that we can rely on the magic string "littlefs" always residing in blocks 0x{0,1}, even if the superblock chain has multiple superblocks. The downside is that earlier superblocks in the superblock chain may contain out-of-date configuration. This is a bit annoying, and risks hard-to-reach bugs, but in theory shouldn't break anything as long as the filesystem is aware of this. Unfortunately this was lost at some point during refactoring in the early v2-alpha work. A lot of code was moving around in this stage, so it's a bit hard to track down the change and if it was intentional. The result is superblock expansion creates a valid linked-list of superblocks, but only the last superblock contains a valid superblock entry: .--------. .--------. .|crc32 |->|littlefs| || | ||bs=4096 | || | ||bc=256 | || | ||root dir| || | ||crc32 | |'--------' |'--------' '--------' '--------' What's interesting is this isn't invalid as far as lfs_mount is concerned. lfs_mount is happy as long as a superblock entry exists anywhere in the superblock chain. This is good for compat flexibility, but is the main reason this has gone unnoticed for so long. --- With the benefit of more time to think about the problem, it may have been more preferable to copy only the "littlefs" magic string and NOT the superblock entry: .--------. .--------. .|littlefs|->|littlefs| ||crc32c | ||bs=4096 | || | ||bc=256 | || | ||root dir| || | ||crc32 | |'--------' |'--------' '--------' '--------' This would allow for simple "littlefs" magic string checks without the risks associated with out-of-date superblock entries. Unfortunately the current implementation errors if it finds a "littlefs" magic string without an associated superblock entry, so such a change would not be compatible with old drivers. --- This commit tweaks superblock expansion to duplicate the superblock entry instead of simply moving it to the new superblock. And adds tests over the magic string "littlefs" both before and after superblock expansion. Found by rojer and Nikola Kosturski --- lfs.c | 7 +++-- tests/test_superblocks.toml | 53 ++++++++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/lfs.c b/lfs.c index a0bd76fd..4e4dd9b4 100644 --- a/lfs.c +++ b/lfs.c @@ -2191,7 +2191,8 @@ static int lfs_dir_splittingcompact(lfs_t *lfs, lfs_mdir_t *dir, // we can do, we'll error later if we've become frozen LFS_WARN("Unable to expand superblock"); } else { - end = begin; + // duplicate the superblock entry into the new superblock + end = 1; } } } @@ -2358,7 +2359,9 @@ fixmlist:; while (d->id >= d->m.count && d->m.split) { // we split and id is on tail now - d->id -= d->m.count; + if (lfs_pair_cmp(d->m.tail, lfs->root) != 0) { + d->id -= d->m.count; + } int err = lfs_dir_fetch(lfs, &d->m, d->m.tail); if (err) { return err; diff --git a/tests/test_superblocks.toml b/tests/test_superblocks.toml index a7c2aff3..e93f02eb 100644 --- a/tests/test_superblocks.toml +++ b/tests/test_superblocks.toml @@ -14,6 +14,24 @@ code = ''' lfs_unmount(&lfs) => 0; ''' +# make sure the magic string "littlefs" is always at offset=8 +[cases.test_superblocks_magic] +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + + // check our magic string + // + // note if we lose power we may not have the magic string in both blocks! + // but we don't lose power in this test so we can assert the magic string + // is present in both + uint8_t magic[lfs_max(16, READ_SIZE)]; + cfg->read(cfg, 0, 0, magic, lfs_max(16, READ_SIZE)) => 0; + assert(memcmp(&magic[8], "littlefs", 8) == 0); + cfg->read(cfg, 1, 0, magic, lfs_max(16, READ_SIZE)) => 0; + assert(memcmp(&magic[8], "littlefs", 8) == 0); +''' + # mount/unmount from interpretting a previous superblock block_count [cases.test_superblocks_mount_unknown_block_count] code = ''' @@ -28,7 +46,6 @@ code = ''' lfs_unmount(&lfs) => 0; ''' - # reentrant format [cases.test_superblocks_reentrant_format] reentrant = true @@ -135,6 +152,39 @@ code = ''' lfs_unmount(&lfs) => 0; ''' +# make sure the magic string "littlefs" is always at offset=8 +[cases.test_superblocks_magic_expand] +defines.BLOCK_CYCLES = [32, 33, 1] +defines.N = [10, 100, 1000] +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + lfs_mount(&lfs, cfg) => 0; + for (int i = 0; i < N; i++) { + lfs_file_t file; + lfs_file_open(&lfs, &file, "dummy", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0; + lfs_file_close(&lfs, &file) => 0; + struct lfs_info info; + lfs_stat(&lfs, "dummy", &info) => 0; + assert(strcmp(info.name, "dummy") == 0); + assert(info.type == LFS_TYPE_REG); + lfs_remove(&lfs, "dummy") => 0; + } + lfs_unmount(&lfs) => 0; + + // check our magic string + // + // note if we lose power we may not have the magic string in both blocks! + // but we don't lose power in this test so we can assert the magic string + // is present in both + uint8_t magic[lfs_max(16, READ_SIZE)]; + cfg->read(cfg, 0, 0, magic, lfs_max(16, READ_SIZE)) => 0; + assert(memcmp(&magic[8], "littlefs", 8) == 0); + cfg->read(cfg, 1, 0, magic, lfs_max(16, READ_SIZE)) => 0; + assert(memcmp(&magic[8], "littlefs", 8) == 0); +''' + # expanding superblock with power cycle [cases.test_superblocks_expand_power_cycle] defines.BLOCK_CYCLES = [32, 33, 1] @@ -221,6 +271,7 @@ code = ''' lfs_unmount(&lfs) => 0; ''' + # mount with unknown block_count [cases.test_superblocks_unknown_blocks] code = ''' From 25ee90fdf1829108babbe34598e1cc5594ae1afe Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 19 Mar 2024 00:20:32 -0500 Subject: [PATCH 2/6] Clarified what is accessible at specific superblock offsets in SPEC.md It used to be the case that the entire superblock entry could be found at specific offsets, but this was only possible while the superblock entry was immutable. Now that the superblock entry is very mutable (block-count changes, lfs2.0 -> lfs2.1 version bumps, etc), the correct superblock entry may end up later in the metadata log. At the very least, the "littlefs" magic string is still immutable and at the specific offset offset=8. This is arguably the most useful fixed-offset item. --- SPEC.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/SPEC.md b/SPEC.md index 2370ea6d..6682c749 100644 --- a/SPEC.md +++ b/SPEC.md @@ -441,9 +441,10 @@ Superblock fields: 7. **Attr max (32-bits)** - Maximum size of file attributes in bytes. -The superblock must always be the first entry (id 0) in a metadata pair as well -as be the first entry written to the block. This means that the superblock -entry can be read from a device using offsets alone. +The superblock must always be the first entry (id 0) in the metadata pair, and +the name tag must always be the first tag in the metadata pair. This makes it +so that the magic string "littlefs" will always reside at offset=8 in a valid +littlefs superblock. --- #### `0x2xx` LFS_TYPE_STRUCT From 11b036cc6c870a30734b3a1ff51f236a022331bd Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 19 Mar 2024 00:31:49 -0500 Subject: [PATCH 3/6] Prevented unnecessary superblock rewrites if old version in superblock chain Because multiple, out-of-date superblocks can exist in our superblock chain, we need to be careful to make sure newer superblock entries override older superblock entries. If we see an older on-disk minor version in the superblock chain, we were correctly overriding the on-disk minor version, but we were also leaving the "needs superblock" bit set in our consistency state. This isn't a hard-error, but would lead to a superblock rewrite every mount. The rewrite would make no progress, as the out-of-date version is effectively immutable at this point, and just waste prog cycles. This should fix that by clearing the "needs superblock" bit if we see a newer on-disk minor version. --- lfs.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lfs.c b/lfs.c index 4e4dd9b4..0e62b5fb 100644 --- a/lfs.c +++ b/lfs.c @@ -4469,6 +4469,7 @@ static int lfs_mount_(lfs_t *lfs, const struct lfs_config *cfg) { // found older minor version? set an in-device only bit in the // gstate so we know we need to rewrite the superblock before // the first write + bool needssuperblock = false; if (minor_version < lfs_fs_disk_version_minor(lfs)) { LFS_DEBUG("Found older minor version " "v%"PRIu16".%"PRIu16" < v%"PRIu16".%"PRIu16, @@ -4476,10 +4477,11 @@ static int lfs_mount_(lfs_t *lfs, const struct lfs_config *cfg) { minor_version, lfs_fs_disk_version_major(lfs), lfs_fs_disk_version_minor(lfs)); - // note this bit is reserved on disk, so fetching more gstate - // will not interfere here - lfs_fs_prepsuperblock(lfs, true); + needssuperblock = true; } + // note this bit is reserved on disk, so fetching more gstate + // will not interfere here + lfs_fs_prepsuperblock(lfs, needssuperblock); // check superblock configuration if (superblock.name_max) { From 8336ecd203dfda1d5327fc6324165784777b4264 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 17 Apr 2024 13:09:04 -0500 Subject: [PATCH 4/6] Added a link to pico-littlefs-usb (FAT12 emulation) Implemented by oyama, pico-littlefs-usb provides an easy interface to littlefs by emulating a FAT12 filesystem over USB. There are some tradeoffs to this, but being able to mount a littlefs device without installing additional drivers is very nice. Maybe in the future devices could provide both a FAT and raw endpoint for easy/advanced filesystem access. --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 34412935..3f0c57a4 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,10 @@ License Identifiers that are here available: http://spdx.org/licenses/ - [mklfs] - A command line tool built by the [Lua RTOS] guys for making littlefs images from a host PC. Supports Windows, Mac OS, and Linux. +- [pico-littlefs-usb] - An interface for littlefs that emulates a FAT12 + filesystem over USB. Allows mounting littlefs on a host PC without additional + drivers. + - [Mbed OS] - The easiest way to get started with littlefs is to jump into Mbed which already has block device drivers for most forms of embedded storage. littlefs is available in Mbed OS as the [LittleFileSystem] class. @@ -278,3 +282,4 @@ License Identifiers that are here available: http://spdx.org/licenses/ [littlefs2-rust]: https://crates.io/crates/littlefs2 [chamelon]: https://github.com/yomimono/chamelon [nim-littlefs]: https://github.com/Graveflo/nim-littlefs +[pico-littlefs-usb]: https://github.com/oyama/pico-littlefs-usb From 78082336e7259541e306292e99c03246c388ede3 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 17 Apr 2024 13:39:11 -0500 Subject: [PATCH 5/6] Added a link to mklittlefs Implemented by earlephilhower, mklittlefs is a command line interface that seems to be used by the ESP8266 and RP2040 ecosystems. It deserves a mention. Also tweaked mklfs's description a bit. --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3f0c57a4..8b6585c0 100644 --- a/README.md +++ b/README.md @@ -234,8 +234,11 @@ License Identifiers that are here available: http://spdx.org/licenses/ - [littlefs-disk-img-viewer] - A memory-efficient web application for viewing littlefs disk images in your web browser. -- [mklfs] - A command line tool built by the [Lua RTOS] guys for making - littlefs images from a host PC. Supports Windows, Mac OS, and Linux. +- [mklfs] - A command line tool for creating littlefs images. Used in the Lua + RTOS ecosystem. + +- [mklittlefs] - A command line tool for creating littlefs images. Used in the + ESP8266 and RP2040 ecosystem. - [pico-littlefs-usb] - An interface for littlefs that emulates a FAT12 filesystem over USB. Allows mounting littlefs on a host PC without additional @@ -272,7 +275,7 @@ License Identifiers that are here available: http://spdx.org/licenses/ [littlefs-js]: https://github.com/geky/littlefs-js [littlefs-js-demo]:http://littlefs.geky.net/demo.html [mklfs]: https://github.com/whitecatboard/Lua-RTOS-ESP32/tree/master/components/mklfs/src -[Lua RTOS]: https://github.com/whitecatboard/Lua-RTOS-ESP32 +[mklittlefs]: https://github.com/earlephilhower/mklittlefs [Mbed OS]: https://github.com/armmbed/mbed-os [LittleFileSystem]: https://os.mbed.com/docs/mbed-os/latest/apis/littlefilesystem.html [SPIFFS]: https://github.com/pellepl/spiffs From 0bbb8bc88b67a6d144a11a18d3a4e6863d056cb6 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 17 Apr 2024 13:46:33 -0500 Subject: [PATCH 6/6] Reorganized external project links a bit These were grouped up a bit better at one point, but that sort of drifted as new project were added: 1. Official repos (mainly littlefs-fuse) 2. Non-C reimplementations/wrappers 3. Utilities 4. Non-littlefs related projects Eventually, maybe when these move out of the README.md, these categories should probably be actually codified as headers or something. --- README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 8b6585c0..5a75df3b 100644 --- a/README.md +++ b/README.md @@ -231,6 +231,13 @@ License Identifiers that are here available: http://spdx.org/licenses/ to use littlefs in a Rust-friendly API, reaping the benefits of Rust's memory safety and other guarantees. +- [nim-littlefs] - A Nim wrapper and API for littlefs. Includes a fuse + implementation based on [littlefs-fuse] + +- [chamelon] - A pure-OCaml implementation of (most of) littlefs, designed for + use with the MirageOS library operating system project. It is interoperable + with the reference implementation, with some caveats. + - [littlefs-disk-img-viewer] - A memory-efficient web application for viewing littlefs disk images in your web browser. @@ -261,28 +268,21 @@ License Identifiers that are here available: http://spdx.org/licenses/ for microcontroller-scale devices. Due to limitations of FAT it can't provide power-loss resilience, but it does allow easy interop with PCs. -- [chamelon] - A pure-OCaml implementation of (most of) littlefs, designed for - use with the MirageOS library operating system project. It is interoperable - with the reference implementation, with some caveats. - -- [nim-littlefs] - A Nim wrapper and API for littlefs. Includes a fuse - implementation based on [littlefs-fuse] - [BSD-3-Clause]: https://spdx.org/licenses/BSD-3-Clause.html -[littlefs-disk-img-viewer]: https://github.com/tniessen/littlefs-disk-img-viewer [littlefs-fuse]: https://github.com/geky/littlefs-fuse [FUSE]: https://github.com/libfuse/libfuse [littlefs-js]: https://github.com/geky/littlefs-js [littlefs-js-demo]:http://littlefs.geky.net/demo.html +[littlefs-python]: https://pypi.org/project/littlefs-python/ +[littlefs2-rust]: https://crates.io/crates/littlefs2 +[nim-littlefs]: https://github.com/Graveflo/nim-littlefs +[chamelon]: https://github.com/yomimono/chamelon +[littlefs-disk-img-viewer]: https://github.com/tniessen/littlefs-disk-img-viewer [mklfs]: https://github.com/whitecatboard/Lua-RTOS-ESP32/tree/master/components/mklfs/src [mklittlefs]: https://github.com/earlephilhower/mklittlefs +[pico-littlefs-usb]: https://github.com/oyama/pico-littlefs-usb [Mbed OS]: https://github.com/armmbed/mbed-os [LittleFileSystem]: https://os.mbed.com/docs/mbed-os/latest/apis/littlefilesystem.html [SPIFFS]: https://github.com/pellepl/spiffs [Dhara]: https://github.com/dlbeer/dhara [ChaN's FatFs]: http://elm-chan.org/fsw/ff/00index_e.html -[littlefs-python]: https://pypi.org/project/littlefs-python/ -[littlefs2-rust]: https://crates.io/crates/littlefs2 -[chamelon]: https://github.com/yomimono/chamelon -[nim-littlefs]: https://github.com/Graveflo/nim-littlefs -[pico-littlefs-usb]: https://github.com/oyama/pico-littlefs-usb