Github/littlefs
1
Fork 0
Code Actions Activity

kv: Implemented a simple key-value API

Browse Source 
This adds a couple functions that treat files as simple key-value pairs:

- lfs3_get    - Read a file
- lfs3_size   - Get the size of a file
- lfs3_set    - Write a file
- lfs3_remove - Remove a file (this one already exists!)

The idea is the only real difference between a filesystem and key-value
store in the microcontroller space is the API, and the key-value API
_is_ much easier to use.

It also opens the door to making the file API opt-out in the future to
trade code cost for feature set. littlefs will probably never be
competitive with other microcontroller-scale key-value stores, but it
may be interesting for systems already using littlefs for other storage.

And don't worry, these are still files, so they can always be opened
with the full file API when more advanced operations are needed.

These APIs also matches the custom attribute APIs, which makes sense
because they're both key-values. Any mismatch should be considered an
API bug, because the best user interface is a consistent one.

This new API is tested in tests/test_kv.toml.

---

At the moment the implementation is naive, just sitting on top of the
file API. This works remarkably well thanks to littlefs's cache
bypassing logic, but does have some downsides:

- lfs3_set always writes two commits: one for the stickynote and one for
  the file sync.

  Unfortunately this is a fundamental limitation of littlefs's file API.
  One nice benefit of lfs3_set is in theory we can bypass this
  limitation, but not if we just sit on top of the file API.

- There may be code savings from more tightly integrating the key-value
  code.

This also highlighted an awkward corner case with per-file cache
configuration in which the buffer needs to be non-null even if zero. Not
the end of the world, but just a bit awkward. Maybe this deserves
revisiting in the config API rework?

---

Code changes were relatively minimal given that this is a whole new API,
unfortunately the stack took quite a hit:

           code          stack          ctx
  before: 37352           2280          636
  after:  37644 (+0.8%)   2448 (+7.4%)  636 (+0.0%)

The stack surprised me, but in hindsight it makes sense. In sitting on
top of the reset of the codebase, the key-value API adds very little
code, but every stack allocation in these functions add to the stack
hot-path.

This isn't the end of the world, and it's actually probably a good thing
to have an lfs3_file_t allocated in the stack hot-path. lfs3_file_t's
size has been a bit difficult to track thanks to struct lfs3_info
dominating ctx measurements...
This commit is contained in:
Christopher Haster
2025-06-09 13:31:46 -05:00
parent 40a8c02604
commit a75537faff
4 changed files with 1574 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs3.c
+130 -29
View File
@@ -11316,7 +11316,7 @@ static inline void lfs3_file_discardbshrub(lfs3_file_t *file) {
static inline lfs3_size_t lfs3_file_cachesize(lfs3_t *lfs3,
const lfs3_file_t *file) {
return (file->cfg->cache_size)
return (file->cfg->cache_buffer || file->cfg->cache_size)
? file->cfg->cache_size
: lfs3->cfg->file_cache_size;
}
@@ -11337,7 +11337,16 @@ static inline lfs3_off_t lfs3_file_size_(const lfs3_file_t *file) {
// file operations
static int lfs3_file_fetch(lfs3_t *lfs3, lfs3_file_t *file, bool trunc) {
static void lfs3_file_init(lfs3_file_t *file, uint32_t flags,
const struct lfs3_file_config *cfg) {
file->cfg = cfg;
file->b.o.flags = lfs3_o_typeflags(LFS3_TYPE_REG) | flags;
file->pos = 0;
// default to no cache
file->cache.size = 0;
}
static int lfs3_file_fetch(lfs3_t *lfs3, lfs3_file_t *file, uint32_t flags) {
// default data state
lfs3_file_discardbshrub(file);
// discard the current cache
@@ -11346,7 +11355,11 @@ static int lfs3_file_fetch(lfs3_t *lfs3, lfs3_file_t *file, bool trunc) {
lfs3_file_discardleaf(file);
// don't bother reading disk if we're not created or truncating
if (!lfs3_o_isuncreat(file->b.o.flags) && !trunc) {
if (lfs3_o_isuncreat(flags) || lfs3_o_istrunc(flags)) {
// but do mark as unsync
file->b.o.flags |= LFS3_o_UNSYNC;
} else {
// lookup the file struct, if there is one
lfs3_tag_t tag;
lfs3_data_t data;
@@ -11399,7 +11412,7 @@ static int lfs3_file_fetch(lfs3_t *lfs3, lfs3_file_t *file, bool trunc) {
}
// don't bother reading disk if we're not created yet
if (lfs3_o_isuncreat(file->b.o.flags)) {
if (lfs3_o_isuncreat(flags)) {
if (file->cfg->attrs[i].size) {
*file->cfg->attrs[i].size = LFS3_ERR_NOATTR;
}
@@ -11479,9 +11492,6 @@ int lfs3_file_opencfg(lfs3_t *lfs3, lfs3_file_t *file,
|| !lfs3_o_isexcl(cfg->attrs[i].flags));
}
// mounted with LFS3_M_FLUSH/SYNC? implies LFS3_O_FLUSH/SYNC
flags |= lfs3->flags & (LFS3_M_FLUSH | LFS3_M_SYNC);
if (!lfs3_o_isrdonly(flags)) {
// prepare our filesystem for writing
#ifndef LFS3_RDONLY
@@ -11493,12 +11503,10 @@ int lfs3_file_opencfg(lfs3_t *lfs3, lfs3_file_t *file,
}
// setup file state
file->cfg = cfg;
file->b.o.flags = flags
| lfs3_o_typeflags(LFS3_TYPE_REG)
// default to unsynced for uncreated/truncated files
| LFS3_o_UNSYNC;
file->pos = 0;
lfs3_file_init(file,
// mounted with LFS3_M_FLUSH/SYNC? implies LFS3_O_FLUSH/SYNC
flags | (lfs3->flags & (LFS3_M_FLUSH | LFS3_M_SYNC)),
cfg);
// lookup our parent
lfs3_tag_t tag;
@@ -11508,14 +11516,14 @@ int lfs3_file_opencfg(lfs3_t *lfs3, lfs3_file_t *file,
if (err && !(err == LFS3_ERR_NOENT && lfs3_path_islast(path))) {
return err;
}
bool exists = err != LFS3_ERR_NOENT;
bool exists = (err != LFS3_ERR_NOENT);
// creating a new entry?
if (!exists || tag == LFS3_TAG_ORPHAN) {
if (!lfs3_o_iscreat(flags)) {
if (!lfs3_o_iscreat(file->b.o.flags)) {
return LFS3_ERR_NOENT;
}
LFS3_ASSERT(!lfs3_o_isrdonly(flags));
LFS3_ASSERT(!lfs3_o_isrdonly(file->b.o.flags));
#ifndef LFS3_RDONLY
// we're a file, don't allow trailing slashes
@@ -11550,10 +11558,13 @@ int lfs3_file_opencfg(lfs3_t *lfs3, lfs3_file_t *file,
}
}
}
// mark as uncreated
file->b.o.flags |= LFS3_o_UNCREAT;
#endif
} else {
// wanted to create a new entry?
if (lfs3_o_isexcl(flags)) {
if (lfs3_o_isexcl(file->b.o.flags)) {
return LFS3_ERR_EXIST;
}
@@ -11564,15 +11575,14 @@ int lfs3_file_opencfg(lfs3_t *lfs3, lfs3_file_t *file,
if (tag == LFS3_TAG_UNKNOWN) {
return LFS3_ERR_NOTSUP;
}
}
// if stickynote, mark as uncreated, we need to convert to reg file
// on first sync
if (!exists
|| tag == LFS3_TAG_STICKYNOTE
|| tag == LFS3_TAG_ORPHAN) {
#ifndef LFS3_RDONLY
// if stickynote, mark as uncreated
if (tag == LFS3_TAG_STICKYNOTE) {
file->b.o.flags |= LFS3_o_UNCREAT;
}
#endif
}
// allocate cache if necessary
if (file->cfg->cache_buffer) {
@@ -11585,14 +11595,15 @@ int lfs3_file_opencfg(lfs3_t *lfs3, lfs3_file_t *file,
}
// fetch the file struct and custom attrs
err = lfs3_file_fetch(lfs3, file, lfs3_o_istrunc(flags));
err = lfs3_file_fetch(lfs3, file, file->b.o.flags);
if (err) {
goto failed;
}
// check metadata/data for errors?
if (lfs3_t_isckmeta(flags) || lfs3_t_isckdata(flags)) {
err = lfs3_file_ck(lfs3, file, flags);
if (lfs3_t_isckmeta(file->b.o.flags)
|| lfs3_t_isckdata(file->b.o.flags)) {
err = lfs3_file_ck(lfs3, file, file->b.o.flags);
if (err) {
goto failed;
}
@@ -11609,11 +11620,12 @@ failed:;
}
// default file config
static const struct lfs3_file_config lfs3_file_defaults = {0};
static const struct lfs3_file_config lfs3_file_defaultcfg = {0};
int lfs3_file_open(lfs3_t *lfs3, lfs3_file_t *file,
const char *path, uint32_t flags) {
return lfs3_file_opencfg(lfs3, file, path, flags, &lfs3_file_defaults);
return lfs3_file_opencfg(lfs3, file, path, flags,
&lfs3_file_defaultcfg);
}
// clean up resources
@@ -13456,7 +13468,9 @@ int lfs3_file_resync(lfs3_t *lfs3, lfs3_file_t *file) {
// do nothing if already in-sync
if (lfs3_o_isunsync(file->b.o.flags)) {
// refetch the file struct from disk
err = lfs3_file_fetch(lfs3, file, false);
err = lfs3_file_fetch(lfs3, file,
// don't truncate again!
file->b.o.flags & ~LFS3_O_TRUNC);
if (err) {
goto failed;
}
@@ -13840,6 +13854,93 @@ int lfs3_file_ckdata(lfs3_t *lfs3, lfs3_file_t *file) {
/// Simple key-value API ///
// a simple key-value API is easier to use if your file fits in RAM, and
// if that's all you need you can potentially compile-out the more
// advanced file operations
// kv file config, we need to explicitly disable the file cache
static const struct lfs3_file_config lfs3_file_kvconfig = {
// TODO is this the best way to do this?
.cache_buffer = (uint8_t*)true,
.cache_size = 0,
};
lfs3_ssize_t lfs3_get(lfs3_t *lfs3, const char *path,
void *buffer, lfs3_size_t size) {
// we just use the file API here, but with no cache so all reads
// bypass the cache
lfs3_file_t file;
int err = lfs3_file_opencfg(lfs3, &file, path, LFS3_O_RDONLY,
&lfs3_file_kvconfig);
if (err) {
return err;
}
lfs3_ssize_t size_ = lfs3_file_read(lfs3, &file, buffer, size);
// unconditionally close
err = lfs3_file_close(lfs3, &file);
// we didn't allocate anything, so this can't fail
LFS3_ASSERT(!err);
return size_;
}
lfs3_ssize_t lfs3_size(lfs3_t *lfs3, const char *path) {
// we just use the file API here, but with no cache so all reads
// bypass the cache
lfs3_file_t file;
int err = lfs3_file_opencfg(lfs3, &file, path, LFS3_O_RDONLY,
&lfs3_file_kvconfig);
if (err) {
return err;
}
lfs3_ssize_t size_ = lfs3_file_size_(&file);
// unconditionally close
err = lfs3_file_close(lfs3, &file);
// we didn't allocate anything, so this can't fail
LFS3_ASSERT(!err);
return size_;
}
#ifndef LFS3_RDONLY
int lfs3_set(lfs3_t *lfs3, const char *path,
const void *buffer, lfs3_size_t size) {
// we just use the file API here, but with no cache so all writes
// bypass the cache
lfs3_file_t file;
int err = lfs3_file_opencfg(lfs3, &file, path,
LFS3_O_WRONLY | LFS3_O_CREAT | LFS3_O_TRUNC,
&lfs3_file_kvconfig);
if (err) {
return err;
}
lfs3_ssize_t size_ = lfs3_file_write(lfs3, &file, buffer, size);
if (size_ < 0) {
err = size_;
}
// unconditionally close
int err_ = lfs3_file_close(lfs3, &file);
if (err_) {
// we didn't allocate anything, and write failing would set the
// desync flag, so only one of write/close can fail
LFS3_ASSERT(!err);
err = err_;
}
return err;
}
#endif
/// High-level filesystem operations ///
lfs3.h
+22 -1
View File
@@ -619,7 +619,7 @@ struct lfs3_file_config {
// Size of the file cache in bytes. In addition to filesystem-wide
// read/prog caches, each file gets its own cache to reduce disk
// accesses. Defaults to file_cache_size.
// accesses. Defaults to file_cache_size if cache_buffer is NULL.
lfs3_size_t cache_size;
// Optional list of custom attributes attached to the file. If readable,
@@ -916,6 +916,27 @@ int lfs3_unmount(lfs3_t *lfs3);
/// General operations ///
// Get the value of a file
//
// Returns the number of bytes read, or a negative error code on failure.
// Note this may be less than the on-disk file size if the buffer is not
// large enough.
lfs3_ssize_t lfs3_get(lfs3_t *lfs3, const char *path,
void *buffer, lfs3_size_t size);
// Get a file's size
//
// Returns the size of the file, or a negative error code on failure.
lfs3_ssize_t lfs3_size(lfs3_t *lfs3, const char *path);
// Set the value of a file
//
// Returns a negative error code on failure.
#ifndef LFS3_RDONLY
int lfs3_set(lfs3_t *lfs3, const char *path,
const void *buffer, lfs3_size_t size);
#endif
// Removes a file or directory
//
// If removing a directory, the directory must be empty.
tests/test_attrs.toml
-20
View File
@@ -223,26 +223,6 @@ code = '''
lfs3_file_close(&lfs3, &file) => 0;
}
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfs3_unmount(&lfs3) => 0;
lfs3_mount(&lfs3, LFS3_M_RDWR, CFG) => 0;
}
// make sure setattr didn't quietly create attrs
// try getting the attr sizes
lfs3_sizeattr(&lfs3, path, 'a') => LFS3_ERR_NOATTR;
lfs3_sizeattr(&lfs3, path, 'b') => LFS3_ERR_NOATTR;
lfs3_sizeattr(&lfs3, path, 'c') => LFS3_ERR_NOATTR;
// try reading attrs
uint8_t rbuf[256];
lfs3_getattr(&lfs3, path, 'a', rbuf, sizeof(rbuf)) => LFS3_ERR_NOATTR;
lfs3_getattr(&lfs3, path, 'b', rbuf, sizeof(rbuf)) => LFS3_ERR_NOATTR;
lfs3_getattr(&lfs3, path, 'c', rbuf, sizeof(rbuf)) => LFS3_ERR_NOATTR;
}
lfs3_unmount(&lfs3) => 0;
'''
tests/test_kv.toml
+1421
View File
File diff suppressed because it is too large Load Diff