Github/littlefs
1
Fork 0
Code Actions Activity

Added bigger_than_expected tests, allow fragments/blocks > weight

Browse Source 
This may be useful for compression in the future, where compression +
noise can result in blocks _larger_ than the expected weight.

Thinking about how compression might be integrated into littlefs, it
would be nice if such a topology did _not_ trigger asserts. This would
allow littlefs images to interact with compressed files at least a
little bit (rename/remove could be very useful), even if the compression
algorithm isn't supported.

Supporting this requires only a single clamp in lfsr_file_lookupleaf,
but it's a little bit more costly than you might expect:

           code          stack          ctx
  before: 35692           2440          640
  after:  35740 (+0.1%)   2440 (+0.0%)  640 (+0.0%)

This is due to internal API awkwardness:

1. LFSR_DATA_TRUNCATE is surprisingly costly
2. We need to create a local weight copy in case the caller's is NULL
This commit is contained in:
Christopher Haster
2025-04-21 20:21:22 -05:00
parent a0b3eccf15
commit 0a485da7c9
2 changed files with 231 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+10 -2
View File
@@ -11505,17 +11505,21 @@ static int lfsr_file_lookupleaf(lfs_t *lfs, const lfsr_file_t *file,
lfsr_bid_t *bid_, lfsr_rbyd_t *rbyd_, lfsr_srid_t *rid_,
lfsr_bid_t *weight_, lfsr_bptr_t *bptr_) {
lfsr_tag_t tag;
lfsr_bid_t weight;
lfsr_data_t data;
int err = lfsr_bshrub_lookupleaf(lfs, &file->b, bid,
bid_, rbyd_, rid_, &tag, weight_, &data);
bid_, rbyd_, rid_, &tag, &weight, &data);
if (err) {
return err;
}
LFS_ASSERT(tag == LFSR_TAG_DATA
|| tag == LFSR_TAG_BLOCK);
// decode bptrs
if (weight_) {
*weight_ = weight;
}
if (bptr_) {
// decode bptrs
if (tag == LFSR_TAG_DATA) {
bptr_->data = data;
} else {
@@ -11524,6 +11528,10 @@ static int lfsr_file_lookupleaf(lfs_t *lfs, const lfsr_file_t *file,
return err;
}
}
// limit bptrs to btree weights, this may be useful for
// compression in the future
bptr_->data = LFSR_DATA_TRUNCATE(bptr_->data, weight);
}
return 0;
}
tests/test_fwrite.toml
+221
View File
@@ -4859,3 +4859,224 @@ code = '''
'''
# test that we don't error on fragments > weight
#
# this may be useful in the future for compression
#
[cases.test_fwrite_bigger_than_expected_fragments]
defines.N = [0, 1, 2, 3, 4]
defines.SIZE = 'N*FRAGMENT_SIZE'
defines.CHUNK = [32, 8, 1]
defines.CRYSTAL_THRESH = -1
if = [
'CHUNK <= SIZE',
'FRAGMENT_SIZE > 1',
]
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, LFS_F_RDWR, CFG) => 0;
lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0;
// create a file
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "hello",
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf[SIZE];
uint32_t prng = 42;
for (lfs_size_t i = 0; i < SIZE; i++) {
wbuf[i] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_close(&lfs, &file) => 0;
// reduce the weight of each btree entry
//
// this should normally never happen, so we need to use the
// internal bshrub APIs to force this
lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0;
lfs_off_t pos = 0;
while (true) {
lfsr_tag_t tag;
lfsr_bid_t weight;
lfsr_data_t data;
int err = lfsr_bshrub_lookupnext(&lfs, &file.b, pos,
&pos, &tag, &weight, &data);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
}
printf("pos = %d, %d\n", pos, weight);
lfsr_bshrub_commit(&lfs, &file.b, pos, LFSR_RATTRS(
LFSR_RATTR_DATA(
LFSR_TAG_GROW | tag, -(weight/2),
&data))) => 0;
pos = pos - (weight/2) + 1;
}
file.b.o.flags |= LFS_o_UNSYNC;
lfsr_file_close(&lfs, &file) => 0;
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0;
}
// check our file with stat
struct lfs_info info;
lfsr_stat(&lfs, "hello", &info) => 0;
assert(strcmp(info.name, "hello") == 0);
assert(info.type == LFS_TYPE_REG);
assert(info.size == SIZE/2);
// and with dir read
lfsr_dir_t dir;
lfsr_dir_open(&lfs, &dir, "/") => 0;
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, ".") == 0);
assert(info.type == LFS_TYPE_DIR);
assert(info.size == 0);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, "..") == 0);
assert(info.type == LFS_TYPE_DIR);
assert(info.size == 0);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, "hello") == 0);
assert(info.type == LFS_TYPE_REG);
assert(info.size == SIZE/2);
lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT;
lfsr_dir_close(&lfs, &dir) => 0;
// try reading our file
lfsr_file_open(&lfs, &file, "hello", LFS_O_RDONLY) => 0;
// is size correct?
lfsr_file_size(&lfs, &file) => SIZE/2;
// try reading
uint8_t rbuf[2*SIZE];
memset(rbuf, 0xaa, 2*SIZE);
lfsr_file_read(&lfs, &file, rbuf, 2*SIZE) => SIZE/2;
for (lfs_size_t i = 0; i < SIZE/FRAGMENT_SIZE; i++) {
assert(memcmp(
&rbuf[i*FRAGMENT_SIZE/2],
&wbuf[i*FRAGMENT_SIZE],
FRAGMENT_SIZE/2) == 0);
}
lfsr_file_close(&lfs, &file) => 0;
}
lfsr_unmount(&lfs) => 0;
'''
# test that we don't error on blocks > weight
#
# this may be useful in the future for compression
#
[cases.test_fwrite_bigger_than_expected_blocks]
defines.N = [0, 1, 2, 3, 4]
defines.SIZE = 'N*BLOCK_SIZE'
defines.CHUNK = [32, 8, 1]
if = [
'CHUNK <= SIZE',
'BLOCK_SIZE > 1',
]
in = 'lfs.c'
code = '''
lfs_t lfs;
lfsr_format(&lfs, LFS_F_RDWR, CFG) => 0;
lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0;
// create a file
lfsr_file_t file;
lfsr_file_open(&lfs, &file, "hello",
LFS_O_WRONLY | LFS_O_CREAT | LFS_O_EXCL) => 0;
uint8_t wbuf[SIZE];
uint32_t prng = 42;
for (lfs_size_t i = 0; i < SIZE; i++) {
wbuf[i] = 'a' + (TEST_PRNG(&prng) % 26);
}
lfsr_file_write(&lfs, &file, wbuf, SIZE) => SIZE;
lfsr_file_close(&lfs, &file) => 0;
// reduce the weight of each btree entry
//
// this should normally never happen, so we need to use the
// internal bshrub APIs to force this
lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0;
lfs_off_t pos = 0;
while (true) {
lfsr_tag_t tag;
lfsr_bid_t weight;
lfsr_data_t data;
int err = lfsr_bshrub_lookupnext(&lfs, &file.b, pos,
&pos, &tag, &weight, &data);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
}
printf("pos = %d, %d\n", pos, weight);
lfsr_bshrub_commit(&lfs, &file.b, pos, LFSR_RATTRS(
LFSR_RATTR_DATA(
LFSR_TAG_GROW | tag, -(weight/2),
&data))) => 0;
pos = pos - (weight/2) + 1;
}
file.b.o.flags |= LFS_o_UNSYNC;
lfsr_file_close(&lfs, &file) => 0;
for (int remount = 0; remount < 2; remount++) {
// remount?
if (remount) {
lfsr_unmount(&lfs) => 0;
lfsr_mount(&lfs, LFS_M_RDWR, CFG) => 0;
}
// check our file with stat
struct lfs_info info;
lfsr_stat(&lfs, "hello", &info) => 0;
assert(strcmp(info.name, "hello") == 0);
assert(info.type == LFS_TYPE_REG);
assert(info.size == SIZE/2);
// and with dir read
lfsr_dir_t dir;
lfsr_dir_open(&lfs, &dir, "/") => 0;
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, ".") == 0);
assert(info.type == LFS_TYPE_DIR);
assert(info.size == 0);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, "..") == 0);
assert(info.type == LFS_TYPE_DIR);
assert(info.size == 0);
lfsr_dir_read(&lfs, &dir, &info) => 0;
assert(strcmp(info.name, "hello") == 0);
assert(info.type == LFS_TYPE_REG);
assert(info.size == SIZE/2);
lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT;
lfsr_dir_close(&lfs, &dir) => 0;
// try reading our file
lfsr_file_open(&lfs, &file, "hello", LFS_O_RDONLY) => 0;
// is size correct?
lfsr_file_size(&lfs, &file) => SIZE/2;
// try reading
uint8_t rbuf[2*SIZE];
memset(rbuf, 0xaa, 2*SIZE);
lfsr_file_read(&lfs, &file, rbuf, 2*SIZE) => SIZE/2;
for (lfs_size_t i = 0; i < SIZE/BLOCK_SIZE; i++) {
assert(memcmp(
&rbuf[i*BLOCK_SIZE/2],
&wbuf[i*BLOCK_SIZE],
BLOCK_SIZE/2) == 0);
}
lfsr_file_close(&lfs, &file) => 0;
}
lfsr_unmount(&lfs) => 0;
'''