Github/littlefs
1
Fork 0
Code Actions Activity

Made data read functions "consume" their data pointers

Browse Source 
Composable parsing functions always feel a bit weird to me in C. I don't
know if this is because of something C lacks, such as multiple return
values, or if composable parsers are just inherently awkward to describe
in procedural languages because of the different levels of state.

But I think the API here is pretty ok. The main idea is that data
parsers can be added as functions in the lfsr_data_* namespace that take
lfsr_data_t as a mutable reference, updating the lfsr_data_t's internal
state as data is parsed.

In practice you only need a couple of primitives, bytes, le32s, leb128s,
that touch the internals of lfsr_data_t, and the other parsers can be
built using these.

This leverages the pointer-like abstraction of lfsr_data_t, and avoids
needing to keep track of offsets. And thanks to lfsr_data_t being
relatively cheap to make copies, this API is relatively flexible.

Some other tweaks:

- Signed leb128 overflow detection is moved up into lfs_fromleb128.
  littlefs now assumes _all_ leb128s are 31-bits, which is useful for
  leveraging the sign bit internally.

  This also fixes the an issue in overflow detection in lfs_fromleb128
  which wouldn't catch overflows in the last byte of a >32-bit leb128.

- Most lfsr_data_t functions now take a pointer. This offered a small
  bit of code savings and feels more natural in C. Though most functions
  that accept lfsr_data_t still take a copy. Most of these functions
  would need to make a copy anyways now that the parsers are consuming,
  and these copies avoid concerns about shared state.

  At 3-words, lfsr_data_t is right at that boundary of questionable
  reasonableness for copying, but copying is a very useful feature of
  this struct.

This ends up with some decent code/stack savings:

            code          stack
  before:  22118           2048
  after:   21722 (-1.8%)   1992 (-2.7%)
This commit is contained in:
Christopher Haster
2023-08-09 02:32:55 -05:00
parent 77de73e39c
commit d8f988a8fc
6 changed files with 898 additions and 920 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+303 -330
View File
File diff suppressed because it is too large Load Diff
lfs_util.c
+9 -4
View File
@@ -16,7 +16,7 @@
// Convert to/from leb128 encoding
ssize_t lfs_toleb128(uint32_t word, void *buffer, size_t size) {
ssize_t lfs_toleb128(int32_t word, void *buffer, size_t size) {
uint8_t *data = buffer;
for (size_t i = 0; i < size; i++) {
@@ -33,14 +33,19 @@ ssize_t lfs_toleb128(uint32_t word, void *buffer, size_t size) {
return LFS_ERR_CORRUPT;
}
ssize_t lfs_fromleb128(uint32_t *word, const void *buffer, size_t size) {
ssize_t lfs_fromleb128(int32_t *word, const void *buffer, size_t size) {
const uint8_t *data = buffer;
uint32_t word_ = 0;
int32_t word_ = 0;
for (size_t i = 0; i < size; i++) {
uint8_t dat = data[i];
int32_t dat = data[i];
word_ |= (dat & 0x7f) << 7*i;
if (!(dat & 0x80)) {
// did we overflow?
if ((word_ >> 7*i) != dat) {
return LFS_ERR_CORRUPT;
}
*word = word_;
return i+1;
}
lfs_util.h
+3 -3
View File
@@ -330,11 +330,11 @@ static inline uint32_t lfs_fromle32_(const void *buffer) {
// Convert to/from leb128 encoding
// TODO should we really be using ssize_t here and not lfs_ssize_t?
ssize_t lfs_toleb128(uint32_t word, void *buffer, size_t size);
ssize_t lfs_toleb128(int32_t word, void *buffer, size_t size);
ssize_t lfs_fromleb128(uint32_t *word, const void *buffer, size_t size);
ssize_t lfs_fromleb128(int32_t *word, const void *buffer, size_t size);
static inline size_t lfs_sizeleb128(uint32_t word) {
static inline size_t lfs_sizeleb128(int32_t word) {
// this is the size of the leb128 after encoding
return (lfs_nlog2(word+1)+7-1) / 7;
}
tests/test_btree.toml
+32 -32
View File
@@ -531,7 +531,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(weight_ == W);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
}
lfsr_btree_lookupnext(&lfs, &btree, bid_+1,
@@ -667,7 +667,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(weight_ == sim_weights[i]);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, &sim[i], 1) == 0);
}
lfsr_btree_lookupnext(&lfs, &btree, bid_+1,
@@ -1080,7 +1080,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(weight_ == W);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, &uppers[i % 26], 1) == 0);
}
lfsr_btree_lookupnext(&lfs, &btree, bid_+1,
@@ -1227,7 +1227,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(weight_ == sim_weights[i]);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, &sim[i], 1) == 0);
}
lfsr_btree_lookupnext(&lfs, &btree, bid_+1,
@@ -1890,7 +1890,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(weight_ == W);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, &alphas[i % 26], 1) == 0);
}
@@ -1900,7 +1900,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(weight_ == W);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, "R", 1) == 0);
lfsr_btree_lookupnext(&lfs, &btree, bid_+1,
@@ -2060,7 +2060,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(weight_ == sim_weights[i]);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, &sim[i], 1) == 0);
}
lfsr_btree_lookupnext(&lfs, &btree, bid_+1,
@@ -2414,7 +2414,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(weight_ == sim_weights[i]);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, &sim[i], 1) == 0);
}
lfsr_btree_lookupnext(&lfs, &btree, bid_+1,
@@ -2707,7 +2707,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(weight_ == sim_weights[i]);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, &sim[i], 1) == 0);
}
lfsr_btree_lookupnext(&lfs, &btree, bid_+1,
@@ -2787,7 +2787,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(bid_ == 0);
assert(weight_ == 1);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, "0", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, 1*DID, "aab", 3,
@@ -2795,7 +2795,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(bid_ == 0);
assert(weight_ == 1);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, "0", 1) == 0);
'''
@@ -2839,7 +2839,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(bid_ == 0);
assert(weight_ == 1);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, "0", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, 0, "aab", 3,
@@ -2847,7 +2847,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(bid_ == 1);
assert(weight_ == 1);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, "1", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, 0, "aac", 3,
@@ -2855,7 +2855,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(bid_ == 1);
assert(weight_ == 1);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, "1", 1) == 0);
'''
@@ -2902,7 +2902,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(bid_ == 0);
assert(weight_ == 1);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, "0", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, 1*DID, "aab", 3,
@@ -2910,7 +2910,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(bid_ == 1);
assert(weight_ == 1);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, "1", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, 2*DID, "aac", 3,
@@ -2918,7 +2918,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(bid_ == 2);
assert(weight_ == 1);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, "2", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, 3*DID, "aad", 3,
@@ -2926,7 +2926,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(bid_ == 2);
assert(weight_ == 1);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, "2", 1) == 0);
'''
@@ -2973,7 +2973,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(bid_ == 0);
assert(weight_ == 1);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, "0", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, 1*DID, "aab", 3,
@@ -2981,7 +2981,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(bid_ == 1);
assert(weight_ == 1);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, "1", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, 2*DID, "aac", 3,
@@ -2989,7 +2989,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(bid_ == 2);
assert(weight_ == 1);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, "2", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, 3*DID, "aad", 3,
@@ -2997,7 +2997,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(bid_ == 2);
assert(weight_ == 1);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, "2", 1) == 0);
'''
@@ -3062,7 +3062,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(bid_ == i);
assert(weight_ == 1);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, &nums[i % 10], 1) == 0);
}
'''
@@ -3168,7 +3168,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(bid_ == i);
assert(weight_ == 1);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, &sim[i], 1) == 0);
}
@@ -3240,7 +3240,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(bid_ == i*W+W-1);
assert(weight_ == W);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, &nums[i % 10], 1) == 0);
}
'''
@@ -3385,7 +3385,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(bid_ == weighted_bid+sim_weights[i]-1);
assert(weight_ == sim_weights[i]);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, &sim[i], 1) == 0);
}
@@ -3463,7 +3463,7 @@ code = '''
lfsr_btree_namelookup(&lfs, &btree, 0, name, 3,
&split_bid, NULL, NULL, &split_data) => 0;
uint8_t split_buf[4];
lfsr_data_read(&lfs, split_data, 0, split_buf, 4) => 1;
lfsr_data_read(&lfs, &split_data, split_buf, 4) => 1;
if (split_bid > bid) {
int err = lfsr_btree_split(&lfs, &btree,
split_bid,
@@ -3562,7 +3562,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(bid_ == i);
assert(weight_ == 1);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, &sim[i], 1) == 0);
}
@@ -3657,7 +3657,7 @@ code = '''
&split_bid, NULL, &split_weight,
&split_data) => 0;
uint8_t split_buf[4];
lfsr_data_read(&lfs, split_data, 0, split_buf, 4) => 1;
lfsr_data_read(&lfs, &split_data, split_buf, 4) => 1;
if (split_bid > weighted_bid+sim_weights[bid]-1) {
int err = lfsr_btree_split(&lfs, &btree,
split_bid, LFSR_DATA_NAME(0, sim_names[bid+1], 3),
@@ -3783,7 +3783,7 @@ code = '''
assert(tag_ == LFSR_TAG_INLINED);
assert(bid_ == weighted_bid+sim_weights[i]-1);
assert(weight_ == sim_weights[i]);
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
lfsr_data_read(&lfs, &data_, buffer, 4) => 1;
assert(memcmp(buffer, &sim[i], 1) == 0);
}
@@ -3884,7 +3884,7 @@ code = '''
bid_,
tag_,
weight_,
lfsr_data_size(data_));
lfsr_data_size(&data_));
}
}
@@ -4032,7 +4032,7 @@ code = '''
bid_,
tag_,
weight_,
lfsr_data_size(data_));
lfsr_data_size(&data_));
}
}
tests/test_mtree.toml
+7 -7
View File
@@ -3520,7 +3520,7 @@ code = '''
mid_.bid,
mid_.rid,
tag_,
lfsr_data_size(data_));
lfsr_data_size(&data_));
assert(false);
}
}
@@ -3645,7 +3645,7 @@ code = '''
mid_.bid,
mid_.rid,
tag_,
lfsr_data_size(data_));
lfsr_data_size(&data_));
assert(false);
}
}
@@ -3781,7 +3781,7 @@ code = '''
mid_.bid,
mid_.rid,
tag_,
lfsr_data_size(data_));
lfsr_data_size(&data_));
assert(false);
}
}
@@ -3909,7 +3909,7 @@ code = '''
mid_.bid,
mid_.rid,
tag_,
lfsr_data_size(data_));
lfsr_data_size(&data_));
assert(false);
}
}
@@ -4046,7 +4046,7 @@ code = '''
mid_.bid,
mid_.rid,
tag_,
lfsr_data_size(data_));
lfsr_data_size(&data_));
assert(false);
}
}
@@ -4210,7 +4210,7 @@ code = '''
mid_.bid,
mid_.rid,
tag_,
lfsr_data_size(data_));
lfsr_data_size(&data_));
assert(false);
}
}
@@ -4310,7 +4310,7 @@ code = '''
mid_.bid,
mid_.rid,
tag_,
lfsr_data_size(data_));
lfsr_data_size(&data_));
assert(false);
}
}
tests/test_rbyd.toml
+544 -544
View File
File diff suppressed because it is too large Load Diff