Implemented ckcksums
Since we already need all the machinery to track ck info for ckparity, I
figured we might as well implement a full ckcksums option as well.
Ckcksums closes the checksum-read-hole by reading enough data to check a
relevant checksum on ever read, even if this ends up being significantly
more data than the initial request. This should always detect detectable
bit-errors, even if they occur between consecutive reads.
If this sounds naive, that's because it is. Performance will be awful.
To be clear, ckcksums should probably never be used in production. I
can't think of a use case that isn't better handled by either ECC in the
block device or the future-planned ckredund feature. Just look at the
runtime complexities:
small-reads rbyd-lookup rbyd-compaction
ckcksums: O(b^2) O(b log b) O(b^2 log b)
ckredund*: O(log_b(n) + xb) O(log b) O(b log b)
eccbd*: O(b) O(log b) O(b log b)
* theoretical
We've already seen that O(b^2) compactions turns a performance problem
into a tractability problem, so I think O(b^2 log b) compactions will be
a bit too much for most applications.
We can already seen this in our test_ck_ckcksums_* tests (which do pass
by the way!). Compare to test_ck_ckprogs_*, which is basically the same
set of tests:
test_ck_ckprogs_*: 6.08s
test_ck_ckcksums_*: 64.88s
Or consider test_rbyd with/without ckcksums:
test_rbyd: 12.21s
test_rbyd+ckcksums: 389.94s
Still, ckcksums is an interesting proof-of-concept, and does manage to
close the checksum-read-hole.
---
Like ckprogs/ckfetches/ckparity/etc, ckcksums is an opt-in feature,
requiring both 1. defining LFS_CKCKSUMS and 2. passing LFS_M_CKCKSUMS at
mount time.
Like ckparity, ckcksums requires a significant code and stack increase
to track ck info in lfsr_data_t:
code stack
before: 36416 2616
yes-ckcksums: 38872 (+6.7%) 3176 (+21.4%)
no-ckcksums: 36416 (+0.0%) 2616 (+0.0%)
It's interesting to note how this compares to all of the current
ck-modes, though each has their own set of tradeoffs:
code stack
default: 36416 2616
ckprogs: 36468 (+0.1%) 2616 (+0.0%)
ckfetches: 36666 (+0.7%) 2648 (+1.2%)
ckparity: 37996 (+4.3%) 3040 (+16.2%)
ckcksums: 38872 (+6.7%) 3176 (+21.4%)
---
Note that even though ckcksums is opt-in, it may still be worth removing
from the codebase in the future, for a couple reasons:
- Every feature, even if unused, adds developer/maintenance burden.
- Ck info is particularly messy with how it interacts with all
lfsr_data_t APIs. Though getting rid of ck info would also require
getting rid of ckparity.
- It's possible for a user to see ckcksums in the codebase,
misunderstand its tradeoffs, enable it, and get the impression that
littlefs itself is just unusably slow.
This commit is contained in:
@@ -162,6 +162,9 @@ enum lfs_type {
|
||||
#ifdef LFS_CKPARITY
|
||||
#define LFS_F_CKPARITY 0x00400000 // Check tag parity bits on reads
|
||||
#endif
|
||||
#ifdef LFS_CKCKSUMS
|
||||
#define LFS_F_CKCKSUMS 0x00800000 // Check checksums on reads (expensive!)
|
||||
#endif
|
||||
|
||||
#define LFS_F_MTREEONLY 0x00000800 // Only traverse the mtree
|
||||
#define LFS_F_COMPACT 0x00008000 // Compact metadata logs
|
||||
@@ -182,6 +185,9 @@ enum lfs_type {
|
||||
#ifdef LFS_CKPARITY
|
||||
#define LFS_M_CKPARITY 0x00400000 // Check tag parity bits on reads
|
||||
#endif
|
||||
#ifdef LFS_CKCKSUMS
|
||||
#define LFS_M_CKCKSUMS 0x00800000 // Check checksums on reads (expensive!)
|
||||
#endif
|
||||
|
||||
#define LFS_M_MTREEONLY 0x00000800 // Only traverse the mtree
|
||||
#define LFS_M_MKCONSISTENT \
|
||||
@@ -204,6 +210,9 @@ enum lfs_type {
|
||||
#ifdef LFS_CKPARITY
|
||||
#define LFS_I_CKPARITY 0x00400000 // Filesystem mounted with LFS_M_CKPARITY
|
||||
#endif
|
||||
#ifdef LFS_CKCKSUMS
|
||||
#define LFS_I_CKCKSUMS 0x00800000 // Filesystem mounted with LFS_M_CKCKSUMS
|
||||
#endif
|
||||
|
||||
#define LFS_I_INCONSISTENT \
|
||||
0x01000000 // Filesystem needs mkconsistent to write
|
||||
@@ -559,7 +568,7 @@ typedef struct lfsr_omdir {
|
||||
// lfs_block_t tail[2];
|
||||
//} lfs_mdir_t;
|
||||
|
||||
#ifdef LFS_CKPARITY
|
||||
#if defined(LFS_CKPARITY) || defined(LFS_CKCKSUMS)
|
||||
// context for validating data
|
||||
typedef struct lfsr_ck {
|
||||
// sign(cksize)=0 => cksum check
|
||||
@@ -583,7 +592,7 @@ typedef struct lfsr_data {
|
||||
struct {
|
||||
lfs_block_t block;
|
||||
lfs_size_t off;
|
||||
#ifdef LFS_CKPARITY
|
||||
#if defined(LFS_CKPARITY) || defined(LFS_CKCKSUMS)
|
||||
lfsr_ck_t ck;
|
||||
#endif
|
||||
} disk;
|
||||
@@ -758,11 +767,11 @@ typedef struct lfsr_grm {
|
||||
lfsr_smid_t mids[2];
|
||||
} lfsr_grm_t;
|
||||
|
||||
#ifdef LFS_CKPARITY
|
||||
#if defined(LFS_CKPARITY) || defined(LFS_CKCKSUMS)
|
||||
typedef struct lfsr_tailck {
|
||||
lfs_block_t ckblock;
|
||||
// sign(ckoff) => tail parity
|
||||
lfs_size_t ckoff;
|
||||
uint32_t cksum;
|
||||
} lfsr_tailck_t;
|
||||
#endif
|
||||
|
||||
@@ -798,7 +807,7 @@ typedef struct lfs {
|
||||
uint8_t *buffer;
|
||||
} pcache;
|
||||
|
||||
#ifdef LFS_CKPARITY
|
||||
#if defined(LFS_CKPARITY) || defined(LFS_CKCKSUMS)
|
||||
lfsr_tailck_t tailck;
|
||||
#endif
|
||||
|
||||
|
||||
Reference in New Issue
Block a user