Github/littlefs
1
Fork 0
Code Actions Activity

Redesigned the inlined topology of files, now using geoxylic btrees

Browse Source 
As a part of the general redesign of files, all files, not just small
files, can inline some data directly in the metadata log. Originally,
this was a single piece of inlined data or an inlined tree (shrub) that
effectively acted as an overlay over the block/btree data.

This is now changed so that when we have a block/btree, the root of the
btree is inlined. In effect making a full btree a sort of extended
shrub.

I'm currently calling this a "geoxylic btree", since that seems to be a
somewhat related botanical term. Geoxylic btrees have, at least on
paper, a number of benefits:

- There is a single lookup path instead of two, this simplifies code a
  bit and decreases lookup costs.

- One data structure instead of two also means lfsr_file_t requires
  less RAM, since all of the on-disk variants can go into one big union.
  Though I'm not sure this is very significant vs stack/buffer costs.

- The write path is much simpler and has less duplication (it was
  difficult to deduplicate the shrub/btree code because of how the
  shrub goes through the mdir).

  In this redesign, lfsr_btree_commit_ leaves root attrs uncommitted,
  allowing lfsr_bshrub_commit to finish the job via lfsr_mdir_commit.

- We don't need to maintain a shrub estimate, we just lazily evict trees
  during mdir compaction. This has a side-effect of allowing shrubs to
  temporarily grow larger than shrub_size before eviction.

  NOTE THIS (fundamentally?) DOESN'T WORK

- There is no awkwardly high overhead for small btrees. The btree root
  for two-block files should be able to comfortably fit in the shrub
  portion of the btree, for example.

- It may be possible to also make the mtree geoxylic, which should
  reduce storage overhead of small mtrees and make better use of the
  mroot.

All of this being said, things aren't working yet. Shrub eviction during
compaction runs into a problem with a single pcache -- how do we write
the new btrees without dropping the compaction pcache? We can't evict
btrees in a separate pass becauce their number is unbounded...
This commit is contained in:
Christopher Haster
2023-11-18 23:23:39 -06:00
parent 7243c0f371
commit c94b5f4767
5 changed files with 948 additions and 1191 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+898 -1140
View File
File diff suppressed because it is too large Load Diff
lfs.h
+22 -31
View File
@@ -293,6 +293,7 @@ struct lfs_config {
// TODO document
lfs_size_t inline_size;
lfs_size_t shrub_size;
lfs_size_t fragment_size;
lfs_size_t crystal_size;
};
@@ -512,31 +513,22 @@ typedef struct lfs_file {
const struct lfs_file_config *cfg;
} lfs_file_t;
typedef struct lfsr_shrub {
union {
// the sign bit indicates if data is a single inlined data, or an
// inlined tree, this works because inlined data is always on disk,
// so data.size always has sign=1
lfs_soff_t weight;
lfsr_data_t data;
lfsr_rbyd_t rbyd;
struct {
lfs_soff_t weight;
lfs_size_t trunk;
lfs_off_t estimate;
} shrub;
} u;
} lfsr_shrub_t;
// bsprouts must always be associated with an mdir
typedef struct lfsr_bsprout {
lfsr_data_t data;
// copy for staging
lfsr_data_t data_;
} lfsr_bsprout_t;
typedef struct lfsr_tree {
union {
// the sign bit indicates if this is a direct block pointer or
// indirect tree of block pointers/inlined datas
lfs_soff_t size;
lfsr_bptr_t bptr;
lfsr_btree_t btree;
} u;
} lfsr_tree_t;
// bshrubs must always be associated with an mdir
//
// btree.block == mdir.blocks[0] => bshrub
// btree.block != mdir.blocks[0] => btree
typedef struct lfsr_bshrub {
lfsr_btree_t btree;
// copy for staging
lfsr_btree_t btree_;
} lfsr_bshrub_t;
typedef struct lfsr_file {
lfsr_openedmdir_t m;
@@ -548,13 +540,12 @@ typedef struct lfsr_file {
uint8_t *buffer;
lfs_size_t buffer_size;
// we need a staging copy of each shrubs during mdir compaction, we put
// this in the file struct directly, since we don't know how many files
// may be opened
lfsr_shrub_t shrub;
lfsr_shrub_t shrub_;
lfsr_tree_t tree;
union {
lfsr_bsprout_t bsprout;
lfsr_bptr_t bptr;
lfsr_bshrub_t bshrub;
lfsr_btree_t btree;
} u;
const struct lfs_file_config *cfg;
} lfsr_file_t;
runners/bench_runner.h
+13 -9
View File
@@ -112,7 +112,7 @@ intmax_t bench_define(size_t define);
// a few preconfigured defines that control how benches run
#define BENCH_IMPLICIT_DEFINE_COUNT 15
#define BENCH_IMPLICIT_DEFINE_COUNT 16
#define BENCH_GEOMETRY_DEFINE_COUNT 3
#define READ_SIZE_i 0
@@ -122,14 +122,15 @@ intmax_t bench_define(size_t define);
#define DISK_SIZE_i 4
#define CACHE_SIZE_i 5
#define INLINE_SIZE_i 6
#define FRAGMENT_SIZE_i 7
#define CRYSTAL_SIZE_i 8
#define LOOKAHEAD_SIZE_i 9
#define BLOCK_CYCLES_i 10
#define ERASE_VALUE_i 11
#define ERASE_CYCLES_i 12
#define BADBLOCK_BEHAVIOR_i 13
#define POWERLOSS_BEHAVIOR_i 14
#define SHRUB_SIZE_i 7
#define FRAGMENT_SIZE_i 8
#define CRYSTAL_SIZE_i 9
#define LOOKAHEAD_SIZE_i 10
#define BLOCK_CYCLES_i 11
#define ERASE_VALUE_i 12
#define ERASE_CYCLES_i 13
#define BADBLOCK_BEHAVIOR_i 14
#define POWERLOSS_BEHAVIOR_i 15
#define READ_SIZE bench_define(READ_SIZE_i)
#define PROG_SIZE bench_define(PROG_SIZE_i)
@@ -138,6 +139,7 @@ intmax_t bench_define(size_t define);
#define DISK_SIZE bench_define(DISK_SIZE_i)
#define CACHE_SIZE bench_define(CACHE_SIZE_i)
#define INLINE_SIZE bench_define(INLINE_SIZE_i)
#define SHRUB_SIZE bench_define(SHRUB_SIZE_i)
#define FRAGMENT_SIZE bench_define(FRAGMENT_SIZE_i)
#define CRYSTAL_SIZE bench_define(CRYSTAL_SIZE_i)
#define LOOKAHEAD_SIZE bench_define(LOOKAHEAD_SIZE_i)
@@ -156,6 +158,7 @@ intmax_t bench_define(size_t define);
BENCH_DEF(DISK_SIZE, 1024*1024 ) \
BENCH_DEF(CACHE_SIZE, lfs_max(16, lfs_max(READ_SIZE, PROG_SIZE))) \
BENCH_DEF(INLINE_SIZE, BLOCK_SIZE/8 ) \
BENCH_DEF(SHRUB_SIZE, INLINE_SIZE ) \
BENCH_DEF(FRAGMENT_SIZE, CACHE_SIZE ) \
BENCH_DEF(CRYSTAL_SIZE, BLOCK_SIZE/8 ) \
BENCH_DEF(LOOKAHEAD_SIZE, 16 ) \
@@ -181,6 +184,7 @@ intmax_t bench_define(size_t define);
.block_cycles = BLOCK_CYCLES, \
.cache_size = CACHE_SIZE, \
.inline_size = INLINE_SIZE, \
.shrub_size = SHRUB_SIZE, \
.fragment_size = FRAGMENT_SIZE, \
.crystal_size = CRYSTAL_SIZE, \
.lookahead_size = LOOKAHEAD_SIZE,
runners/test_runner.h
+13 -9
View File
@@ -98,7 +98,7 @@ intmax_t test_define(size_t define);
// a few preconfigured defines that control how tests run
#define TEST_IMPLICIT_DEFINE_COUNT 15
#define TEST_IMPLICIT_DEFINE_COUNT 16
#define TEST_GEOMETRY_DEFINE_COUNT 3
#define READ_SIZE_i 0
@@ -108,14 +108,15 @@ intmax_t test_define(size_t define);
#define DISK_SIZE_i 4
#define CACHE_SIZE_i 5
#define INLINE_SIZE_i 6
#define FRAGMENT_SIZE_i 7
#define CRYSTAL_SIZE_i 8
#define LOOKAHEAD_SIZE_i 9
#define BLOCK_CYCLES_i 10
#define ERASE_VALUE_i 11
#define ERASE_CYCLES_i 12
#define BADBLOCK_BEHAVIOR_i 13
#define POWERLOSS_BEHAVIOR_i 14
#define SHRUB_SIZE_i 7
#define FRAGMENT_SIZE_i 8
#define CRYSTAL_SIZE_i 9
#define LOOKAHEAD_SIZE_i 10
#define BLOCK_CYCLES_i 11
#define ERASE_VALUE_i 12
#define ERASE_CYCLES_i 13
#define BADBLOCK_BEHAVIOR_i 14
#define POWERLOSS_BEHAVIOR_i 15
#define READ_SIZE TEST_DEFINE(READ_SIZE_i)
#define PROG_SIZE TEST_DEFINE(PROG_SIZE_i)
@@ -124,6 +125,7 @@ intmax_t test_define(size_t define);
#define DISK_SIZE TEST_DEFINE(DISK_SIZE_i)
#define CACHE_SIZE TEST_DEFINE(CACHE_SIZE_i)
#define INLINE_SIZE TEST_DEFINE(INLINE_SIZE_i)
#define SHRUB_SIZE TEST_DEFINE(SHRUB_SIZE_i)
#define FRAGMENT_SIZE TEST_DEFINE(FRAGMENT_SIZE_i)
#define CRYSTAL_SIZE TEST_DEFINE(CRYSTAL_SIZE_i)
#define LOOKAHEAD_SIZE TEST_DEFINE(LOOKAHEAD_SIZE_i)
@@ -142,6 +144,7 @@ intmax_t test_define(size_t define);
TEST_DEF(DISK_SIZE, 1024*1024 ) \
TEST_DEF(CACHE_SIZE, lfs_max(16, lfs_max(READ_SIZE, PROG_SIZE)) ) \
TEST_DEF(INLINE_SIZE, BLOCK_SIZE/8 ) \
TEST_DEF(SHRUB_SIZE, INLINE_SIZE ) \
TEST_DEF(FRAGMENT_SIZE, CACHE_SIZE ) \
TEST_DEF(CRYSTAL_SIZE, BLOCK_SIZE/8 ) \
TEST_DEF(LOOKAHEAD_SIZE, 16 ) \
@@ -167,6 +170,7 @@ intmax_t test_define(size_t define);
.block_cycles = BLOCK_CYCLES, \
.cache_size = CACHE_SIZE, \
.inline_size = INLINE_SIZE, \
.shrub_size = SHRUB_SIZE, \
.fragment_size = FRAGMENT_SIZE, \
.crystal_size = CRYSTAL_SIZE, \
.lookahead_size = LOOKAHEAD_SIZE,
tests/test_btree.toml
+2 -2
View File
@@ -4295,7 +4295,7 @@ code = '''
assert(i <= 2*N);
lfsr_binfo_t binfo;
int err = lfsr_btraversal_read(&lfs, &btree, &traversal, &binfo);
int err = lfsr_btree_traversalread(&lfs, &btree, &traversal, &binfo);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;
@@ -4447,7 +4447,7 @@ code = '''
assert(i <= 2*N);
lfsr_binfo_t binfo;
int err = lfsr_btraversal_read(&lfs, &btree, &traversal, &binfo);
int err = lfsr_btree_traversalread(&lfs, &btree, &traversal, &binfo);
assert(!err || err == LFS_ERR_NOENT);
if (err == LFS_ERR_NOENT) {
break;