From c5efe35ab23208e72597af377f9bd39a4b739a93 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 18 Apr 2025 15:43:38 -0500 Subject: [PATCH] Split crystal_thresh into crystal_thresh + fragment_thresh So now crystal_thresh only controls when fragments are compacted into blocks, while fragment_thresh controls when blocks are broken into fragments. Setting fragment_thresh=-1 will follow crystal_thresh and keeps the previous behavior. These were already two separate pieces of logic, so it makes sense to provide two separate knobs for tuning. Setting fragment_thresh lower than crystal_thresh has some potential to reduce hysteresis in cases where random writes push blocks close to crystal_thresh. It will be interesting to explore this more when benchmarking. --- The additional config option adds a bit of code/ctx, but hopefully that will go away in the future config rework: code stack ctx before: 35584 2480 636 after: 35600 (+0.0%) 2480 (+0.0%) 640 (+0.6%) --- lfs.c | 15 +++++++++++---- lfs.h | 17 +++++++++++++++-- runners/bench_runner.h | 4 +++- runners/test_runner.h | 4 +++- tests/test_fwrite.toml | 7 +++++-- 5 files changed, 37 insertions(+), 10 deletions(-) diff --git a/lfs.c b/lfs.c index ada025c2..338005ab 100644 --- a/lfs.c +++ b/lfs.c @@ -11614,10 +11614,12 @@ static int lfsr_file_carve(lfs_t *lfs, lfsr_file_t *file, -1); // left sibling needs carving but falls underneath our - // crystallization threshold? break into fragments + // fragment threshold? break into fragments while (lfsr_bptr_isbptr(&bptr_) && lfsr_data_size(l.data) > lfs->cfg->fragment_size - && lfsr_data_size(l.data) < lfs->cfg->crystal_thresh) { + && lfsr_data_size(l.data) < lfs_min( + lfs->cfg->fragment_thresh, + lfs->cfg->crystal_thresh)) { bptr_.data = LFSR_DATA_SLICE(bptr_.data, lfs->cfg->fragment_size, -1); @@ -11643,10 +11645,12 @@ static int lfsr_file_carve(lfs_t *lfs, lfsr_file_t *file, } // right sibling needs carving but falls underneath our - // crystallization threshold? break into fragments + // fragment threshold? break into fragments while (lfsr_bptr_isbptr(&bptr_) && lfsr_data_size(r.data) > lfs->cfg->fragment_size - && lfsr_data_size(r.data) < lfs->cfg->crystal_thresh) { + && lfsr_data_size(r.data) < lfs_min( + lfs->cfg->fragment_thresh, + lfs->cfg->crystal_thresh)) { bptr_.data = LFSR_DATA_SLICE(bptr_.data, -1, lfsr_data_size(bptr_.data) - lfs->cfg->fragment_size); @@ -13093,6 +13097,9 @@ static int lfs_init(lfs_t *lfs, uint32_t flags, LFS_ASSERT(lfs->cfg->inline_size <= lfs->cfg->block_size/4); // fragment_size must be <= block_size/4 LFS_ASSERT(lfs->cfg->fragment_size <= lfs->cfg->block_size/4); + // fragment_thresh > crystal_thresh is probably a mistake + LFS_ASSERT(lfs->cfg->fragment_thresh == (lfs_size_t)-1 + || lfs->cfg->fragment_thresh <= lfs->cfg->crystal_thresh); // setup flags lfs->flags = flags diff --git a/lfs.h b/lfs.h index 06973487..ca81c976 100644 --- a/lfs.h +++ b/lfs.h @@ -446,12 +446,25 @@ struct lfs_config { lfs_size_t fragment_size; // Threshold for compacting multiple fragments into a block. Smaller - // values will compact more frequently, reducing disk usage, but - // increasing the cost of random-writes. + // values will compact more eagerly, reducing disk usage, but increasing + // the cost of random-writes. // // 0 only writes blocks, minimizing disk usage, while -1 or any value >= // block_size only writes fragments, minimizing random-write cost. lfs_size_t crystal_thresh; + + // Threshold for breaking a block into fragments. Smaller values will + // break more lazily, reducing random-write cost, but risk leaving blocks + // around with wasted storage. + // + // This can be set lower than crystal_thresh to prevent repeated + // compact/break operations in files with heavy random writes, at a + // storage cost. Setting this higher than crystal_thresh is probably not + // a good idea. + // + // 0 will never fragment a block once compacted, while -1 will fragment + // as soon as a block drops below crystal_thresh. + lfs_size_t fragment_thresh; }; // File info structure diff --git a/runners/bench_runner.h b/runners/bench_runner.h index 1c4cf9d5..de3af28d 100644 --- a/runners/bench_runner.h +++ b/runners/bench_runner.h @@ -120,6 +120,7 @@ void bench_permutation(size_t i, uint32_t *buffer, size_t size); BENCH_DEFINE(INLINE_SIZE, BLOCK_SIZE/4 ) \ BENCH_DEFINE(FRAGMENT_SIZE, BLOCK_SIZE/8 ) \ BENCH_DEFINE(CRYSTAL_THRESH, BLOCK_SIZE/8 ) \ + BENCH_DEFINE(FRAGMENT_THRESH, -1 ) \ BENCH_DEFINE(ERASE_VALUE, 0xff ) \ BENCH_DEFINE(ERASE_CYCLES, 0 ) \ BENCH_DEFINE(BADBLOCK_BEHAVIOR, LFS_EMUBD_BADBLOCK_PROGERROR ) \ @@ -148,7 +149,8 @@ void bench_permutation(size_t i, uint32_t *buffer, size_t size); .gc_compact_thresh = GC_COMPACT_THRESH, \ .inline_size = INLINE_SIZE, \ .fragment_size = FRAGMENT_SIZE, \ - .crystal_thresh = CRYSTAL_THRESH, + .crystal_thresh = CRYSTAL_THRESH, \ + .fragment_thresh = FRAGMENT_THRESH, #ifdef LFS_GC #define BENCH_GC_CFG \ diff --git a/runners/test_runner.h b/runners/test_runner.h index ad1b896f..2efeb6a8 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -111,6 +111,7 @@ void test_permutation(size_t i, uint32_t *buffer, size_t size); TEST_DEFINE(INLINE_SIZE, BLOCK_SIZE/4 ) \ TEST_DEFINE(FRAGMENT_SIZE, BLOCK_SIZE/8 ) \ TEST_DEFINE(CRYSTAL_THRESH, BLOCK_SIZE/8 ) \ + TEST_DEFINE(FRAGMENT_THRESH, -1 ) \ TEST_DEFINE(ERASE_VALUE, 0xff ) \ TEST_DEFINE(ERASE_CYCLES, 0 ) \ TEST_DEFINE(BADBLOCK_BEHAVIOR, LFS_EMUBD_BADBLOCK_PROGERROR ) \ @@ -139,7 +140,8 @@ void test_permutation(size_t i, uint32_t *buffer, size_t size); .gc_compact_thresh = GC_COMPACT_THRESH, \ .inline_size = INLINE_SIZE, \ .fragment_size = FRAGMENT_SIZE, \ - .crystal_thresh = CRYSTAL_THRESH + .crystal_thresh = CRYSTAL_THRESH, \ + .fragment_thresh = FRAGMENT_THRESH, #ifdef LFS_GC #define TEST_GC_CFG \ diff --git a/tests/test_fwrite.toml b/tests/test_fwrite.toml index 12933541..bb13cf3a 100644 --- a/tests/test_fwrite.toml +++ b/tests/test_fwrite.toml @@ -5,8 +5,11 @@ after = 'test_files' # test with different fragment sizes defines.FRAGMENT_SIZE = [1, 16, 64] -# test with different crystal sizes -defines.CRYSTAL_SIZE = [512] +# test with different crystallization thresholds +defines.CRYSTAL_THRESH = [512] + +# test with different fragment thresholds +defines.FRAGMENT_THRESH = [-1] # test with different prog sizes defines.PROG_SIZE = [1, 16]