From 23aab1a238384dd3019fe9d711ba303e384c5667 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 26 Feb 2024 01:28:15 -0600 Subject: [PATCH] Increased mleaf-bits to account for better compaction algorithms As defined previously, mleaf-bits depended on the attr estimate, which depended on the details of our compaction algorithm: block_size m = ---------- a_0 Assuming t=4, the _minimum_ tag encoding: block_size block_size m = ---------- = ---------- 3*4 + 4 16 However, with our new compaction algorithm, our attr estimate changes: block_size block_size block_size m = ---------- = ----------- = ---------- a_1 (5/2)*4 + 2 12 But tying our mleaf-bits to our attr estimate is a bit fragile. Unlike attr estimate, the calculated mleaf-bits MUST be the same across all littlefs implementations, or else the filesystem may not be mountable. We _could_ store mleaf-bits as an fs attr in the mroot, like we do with name-limit, size-limit, block-size, etc, but I'd prefer to not add fs attrs unless strictly required. Each fs attr adds complexity to mounting, which has a non-zero cost and headache. Instead, we can assume our compaction algorithm is perfect: block_size block_size block_size m = ---------- = ---------- = ---------- a_inf 2*4 8 This isn't actually achievable without unbounded RAM. But just because our current implementation is limited to bounded RAM, does not prevent some other implementation from pushing things further with unbounded RAM. In theory, since this is a perfect compaction algorithm, and builds perfect rbyd trunks, this should be the maximum possible mleaf-bits achievable in littlefs's current design, and should be compatible with any future implementation. --- Worst case, we can always add mleaf-bits as an fs attr retroactively without breaking backwards compatibility. You would just need to assume the above block_size-dependent value if the hypothetical mleaf-bits attr is missing. This is one nice thing about our fs attr system, it's very flexible. --- lfs.c | 31 +++++++++++++++++++++---------- scripts/dbgbmap.py | 2 +- scripts/dbglfs.py | 2 +- scripts/dbgmtree.py | 2 +- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/lfs.c b/lfs.c index 2ac8aab7..94dfb802 100644 --- a/lfs.c +++ b/lfs.c @@ -15137,23 +15137,34 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->hasorphans = false; - // compute the number of bits we need to reserve for metadata rids + // compute the number of bits we need to reserve for mdir rids // - // This is equivalent to the nlog2 of the maximum number of rids we can - // ever have in a single mdir. With some knowledge of our system we can - // find a conservative, but useful, limit to this upper bound: + // Worst case (or best case?) each metadata entry is a single tag. In + // theory each entry also needs a name, but with power-of-two rounding, + // this is negligible // - // - Each tag needs <=2 alts+null with our current compaction strategy - // - Each tag/alt encodes to a minimum of 4 bytes + // Assuming a _perfect_ compaction algorithm (requires unbounded RAM), + // each tag also needs ~1 alt, this gives us: // - // This gives us ~4*4 or ~16 bytes per mid at minimum. If we cram an mdir - // with the smallest possible mids, this gives us at most ~block_size/16 - // mids in a single mdir before the mdir runs out of space. + // block_size + // m = ---------- + // 2t + // + // Assuming t=4 bytes, the minimum tag encoding: + // + // block_size block_size + // m = ---------- = ---------- + // 2*4 8 // // Note we can't assume ~1/2 block utilization here, as an mdir may // temporarily fill with more mids before compaction occurs. // - lfs->mleaf_bits = lfs_nlog2(lfs->cfg->block_size/16); + // Note note our actual compaction algorithm is not perfect, and + // requires (5/2)t+2 bytes per tag, or with t=4 bytes => ~block_size/12 + // metadata entries per block. But we intentionally don't leverage this + // to maintain compatibility with a theoretical perfect implementation. + // + lfs->mleaf_bits = lfs_nlog2(lfs->cfg->block_size/8); // zero linked-list of opened mdirs lfs->opened = NULL; diff --git a/scripts/dbgbmap.py b/scripts/dbgbmap.py index 83f6bbd5..5cf465d0 100755 --- a/scripts/dbgbmap.py +++ b/scripts/dbgbmap.py @@ -1344,7 +1344,7 @@ def main(disk, mroots=None, *, # determine the mleaf_weight from the block_size, this is just for # printing purposes if mleaf_weight is None: - mleaf_weight = 1 << m.ceil(m.log2(block_size // 16)) + mleaf_weight = 1 << m.ceil(m.log2(block_size // 8)) #### traverse the filesystem diff --git a/scripts/dbglfs.py b/scripts/dbglfs.py index 31e16da8..39ad1042 100755 --- a/scripts/dbglfs.py +++ b/scripts/dbglfs.py @@ -1687,7 +1687,7 @@ def main(disk, mroots=None, *, # determine the mleaf_weight from the block_size, this is just for # printing purposes if mleaf_weight is None: - mleaf_weight = 1 << m.ceil(m.log2(block_size // 16)) + mleaf_weight = 1 << m.ceil(m.log2(block_size // 8)) # before we print, we need to do a pass for a few things: # - find the actual mroot diff --git a/scripts/dbgmtree.py b/scripts/dbgmtree.py index e4c6d1b9..e8ab8ad4 100755 --- a/scripts/dbgmtree.py +++ b/scripts/dbgmtree.py @@ -844,7 +844,7 @@ def main(disk, mroots=None, *, # determine the mleaf_weight from the block_size, this is just for # printing purposes if mleaf_weight is None: - mleaf_weight = 1 << m.ceil(m.log2(block_size // 16)) + mleaf_weight = 1 << m.ceil(m.log2(block_size // 8)) # before we print, we need to do a pass for a few things: # - find the actual mroot