Github/littlefs
1
Fork 0
Code Actions Activity

Adopted a different strategy for mdir split threshold estimation

Browse Source 
This approach is simpler: fall back to using two passes if we split a
supermdir.

This trades off code complexity with runtime, but I think we really don't
care about the runtime here, since this operation should really only happen
once in a filesystem's entire lifetime.
This commit is contained in:
Christopher Haster
2023-05-07 16:24:45 -05:00
parent 06b04bda6b
commit a3bfa3488f
1 changed files with 221 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+221 -110
View File
@@ -2667,14 +2667,14 @@ failed:;
// exist in the rbyd namespace
enum {
LFSR_INTHRESH_NO = 0,
LFSR_INTHRESH_YES = 1,
LFSR_INTHRESH_UNINLINED = 2,
LFSR_INTHRESH_DEGENERATE = 3,
LFSR_ESTIMATE_OVERFLOWS = 0,
LFSR_ESTIMATE_FITS = 1,
LFSR_ESTIMATE_DEGENERATE = 2,
};
static int lfsr_rbyd_inthresh(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
bool inlined, lfs_ssize_t cutoff, lfs_size_t dcount, lfs_size_t dsize,
static int lfsr_rbyd_estimate(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
lfs_ssize_t id, lfs_ssize_t cutoff,
lfs_size_t init_tcount, lfs_size_t init_dsize,
lfs_size_t *lower_id_, lfs_size_t *lower_dsize_) {
// determine if a given rbyd will be within the compaction threshold (1/2)
// after compaction, note this uses a conservative estimate so the actual
@@ -2689,12 +2689,15 @@ static int lfsr_rbyd_inthresh(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
const lfs_size_t tag_dsize = 2
+ 5
+ (lfs_nlog2(lfs->cfg->block_size)+7-1)/7;
lfs_size_t count = 0;
lfs_size_t uninlined_dcount = 0;
lfs_size_t uninlined_dsize = 0;
lfs_size_t altless_dsize = 0;
lfs_size_t tcount = init_tcount;
lfs_size_t dsize = 0;
lfs_size_t real_dsize = init_dsize;
// note id count != tag count, and id count != id+1
//
// we use this to test for degenerate rbyds, where a degenerate rbyd is
// defined as an rbyd with id count < cutoff, irregardless of weight
lfs_size_t idcount = 0;
lfs_ssize_t id = -1;
lfsr_tag_t tag = 0;
while (true) {
lfs_size_t w;
@@ -2705,15 +2708,15 @@ static int lfsr_rbyd_inthresh(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
return err;
}
if (err == LFS_ERR_NOENT) {
if (cutoff >= 0 && count <= (lfs_size_t)cutoff) {
return LFSR_INTHRESH_DEGENERATE;
if (cutoff >= 0 && idcount <= (lfs_size_t)cutoff) {
return LFSR_ESTIMATE_DEGENERATE;
}
return LFSR_INTHRESH_YES;
return LFSR_ESTIMATE_FITS;
}
// count ids to determine if we're without our cutoff
if (w > 0) {
count += 1;
idcount += 1;
}
// Exhibit A. Why I really didn't want to estimate the rbyd threshold:
@@ -2721,51 +2724,137 @@ static int lfsr_rbyd_inthresh(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
// TODO do we really need this tight a bound? this might be the only
// place we divide by a non-power-of-two
// keep track of alt-less tcount and dsize
//
// this is used as a heuristic for split, so the exactness matters less,
// but we need to be able to subtract tags from the result so we can't
// use the estimate with alt pointers
tcount += 1;
dsize += LFSR_TAG_DSIZE + lfsr_data_size(data);
// determine the upper-bound of our alt pointers, tag, and data
//
// fortunately rybd gives us a tight bound on the number of alt
// pointers
dcount += 1;
dsize += (2*lfs_nlog2(dcount+1)+1) * tag_dsize
// fortunately the self-balancing nature of rybds give us a tight
// bound on the number of alt pointers
real_dsize
+= (2*lfs_nlog2(tcount+1)+1) * tag_dsize
+ tag_dsize
+ lfsr_data_size(data);
// also keep track of upper-bound if we ignore any -1 ids, this
// is useful for lfsr_mdir_commit to see if we don't need to split
// when uninlining
if (id >= 0) {
uninlined_dcount += 1;
uninlined_dsize += (2*lfs_nlog2(uninlined_dcount+1)+1) * tag_dsize
+ tag_dsize
+ lfsr_data_size(data);
}
// also keep track of alt-less dsize in case we need to split,
// assume a worst-case tag size because it's cheaper and this
// matters less
if (!inlined || id >= 0) {
altless_dsize += LFSR_TAG_DSIZE + lfsr_data_size(data);
}
// exceeded our compaction threshold?
if (dsize > lfs->cfg->block_size/2) {
if (real_dsize > lfs->cfg->block_size/2) {
// TODO do these need to be conditional?
if (lower_id_) {
*lower_id_ = id;
}
if (lower_dsize_) {
*lower_dsize_ = altless_dsize;
*lower_dsize_ = dsize;
}
if (inlined && uninlined_dsize <= lfs->cfg->block_size/2) {
return LFSR_INTHRESH_UNINLINED;
} else {
return LFSR_INTHRESH_NO;
}
return LFSR_ESTIMATE_OVERFLOWS;
}
}
}
//enum {
// LFSR_INTHRESH_NO = 0,
// LFSR_INTHRESH_YES = 1,
// LFSR_INTHRESH_UNINLINED = 2,
// LFSR_INTHRESH_DEGENERATE = 3,
//};
//
//static int lfsr_rbyd_inthresh(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
// bool inlined, lfs_ssize_t cutoff, lfs_size_t dcount, lfs_size_t dsize,
// lfs_size_t *lower_id_, lfs_size_t *lower_dsize_) {
// // determine if a given rbyd will be within the compaction threshold (1/2)
// // after compaction, note this uses a conservative estimate so the actual
// // on-disk cost may be smaller
// //
// // returns the id/dsize where the threshold failed, this isn't that useful
// // on its own, but can be used to find a good split_id with lfsr_rbyd_bisect
//
// // TODO should we store this in lfs_t somewhere?
// // assume a tighter bound on size/jump leb128 encoding if we know
// // our block_size
// const lfs_size_t tag_dsize = 2
// + 5
// + (lfs_nlog2(lfs->cfg->block_size)+7-1)/7;
// lfs_size_t count = 0;
// lfs_size_t uninlined_dcount = 0;
// lfs_size_t uninlined_dsize = 0;
// lfs_size_t altless_dsize = 0;
//
// lfs_ssize_t id = -1;
// lfsr_tag_t tag = 0;
// while (true) {
// lfs_size_t w;
// lfsr_data_t data;
// int err = lfsr_rbyd_lookupnext(lfs, rbyd, id, lfsr_tag_next(tag),
// &id, &tag, &w, &data);
// if (err && err != LFS_ERR_NOENT) {
// return err;
// }
// if (err == LFS_ERR_NOENT) {
// if (cutoff >= 0 && count <= (lfs_size_t)cutoff) {
// return LFSR_INTHRESH_DEGENERATE;
// }
// return LFSR_INTHRESH_YES;
// }
//
// // count ids to determine if we're without our cutoff
// if (w > 0) {
// count += 1;
// }
//
// // Exhibit A. Why I really didn't want to estimate the rbyd threshold:
//
// // TODO do we really need this tight a bound? this might be the only
// // place we divide by a non-power-of-two
//
// // determine the upper-bound of our alt pointers, tag, and data
// //
// // fortunately rybd gives us a tight bound on the number of alt
// // pointers
// dcount += 1;
// dsize += (2*lfs_nlog2(dcount+1)+1) * tag_dsize
// + tag_dsize
// + lfsr_data_size(data);
//
// // also keep track of upper-bound if we ignore any -1 ids, this
// // is useful for lfsr_mdir_commit to see if we don't need to split
// // when uninlining
// if (id >= 0) {
// uninlined_dcount += 1;
// uninlined_dsize += (2*lfs_nlog2(uninlined_dcount+1)+1) * tag_dsize
// + tag_dsize
// + lfsr_data_size(data);
// }
//
// // also keep track of alt-less dsize in case we need to split,
// // assume a worst-case tag size because it's cheaper and this
// // matters less
// if (!inlined || id >= 0) {
// altless_dsize += LFSR_TAG_DSIZE + lfsr_data_size(data);
// }
//
// // exceeded our compaction threshold?
// if (dsize > lfs->cfg->block_size/2) {
// // TODO do these need to be conditional?
// if (lower_id_) {
// *lower_id_ = id;
// }
// if (lower_dsize_) {
// *lower_dsize_ = altless_dsize;
// }
//
// if (inlined && uninlined_dsize <= lfs->cfg->block_size/2) {
// return LFSR_INTHRESH_UNINLINED;
// } else {
// return LFSR_INTHRESH_NO;
// }
// }
// }
//}
static lfs_ssize_t lfsr_rbyd_bisect(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
lfs_size_t lower_id, lfs_size_t lower_dsize) {
// find the best id to split the rbyd evenly
@@ -2837,39 +2926,39 @@ static lfs_ssize_t lfsr_rbyd_bisect(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
return lower_id_;
}
static int lfsr_rbyd_incutoff(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
lfs_ssize_t cutoff) {
// determine if there are fewer than "cutoff" unique ids in the rbyd,
// this is used to determine if the underlying rbyd is degenerate and can
// be reverted to an inlined btree
//
// note cutoff is expected to be quite small, <= 2, so we should make sure
// to exit our traverse early
// cutoff=-1 => no cutoff
if (cutoff < 0) {
return false;
}
// count ids until we exceed our cutoff
lfs_ssize_t id = -1;
lfs_size_t count = 0;
while (true) {
int err = lfsr_rbyd_lookupnext(lfs, rbyd, id+1, 0,
&id, NULL, NULL, NULL);
if (err && err != LFS_ERR_NOENT) {
return err;
}
if (err == LFS_ERR_NOENT) {
return true;
}
count += 1;
if (count > (lfs_size_t)cutoff) {
return false;
}
}
}
//static int lfsr_rbyd_incutoff(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
// lfs_ssize_t cutoff) {
// // determine if there are fewer than "cutoff" unique ids in the rbyd,
// // this is used to determine if the underlying rbyd is degenerate and can
// // be reverted to an inlined btree
// //
// // note cutoff is expected to be quite small, <= 2, so we should make sure
// // to exit our traverse early
//
// // cutoff=-1 => no cutoff
// if (cutoff < 0) {
// return false;
// }
//
// // count ids until we exceed our cutoff
// lfs_ssize_t id = -1;
// lfs_size_t count = 0;
// while (true) {
// int err = lfsr_rbyd_lookupnext(lfs, rbyd, id+1, 0,
// &id, NULL, NULL, NULL);
// if (err && err != LFS_ERR_NOENT) {
// return err;
// }
// if (err == LFS_ERR_NOENT) {
// return true;
// }
//
// count += 1;
// if (count > (lfs_size_t)cutoff) {
// return false;
// }
// }
//}
@@ -3472,9 +3561,9 @@ static int lfsr_btree_commit(lfs_t *lfs,
#ifndef LFSR_BTREE_NOTHRESH
// can't commit, try to compact
lfsr_rbyd_t rbyd_;
lfs_size_t lower_id = 0;
lfs_size_t lower_dsize = 0;
lfs_size_t dcount = 0;
lfs_size_t lower_id;
lfs_size_t lower_dsize;
lfs_size_t tcount;
if (err) {
// // TODO can we combine this with lfsr_rbyd_inthresh?
// //
@@ -3506,20 +3595,35 @@ static int lfsr_btree_commit(lfs_t *lfs,
// attributes up to upper layers
//
// note we account for the revision count here
int inthresh = lfsr_rbyd_inthresh(lfs, rbyd,
false, (pid == -1 ? cutoff : -1), 0, sizeof(uint32_t),
int estimate = lfsr_rbyd_estimate(lfs, rbyd,
-1, (pid == -1 ? cutoff : -1), 0, sizeof(uint32_t),
&lower_id, &lower_dsize);
if (inthresh < 0) {
return inthresh;
if (estimate < 0) {
return estimate;
}
if (inthresh == LFSR_INTHRESH_DEGENERATE) {
if (estimate == LFSR_ESTIMATE_DEGENERATE) {
// TODO LFSR_BTREE_DEGENERATE? just propagate LFSR_ESTIMATE_DEGENERATE?
return true;
} else if (!inthresh) {
} else if (estimate == LFSR_ESTIMATE_OVERFLOWS) {
LFS_ASSERT(lower_id > 0);
goto split;
}
// int inthresh = lfsr_rbyd_inthresh(lfs, rbyd,
// false, (pid == -1 ? cutoff : -1), 0, sizeof(uint32_t),
// &lower_id, &lower_dsize);
// if (inthresh < 0) {
// return inthresh;
// }
//
// if (inthresh == LFSR_INTHRESH_DEGENERATE) {
// return true;
// } else if (!inthresh) {
// LFS_ASSERT(lower_id > 0);
// goto split;
// }
// TODO were we doing something funky with rev?
// allocate a new rbyd
err = lfsr_rbyd_alloc(lfs, &rbyd_, rbyd->rev+1);
@@ -3572,7 +3676,7 @@ static int lfsr_btree_commit(lfs_t *lfs,
// keep track of the number of tags we've written in case we
// try to merge
dcount += 1;
tcount += 1;
}
// append any pending attrs, it's up to upper
@@ -3588,7 +3692,7 @@ static int lfsr_btree_commit(lfs_t *lfs,
// keep track of the number of tags we've written in case we
// try to merge
dcount += 1;
tcount += 1;
}
// TODO do we really need a threshold for this? should we just
@@ -3924,15 +4028,15 @@ static int lfsr_btree_commit(lfs_t *lfs,
LFS_ASSERT(sibling.weight == sweight);
// estimate if our sibling will fit
lfs_ssize_t inthresh = lfsr_rbyd_inthresh(lfs, &sibling,
false, -1, dcount, rbyd_.off,
int estimate = lfsr_rbyd_estimate(lfs, &sibling,
-1, -1, tcount, rbyd_.off,
NULL, NULL);
if (inthresh < 0) {
return inthresh;
if (estimate < 0) {
return estimate;
}
// don't fit? can't merge
if (!inthresh) {
if (estimate == LFSR_ESTIMATE_OVERFLOWS) {
continue;
}
@@ -5049,31 +5153,38 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir,
lfsr_mdir_t mdir_;
bool issupermdirsplit = false; // TODO do this differently?
bool uninlining = false;
lfs_size_t lower_id = 0;
lfs_size_t lower_dsize = 0;
// check if we're within our compaction threshold, otherwise we
// need to split
//
// note we account for the revision count here
int inthresh = lfsr_rbyd_inthresh(lfs, &mdir->rbyd,
lfsr_btree_isnull(&lfs->mtree), -1, 0, sizeof(uint32_t),
lfs_size_t lower_id;
lfs_size_t lower_dsize;
int estimate = lfsr_rbyd_estimate(lfs, &mdir->rbyd,
-1, -1, 0, sizeof(uint32_t),
&lower_id, &lower_dsize);
if (inthresh < 0) {
return inthresh;
if (estimate < 0) {
return estimate;
}
if (inthresh != LFSR_INTHRESH_YES) {
// TODO this is a bit weird, because lfsr_rbyd_inthresh inlining
// parameter above, better way to structure this?
if (estimate != LFSR_ESTIMATE_FITS) {
// are we inlined into the supermdir? we need to uninline
// before we split, and it's possible uninlining makes the mdir
// small enough that we don't even need to split
if (lfsr_btree_isnull(&lfs->mtree)) {
uninlining = true;
// are we inlined? no matter what we do we need to uninline, but
// we may not need to split if we fit our compaction threshold
// without the superattrs
uninlining = lfsr_btree_isnull(&lfs->mtree);
estimate = lfsr_rbyd_estimate(lfs, &mdir->rbyd,
// note id was changed to 0 here
0, -1, 0, sizeof(uint32_t),
&lower_id, &lower_dsize);
if (estimate < 0) {
return estimate;
}
}
// need to split
if (inthresh != LFSR_INTHRESH_UNINLINED) {
if (LFSR_ESTIMATE_OVERFLOWS) {
// needs a split
LFS_ASSERT(lower_id > 0);
goto split;
}
@@ -5102,7 +5213,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir,
// estimate the worst-case rbyd size
// TODO function for this?
lfs_size_t dsize = 4; // 4 bytes for rev
lfs_size_t dcount = 0;
lfs_size_t tcount = 0;
lfs_ssize_t id = -1;
lfsr_tag_t tag = 0;
while (true) {
@@ -5120,12 +5231,12 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir,
// keep track of size and count of tags
dsize += LFSR_TAG_DSIZE + lfsr_data_size(data);
dcount += 1;
tcount += 1;
}
// TODO account for block_size limits in attr size dsizes?
// account for alt pointers
dsize += dcount*(LFSR_TAG_DSIZE * (2*lfs_nlog2(dcount)+1));
dsize += tcount*(LFSR_TAG_DSIZE * (2*lfs_nlog2(tcount)+1));
// keep rbyd < our compaction threshold (1/2) to avoid
// degenerate cases