Changed namelookup functions to include a directory-id

The plan is that names in littlefs now include a directory-id prefixed
as a single leb128.

  01 66 69 6c 65 2e 74 78 74  .file.txt
   ^ '----------+----------'
   '------------|------------ leb128 directory-id
                '------------ ascii/utf8 name

Unfortunately, while this is easy for read/compare operations to implement,
it creates a bit of a problem for writes. We can't allocate a new buffer
for each name, so we need some sort of extra mechanism.

The solution here is to just add a did member to lfsr_data_t that is
written when non-negative. This works, though it does introduce some
complexity.

Fortunately, did in lfsr_data_t is somewhat free when
sizeof(void*) == sizeof(lfs_size_t), due to the union with disk
references.
This commit is contained in:
Christopher Haster
2023-07-03 02:52:02 -05:00
parent 2fe2078f50
commit 0bb1e0b8b5
2 changed files with 113 additions and 45 deletions
+62 -10
View File
@@ -1001,6 +1001,7 @@ typedef union lfsr_data {
struct {
lfs_size_t size;
const uint8_t *buffer;
lfs_ssize_t did;
} buf;
struct {
lfs_size_t size;
@@ -1009,13 +1010,20 @@ typedef union lfsr_data {
} disk;
} lfsr_data_t;
#define LFSR_DATA_NULL \
((lfsr_data_t){.size=0})
#define LFSR_DATA_NULL LFSR_DATA_BUF(NULL, 0)
#define LFSR_DATA_BUF(_buffer, _size) \
((lfsr_data_t){.buf={ \
.size=_size, \
.buffer=(const void*)(_buffer)}})
.buffer=(const void*)(_buffer), \
.did=-1}})
#define LFSR_DATA_DNAME(_did, _buffer, _size) \
((lfsr_data_t){.buf={ \
/* note this find the effective leb128 size */ \
.size=_size + lfs_min32(lfs_nlog2(_did)/7, 1), \
.buffer=(const void*)(_buffer), \
.did=_did}})
#define LFSR_DATA_DISK(_block, _off, _size) \
((lfsr_data_t){.disk={ \
@@ -1114,7 +1122,26 @@ static lfs_scmp_t lfsr_data_cmp(lfs_t *lfs, lfsr_data_t data,
}
}
static lfs_ssize_t lfsr_bd_progdata(lfs_t *lfs,
static lfs_scmp_t lfsr_data_dnamecmp(lfs_t *lfs, lfsr_data_t data,
lfs_off_t off, lfs_size_t did, const char *name, lfs_size_t name_size) {
// first compare the did
lfs_size_t did_;
lfs_ssize_t d = lfsr_data_readleb128(lfs, data, off, &did_);
if (d < 0) {
return d;
}
if (did_ < did) {
return LFS_CMP_LT;
} else if (did_ > did) {
return LFS_CMP_GT;
}
// next compare the actual name
return lfsr_data_cmp(lfs, data, off+d, name, name_size);
}
static int lfsr_bd_progdata(lfs_t *lfs,
lfs_block_t block, lfs_off_t off,
lfsr_data_t data,
uint32_t *csum_) {
@@ -1140,6 +1167,25 @@ static lfs_ssize_t lfsr_bd_progdata(lfs_t *lfs,
}
} else {
// this is kind of a hack, but when lfsr_data_t is in buffer mode, it
// can also contain a leb128 encoded directory-id prefix
if (data.buf.did != -1) {
// TODO should progleb128 be its own function? rely on caching?
uint8_t buf[5];
lfs_ssize_t d = lfs_toleb128(data.buf.did, buf, 5);
if (d < 0) {
return d;
}
int err = lfsr_bd_prog(lfs, block, off, buf, d, csum_);
if (err) {
return err;
}
off += d;
data.buf.size -= d;
}
int err = lfsr_bd_prog(lfs, block, off,
data.buf.buffer, lfsr_data_size(data),
csum_);
@@ -3068,10 +3114,16 @@ static int lfsr_rbyd_isdegenerate(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
}
}
// some low-level dname things
//
// dnames in littlefs are tuples of directory-ids + ascii/utf8 strings
// binary search an rbyd for a name, leaving the id_/weight_ with the best
// matching name if not found
static int lfsr_rbyd_namelookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
const char *name, lfs_size_t name_size,
static int lfsr_rbyd_dnamelookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
lfs_size_t did, const char *name, lfs_size_t name_size,
lfs_ssize_t *id_, lfsr_tag_t *tag_, lfs_size_t *weight_,
lfsr_data_t *data_) {
// binary search for our name
@@ -3101,7 +3153,7 @@ static int lfsr_rbyd_namelookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
// compare names
} else {
cmp = lfsr_data_cmp(lfs, data__, 0, name, name_size);
cmp = lfsr_data_dnamecmp(lfs, data__, 0, did, name, name_size);
if (cmp < 0) {
return cmp;
}
@@ -3505,8 +3557,8 @@ static int lfsr_btree_parent(lfs_t *lfs,
}
}
static lfs_ssize_t lfsr_btree_namelookup(lfs_t *lfs, const lfsr_btree_t *btree,
const char *name, lfs_size_t name_size,
static lfs_ssize_t lfsr_btree_dnamelookup(lfs_t *lfs, const lfsr_btree_t *btree,
lfs_size_t did, const char *name, lfs_size_t name_size,
lfs_size_t *bid_, lfsr_tag_t *tag_, lfs_size_t *weight_,
lfsr_data_t *data_) {
// an empty tree?
@@ -3539,7 +3591,7 @@ static lfs_ssize_t lfsr_btree_namelookup(lfs_t *lfs, const lfsr_btree_t *btree,
// lookup our name in the rbyd via binary search
lfs_ssize_t rid__;
lfs_size_t weight__;
int err = lfsr_rbyd_namelookup(lfs, &branch, name, name_size,
int err = lfsr_rbyd_dnamelookup(lfs, &branch, did, name, name_size,
&rid__, NULL, &weight__, NULL);
if (err && err != LFS_ERR_NOENT) {
return err;
+51 -35
View File
@@ -2853,11 +2853,13 @@ code = '''
lfs_size_t weight_;
lfsr_data_t data_;
lfsr_btree_namelookup(&lfs, &btree, "aaa", 3,
lfsr_btree_dnamelookup(&lfs, &btree, 0, "aaa", 3,
&id_, &tag_, &weight_, &data_) => LFS_ERR_NOENT;
'''
[cases.t2_btree_find_one]
# true or false for if we should use dids vs names
defines.DID = [false, true]
in = 'lfs.c'
code = '''
lfs_t lfs;
@@ -2887,7 +2889,7 @@ code = '''
lfs_size_t weight_;
lfsr_data_t data_;
lfsr_btree_namelookup(&lfs, &btree, "aaa", 3,
lfsr_btree_dnamelookup(&lfs, &btree, 0*DID, "aaa", 3,
&id_, &tag_, &weight_, &data_) => 0;
assert(tag_ == LFSR_TAG_INLINED);
assert(id_ == 0);
@@ -2895,7 +2897,7 @@ code = '''
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
assert(memcmp(buffer, "0", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, "aab", 3,
lfsr_btree_dnamelookup(&lfs, &btree, 1*DID, "aab", 3,
&id_, &tag_, &weight_, &data_) => 0;
assert(tag_ == LFSR_TAG_INLINED);
assert(id_ == 0);
@@ -2905,6 +2907,8 @@ code = '''
'''
[cases.t2_btree_find_two]
# true or false for if we should use dids vs names
defines.DID = [false, true]
in = 'lfs.c'
code = '''
lfs_t lfs;
@@ -2921,7 +2925,7 @@ code = '''
lfsr_btree_t btree = LFSR_BTREE_NULL;
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1,
LFSR_DATA_BUF("0", 1)) => 0;
lfsr_btree_split(&lfs, &btree, 0, LFSR_DATA_BUF("aab", 3),
lfsr_btree_split(&lfs, &btree, 0, LFSR_DATA_DNAME(0*DID, "aab", 3),
LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("0", 1),
LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("1", 1)) => 0;
printf("btree: w%d 0x%x.%x\n",
@@ -2937,7 +2941,7 @@ code = '''
lfs_size_t weight_;
lfsr_data_t data_;
lfsr_btree_namelookup(&lfs, &btree, "aaa", 3,
lfsr_btree_dnamelookup(&lfs, &btree, 0, "aaa", 3,
&id_, &tag_, &weight_, &data_) => 0;
assert(tag_ == LFSR_TAG_INLINED);
assert(id_ == 0);
@@ -2945,7 +2949,7 @@ code = '''
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
assert(memcmp(buffer, "0", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, "aab", 3,
lfsr_btree_dnamelookup(&lfs, &btree, 0, "aab", 3,
&id_, &tag_, &weight_, &data_) => 0;
assert(tag_ == LFSR_TAG_INLINED);
assert(id_ == 1);
@@ -2953,7 +2957,7 @@ code = '''
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
assert(memcmp(buffer, "1", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, "aac", 3,
lfsr_btree_dnamelookup(&lfs, &btree, 0, "aac", 3,
&id_, &tag_, &weight_, &data_) => 0;
assert(tag_ == LFSR_TAG_INLINED);
assert(id_ == 1);
@@ -2964,6 +2968,8 @@ code = '''
[cases.t2_btree_find_three]
in = 'lfs.c'
# true or false for if we should use dids vs names
defines.DID = [false, true]
code = '''
lfs_t lfs;
lfs_init(&lfs, cfg) => 0;
@@ -2979,10 +2985,10 @@ code = '''
lfsr_btree_t btree = LFSR_BTREE_NULL;
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1,
LFSR_DATA_BUF("0", 1)) => 0;
lfsr_btree_split(&lfs, &btree, 0, LFSR_DATA_BUF("aab", 3),
lfsr_btree_split(&lfs, &btree, 0, LFSR_DATA_DNAME(1*DID, "aab", 3),
LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("0", 1),
LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("1", 1)) => 0;
lfsr_btree_split(&lfs, &btree, 1, LFSR_DATA_BUF("aac", 3),
lfsr_btree_split(&lfs, &btree, 1, LFSR_DATA_DNAME(2*DID, "aac", 3),
LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("1", 1),
LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("2", 1)) => 0;
printf("btree: w%d 0x%x.%x\n",
@@ -2998,7 +3004,7 @@ code = '''
lfs_size_t weight_;
lfsr_data_t data_;
lfsr_btree_namelookup(&lfs, &btree, "aaa", 3,
lfsr_btree_dnamelookup(&lfs, &btree, 0*DID, "aaa", 3,
&id_, &tag_, &weight_, &data_) => 0;
assert(tag_ == LFSR_TAG_INLINED);
assert(id_ == 0);
@@ -3006,7 +3012,7 @@ code = '''
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
assert(memcmp(buffer, "0", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, "aab", 3,
lfsr_btree_dnamelookup(&lfs, &btree, 1*DID, "aab", 3,
&id_, &tag_, &weight_, &data_) => 0;
assert(tag_ == LFSR_TAG_INLINED);
assert(id_ == 1);
@@ -3014,7 +3020,7 @@ code = '''
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
assert(memcmp(buffer, "1", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, "aac", 3,
lfsr_btree_dnamelookup(&lfs, &btree, 2*DID, "aac", 3,
&id_, &tag_, &weight_, &data_) => 0;
assert(tag_ == LFSR_TAG_INLINED);
assert(id_ == 2);
@@ -3022,7 +3028,7 @@ code = '''
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
assert(memcmp(buffer, "2", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, "aad", 3,
lfsr_btree_dnamelookup(&lfs, &btree, 3*DID, "aad", 3,
&id_, &tag_, &weight_, &data_) => 0;
assert(tag_ == LFSR_TAG_INLINED);
assert(id_ == 2);
@@ -3032,6 +3038,8 @@ code = '''
'''
[cases.t2_btree_find_three_backwards]
# true or false for if we should use dids vs names
defines.DID = [false, true]
in = 'lfs.c'
code = '''
lfs_t lfs;
@@ -3048,10 +3056,10 @@ code = '''
lfsr_btree_t btree = LFSR_BTREE_NULL;
lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1,
LFSR_DATA_BUF("0", 1)) => 0;
lfsr_btree_split(&lfs, &btree, 0, LFSR_DATA_BUF("aac", 3),
lfsr_btree_split(&lfs, &btree, 0, LFSR_DATA_DNAME(2*DID, "aac", 3),
LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("1", 1),
LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("2", 1)) => 0;
lfsr_btree_split(&lfs, &btree, 0, LFSR_DATA_BUF("aab", 3),
lfsr_btree_split(&lfs, &btree, 0, LFSR_DATA_DNAME(1*DID, "aab", 3),
LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("0", 1),
LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("1", 1)) => 0;
printf("btree: w%d 0x%x.%x\n",
@@ -3067,7 +3075,7 @@ code = '''
lfs_size_t weight_;
lfsr_data_t data_;
lfsr_btree_namelookup(&lfs, &btree, "aaa", 3,
lfsr_btree_dnamelookup(&lfs, &btree, 0*DID, "aaa", 3,
&id_, &tag_, &weight_, &data_) => 0;
assert(tag_ == LFSR_TAG_INLINED);
assert(id_ == 0);
@@ -3075,7 +3083,7 @@ code = '''
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
assert(memcmp(buffer, "0", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, "aab", 3,
lfsr_btree_dnamelookup(&lfs, &btree, 1*DID, "aab", 3,
&id_, &tag_, &weight_, &data_) => 0;
assert(tag_ == LFSR_TAG_INLINED);
assert(id_ == 1);
@@ -3083,7 +3091,7 @@ code = '''
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
assert(memcmp(buffer, "1", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, "aac", 3,
lfsr_btree_dnamelookup(&lfs, &btree, 2*DID, "aac", 3,
&id_, &tag_, &weight_, &data_) => 0;
assert(tag_ == LFSR_TAG_INLINED);
assert(id_ == 2);
@@ -3091,7 +3099,7 @@ code = '''
lfsr_data_read(&lfs, data_, 0, buffer, 4) => 1;
assert(memcmp(buffer, "2", 1) == 0);
lfsr_btree_namelookup(&lfs, &btree, "aad", 3,
lfsr_btree_dnamelookup(&lfs, &btree, 3*DID, "aad", 3,
&id_, &tag_, &weight_, &data_) => 0;
assert(tag_ == LFSR_TAG_INLINED);
assert(id_ == 2);
@@ -3102,6 +3110,8 @@ code = '''
[cases.t2_btree_find]
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
# true or false for if we should use dids vs names
defines.DID = [false, true]
in = 'lfs.c'
code = '''
lfs_t lfs;
@@ -3125,7 +3135,8 @@ code = '''
char name[3] = {
alphas[(i/26/26) % 26], alphas[(i/26) % 26], alphas[i % 26]
};
int err = lfsr_btree_split(&lfs, &btree, i-1, LFSR_DATA_BUF(name, 3),
int err = lfsr_btree_split(&lfs, &btree, i-1,
LFSR_DATA_DNAME(i*DID, name, 3),
LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(&nums[(i-1) % 10], 1),
LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(&nums[(i-0) % 10], 1));
// ignore space issues
@@ -3153,7 +3164,7 @@ code = '''
alphas[(i/26/26) % 26], alphas[(i/26) % 26], alphas[i % 26]
};
lfsr_btree_namelookup(&lfs, &btree, name, 3,
lfsr_btree_dnamelookup(&lfs, &btree, i*DID, name, 3,
&id_, &tag_, &weight_, &data_) => 0;
assert(tag_ == LFSR_TAG_INLINED);
assert(id_ == i);
@@ -3227,7 +3238,8 @@ code = '''
}
// split btree
int err = lfsr_btree_split(&lfs, &btree, id, LFSR_DATA_BUF(name, 3),
int err = lfsr_btree_split(&lfs, &btree, id,
LFSR_DATA_DNAME(0, name, 3),
LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(&nums[i % 10], 1),
LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(&nums[i % 10], 1));
// ignore space issues
@@ -3268,7 +3280,7 @@ code = '''
lfs_size_t weight_;
lfsr_data_t data_;
for (lfs_size_t i = 0; i < sim_size; i++) {
lfsr_btree_namelookup(&lfs, &btree, sim_names[i], 3,
lfsr_btree_dnamelookup(&lfs, &btree, 0, sim_names[i], 3,
&id_, &tag_, &weight_, &data_) => 0;
assert(tag_ == LFSR_TAG_INLINED);
assert(id_ == i);
@@ -3287,6 +3299,8 @@ code = '''
[cases.t2_btree_find_sparse]
defines.N = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
defines.W = 5
# true or false for if we should use dids vs names
defines.DID = [false, true]
in = 'lfs.c'
code = '''
lfs_t lfs;
@@ -3311,7 +3325,7 @@ code = '''
alphas[(i/26/26) % 26], alphas[(i/26) % 26], alphas[i % 26]
};
int err = lfsr_btree_split(&lfs, &btree,
(i-1)*W+W-1, LFSR_DATA_BUF(name, 3),
(i-1)*W+W-1, LFSR_DATA_DNAME(i*DID, name, 3),
LFSR_TAG_INLINED, W, LFSR_DATA_BUF(&nums[(i-1) % 10], 1),
LFSR_TAG_INLINED, W, LFSR_DATA_BUF(&nums[(i-0) % 10], 1));
// ignore space issues
@@ -3339,7 +3353,7 @@ code = '''
alphas[(i/26/26) % 26], alphas[(i/26) % 26], alphas[i % 26]
};
lfsr_btree_namelookup(&lfs, &btree, name, 3,
lfsr_btree_dnamelookup(&lfs, &btree, i*DID, name, 3,
&id_, &tag_, &weight_, &data_) => 0;
assert(tag_ == LFSR_TAG_INLINED);
assert(id_ == i*W+W-1);
@@ -3427,7 +3441,8 @@ code = '''
// split btree
int err = lfsr_btree_split(&lfs, &btree,
weighted_id+sim_weights[id]-1, LFSR_DATA_BUF(name, 3),
weighted_id+sim_weights[id]-1,
LFSR_DATA_DNAME(0, name, 3),
LFSR_TAG_INLINED, weight1, LFSR_DATA_BUF(&nums[i % 10], 1),
LFSR_TAG_INLINED, weight2, LFSR_DATA_BUF(&nums[i % 10], 1));
// ignore space issues
@@ -3493,7 +3508,7 @@ code = '''
weighted_id += sim_weights[j];
}
lfsr_btree_namelookup(&lfs, &btree, sim_names[i], 3,
lfsr_btree_dnamelookup(&lfs, &btree, 0, sim_names[i], 3,
&id_, &tag_, &weight_, &data_) => 0;
assert(tag_ == LFSR_TAG_INLINED);
assert(id_ == weighted_id+sim_weights[i]-1);
@@ -3584,13 +3599,14 @@ code = '''
// split btree
lfs_size_t split_id;
lfsr_data_t split_data;
lfsr_btree_namelookup(&lfs, &btree, name, 3,
lfsr_btree_dnamelookup(&lfs, &btree, 0, name, 3,
&split_id, NULL, NULL, &split_data) => 0;
uint8_t split_buf[4];
lfsr_data_read(&lfs, split_data, 0, split_buf, 4) => 1;
if (split_id > id) {
int err = lfsr_btree_split(&lfs, &btree,
split_id, LFSR_DATA_BUF(sim_names[id+1], 3),
split_id,
LFSR_DATA_DNAME(0, sim_names[id+1], 3),
LFSR_TAG_INLINED, 1,
LFSR_DATA_BUF(&nums[i % 10], 1),
LFSR_TAG_INLINED, 1,
@@ -3602,7 +3618,7 @@ code = '''
assert(err == 0);
} else {
int err = lfsr_btree_split(&lfs, &btree,
split_id, LFSR_DATA_BUF(name, 3),
split_id, LFSR_DATA_DNAME(0, name, 3),
LFSR_TAG_INLINED, 1,
LFSR_DATA_BUF(split_buf, 1),
LFSR_TAG_INLINED, 1,
@@ -3680,7 +3696,7 @@ code = '''
lfs_size_t weight_;
lfsr_data_t data_;
for (lfs_size_t i = 0; i < sim_size; i++) {
lfsr_btree_namelookup(&lfs, &btree, sim_names[i], 3,
lfsr_btree_dnamelookup(&lfs, &btree, 0, sim_names[i], 3,
&id_, &tag_, &weight_, &data_) => 0;
assert(tag_ == LFSR_TAG_INLINED);
assert(id_ == i);
@@ -3787,14 +3803,14 @@ code = '''
lfs_size_t split_id;
lfs_size_t split_weight;
lfsr_data_t split_data;
lfsr_btree_namelookup(&lfs, &btree, name, 3,
lfsr_btree_dnamelookup(&lfs, &btree, 0, name, 3,
&split_id, NULL, &split_weight,
&split_data) => 0;
uint8_t split_buf[4];
lfsr_data_read(&lfs, split_data, 0, split_buf, 4) => 1;
if (split_id > weighted_id+sim_weights[id]-1) {
int err = lfsr_btree_split(&lfs, &btree,
split_id, LFSR_DATA_BUF(sim_names[id+1], 3),
split_id, LFSR_DATA_DNAME(0, sim_names[id+1], 3),
LFSR_TAG_INLINED, weight,
LFSR_DATA_BUF(&nums[i % 10], 1),
LFSR_TAG_INLINED, split_weight,
@@ -3806,7 +3822,7 @@ code = '''
assert(err == 0);
} else {
int err = lfsr_btree_split(&lfs, &btree,
split_id, LFSR_DATA_BUF(name, 3),
split_id, LFSR_DATA_DNAME(0, name, 3),
LFSR_TAG_INLINED, split_weight,
LFSR_DATA_BUF(split_buf, 1),
LFSR_TAG_INLINED, weight,
@@ -3912,7 +3928,7 @@ code = '''
weighted_id += sim_weights[j];
}
lfsr_btree_namelookup(&lfs, &btree, sim_names[i], 3,
lfsr_btree_dnamelookup(&lfs, &btree, 0, sim_names[i], 3,
&id_, &tag_, &weight_, &data_) => 0;
assert(tag_ == LFSR_TAG_INLINED);
assert(id_ == weighted_id+sim_weights[i]-1);