From 02ae6050de12ad1b490b93aab8202cef3e8cb292 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 22 Sep 2023 22:19:37 -0500 Subject: [PATCH] Changed lfsr_data_t internals, added LFSR_DATA_CAT The main purpose of this change is to introduce LFSR_DATA_CAT, a generalized way to concatenated various data references internally. As a side-effect lfsr_data_t has been completely restructured. Now, lfsr_data_t can be in one of 4 modes: If the size field's sign bit=0, the lfsr_data_t points in-device. A new, count field, determines the encoding: sign(size)=0, count=0 => inlined: .---+---+---+---. | size | |---+---+---+---| |c=0| inlined d | note inlined data is just enough to hold |---+ | one encoded leb128 | ata... | '---------------' sign(size)=1, count=1 => direct: .---+---+---+---. .---+---+---+---. | size | .>| data... | |---+---+---+---| | | . | |c=1| | | . . . |---+---+---+---| | . . . | direct ptr -----' . . '---------------' sign(size)=1, count>=2 => indirect: .---+---+---+---. .---+---+---+---. .---+---+---+---. | size | .>| size | .>| data... | |---+---+---+---| | |---+---+---+---| | | . | |c>1| | | |c=1| | | . . . |---+---+---+---| | |---+---+---+---| | . . . | indirect ptr ---' | direct ptr -----' . . '---------------' '---------------' .---+---+---+---. | size | .>| data... | |---+---+---+---| | | . | |c=1| | | . . . |---+---+---+---| | . . . | direct ptr -----' . . '---+---+---+---' | . | | . | . . . . . . . note only one indirect layer is allowed due to no recursion If the size field's sign bit=1, the lfsr_data_t points on-disk: sign(size)=0 => on-disk: .---+---+---+---. ..... | size | ..'' ''.. |---+---+---+---| : : : | block ------+->| ..:| |---+---+---+---| | |......( )::::::| | off -------' |:::' : | '---------------' :' : : ''.. :.'' ''''' My goal with this commit was to test the new implementation and see how it would impact code/RAM size before adopting it in the actual file handling code, and the results are... not great... code stack before: 24668 1840 after: 25552 (+3.5%) 1920 (+4.2%) I think most of the new cost comes from the now correct handling of read/cmp with concatentated datas, which previously would just assert. This change gives us LFSR_DATA_CAT, so I will be working with it for now, but this may be worth looking at again in the future. Maybe the correct handling of read/cmp should just be reverted to an assert... --- lfs.c | 411 ++++++++++++++++++++++++++++++++---------- lfs.h | 31 +++- lfs_util.h | 5 - tests/test_btree.toml | 311 ++++++++++++++++---------------- tests/test_mtree.toml | 28 +-- 5 files changed, 510 insertions(+), 276 deletions(-) diff --git a/lfs.c b/lfs.c index 84282b75..56bc8206 100644 --- a/lfs.c +++ b/lfs.c @@ -980,20 +980,23 @@ static lfs_ssize_t lfsr_bd_progtag(lfs_t *lfs, // use the sign bit to indicate on-disk vs in-device #define LFSR_DATA_ONDISK 0x80000000 -// the most common use is to pass a buffer with explicit size -#define LFSR_DATA(_buffer, _size) \ - ((lfsr_data_t){ \ - .u.buffer.size=_size, \ - .u.buffer.leb128=0, \ - .u.buffer.buffer=(const void*)(_buffer)}) - // LFSR_DATA_DATA just provides and escape hatch to pass raw datas // through the LFSR_ATTR macro #define LFSR_DATA_DATA(_data) (_data) -#define LFSR_DATA_NULL LFSR_DATA(NULL, 0) +#define LFSR_DATA_NULL LFSR_DATA_BUF(NULL, 0) -#define LFSR_DATA_BUF(_buffer, _size) LFSR_DATA(_buffer, _size) +#define LFSR_DATA_IMM(_buffer, _size) \ + lfsr_data_fromimm(_buffer, _size) + +#define LFSR_DATA_LEB128(_word) \ + lfsr_data_fromleb128(_word) + +#define LFSR_DATA_BUF(_buffer, _size) \ + ((lfsr_data_t){ \ + .u.direct.size=_size, \ + .u.direct.count=1, \ + .u.direct.buffer=(const void*)(_buffer)}) #define LFSR_DATA_DISK(_block, _off, _size) \ ((lfsr_data_t){ \ @@ -1001,17 +1004,11 @@ static lfs_ssize_t lfsr_bd_progtag(lfs_t *lfs, .u.disk.block=_block, \ .u.disk.off=_off}) -#define LFSR_DATA_LEB128(_leb) \ - ((lfsr_data_t){ \ - .u.buffer.size=lfs_sizeleb128(_leb), \ - .u.buffer.leb128=(0x80000000 | (_leb)), \ - .u.buffer.buffer=NULL}) - -#define LFSR_DATA_NAME(_did, _buffer, _size) \ - ((lfsr_data_t){ \ - .u.buffer.size=lfs_sizeleb128(_did) + (_size), \ - .u.buffer.leb128=(0x80000000 | (_did)), \ - .u.buffer.buffer=(const void*)(_buffer)}) +// these rely on temporary allocations which is a bit precarious... +#define LFSR_DATA_CAT(...) \ + lfsr_data_fromcat( \ + (const lfsr_data_t[]){__VA_ARGS__}, \ + sizeof((const lfsr_data_t[]){__VA_ARGS__}) / sizeof(lfsr_data_t)) // These aren't true runtime-typed datas, but allows some special cases to // bypass data encoding. External context is required to access these @@ -1019,16 +1016,16 @@ static lfs_ssize_t lfsr_bd_progtag(lfs_t *lfs, // a move of all attrs from an mdir entry #define LFSR_DATA_MOVE(_mdir) \ - ((lfsr_data_t){.u.buffer.buffer=(const void*)(const lfsr_mdir_t*){_mdir}}) + ((lfsr_data_t){.u.direct.buffer=(const void*)(const lfsr_mdir_t*){_mdir}}) // a grm update, note this is mutable! we may update the grm during // mdir commits #define LFSR_DATA_GRM(_grm) \ - ((lfsr_data_t){.u.buffer.buffer=(const void*)(lfsr_grm_t*){_grm}}) + ((lfsr_data_t){.u.direct.buffer=(const void*)(lfsr_grm_t*){_grm}}) // writing to an unrelated trunk in the rbyd #define LFSR_DATA_SHRUBATTRS(_rbyd, ...) \ - ((lfsr_data_t){.u.buffer.buffer=(const void*)&(const lfsr_shrubattrs_t){ \ + ((lfsr_data_t){.u.direct.buffer=(const void*)&(const lfsr_shrubattrs_t){ \ .rbyd=_rbyd, \ .attrs=(const lfsr_attr_t[]){__VA_ARGS__}, \ .attr_count=sizeof((const lfsr_attr_t[]){__VA_ARGS__}) \ @@ -1037,7 +1034,7 @@ static lfs_ssize_t lfsr_bd_progtag(lfs_t *lfs, // the reason for lazily encoding inlined trunks is because they can change // underneath us during mdir compaction, the horror #define LFSR_DATA_TRUNK(_rbyd) \ - ((lfsr_data_t){.u.buffer.buffer=(const void*)(lfsr_rbyd_t*){_rbyd}}) + ((lfsr_data_t){.u.direct.buffer=(const void*)(lfsr_rbyd_t*){_rbyd}}) static inline bool lfsr_data_ondisk(const lfsr_data_t *data) { return data->u.size & LFSR_DATA_ONDISK; @@ -1047,12 +1044,44 @@ static inline lfs_size_t lfsr_data_size(const lfsr_data_t *data) { return data->u.size & ~LFSR_DATA_ONDISK; } -static inline bool lfsr_data_hasleb128(const lfsr_data_t *data) { - return data->u.buffer.leb128; +// some data initializers just can't be macros, we at least make these inline +// so most of the internal logic gets elided +static inline lfsr_data_t lfsr_data_fromimm( + const void *buffer, lfs_size_t size) { + LFS_ASSERT(size <= 5); + + lfsr_data_t data; + memcpy(data.u.inlined.buf, buffer, size); + data.u.inlined.size = size; + data.u.inlined.count = 0; + return data; } -static inline int32_t lfsr_data_leb128(const lfsr_data_t *data) { - return data->u.buffer.leb128 & 0x7fffffff; +static inline lfsr_data_t lfsr_data_fromleb128(int32_t word) { + lfsr_data_t data; + lfs_ssize_t size = lfs_toleb128(word, data.u.inlined.buf, 5); + LFS_ASSERT(size >= 0); + LFS_ASSERT(size <= 5); + data.u.inlined.size = size; + data.u.inlined.count = 0; + return data; +} + +static inline lfsr_data_t lfsr_data_fromcat( + const lfsr_data_t *datas, lfs_size_t count) { + LFS_ASSERT(count >= 2); + LFS_ASSERT(count <= 255); + + // find total size + lfs_size_t size = 0; + for (uint8_t i = 0; i < count; i++) { + size += lfsr_data_size(&datas[i]); + } + + return (lfsr_data_t){ + .u.indirect.size=size, + .u.indirect.count=count, + .u.indirect.datas=datas}; } static void lfsr_data_add(lfsr_data_t *data, lfs_size_t off) { @@ -1060,14 +1089,26 @@ static void lfsr_data_add(lfsr_data_t *data, lfs_size_t off) { // limit our off to data range lfs_size_t off_ = lfs_min32(off, lfsr_data_size(&data_)); - // note both these paths are the same! though I'm not sure the compiler - // is able to notice this... + // on-disk? increment if (lfsr_data_ondisk(&data_)) { data_.u.disk.off += off_; data_.u.disk.size -= off_; + + // inlined? internal memmove + } else if (data_.u.indirect.count == 0) { + memmove(data_.u.inlined.buf, + data_.u.inlined.buf + off_, + data_.u.inlined.size - off_); + data_.u.inlined.size -= off_; + + // direct? increment + } else if (data_.u.indirect.count == 1) { + data_.u.direct.buffer += off_; + data_.u.direct.size -= off_; + + // indirect? just update size, more on how this works in lfsr_data_read } else { - data_.u.buffer.buffer += off_; - data_.u.buffer.size -= off_; + data_.u.indirect.size -= off_; } *data = data_; @@ -1077,25 +1118,91 @@ static void lfsr_data_add(lfsr_data_t *data, lfs_size_t off) { // lfsr_data_read* operations update the lfsr_data_t, effectively // consuming the data -static lfs_ssize_t lfsr_data_read(lfs_t *lfs, lfsr_data_t *data, + +static lfs_ssize_t lfsr_data_read_(lfs_t *lfs, const lfsr_data_t *data, void *buffer, lfs_size_t size) { // limit our size to data range - lfsr_data_t data_ = *data; - lfs_size_t d = lfs_min32(size, lfsr_data_size(&data_)); + lfs_size_t d = lfs_min32(size, lfsr_data_size(data)); - if (lfsr_data_ondisk(&data_)) { - int err = lfsr_bd_read(lfs, data_.u.disk.block, data_.u.disk.off, + // on-disk? + if (lfsr_data_ondisk(data)) { + int err = lfsr_bd_read(lfs, data->u.disk.block, data->u.disk.off, // note our hint includes the full data range - lfsr_data_size(&data_), + lfsr_data_size(data), buffer, d); if (err) { return err; } - } else { - // leb128 prefixes not supported here - LFS_ASSERT(!lfsr_data_hasleb128(&data_)); - memcpy(buffer, data_.u.buffer.buffer, d); + // inlined? + } else if (data->u.indirect.count == 0) { + memcpy(buffer, data->u.inlined.buf, d); + + // direct? + } else if (data->u.indirect.count == 1) { + memcpy(buffer, data->u.direct.buffer, d); + + // indirect? we shouldn't handle this here + } else { + LFS_UNREACHABLE(); + } + + return d; +} + +static lfs_size_t lfsr_data_indirectoff(const lfsr_data_t *data) { + LFS_ASSERT(!lfsr_data_ondisk(data)); + LFS_ASSERT(data->u.indirect.count >= 2); + + lfs_size_t indirect_size = 0; + for (uint8_t i = 0; i < data->u.indirect.count; i++) { + indirect_size += lfsr_data_size(&data->u.indirect.datas[i]); + } + + return indirect_size - data->u.indirect.size; +} + +static lfs_ssize_t lfsr_data_read(lfs_t *lfs, lfsr_data_t *data, + void *buffer, lfs_size_t size) { + // handle indirect data specially to avoid recursion + lfs_ssize_t d; + if (!lfsr_data_ondisk(data) && data->u.indirect.count >= 2) { + // Indirect data is a bit complicated because we don't want to modify + // the indirect datas themselves. It's tempting to just make them + // mutable, but this breaks the common pattern of using shallow copies + // of lfsr_data_t to cheaply track progress. + // + // The solution here is to use the redundant sizes to figure out what + // we've read so far. This risks O(n^2) performance, but we usually + // only have a couple indirect datas at most + // + lfs_size_t indirect_off = lfsr_data_indirectoff(data); + uint8_t *buffer_ = buffer; + d = 0; + for (uint8_t i = 0; i < data->u.indirect.count && size > 0; i++) { + // skip consumed data + lfsr_data_t indirect_data = data->u.indirect.datas[i]; + lfsr_data_add(&indirect_data, indirect_off); + indirect_off -= lfs_min32( + lfsr_data_size(&indirect_data), + indirect_off); + + // read unconsumed data + lfs_ssize_t d_ = lfsr_data_read_(lfs, &indirect_data, + buffer_, size); + if (d_ < 0) { + return d_; + } + + buffer_ += d_; + size -= d_; + d += d_; + } + } else { + d = lfsr_data_read_(lfs, data, buffer, size); + if (d < 0) { + return d; + } } lfsr_data_add(data, d); @@ -1141,28 +1248,40 @@ static int lfsr_data_readleb128(lfs_t *lfs, lfsr_data_t *data, return 0; } -static lfs_scmp_t lfsr_data_cmp(lfs_t *lfs, const lfsr_data_t *data, +static lfs_scmp_t lfsr_data_cmp_(lfs_t *lfs, const lfsr_data_t *data, const void *buffer, lfs_size_t size) { - // limit our off/size to data range + // limit our size to data range lfs_size_t d = lfs_min32(size, lfsr_data_size(data)); - // compare our data + // on-disk? if (lfsr_data_ondisk(data)) { int cmp = lfsr_bd_cmp(lfs, data->u.disk.block, data->u.disk.off, 0, buffer, d); if (cmp != LFS_CMP_EQ) { return cmp; } - } else { - // leb128 prefixes not supported here - LFS_ASSERT(!lfsr_data_hasleb128(data)); - int cmp = memcmp(data->u.buffer.buffer, buffer, d); + // inlined? + } else if (data->u.indirect.count == 0) { + int cmp = memcmp(data->u.inlined.buf, buffer, d); if (cmp < 0) { return LFS_CMP_LT; } else if (cmp > 0) { return LFS_CMP_GT; } + + // direct? + } else if (data->u.indirect.count == 1) { + int cmp = memcmp(data->u.direct.buffer, buffer, d); + if (cmp < 0) { + return LFS_CMP_LT; + } else if (cmp > 0) { + return LFS_CMP_GT; + } + + // indirect? we shouldn't handle this here + } else { + LFS_UNREACHABLE(); } // if data is equal, check for size mismatch @@ -1175,6 +1294,52 @@ static lfs_scmp_t lfsr_data_cmp(lfs_t *lfs, const lfsr_data_t *data, } } +static lfs_scmp_t lfsr_data_cmp(lfs_t *lfs, const lfsr_data_t *data, + const void *buffer, lfs_size_t size) { + // handle indirect data specially to avoid recursion + if (!lfsr_data_ondisk(data) && data->u.indirect.count >= 2) { + // Indirect data is a bit complicated because we don't want to modify + // the indirect datas themselves. It's tempting to just make them + // mutable, but this breaks the common pattern of using shallow copies + // of lfsr_data_t to cheaply track progress. + // + // The solution here is to use the redundant sizes to figure out what + // we've read so far. This risks O(n^2) performance, but we usually + // only have a couple indirect datas at most + // + lfs_size_t indirect_off = lfsr_data_indirectoff(data); + const uint8_t *buffer_ = buffer; + for (uint8_t i = 0; i < data->u.indirect.count && size > 0; i++) { + // skip consumed data + lfsr_data_t indirect_data = data->u.indirect.datas[i]; + lfsr_data_add(&indirect_data, indirect_off); + indirect_off -= lfs_min32( + lfsr_data_size(&indirect_data), + indirect_off); + + // compare against unconsumed data + lfs_size_t d = lfs_min32(size, lfsr_data_size(&indirect_data)); + lfs_scmp_t cmp = lfsr_data_cmp_(lfs, &indirect_data, buffer_, d); + if (cmp != LFS_CMP_EQ) { + return cmp; + } + + buffer_ += d; + size -= d; + } + + // data is equal, the only remaining condition to check for is size + // remaining + if (size > 0) { + return LFS_CMP_LT; + } else { + return LFS_CMP_EQ; + } + } else { + return lfsr_data_cmp_(lfs, data, buffer, size); + } +} + static lfs_scmp_t lfsr_data_namecmp(lfs_t *lfs, const lfsr_data_t *data, lfsr_did_t did, const char *name, lfs_size_t name_size) { // first compare the did @@ -1191,13 +1356,14 @@ static lfs_scmp_t lfsr_data_namecmp(lfs_t *lfs, const lfsr_data_t *data, return LFS_CMP_GT; } - // next compare the actual name + // then compare the actual name return lfsr_data_cmp(lfs, &data_, name, name_size); } -static int lfsr_bd_progdata(lfs_t *lfs, +static int lfsr_bd_progdata_(lfs_t *lfs, lfs_block_t block, lfs_size_t off, lfsr_data_t data, uint32_t *cksum_) { + // on-disk? if (lfsr_data_ondisk(&data)) { // TODO byte-level copies have been a pain point, works for prototyping // but can this be better? configurable? leverage @@ -1219,32 +1385,66 @@ static int lfsr_bd_progdata(lfs_t *lfs, } } - } else { - // This is a bit of a hack, but lfsr_data_t can inject a single leb128 - // into lfsr_bd_progdata. This is needed for directory-id prefixes, - // but can also be used for programming single leb128s cheaply. - if (lfsr_data_hasleb128(&data)) { - uint8_t leb_buf[5]; - lfs_ssize_t leb_dsize = lfs_toleb128( - lfsr_data_leb128(&data), - leb_buf, 5); - if (leb_dsize < 0) { - return leb_dsize; - } + // inlined? + } else if (data.u.indirect.count == 0) { + int err = lfsr_bd_prog(lfs, block, off, + data.u.inlined.buf, data.u.inlined.size, + cksum_); + if (err) { + return err; + } - int err = lfsr_bd_prog(lfs, block, off, - leb_buf, leb_dsize, cksum_); + // direct? + } else if (data.u.indirect.count == 1) { + int err = lfsr_bd_prog(lfs, block, off, + data.u.direct.buffer, data.u.direct.size, + cksum_); + if (err) { + return err; + } + + // indirect? we shouldn't handle this here + } else { + LFS_UNREACHABLE(); + } + + return 0; +} + +static int lfsr_bd_progdata(lfs_t *lfs, + lfs_block_t block, lfs_size_t off, lfsr_data_t data, + uint32_t *cksum_) { + // handle indirect data specially to avoid recursion + if (!lfsr_data_ondisk(&data) && data.u.indirect.count >= 2) { + // Indirect data is a bit complicated because we don't want to modify + // the indirect datas themselves. It's tempting to just make them + // mutable, but this breaks the common pattern of using shallow copies + // of lfsr_data_t to cheaply track progress. + // + // The solution here is to use the redundant sizes to figure out what + // we've read so far. This risks O(n^2) performance, but we usually + // only have a couple indirect datas at most + // + lfs_size_t indirect_off = lfsr_data_indirectoff(&data); + for (uint8_t i = 0; i < data.u.indirect.count; i++) { + // skip consumed data + lfsr_data_t indirect_data = data.u.indirect.datas[i]; + lfsr_data_add(&indirect_data, indirect_off); + indirect_off -= lfs_min32( + lfsr_data_size(&indirect_data), + indirect_off); + + // prog unconsumed data + int err = lfsr_bd_progdata_(lfs, block, off, indirect_data, + cksum_); if (err) { return err; } - off += leb_dsize; - LFS_ASSERT(data.u.buffer.size >= leb_dsize); - data.u.buffer.size -= leb_dsize; + off += lfsr_data_size(&indirect_data); } - - int err = lfsr_bd_prog(lfs, block, off, - data.u.buffer.buffer, lfsr_data_size(&data), + } else { + int err = lfsr_bd_progdata_(lfs, block, off, data, cksum_); if (err) { return err; @@ -3132,7 +3332,7 @@ static int lfsr_rbyd_appendgdelta(lfs_t *lfs, lfsr_rbyd_t *rbyd) { } } - err = lfsr_grm_xor(lfs, grm_buf, LFSR_DATA( + err = lfsr_grm_xor(lfs, grm_buf, LFSR_DATA_BUF( &lfs->dgrm, LFSR_GRM_DSIZE)); if (err) { return err; @@ -3143,7 +3343,7 @@ static int lfsr_rbyd_appendgdelta(lfs_t *lfs, lfsr_rbyd_t *rbyd) { err = lfsr_rbyd_appendattr(lfs, rbyd, -1, // opportunistically remove this tag if delta is all zero (size == 0 ? LFSR_TAG_RM(GRM) : LFSR_TAG_GRM), 0, - LFSR_DATA(grm_buf, size)); + LFSR_DATA_BUF(grm_buf, size)); if (err) { return err; } @@ -3521,7 +3721,7 @@ static int lfsr_btree_lookupnext_(lfs_t *lfs, *weight_ = lfsr_btree_weight(btree); } if (data_) { - *data_ = LFSR_DATA(btree->u.inlined.buf, btree->u.inlined.size); + *data_ = LFSR_DATA_BUF(btree->u.inlined.buf, btree->u.inlined.size); } return 0; } @@ -4314,7 +4514,7 @@ static int lfsr_btree_namelookup(lfs_t *lfs, const lfsr_btree_t *btree, *weight_ = lfsr_btree_weight(btree); } if (data_) { - *data_ = LFSR_DATA(btree->u.inlined.buf, btree->u.inlined.size); + *data_ = LFSR_DATA_BUF(btree->u.inlined.buf, btree->u.inlined.size); } return 0; } @@ -4416,7 +4616,9 @@ static int lfsr_btree_traversal_next(lfs_t *lfs, const lfsr_btree_t *btree, *weight_ = lfsr_btree_weight(btree); } if (data_) { - *data_ = LFSR_DATA(btree->u.inlined.buf, btree->u.inlined.size); + *data_ = LFSR_DATA_BUF( + btree->u.inlined.buf, + btree->u.inlined.size); } return 0; } @@ -4440,7 +4642,9 @@ static int lfsr_btree_traversal_next(lfs_t *lfs, const lfsr_btree_t *btree, } if (data_) { // note btrees are returned decoded - *data_ = LFSR_DATA(&traversal->branch, sizeof(lfsr_rbyd_t)); + *data_ = LFSR_DATA_BUF( + &traversal->branch, + sizeof(lfsr_rbyd_t)); } return 0; } @@ -4499,7 +4703,9 @@ static int lfsr_btree_traversal_next(lfs_t *lfs, const lfsr_btree_t *btree, } if (data_) { // note btrees are returned decoded - *data_ = LFSR_DATA(&traversal->branch, sizeof(lfsr_rbyd_t)); + *data_ = LFSR_DATA_BUF( + &traversal->branch, + sizeof(lfsr_rbyd_t)); } return 0; } @@ -5005,7 +5211,7 @@ static int lfsr_mdir_commit__(lfs_t *lfs, lfsr_mdir_t *mdir, // weighted moves are not supported LFS_ASSERT(attrs[i].delta == 0); const lfsr_mdir_t *mdir__ - = (const lfsr_mdir_t*)attrs[i].data.u.buffer.buffer; + = (const lfsr_mdir_t*)attrs[i].data.u.direct.buffer; // skip the name tag, this is always replaced by upper layers lfsr_tag_t tag = LFSR_TAG_STRUCT-1; @@ -5035,7 +5241,7 @@ static int lfsr_mdir_commit__(lfs_t *lfs, lfsr_mdir_t *mdir, } else if (attrs[i].tag == LFSR_TAG_SHRUBATTRS) { const lfsr_shrubattrs_t *shrubattrs = (const lfsr_shrubattrs_t*) - attrs[i].data.u.buffer.buffer; + attrs[i].data.u.direct.buffer; // swap out our trunk/weight temporarily, note we're operating // on a copy so if this fails not _too_ many things will get @@ -5072,7 +5278,7 @@ static int lfsr_mdir_commit__(lfs_t *lfs, lfsr_mdir_t *mdir, // // TODO should we preserve mode for all of these? } else if (lfsr_tag_key(attrs[i].tag) == LFSR_TAG_SHRUBTRUNK) { - lfsr_rbyd_t *rbyd = (lfsr_rbyd_t*)attrs[i].data.u.buffer.buffer; + lfsr_rbyd_t *rbyd = (lfsr_rbyd_t*)attrs[i].data.u.direct.buffer; uint8_t trunk_buf[LFSR_TRUNK_DSIZE]; int err = lfsr_rbyd_appendattr(lfs, &mdir_.u.rbyd, @@ -5435,12 +5641,12 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, for (lfs_size_t i = 0; i < attr_count; i++) { if (attrs[i].tag == LFSR_TAG_GRM) { // encode to disk - lfsr_grm_t *grm = (lfsr_grm_t*)attrs[i].data.u.buffer.buffer; + lfsr_grm_t *grm = (lfsr_grm_t*)attrs[i].data.u.direct.buffer; lfsr_data_fromgrm(grm, lfs->dgrm); // xor with our current gstate to find our initial gdelta int err = lfsr_grm_xor(lfs, lfs->dgrm, - LFSR_DATA(lfs->ggrm, LFSR_GRM_DSIZE)); + LFSR_DATA_BUF(lfs->ggrm, LFSR_GRM_DSIZE)); if (err) { return err; } @@ -5687,7 +5893,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, // // gd' = gd xor (grm' xor grm) // - lfsr_grm_t *grm = (lfsr_grm_t*)attrs[i].data.u.buffer.buffer; + lfsr_grm_t *grm = (lfsr_grm_t*)attrs[i].data.u.direct.buffer; uint8_t grm_buf[LFSR_GRM_DSIZE]; err = lfsr_grm_xor(lfs, lfs->dgrm, lfsr_data_fromgrm(grm, grm_buf)); @@ -5862,7 +6068,7 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, for (lfs_size_t i = 0; i < attr_count; i++) { // update gstate if (attrs[i].tag == LFSR_TAG_GRM) { - lfs->grm = *(lfsr_grm_t*)attrs[i].data.u.buffer.buffer; + lfs->grm = *(lfsr_grm_t*)attrs[i].data.u.direct.buffer; // keep track of the exact encoding on-disk lfsr_data_fromgrm(&lfs->grm, lfs->ggrm); @@ -6284,7 +6490,7 @@ static int lfsr_mtree_traversal_next(lfs_t *lfs, *tag_ = LFSR_TAG_MDIR; } if (data_) { - *data_ = LFSR_DATA(&traversal->mdir, sizeof(lfsr_mdir_t)); + *data_ = LFSR_DATA_BUF(&traversal->mdir, sizeof(lfsr_mdir_t)); } return 0; @@ -6348,7 +6554,7 @@ static int lfsr_mtree_traversal_next(lfs_t *lfs, *tag_ = LFSR_TAG_MDIR; } if (data_) { - *data_ = LFSR_DATA(&traversal->mdir, sizeof(lfsr_mdir_t)); + *data_ = LFSR_DATA_BUF(&traversal->mdir, sizeof(lfsr_mdir_t)); } return 0; @@ -6398,7 +6604,7 @@ static int lfsr_mtree_traversal_next(lfs_t *lfs, // validate our btree nodes if requested, this just means we need // to do a full rbyd fetch and make sure the checksums match if (traversal->flags & LFSR_MTREE_TRAVERSAL_VALIDATE) { - lfsr_rbyd_t *branch = (lfsr_rbyd_t*)data.u.buffer.buffer; + lfsr_rbyd_t *branch = (lfsr_rbyd_t*)data.u.direct.buffer; lfsr_rbyd_t branch_; err = lfsr_rbyd_fetch(lfs, &branch_, branch->block, branch->trunk); @@ -6468,7 +6674,7 @@ static int lfsr_mtree_traversal_next(lfs_t *lfs, *tag_ = tag; } if (data_) { - *data_ = LFSR_DATA(&traversal->mdir, sizeof(lfsr_mdir_t)); + *data_ = LFSR_DATA_BUF(&traversal->mdir, sizeof(lfsr_mdir_t)); } return 0; @@ -6594,7 +6800,7 @@ static int lfsr_mountinited(lfs_t *lfs) { continue; } - lfsr_mdir_t *mdir = (lfsr_mdir_t*)data.u.buffer.buffer; + lfsr_mdir_t *mdir = (lfsr_mdir_t*)data.u.direct.buffer; // found an mroot? if (mdir->mid == -1) { // has magic string? @@ -6943,7 +7149,7 @@ static int lfsr_mountinited(lfs_t *lfs) { memcpy(lfs->ggrm, lfs->dgrm, LFSR_GRM_DSIZE); // decode grm so we can report any removed files as missing - int err = lfsr_data_readgrm(lfs, &LFSR_DATA(lfs->ggrm, LFSR_GRM_DSIZE), + int err = lfsr_data_readgrm(lfs, &LFSR_DATA_BUF(lfs->ggrm, LFSR_GRM_DSIZE), &lfs->grm); if (err) { return err; @@ -6992,7 +7198,7 @@ static int lfsr_formatinited(lfs_t *lfs) { // - the root's bookmark tag, which reserves did = 0 for the root err = lfsr_rbyd_commit(lfs, &rbyd, LFSR_ATTRS( LFSR_ATTR(-1, MAGIC, 0, BUF("littlefs", 8)), - LFSR_ATTR(-1, VERSION, 0, BUF(((const uint8_t[2]){ + LFSR_ATTR(-1, VERSION, 0, IMM(((const uint8_t[2]){ LFS_DISK_VERSION_MAJOR, LFS_DISK_VERSION_MINOR}), 2)), LFSR_ATTR(-1, BLOCKLIMIT, 0, LEB128(lfs->cfg->block_size-1)), @@ -7000,7 +7206,7 @@ static int lfsr_formatinited(lfs_t *lfs) { LFSR_ATTR(-1, MLEAFLIMIT, 0, LEB128(lfsr_mleafweight(lfs)-1)), LFSR_ATTR(-1, SIZELIMIT, 0, LEB128(0x7fffffff)), LFSR_ATTR(-1, NAMELIMIT, 0, LEB128(0xff)), - LFSR_ATTR(0, BOOKMARK, +1, NAME(0, NULL, 0)))); + LFSR_ATTR(0, BOOKMARK, +1, LEB128(0)))); if (err) { return err; } @@ -7161,12 +7367,12 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { // mark any blocks we see at in-use, including any btree/mdir blocks if (tag == LFSR_TAG_MDIR) { - lfsr_mdir_t *mdir = (lfsr_mdir_t*)data.u.buffer.buffer; + lfsr_mdir_t *mdir = (lfsr_mdir_t*)data.u.direct.buffer; lfs_alloc_setinuse(lfs, mdir->u.m.blocks[1]); lfs_alloc_setinuse(lfs, mdir->u.m.blocks[0]); } else if (tag == LFSR_TAG_BTREE) { - lfsr_rbyd_t *branch = (lfsr_rbyd_t*)data.u.buffer.buffer; + lfsr_rbyd_t *branch = (lfsr_rbyd_t*)data.u.direct.buffer; lfs_alloc_setinuse(lfs, branch->block); } } @@ -7344,7 +7550,10 @@ int lfsr_mkdir(lfs_t *lfs, const char *path) { // commit our new directory into our parent, creating a grm to self-remove // in case of powerloss err = lfsr_mdir_commit(lfs, &mdir, LFSR_ATTRS( - LFSR_ATTR(mdir.mid, DIR, +1, NAME(did, name, name_size)), + LFSR_ATTR(mdir.mid, + DIR, +1, CAT( + LFSR_DATA_LEB128(did), + LFSR_DATA_BUF(name, name_size))), LFSR_ATTR(mdir.mid, DID, 0, LEB128(did_)), LFSR_ATTR(-1, GRM, 0, GRM(&((lfsr_grm_t){{mdir.mid, -1}}))))); if (err) { @@ -7356,7 +7565,7 @@ int lfsr_mkdir(lfs_t *lfs, const char *path) { // commit our bookmark and zero the grm, the bookmark tag is an empty // entry that marks our did as allocated err = lfsr_mdir_commit(lfs, &bookmark.mdir, LFSR_ATTRS( - LFSR_ATTR(bookmark.mdir.mid, BOOKMARK, +1, NAME(did_, NULL, 0)), + LFSR_ATTR(bookmark.mdir.mid, BOOKMARK, +1, LEB128(did_)), LFSR_ATTR(-1, GRM, 0, GRM(&((lfsr_grm_t){{-1, -1}}))))); if (err) { return err; @@ -7578,7 +7787,9 @@ int lfsr_rename(lfs_t *lfs, const char *old_path, const char *new_path) { ? LFSR_ATTR(new_mdir.mid, RM, -1, NULL) : LFSR_ATTR_NOOP), LFSR_ATTR(new_mdir.mid, - TAG(old_tag), +1, NAME(new_did, new_name, new_name_size)), + TAG(old_tag), +1, CAT( + LFSR_DATA_LEB128(new_did), + LFSR_DATA_BUF(new_name, new_name_size))), LFSR_ATTR(new_mdir.mid, MOVE, 0, MOVE(&old_mdir)), LFSR_ATTR(-1, GRM, 0, GRM(&grm)))); if (err) { @@ -7941,7 +8152,9 @@ int lfsr_file_opencfg(lfs_t *lfs, lfsr_file_t *file, // TODO or is it? ;) err = lfsr_mdir_commit(lfs, &file->m.mdir, LFSR_ATTRS( LFSR_ATTR(file->m.mdir.mid, - REG, +1, NAME(did, name, name_size)))); + REG, +1, CAT( + LFSR_DATA_LEB128(did), + LFSR_DATA_BUF(name, name_size))))); if (err) { return err; } diff --git a/lfs.h b/lfs.h index 89d09bbe..1eadf93c 100644 --- a/lfs.h +++ b/lfs.h @@ -432,19 +432,36 @@ typedef struct lfsr_data { // After removing the sign bit, the size always encodes the resulting // size on-disk. // + // After this the count field indicates the in-device representation, + // which has a few forms: + // - count == 0 => data inlined in data struct + // - count == 1 => direct pointer to data + // - count >= 2 => indirect pointer to array of datas + // + // The indirect pointer can point to inlined/direct datas or even + // on-disk datas, but not more indirect datas as that would require + // recursion. + // lfs_ssize_t size; struct { lfs_ssize_t size; - // This leb128 field is a bit of a hack that allows a single leb128 - // to be injected into lfsr_bd_progdata. Outside of - // lfsr_bd_progdata, this field is invalid! - int32_t leb128; - const uint8_t *buffer; - } buffer; + uint8_t count; + uint8_t buf[5]; + } inlined; + struct { + lfs_ssize_t size; + uint8_t count; + const uint8_t *buffer; + } direct; + struct { + lfs_ssize_t size; + uint8_t count; + const struct lfsr_data *datas; + } indirect; struct { lfs_ssize_t size; - lfs_size_t off; lfs_block_t block; + lfs_size_t off; } disk; } u; } lfsr_data_t; diff --git a/lfs_util.h b/lfs_util.h index 2ca9c695..555569e0 100644 --- a/lfs_util.h +++ b/lfs_util.h @@ -344,11 +344,6 @@ ssize_t lfs_toleb128(int32_t word, void *buffer, size_t size); ssize_t lfs_fromleb128(int32_t *word, const void *buffer, size_t size); -static inline size_t lfs_sizeleb128(int32_t word) { - // this is the size of the leb128 after encoding - return (lfs_nlog2(word+1)+7-1) / 7; -} - // Calculate CRC-32 with polynomial = 0x04c11db7 diff --git a/tests/test_btree.toml b/tests/test_btree.toml index 5d5620ba..fba7c733 100644 --- a/tests/test_btree.toml +++ b/tests/test_btree.toml @@ -147,7 +147,7 @@ code = ''' // create a single-entry tree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("a", 1)) => 0; + LFSR_DATA_BUF("a", 1)) => 0; printf("btree: w%d 0x%x.%x\n", btree.u.rbyd.weight, btree.u.rbyd.block, @@ -186,9 +186,9 @@ code = ''' // create a two-entry tree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("a", 1)) => 0; + LFSR_DATA_BUF("a", 1)) => 0; lfsr_btree_push(&lfs, &btree, 1, LFSR_TAG_INLINED, 1, - LFSR_DATA("b", 1)) => 0; + LFSR_DATA_BUF("b", 1)) => 0; printf("btree: w%d 0x%x.%x\n", btree.u.rbyd.weight, btree.u.rbyd.block, @@ -232,9 +232,9 @@ code = ''' // create a two-entry tree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("b", 1)) => 0; + LFSR_DATA_BUF("b", 1)) => 0; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("a", 1)) => 0; + LFSR_DATA_BUF("a", 1)) => 0; printf("btree: w%d 0x%x.%x\n", btree.u.rbyd.weight, btree.u.rbyd.block, @@ -279,11 +279,11 @@ code = ''' // create a two-entry tree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("a", 1)) => 0; + LFSR_DATA_BUF("a", 1)) => 0; lfsr_btree_push(&lfs, &btree, 1, LFSR_TAG_INLINED, 1, - LFSR_DATA("b", 1)) => 0; + LFSR_DATA_BUF("b", 1)) => 0; lfsr_btree_push(&lfs, &btree, 2, LFSR_TAG_INLINED, 1, - LFSR_DATA("c", 1)) => 0; + LFSR_DATA_BUF("c", 1)) => 0; printf("btree: w%d 0x%x.%x\n", btree.u.rbyd.weight, btree.u.rbyd.block, @@ -333,11 +333,11 @@ code = ''' // create a two-entry tree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("c", 1)) => 0; + LFSR_DATA_BUF("c", 1)) => 0; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("b", 1)) => 0; + LFSR_DATA_BUF("b", 1)) => 0; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("a", 1)) => 0; + LFSR_DATA_BUF("a", 1)) => 0; printf("btree: w%d 0x%x.%x\n", btree.u.rbyd.weight, btree.u.rbyd.block, @@ -393,7 +393,7 @@ code = ''' lfs_size_t n = 0; for (lfs_size_t i = 0; i < N; i++) { int err = lfsr_btree_push(&lfs, &btree, i, LFSR_TAG_INLINED, 1, - LFSR_DATA(&alphas[i % 26], 1)); + LFSR_DATA_BUF(&alphas[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -445,7 +445,7 @@ code = ''' lfs_size_t n = 0; for (lfs_size_t i = 0; i < N; i++) { int err = lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA(&alphas[(N-1-i) % 26], 1)); + LFSR_DATA_BUF(&alphas[(N-1-i) % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -512,7 +512,7 @@ code = ''' // add to btree int err = lfsr_btree_push(&lfs, &btree, bid, LFSR_TAG_INLINED, 1, - LFSR_DATA(&alphas[i % 26], 1)); + LFSR_DATA_BUF(&alphas[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -583,7 +583,7 @@ code = ''' lfs_size_t n = 0; for (lfs_size_t i = 0; i < N; i++) { int err = lfsr_btree_push(&lfs, &btree, i*W, LFSR_TAG_INLINED, W, - LFSR_DATA(&alphas[i % 26], 1)); + LFSR_DATA_BUF(&alphas[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -678,7 +678,7 @@ code = ''' // add to btree int err = lfsr_btree_push(&lfs, &btree, weighted_bid, LFSR_TAG_INLINED, weight, - LFSR_DATA(&alphas[i % 26], 1)); + LFSR_DATA_BUF(&alphas[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -790,10 +790,10 @@ code = ''' // create a single-entry tree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("a", 1)) => 0; + LFSR_DATA_BUF("a", 1)) => 0; // update the tree lfsr_btree_set(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("A", 1)) => 0; + LFSR_DATA_BUF("A", 1)) => 0; printf("btree: w%d 0x%x.%x\n", btree.u.rbyd.weight, btree.u.rbyd.block, @@ -831,14 +831,14 @@ code = ''' // create a two-entry tree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("a", 1)) => 0; + LFSR_DATA_BUF("a", 1)) => 0; lfsr_btree_push(&lfs, &btree, 1, LFSR_TAG_INLINED, 1, - LFSR_DATA("b", 1)) => 0; + LFSR_DATA_BUF("b", 1)) => 0; // update the tree lfsr_btree_set(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("A", 1)) => 0; + LFSR_DATA_BUF("A", 1)) => 0; lfsr_btree_set(&lfs, &btree, 1, LFSR_TAG_INLINED, 1, - LFSR_DATA("B", 1)) => 0; + LFSR_DATA_BUF("B", 1)) => 0; printf("btree: w%d 0x%x.%x\n", btree.u.rbyd.weight, btree.u.rbyd.block, @@ -882,18 +882,18 @@ code = ''' // create a two-entry tree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("a", 1)) => 0; + LFSR_DATA_BUF("a", 1)) => 0; lfsr_btree_push(&lfs, &btree, 1, LFSR_TAG_INLINED, 1, - LFSR_DATA("b", 1)) => 0; + LFSR_DATA_BUF("b", 1)) => 0; lfsr_btree_push(&lfs, &btree, 2, LFSR_TAG_INLINED, 1, - LFSR_DATA("c", 1)) => 0; + LFSR_DATA_BUF("c", 1)) => 0; // update the tree lfsr_btree_set(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("A", 1)) => 0; + LFSR_DATA_BUF("A", 1)) => 0; lfsr_btree_set(&lfs, &btree, 1, LFSR_TAG_INLINED, 1, - LFSR_DATA("B", 1)) => 0; + LFSR_DATA_BUF("B", 1)) => 0; lfsr_btree_set(&lfs, &btree, 2, LFSR_TAG_INLINED, 1, - LFSR_DATA("C", 1)) => 0; + LFSR_DATA_BUF("C", 1)) => 0; printf("btree: w%d 0x%x.%x\n", btree.u.rbyd.weight, btree.u.rbyd.block, @@ -947,7 +947,7 @@ code = ''' const char *uppers = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; for (lfs_size_t i = 0; i < N; i++) { int err = lfsr_btree_push(&lfs, &btree, i, LFSR_TAG_INLINED, 1, - LFSR_DATA(&alphas[i % 26], 1)); + LFSR_DATA_BUF(&alphas[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { printf("btree: w%d 0x%x.%x\n", @@ -961,7 +961,7 @@ code = ''' // update the tree for (lfs_size_t i = 0; i < N; i++) { int err = lfsr_btree_set(&lfs, &btree, i, LFSR_TAG_INLINED, 1, - LFSR_DATA(&uppers[i % 26], 1)); + LFSR_DATA_BUF(&uppers[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { printf("btree: w%d 0x%x.%x\n", @@ -1019,7 +1019,7 @@ code = ''' lfsr_btree_t btree = LFSR_BTREE_NULL; for (lfs_size_t i = 0; i < N; i++) { int err = lfsr_btree_push(&lfs, &btree, i, LFSR_TAG_INLINED, 1, - LFSR_DATA(&alphas[i % 26], 1)); + LFSR_DATA_BUF(&alphas[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { printf("btree: w%d 0x%x.%x\n", @@ -1047,7 +1047,7 @@ code = ''' // update btree int err = lfsr_btree_set(&lfs, &btree, bid, LFSR_TAG_INLINED, 1, - LFSR_DATA(&uppers[i % 26], 1)); + LFSR_DATA_BUF(&uppers[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -1115,7 +1115,7 @@ code = ''' const char *uppers = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; for (lfs_size_t i = 0; i < N; i++) { int err = lfsr_btree_push(&lfs, &btree, i*W, LFSR_TAG_INLINED, W, - LFSR_DATA(&alphas[i % 26], 1)); + LFSR_DATA_BUF(&alphas[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { printf("btree: w%d 0x%x.%x\n", @@ -1129,7 +1129,7 @@ code = ''' // update the tree for (lfs_size_t i = 0; i < N; i++) { int err = lfsr_btree_set(&lfs, &btree, i*W+W-1, LFSR_TAG_INLINED, W, - LFSR_DATA(&uppers[i % 26], 1)); + LFSR_DATA_BUF(&uppers[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { printf("btree: w%d 0x%x.%x\n", @@ -1203,7 +1203,7 @@ code = ''' lfsr_btree_t btree = LFSR_BTREE_NULL; for (lfs_size_t i = 0; i < N; i++) { int err = lfsr_btree_push(&lfs, &btree, i*W, LFSR_TAG_INLINED, W, - LFSR_DATA(&alphas[i % 26], 1)); + LFSR_DATA_BUF(&alphas[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { printf("btree: w%d 0x%x.%x\n", @@ -1242,7 +1242,7 @@ code = ''' // update btree int err = lfsr_btree_set(&lfs, &btree, weighted_bid+sim_weights[bid]-1, LFSR_TAG_INLINED, weight, - LFSR_DATA(&uppers[i % 26], 1)); + LFSR_DATA_BUF(&uppers[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -1351,7 +1351,7 @@ code = ''' // create a single-entry tree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("a", 1)) => 0; + LFSR_DATA_BUF("a", 1)) => 0; // pop! lfsr_btree_pop(&lfs, &btree, 0) => 0; printf("btree: w%d 0x%x.%x\n", @@ -1370,7 +1370,7 @@ code = ''' // try to putting it back to see if things still work lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("A", 1)) => 0; + LFSR_DATA_BUF("A", 1)) => 0; printf("btree: w%d 0x%x.%x\n", btree.u.rbyd.weight, btree.u.rbyd.block, @@ -1404,9 +1404,9 @@ code = ''' // create a single-entry tree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("a", 1)) => 0; + LFSR_DATA_BUF("a", 1)) => 0; lfsr_btree_push(&lfs, &btree, 1, LFSR_TAG_INLINED, 1, - LFSR_DATA("b", 1)) => 0; + LFSR_DATA_BUF("b", 1)) => 0; // pop! lfsr_btree_pop(&lfs, &btree, 1) => 0; printf("btree: w%d 0x%x.%x\n", @@ -1431,7 +1431,7 @@ code = ''' // try to putting it back to see if things still work lfsr_btree_push(&lfs, &btree, 1, LFSR_TAG_INLINED, 1, - LFSR_DATA("B", 1)) => 0; + LFSR_DATA_BUF("B", 1)) => 0; printf("btree: w%d 0x%x.%x\n", btree.u.rbyd.weight, btree.u.rbyd.block, @@ -1471,9 +1471,9 @@ code = ''' // create a single-entry tree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("a", 1)) => 0; + LFSR_DATA_BUF("a", 1)) => 0; lfsr_btree_push(&lfs, &btree, 1, LFSR_TAG_INLINED, 1, - LFSR_DATA("b", 1)) => 0; + LFSR_DATA_BUF("b", 1)) => 0; // pop! lfsr_btree_pop(&lfs, &btree, 0) => 0; printf("btree: w%d 0x%x.%x\n", @@ -1498,7 +1498,7 @@ code = ''' // try to putting it back to see if things still work lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("A", 1)) => 0; + LFSR_DATA_BUF("A", 1)) => 0; printf("btree: w%d 0x%x.%x\n", btree.u.rbyd.weight, btree.u.rbyd.block, @@ -1538,11 +1538,11 @@ code = ''' // create a single-entry tree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("a", 1)) => 0; + LFSR_DATA_BUF("a", 1)) => 0; lfsr_btree_push(&lfs, &btree, 1, LFSR_TAG_INLINED, 1, - LFSR_DATA("b", 1)) => 0; + LFSR_DATA_BUF("b", 1)) => 0; lfsr_btree_push(&lfs, &btree, 2, LFSR_TAG_INLINED, 1, - LFSR_DATA("c", 1)) => 0; + LFSR_DATA_BUF("c", 1)) => 0; // pop! lfsr_btree_pop(&lfs, &btree, 2) => 0; printf("btree: w%d 0x%x.%x\n", @@ -1573,7 +1573,7 @@ code = ''' // try to putting it back to see if things still work lfsr_btree_push(&lfs, &btree, 2, LFSR_TAG_INLINED, 1, - LFSR_DATA("C", 1)) => 0; + LFSR_DATA_BUF("C", 1)) => 0; printf("btree: w%d 0x%x.%x\n", btree.u.rbyd.weight, btree.u.rbyd.block, @@ -1624,7 +1624,7 @@ code = ''' const char *alphas = "abcdefghijklmnopqrstuvwxyz"; for (lfs_size_t i = 0; i < N; i++) { int err = lfsr_btree_push(&lfs, &btree, i, LFSR_TAG_INLINED, 1, - LFSR_DATA(&alphas[i % 26], 1)); + LFSR_DATA_BUF(&alphas[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { printf("btree: w%d 0x%x.%x\n", @@ -1674,7 +1674,7 @@ code = ''' // try recovering lfsr_btree_push(&lfs, &btree, REMAINING, LFSR_TAG_INLINED, 1, - LFSR_DATA("R", 1)) => 0; + LFSR_DATA_BUF("R", 1)) => 0; for (lfs_size_t i = 0; i < REMAINING; i++) { lfsr_btree_get(&lfs, &btree, i, @@ -1715,7 +1715,7 @@ code = ''' const char *alphas = "abcdefghijklmnopqrstuvwxyz"; for (lfs_size_t i = 0; i < N; i++) { int err = lfsr_btree_push(&lfs, &btree, i, LFSR_TAG_INLINED, 1, - LFSR_DATA(&alphas[i % 26], 1)); + LFSR_DATA_BUF(&alphas[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { printf("btree: w%d 0x%x.%x\n", @@ -1764,7 +1764,7 @@ code = ''' // try recovering lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("R", 1)) => 0; + LFSR_DATA_BUF("R", 1)) => 0; lfsr_btree_get(&lfs, &btree, 0, &tag_, &weight_, buffer, 4) => 1; @@ -1807,7 +1807,7 @@ code = ''' lfsr_btree_t btree = LFSR_BTREE_NULL; for (lfs_size_t i = 0; i < N; i++) { int err = lfsr_btree_push(&lfs, &btree, i, LFSR_TAG_INLINED, 1, - LFSR_DATA(&alphas[i % 26], 1)); + LFSR_DATA_BUF(&alphas[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { printf("btree: w%d 0x%x.%x\n", @@ -1905,7 +1905,7 @@ code = ''' const char *alphas = "abcdefghijklmnopqrstuvwxyz"; for (lfs_size_t i = 0; i < N; i++) { int err = lfsr_btree_push(&lfs, &btree, i*W, LFSR_TAG_INLINED, W, - LFSR_DATA(&alphas[i % 26], 1)); + LFSR_DATA_BUF(&alphas[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { printf("btree: w%d 0x%x.%x\n", @@ -1954,7 +1954,7 @@ code = ''' // try recovering lfsr_btree_push(&lfs, &btree, REMAINING*W, LFSR_TAG_INLINED, W, - LFSR_DATA("R", 1)) => 0; + LFSR_DATA_BUF("R", 1)) => 0; for (lfs_size_t i = 0; i < REMAINING; i++) { lfsr_btree_get(&lfs, &btree, i*W+W-1, @@ -2045,7 +2045,7 @@ code = ''' int err = lfsr_btree_push(&lfs, &btree, weighted_bid, LFSR_TAG_INLINED, weight, - LFSR_DATA(&alphas[i % 26], 1)); + LFSR_DATA_BUF(&alphas[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { printf("btree: w%d 0x%x.%x\n", @@ -2183,12 +2183,12 @@ code = ''' lfsr_btree_t btree = LFSR_BTREE_NULL; const char *alphas = "abcdefghijklmnopqrstuvwxyz"; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA(&alphas[0 % 26], 1)) => 0; + LFSR_DATA_BUF(&alphas[0 % 26], 1)) => 0; lfs_size_t n = 1; for (lfs_size_t i = 1; i < N; i++) { int err = lfsr_btree_split(&lfs, &btree, i-1, LFSR_DATA_NULL, - LFSR_TAG_INLINED, 1, LFSR_DATA(&alphas[(i-1) % 26], 1), - LFSR_TAG_INLINED, 1, LFSR_DATA(&alphas[(i-0) % 26], 1)); + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(&alphas[(i-1) % 26], 1), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(&alphas[(i-0) % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -2241,7 +2241,7 @@ code = ''' // create a btree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("_", 1)) => 0; + LFSR_DATA_BUF("_", 1)) => 0; // set up a simulation to compare against // @@ -2259,8 +2259,8 @@ code = ''' // split btree int err = lfsr_btree_split(&lfs, &btree, bid, LFSR_DATA_NULL, - LFSR_TAG_INLINED, 1, LFSR_DATA(&alphas[i % 26], 1), - LFSR_TAG_INLINED, 1, LFSR_DATA(&uppers[i % 26], 1)); + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(&alphas[i % 26], 1), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(&uppers[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -2330,12 +2330,12 @@ code = ''' lfsr_btree_t btree = LFSR_BTREE_NULL; const char *alphas = "abcdefghijklmnopqrstuvwxyz"; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, W, - LFSR_DATA(&alphas[0 % 26], 1)) => 0; + LFSR_DATA_BUF(&alphas[0 % 26], 1)) => 0; lfs_size_t n = 1; for (lfs_size_t i = 1; i < N; i++) { int err = lfsr_btree_split(&lfs, &btree, (i-1)*W+W-1, LFSR_DATA_NULL, - LFSR_TAG_INLINED, W, LFSR_DATA(&alphas[(i-1) % 26], 1), - LFSR_TAG_INLINED, W, LFSR_DATA(&alphas[(i-0) % 26], 1)); + LFSR_TAG_INLINED, W, LFSR_DATA_BUF(&alphas[(i-1) % 26], 1), + LFSR_TAG_INLINED, W, LFSR_DATA_BUF(&alphas[(i-0) % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -2389,7 +2389,7 @@ code = ''' // create a btree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, W, - LFSR_DATA("_", 1)) => 0; + LFSR_DATA_BUF("_", 1)) => 0; // set up a simulation to compare against // @@ -2421,9 +2421,9 @@ code = ''' int err = lfsr_btree_split(&lfs, &btree, weighted_bid+sim_weights[bid]-1, LFSR_DATA_NULL, LFSR_TAG_INLINED, weight1, - LFSR_DATA(&alphas[i % 26], 1), + LFSR_DATA_BUF(&alphas[i % 26], 1), LFSR_TAG_INLINED, weight2, - LFSR_DATA(&uppers[i % 26], 1)); + LFSR_DATA_BUF(&uppers[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -2566,19 +2566,19 @@ code = ''' // the extra push here avoids trying to inline the big entry lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("_", 1)) => 0; + LFSR_DATA_BUF("_", 1)) => 0; uint8_t buf1[SIZE]; memset(buf1, 'a', SIZE); uint8_t buf2[SIZE]; memset(buf2, 'b', SIZE); lfsr_btree_split(&lfs, &btree, 0, LFSR_DATA_NULL, - LFSR_TAG_INLINED, 1, LFSR_DATA(buf1, SIZE), - LFSR_TAG_INLINED, 1, LFSR_DATA(buf2, SIZE)) => 0; + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(buf1, SIZE), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(buf2, SIZE)) => 0; // force compaction btree.u.rbyd.eoff = -1; memset(buf2, 'b', SIZE); lfsr_btree_set(&lfs, &btree, 1, LFSR_TAG_INLINED, 1, - LFSR_DATA(buf2, SIZE)) => 0; + LFSR_DATA_BUF(buf2, SIZE)) => 0; assert(lfsr_btree_weight(&btree) == 2); // now remove one entry, since this brings the rbyd down to zero, @@ -2628,19 +2628,19 @@ code = ''' // the extra push here avoids trying to inline the big entry lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("_", 1)) => 0; + LFSR_DATA_BUF("_", 1)) => 0; uint8_t buf1[SIZE]; memset(buf1, 'a', SIZE); uint8_t buf2[SIZE]; memset(buf2, 'b', SIZE); lfsr_btree_split(&lfs, &btree, 0, LFSR_DATA_NULL, - LFSR_TAG_INLINED, 1, LFSR_DATA(buf1, SIZE), - LFSR_TAG_INLINED, 1, LFSR_DATA(buf2, SIZE)) => 0; + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(buf1, SIZE), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(buf2, SIZE)) => 0; // force compaction btree.u.rbyd.eoff = -1; memset(buf2, 'b', SIZE); lfsr_btree_set(&lfs, &btree, 1, LFSR_TAG_INLINED, 1, - LFSR_DATA(buf2, SIZE)) => 0; + LFSR_DATA_BUF(buf2, SIZE)) => 0; assert(lfsr_btree_weight(&btree) == 2); // now remove one entry, since this brings the rbyd down this zero, @@ -2693,14 +2693,14 @@ code = ''' // the extra push here avoids trying to inline the big entry lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("_", 1)) => 0; + LFSR_DATA_BUF("_", 1)) => 0; uint8_t buf1[SIZE]; memset(buf1, 'a', SIZE); uint8_t buf2[SIZE]; memset(buf2, 'b', SIZE); lfsr_btree_split(&lfs, &btree, 0, LFSR_DATA_NULL, - LFSR_TAG_INLINED, 1, LFSR_DATA(buf1, SIZE), - LFSR_TAG_INLINED, 1, LFSR_DATA(buf2, SIZE)) => 0; + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(buf1, SIZE), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(buf2, SIZE)) => 0; // force compaction, causing a split, but while we're splitting, // also remove an entry, bringing the split rbyd down to zero mid split @@ -2753,26 +2753,26 @@ code = ''' // the extra push here avoids trying to inline the big entry lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("_", 1)) => 0; + LFSR_DATA_BUF("_", 1)) => 0; uint8_t buf1[SIZE]; memset(buf1, 'a', SIZE); uint8_t buf2[SIZE]; memset(buf2, 'b', SIZE); lfsr_btree_split(&lfs, &btree, 0, LFSR_DATA_NULL, - LFSR_TAG_INLINED, 1, LFSR_DATA(buf1, SIZE), - LFSR_TAG_INLINED, 1, LFSR_DATA(buf2, SIZE)) => 0; + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(buf1, SIZE), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(buf2, SIZE)) => 0; // force compaction btree.u.rbyd.eoff = -1; memset(buf2, 'b', SIZE); lfsr_btree_set(&lfs, &btree, 1, LFSR_TAG_INLINED, 1, - LFSR_DATA(buf2, SIZE)) => 0; + LFSR_DATA_BUF(buf2, SIZE)) => 0; assert(lfsr_btree_weight(&btree) == 2); // now make both entries small so they should be merged if either compacts lfsr_btree_set(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("a", 1)) => 0; + LFSR_DATA_BUF("a", 1)) => 0; lfsr_btree_set(&lfs, &btree, 1, LFSR_TAG_INLINED, 1, - LFSR_DATA("b", 1)) => 0; + LFSR_DATA_BUF("b", 1)) => 0; // force compaction, while removing one entry, this drops the rbyd // down to zero while also triggering a merge @@ -2840,7 +2840,7 @@ code = ''' // push to btree int err = lfsr_btree_push(&lfs, &btree, bid, LFSR_TAG_INLINED, 1, - LFSR_DATA(&alphas[i % 26], 1)); + LFSR_DATA_BUF(&alphas[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -2856,7 +2856,7 @@ code = ''' // update btree int err = lfsr_btree_set(&lfs, &btree, bid, LFSR_TAG_INLINED, 1, - LFSR_DATA(&alphas[i % 26], 1)); + LFSR_DATA_BUF(&alphas[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -2967,7 +2967,7 @@ code = ''' // push to btree int err = lfsr_btree_push(&lfs, &btree, weighted_bid, LFSR_TAG_INLINED, weight, - LFSR_DATA(&alphas[i % 26], 1)); + LFSR_DATA_BUF(&alphas[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -2986,7 +2986,7 @@ code = ''' // update btree int err = lfsr_btree_set(&lfs, &btree, weighted_bid+sim_weights[bid]-1, LFSR_TAG_INLINED, weight, - LFSR_DATA(&alphas[i % 26], 1)); + LFSR_DATA_BUF(&alphas[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -3142,7 +3142,7 @@ code = ''' // create a single-entry tree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("0", 1)) => 0; + LFSR_DATA_BUF("0", 1)) => 0; printf("btree: w%d 0x%x.%x\n", btree.u.rbyd.weight, btree.u.rbyd.block, @@ -3191,10 +3191,11 @@ code = ''' // create a two-entry tree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("0", 1)) => 0; - lfsr_btree_split(&lfs, &btree, 0, LFSR_DATA_NAME(0, "aab", 3), - LFSR_TAG_INLINED, 1, LFSR_DATA("0", 1), - LFSR_TAG_INLINED, 1, LFSR_DATA("1", 1)) => 0; + LFSR_DATA_BUF("0", 1)) => 0; + lfsr_btree_split(&lfs, &btree, 0, + LFSR_DATA_CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF("aab", 3)), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("0", 1), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("1", 1)) => 0; printf("btree: w%d 0x%x.%x\n", btree.u.rbyd.weight, btree.u.rbyd.block, @@ -3251,13 +3252,15 @@ code = ''' // create a two-entry tree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("0", 1)) => 0; - lfsr_btree_split(&lfs, &btree, 0, LFSR_DATA_NAME(1*DID, "aab", 3), - LFSR_TAG_INLINED, 1, LFSR_DATA("0", 1), - LFSR_TAG_INLINED, 1, LFSR_DATA("1", 1)) => 0; - lfsr_btree_split(&lfs, &btree, 1, LFSR_DATA_NAME(2*DID, "aac", 3), - LFSR_TAG_INLINED, 1, LFSR_DATA("1", 1), - LFSR_TAG_INLINED, 1, LFSR_DATA("2", 1)) => 0; + LFSR_DATA_BUF("0", 1)) => 0; + lfsr_btree_split(&lfs, &btree, 0, + LFSR_DATA_CAT(LFSR_DATA_LEB128(1*DID), LFSR_DATA_BUF("aab", 3)), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("0", 1), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("1", 1)) => 0; + lfsr_btree_split(&lfs, &btree, 1, + LFSR_DATA_CAT(LFSR_DATA_LEB128(2*DID), LFSR_DATA_BUF("aac", 3)), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("1", 1), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("2", 1)) => 0; printf("btree: w%d 0x%x.%x\n", btree.u.rbyd.weight, btree.u.rbyd.block, @@ -3322,13 +3325,15 @@ code = ''' // create a two-entry tree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("0", 1)) => 0; - lfsr_btree_split(&lfs, &btree, 0, LFSR_DATA_NAME(2*DID, "aac", 3), - LFSR_TAG_INLINED, 1, LFSR_DATA("1", 1), - LFSR_TAG_INLINED, 1, LFSR_DATA("2", 1)) => 0; - lfsr_btree_split(&lfs, &btree, 0, LFSR_DATA_NAME(1*DID, "aab", 3), - LFSR_TAG_INLINED, 1, LFSR_DATA("0", 1), - LFSR_TAG_INLINED, 1, LFSR_DATA("1", 1)) => 0; + LFSR_DATA_BUF("0", 1)) => 0; + lfsr_btree_split(&lfs, &btree, 0, + LFSR_DATA_CAT(LFSR_DATA_LEB128(2*DID), LFSR_DATA_BUF("aac", 3)), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("1", 1), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("2", 1)) => 0; + lfsr_btree_split(&lfs, &btree, 0, + LFSR_DATA_CAT(LFSR_DATA_LEB128(1*DID), LFSR_DATA_BUF("aab", 3)), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("0", 1), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF("1", 1)) => 0; printf("btree: w%d 0x%x.%x\n", btree.u.rbyd.weight, btree.u.rbyd.block, @@ -3396,16 +3401,16 @@ code = ''' const char *alphas = "abcdefghijklmnopqrstuvwxyz"; const char *nums = "0123456789"; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA(&nums[0 % 10], 1)) => 0; + LFSR_DATA_BUF(&nums[0 % 10], 1)) => 0; lfs_size_t n = 1; for (lfs_size_t i = 1; i < N; i++) { char name[3] = { alphas[(i/26/26) % 26], alphas[(i/26) % 26], alphas[i % 26] }; int err = lfsr_btree_split(&lfs, &btree, i-1, - LFSR_DATA_NAME(i*DID, name, 3), - LFSR_TAG_INLINED, 1, LFSR_DATA(&nums[(i-1) % 10], 1), - LFSR_TAG_INLINED, 1, LFSR_DATA(&nums[(i-0) % 10], 1)); + LFSR_DATA_CAT(LFSR_DATA_LEB128(i*DID), LFSR_DATA_BUF(name, 3)), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(&nums[(i-1) % 10], 1), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(&nums[(i-0) % 10], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -3462,7 +3467,7 @@ code = ''' // create a btree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("_", 1)) => 0; + LFSR_DATA_BUF("_", 1)) => 0; // set up a simulation to compare against // @@ -3496,9 +3501,9 @@ code = ''' // split btree int err = lfsr_btree_split(&lfs, &btree, bid, - LFSR_DATA_NAME(0, name, 3), - LFSR_TAG_INLINED, 1, LFSR_DATA(&nums[i % 10], 1), - LFSR_TAG_INLINED, 1, LFSR_DATA(&nums[i % 10], 1)); + LFSR_DATA_CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF(name, 3)), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(&nums[i % 10], 1), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(&nums[i % 10], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -3574,16 +3579,16 @@ code = ''' const char *alphas = "abcdefghijklmnopqrstuvwxyz"; const char *nums = "0123456789"; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, W, - LFSR_DATA(&nums[0 % 10], 1)) => 0; + LFSR_DATA_BUF(&nums[0 % 10], 1)) => 0; lfs_size_t n = 1; for (lfs_size_t i = 1; i < N; i++) { char name[3] = { alphas[(i/26/26) % 26], alphas[(i/26) % 26], alphas[i % 26] }; - int err = lfsr_btree_split(&lfs, &btree, - (i-1)*W+W-1, LFSR_DATA_NAME(i*DID, name, 3), - LFSR_TAG_INLINED, W, LFSR_DATA(&nums[(i-1) % 10], 1), - LFSR_TAG_INLINED, W, LFSR_DATA(&nums[(i-0) % 10], 1)); + int err = lfsr_btree_split(&lfs, &btree, (i-1)*W+W-1, + LFSR_DATA_CAT(LFSR_DATA_LEB128(i*DID), LFSR_DATA_BUF(name, 3)), + LFSR_TAG_INLINED, W, LFSR_DATA_BUF(&nums[(i-1) % 10], 1), + LFSR_TAG_INLINED, W, LFSR_DATA_BUF(&nums[(i-0) % 10], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -3641,7 +3646,7 @@ code = ''' // create a btree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, W, - LFSR_DATA("_", 1)) => 0; + LFSR_DATA_BUF("_", 1)) => 0; // set up a simulation to compare against // @@ -3688,9 +3693,9 @@ code = ''' // split btree int err = lfsr_btree_split(&lfs, &btree, weighted_bid+sim_weights[bid]-1, - LFSR_DATA_NAME(0, name, 3), - LFSR_TAG_INLINED, weight1, LFSR_DATA(&nums[i % 10], 1), - LFSR_TAG_INLINED, weight2, LFSR_DATA(&nums[i % 10], 1)); + LFSR_DATA_CAT(LFSR_DATA_LEB128(0), LFSR_DATA_BUF(name, 3)), + LFSR_TAG_INLINED, weight1, LFSR_DATA_BUF(&nums[i % 10], 1), + LFSR_TAG_INLINED, weight2, LFSR_DATA_BUF(&nums[i % 10], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -3792,7 +3797,7 @@ code = ''' // create a btree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, 1, - LFSR_DATA("_", 1)) => 0; + LFSR_DATA_BUF("_", 1)) => 0; // set up a simulation to compare against // @@ -3841,11 +3846,11 @@ code = ''' if (split_bid > bid) { int err = lfsr_btree_split(&lfs, &btree, split_bid, - LFSR_DATA_NAME(0, sim_names[bid+1], 3), - LFSR_TAG_INLINED, 1, - LFSR_DATA(&nums[i % 10], 1), - LFSR_TAG_INLINED, 1, - LFSR_DATA(split_buf, 1)); + LFSR_DATA_CAT( + LFSR_DATA_LEB128(0), + LFSR_DATA_BUF(sim_names[bid+1], 3)), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(&nums[i % 10], 1), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(split_buf, 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -3853,11 +3858,11 @@ code = ''' assert(err == 0); } else { int err = lfsr_btree_split(&lfs, &btree, - split_bid, LFSR_DATA_NAME(0, name, 3), - LFSR_TAG_INLINED, 1, - LFSR_DATA(split_buf, 1), - LFSR_TAG_INLINED, 1, - LFSR_DATA(&nums[i % 10], 1)); + split_bid, LFSR_DATA_CAT( + LFSR_DATA_LEB128(0), + LFSR_DATA_BUF(name, 3)), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(split_buf, 1), + LFSR_TAG_INLINED, 1, LFSR_DATA_BUF(&nums[i % 10], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -3876,7 +3881,7 @@ code = ''' // update btree int err = lfsr_btree_set(&lfs, &btree, bid, LFSR_TAG_INLINED, 1, - LFSR_DATA(&nums[i % 10], 1)); + LFSR_DATA_BUF(&nums[i % 10], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -3967,7 +3972,7 @@ code = ''' // create a btree lfsr_btree_t btree = LFSR_BTREE_NULL; lfsr_btree_push(&lfs, &btree, 0, LFSR_TAG_INLINED, W, - LFSR_DATA("_", 1)) => 0; + LFSR_DATA_BUF("_", 1)) => 0; // set up a simulation to compare against // @@ -4033,24 +4038,28 @@ code = ''' uint8_t split_buf[4]; lfsr_data_read(&lfs, &split_data, split_buf, 4) => 1; if (split_bid > weighted_bid+sim_weights[bid]-1) { - int err = lfsr_btree_split(&lfs, &btree, - split_bid, LFSR_DATA_NAME(0, sim_names[bid+1], 3), + int err = lfsr_btree_split(&lfs, &btree, split_bid, + LFSR_DATA_CAT( + LFSR_DATA_LEB128(0), + LFSR_DATA_BUF(sim_names[bid+1], 3)), LFSR_TAG_INLINED, weight, - LFSR_DATA(&nums[i % 10], 1), + LFSR_DATA_BUF(&nums[i % 10], 1), LFSR_TAG_INLINED, split_weight, - LFSR_DATA(split_buf, 1)); + LFSR_DATA_BUF(split_buf, 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; } assert(err == 0); } else { - int err = lfsr_btree_split(&lfs, &btree, - split_bid, LFSR_DATA_NAME(0, name, 3), + int err = lfsr_btree_split(&lfs, &btree, split_bid, + LFSR_DATA_CAT( + LFSR_DATA_LEB128(0), + LFSR_DATA_BUF(name, 3)), LFSR_TAG_INLINED, split_weight, - LFSR_DATA(split_buf, 1), + LFSR_DATA_BUF(split_buf, 1), LFSR_TAG_INLINED, weight, - LFSR_DATA(&nums[i % 10], 1)); + LFSR_DATA_BUF(&nums[i % 10], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -4072,7 +4081,7 @@ code = ''' // update btree int err = lfsr_btree_set(&lfs, &btree, weighted_bid+sim_weights[bid]-1, LFSR_TAG_INLINED, weight, - LFSR_DATA(&nums[i % 10], 1)); + LFSR_DATA_BUF(&nums[i % 10], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -4191,7 +4200,7 @@ code = ''' lfs_size_t n = 0; for (lfs_size_t i = 0; i < N; i++) { int err = lfsr_btree_push(&lfs, &btree, i, LFSR_TAG_INLINED, 1, - LFSR_DATA(&alphas[i % 26], 1)); + LFSR_DATA_BUF(&alphas[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -4244,7 +4253,7 @@ code = ''' } if (tag_ == LFSR_TAG_BTREE) { - lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.buffer.buffer; + lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.direct.buffer; printf("traversal: %d 0x%x w%d btree 0x%x.%x\n", bid_, tag_, @@ -4324,7 +4333,7 @@ code = ''' // add to btree int err = lfsr_btree_push(&lfs, &btree, bid, LFSR_TAG_INLINED, 1, - LFSR_DATA(&alphas[i % 26], 1)); + LFSR_DATA_BUF(&alphas[i % 26], 1)); // ignore space issues if (err == LFS_ERR_NOSPC) { break; @@ -4392,7 +4401,7 @@ code = ''' } if (tag_ == LFSR_TAG_BTREE) { - lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.buffer.buffer; + lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.direct.buffer; printf("traversal: %d 0x%x w%d btree 0x%x.%x\n", bid_, tag_, diff --git a/tests/test_mtree.toml b/tests/test_mtree.toml index 2dd46cfb..77fdfa31 100644 --- a/tests/test_mtree.toml +++ b/tests/test_mtree.toml @@ -3465,7 +3465,7 @@ code = ''' } if (tag_ == LFSR_TAG_BTREE) { - lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.buffer.buffer; + lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.direct.buffer; printf("traversal: %d.%d 0x%x btree 0x%x.%x\n", mid_ >> lfs.mleaf_bits, mid_ & lfsr_midrmask(&lfs), @@ -3475,7 +3475,7 @@ code = ''' // keep track of seen blocks seen[branch->block / 8] |= 1 << (branch->block % 8); } else if (tag_ == LFSR_TAG_MDIR) { - lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.buffer.buffer; + lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.direct.buffer; printf("traversal: %d.%d 0x%x mdir 0x{%x,%x}\n", mid_ >> lfs.mleaf_bits, mid_ & lfsr_midrmask(&lfs), @@ -3589,7 +3589,7 @@ code = ''' } if (tag_ == LFSR_TAG_BTREE) { - lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.buffer.buffer; + lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.direct.buffer; printf("traversal: %d.%d 0x%x btree 0x%x.%x\n", mid_ >> lfs.mleaf_bits, mid_ & lfsr_midrmask(&lfs), @@ -3599,7 +3599,7 @@ code = ''' // keep track of seen blocks seen[branch->block / 8] |= 1 << (branch->block % 8); } else if (tag_ == LFSR_TAG_MDIR) { - lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.buffer.buffer; + lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.direct.buffer; printf("traversal: %d.%d 0x%x mdir 0x{%x,%x}\n", mid_ >> lfs.mleaf_bits, mid_ & lfsr_midrmask(&lfs), @@ -3724,7 +3724,7 @@ code = ''' } if (tag_ == LFSR_TAG_BTREE) { - lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.buffer.buffer; + lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.direct.buffer; printf("traversal: %d.%d 0x%x btree 0x%x.%x\n", mid_ >> lfs.mleaf_bits, mid_ & lfsr_midrmask(&lfs), @@ -3734,7 +3734,7 @@ code = ''' // keep track of seen blocks seen[branch->block / 8] |= 1 << (branch->block % 8); } else if (tag_ == LFSR_TAG_MDIR) { - lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.buffer.buffer; + lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.direct.buffer; printf("traversal: %d.%d 0x%x mdir 0x{%x,%x}\n", mid_ >> lfs.mleaf_bits, mid_ & lfsr_midrmask(&lfs), @@ -3851,7 +3851,7 @@ code = ''' } if (tag_ == LFSR_TAG_BTREE) { - lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.buffer.buffer; + lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.direct.buffer; printf("traversal: %d.%d 0x%x btree 0x%x.%x\n", mid_ >> lfs.mleaf_bits, mid_ & lfsr_midrmask(&lfs), @@ -3861,7 +3861,7 @@ code = ''' // keep track of seen blocks seen[branch->block / 8] |= 1 << (branch->block % 8); } else if (tag_ == LFSR_TAG_MDIR) { - lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.buffer.buffer; + lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.direct.buffer; printf("traversal: %d.%d 0x%x mdir 0x{%x,%x}\n", mid_ >> lfs.mleaf_bits, mid_ & lfsr_midrmask(&lfs), @@ -3989,7 +3989,7 @@ code = ''' } if (tag_ == LFSR_TAG_BTREE) { - lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.buffer.buffer; + lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.direct.buffer; printf("traversal: %d.%d 0x%x btree 0x%x.%x\n", mid_ >> lfs.mleaf_bits, mid_ & lfsr_midrmask(&lfs), @@ -3999,7 +3999,7 @@ code = ''' // keep track of seen blocks seen[branch->block / 8] |= 1 << (branch->block % 8); } else if (tag_ == LFSR_TAG_MDIR) { - lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.buffer.buffer; + lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.direct.buffer; printf("traversal: %d.%d 0x%x mdir 0x{%x,%x}\n", mid_ >> lfs.mleaf_bits, mid_ & lfsr_midrmask(&lfs), @@ -4159,7 +4159,7 @@ code = ''' } if (tag_ == LFSR_TAG_BTREE) { - lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.buffer.buffer; + lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.direct.buffer; printf("traversal: %d.%d 0x%x btree 0x%x.%x\n", mid_ >> lfs.mleaf_bits, mid_ & lfsr_midrmask(&lfs), @@ -4169,7 +4169,7 @@ code = ''' // keep track of seen blocks seen[branch->block / 8] |= 1 << (branch->block % 8); } else if (tag_ == LFSR_TAG_MDIR) { - lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.buffer.buffer; + lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.direct.buffer; printf("traversal: %d.%d 0x%x mdir 0x{%x,%x}\n", mid_ >> lfs.mleaf_bits, mid_ & lfsr_midrmask(&lfs), @@ -4267,14 +4267,14 @@ code = ''' } if (tag_ == LFSR_TAG_BTREE) { - lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.buffer.buffer; + lfsr_rbyd_t *branch = (lfsr_rbyd_t *)data_.u.direct.buffer; printf("traversal: %d.%d 0x%x btree 0x%x.%x\n", mid_ >> lfs.mleaf_bits, mid_ & lfsr_midrmask(&lfs), tag_, branch->block, branch->trunk); } else if (tag_ == LFSR_TAG_MDIR) { - lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.buffer.buffer; + lfsr_mdir_t *mdir = (lfsr_mdir_t*)data_.u.direct.buffer; printf("traversal: %d.%d 0x%x mdir 0x{%x,%x}\n", mid_ >> lfs.mleaf_bits, mid_ & lfsr_midrmask(&lfs),