diff --git a/lfs.c b/lfs.c index 18b2568b..0dcf3934 100644 --- a/lfs.c +++ b/lfs.c @@ -8524,8 +8524,7 @@ int lfsr_file_read_(lfs_t *lfs, const lfsr_file_t *file, } // is the data inlined? - if (lfsr_file_hassprout(file) - && pos < lfsr_file_inlinedsize(file)) { + if (lfsr_file_hassprout(file) && pos < lfsr_file_inlinedsize(file)) { lfsr_data_t data = file->inlined.u.data; lfsr_data_add(&data, pos); d = lfsr_data_read(lfs, &data, buffer_, d); @@ -8540,19 +8539,19 @@ int lfsr_file_read_(lfs_t *lfs, const lfsr_file_t *file, } // is the data in an inlined tree? - if (lfsr_file_hasshrub(file) - && pos < lfsr_file_inlinedsize(file)) { + if (lfsr_file_hasshrub(file) && pos < lfsr_file_inlinedsize(file)) { lfsr_srid_t rid; lfsr_tag_t tag; lfsr_rid_t weight; lfsr_data_t data; int err = lfsr_rbyd_lookupnext(lfs, &file->inlined.u.rbyd, pos, 0, &rid, &tag, &weight, &data); - if (err && err != LFS_ERR_NOENT) { + if (err) { + LFS_ASSERT(err != LFS_ERR_NOENT); return err; } - LFS_ASSERT(err == LFS_ERR_NOENT - || tag == LFSR_TAG_SHRUB(INLINED)); + LFS_ASSERT(tag == LFSR_TAG_SHRUB(INLINED)); + LFS_ASSERT(lfsr_data_size(&data) <= weight); if (pos < rid-(weight-1) + lfsr_data_size(&data)) { lfsr_data_add(&data, pos - (rid-(weight-1))); @@ -8655,11 +8654,12 @@ static int lfsr_file_flushbuffer(lfs_t *lfs, lfsr_file_t *file) { file->buffer_pos, file->inlined.u.rbyd.weight)-1, 0, &left_rid, &left_tag, &left_weight, &left_data); - if (err && err != LFS_ERR_NOENT) { + if (err) { + LFS_ASSERT(err != LFS_ERR_NOENT); return err; } - LFS_ASSERT(err != LFS_ERR_NOENT); LFS_ASSERT(left_tag == LFSR_TAG_SHRUB(INLINED)); + LFS_ASSERT(lfsr_data_size(&left_data) <= left_weight); // this can be negative! left_overlap = (left_rid+1) - file->buffer_pos; @@ -8694,40 +8694,44 @@ static int lfsr_file_flushbuffer(lfs_t *lfs, lfsr_file_t *file) { < (lfs_off_t)file->inlined.u.rbyd.weight) { lfsr_srid_t right_rid; lfsr_tag_t right_tag; + lfsr_rid_t right_weight_; + lfsr_data_t right_data_; int err = lfsr_rbyd_lookupnext(lfs, &file->inlined.u.rbyd, file->buffer_pos + file->buffer_size, 0, - &right_rid, &right_tag, &right_weight, &right_data); - if (err && err != LFS_ERR_NOENT) { + &right_rid, &right_tag, &right_weight_, &right_data_); + if (err) { + LFS_ASSERT(err != LFS_ERR_NOENT); return err; } - LFS_ASSERT(err != LFS_ERR_NOENT); LFS_ASSERT(right_tag == LFSR_TAG_SHRUB(INLINED)); + LFS_ASSERT(lfsr_data_size(&right_data) <= right_weight); lfs_soff_t right_overlap = file->buffer_pos + file->buffer_size - - (right_rid-(right_weight-1)); + - (right_rid-(right_weight_-1)); - // need to carve out right data? - if (right_overlap > 0) { + // need to carve out right data? note we eagerly merge with + // data-less holes + if (right_overlap > 0 || lfsr_data_size(&right_data_) == 0) { right_data = LFSR_DATA_DISK( - right_data.u.disk.block, - right_data.u.disk.off + right_overlap, - lfsr_data_size(&right_data) - lfs_min32( + right_data_.u.disk.block, + right_data_.u.disk.off + right_overlap, + lfsr_data_size(&right_data_) - lfs_min32( right_overlap, - lfsr_data_size(&right_data))); - right_weight -= right_overlap; + lfsr_data_size(&right_data_))); + right_weight = right_weight_ - right_overlap; } } // remove any data we're overwriting, note we need to account for // left_sibling changes - lfs_off_t rm_size = lfs_min32( + lfs_off_t rm = lfs_min32( file->buffer_pos + file->buffer_size + right_weight - left_overlap, file->inlined.u.rbyd.weight - left_overlap) - file->buffer_pos; - *attrs_++ = LFSR_ATTR(file->buffer_pos + rm_size - 1, - SHRUB(RM), -rm_size, NULL); + *attrs_++ = LFSR_ATTR(file->buffer_pos + rm - 1, + SHRUB(RM), -rm, NULL); if (lfsr_data_size(&right_data) == 0) { // append our buffer with any remaining weight @@ -9008,7 +9012,6 @@ int lfsr_file_truncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size) { // this should never happen, every route to zero-weight shrub // should revert to an inlined file LFS_ASSERT(file->inlined.u.rbyd.weight > 0); - LFS_ASSERT(size > 0); // left sibling? lfs_soff_t left_overlap = 0; @@ -9021,11 +9024,12 @@ int lfsr_file_truncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size) { size, file->inlined.u.rbyd.weight)-1, 0, &left_rid, &left_tag, &left_weight, &left_data); - if (err && err != LFS_ERR_NOENT) { + if (err) { + LFS_ASSERT(err != LFS_ERR_NOENT); return err; } - LFS_ASSERT(err != LFS_ERR_NOENT); LFS_ASSERT(left_tag == LFSR_TAG_SHRUB(INLINED)); + LFS_ASSERT(lfsr_data_size(&left_data) <= left_weight); // this can be negative! left_overlap = (left_rid+1) - size; @@ -9073,9 +9077,6 @@ int lfsr_file_truncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size) { file->buffer_size = buffer_size; // update our internal file size - // - // if this grows our file, leave it up to lfsr_file_sync to update - // on on-disk metadata file->size = size; return 0; @@ -9087,17 +9088,133 @@ int lfsr_file_fruncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size) { return LFS_ERR_FBIG; } -// // TODO does this risk overflow? -// // if we're increasing the file size, just update our internal size/off -// // and leave it to lfsr_file_sync to do most of the work -// if (size > file->size) { -// file->off += -// file->size = size; -// return 0; -// } + // do nothing if our size does not change + if (file->size == size) { + return 0; + } - // TODO - LFS_ASSERT(false); + // TODO make this recoverable on failure + + // mark as unsynced before we commit anything + file->flags |= LFS_F_UNSYNCED; + + lfsr_attr_t scratch_attrs[2]; + lfsr_attr_t *attrs_ = scratch_attrs; + + // if our truncated file is contained entirely in our buffer, + // revert to a sprout + lfs_size_t buffer_size = file->buffer_size - lfs_min32( + lfs_smax32(file->size - size - file->buffer_pos, 0), + file->buffer_size); + if (size < file->size + && file->size - size >= lfsr_file_inlinedsize(file)) { + file->inlined.u.data = LFSR_DATA_DISK(0, 0, 0); + + // have a sprout/null? convert to a shrub + } else if (!lfsr_file_hasshrub(file)) { + if (lfsr_data_size(&file->inlined.u.data) != 0) { + *attrs_++ = LFSR_ATTR(0, + SHRUB(INLINED), +lfsr_data_size(&file->inlined.u.data) + - lfs_smax32(file->size - size, 0), + DISK( + file->inlined.u.data.u.disk.block, + file->inlined.u.data.u.disk.off + + lfs_smax32(file->size - size, 0), + lfsr_data_size(&file->inlined.u.data) + - lfs_smax32(file->size - size, 0))); + } + + // need to prefix with a hole? + if (size > file->size) { + *attrs_++ = LFSR_ATTR(0, + SHRUB(INLINED), +size - file->size, NULL); + } + + // have a shrub? + } else if (lfsr_file_hasshrub(file)) { + // TODO can this be deduplicated with flushbuffer? some sort of + // lfsr_file_shrubcarveright? + + // this should never happen, every route to zero-weight shrub + // should revert to an inlined file + LFS_ASSERT(file->inlined.u.rbyd.weight > 0); + + // TODO wait a second... are we always copying right sibling in + // flushbuffer? even with no carve? + + // right sibling? + lfsr_rid_t right_weight = 0; + lfsr_data_t right_data = LFSR_DATA_NULL; + if (lfs_smax32(file->size - size, 0) < file->inlined.u.rbyd.weight) { + lfsr_srid_t right_rid; + lfsr_tag_t right_tag; + lfsr_rid_t right_weight_; + lfsr_data_t right_data_; + int err = lfsr_rbyd_lookupnext(lfs, &file->inlined.u.rbyd, + lfs_smax32(file->size - size, 0), 0, + &right_rid, &right_tag, &right_weight_, &right_data_); + if (err) { + LFS_ASSERT(err != LFS_ERR_NOENT); + return err; + } + LFS_ASSERT(right_tag == LFSR_TAG_SHRUB(INLINED)); + LFS_ASSERT(lfsr_data_size(&right_data) <= right_weight); + + lfs_soff_t right_overlap + = lfs_smax32(file->size - size, 0) + - (right_rid-(right_weight_-1)); + + // need to carve out right data? note we eagerly merge with + // data-less holes + if (right_overlap > 0 || lfsr_data_size(&right_data_) == 0) { + right_data = LFSR_DATA_DISK( + right_data_.u.disk.block, + right_data_.u.disk.off + right_overlap, + lfsr_data_size(&right_data_) - lfs_min32( + right_overlap, + lfsr_data_size(&right_data_))); + right_weight = right_weight_ - right_overlap; + } + } + + // remove any data we're fruncating + *attrs_++ = LFSR_ATTR(lfs_smax32(file->size - size, 0) + + right_weight - 1, + SHRUB(RM), -(lfs_smax32(file->size - size, 0) + + right_weight), NULL); + + // write any carved data or a hole, note we take care to consume any + // existing data-less holes to avoid fragmenting literally nothing + if (lfs_smax32(size - file->size, 0) + right_weight > 0) { + *attrs_++ = LFSR_ATTR(0, + SHRUB(INLINED), +lfs_smax32(size - file->size, 0) + + right_weight, + DATA(right_data)); + } + } + + // commit any shrub changes + if (attrs_ > scratch_attrs) { + int err = lfsr_mdir_commit(lfs, &file->m.mdir, LFSR_ATTRS( + LFSR_ATTR_(file->m.mdir.mid, SHRUBATTRS, 0, SHRUBATTRS(file, + scratch_attrs, + attrs_ - scratch_attrs)))); + if (err) { + return err; + } + } + + // TODO update btree + + // update our buffer + file->buffer_pos -= lfs_smin32(file->size - size, file->buffer_pos); + memmove(file->buffer, + file->buffer + (file->buffer_size - buffer_size), + buffer_size); + file->buffer_size = buffer_size; + + // update our internal file size + file->size = size; return 0; } diff --git a/tests/test_files.toml b/tests/test_files.toml index 52eab57b..2f6970df 100644 --- a/tests/test_files.toml +++ b/tests/test_files.toml @@ -988,6 +988,9 @@ code = ''' // truncate to new size lfsr_file_truncate(&lfs, &file, TO) => 0; + if (TO < FROM) { + memset(sim+TO, 0, FROM-TO); + } // close lfsr_file_close(&lfs, &file) => 0; @@ -1035,7 +1038,7 @@ code = ''' lfsr_unmount(&lfs) => 0; ''' -# the main purpose of this test is to check that data is not hidden +# one purpose of this test is to check that data is not hidden # and then revealed by truncate, that would be bad [cases.test_files_truncate_2] defines.FROM = ['0', 'CACHE_SIZE/2', '2*CACHE_SIZE'] @@ -1053,6 +1056,119 @@ code = ''' lfsr_mount(&lfs, CFG) => 0; } + // create a file, truncating in case of powerloss + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "hello", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; + // simulate our file in ram + uint8_t sim[lfs_max32(FROM,lfs_max32(AND,TO))]; + memset(sim, 0, lfs_max32(FROM,lfs_max32(AND,TO))); + uint32_t prng = 42; + for (lfs_size_t i = 0; i < FROM; i++) { + sim[i] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, sim, FROM) => FROM; + + // sync? + if (SYNC) { + lfsr_file_sync(&lfs, &file) => 0; + } + + // remount? + if (REMOUNT) { + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0; + } + + // truncate to intermediate size + lfsr_file_truncate(&lfs, &file, AND) => 0; + if (AND < FROM) { + memset(sim+AND, 0, FROM-AND); + } + + // sync? + if (SYNC) { + lfsr_file_sync(&lfs, &file) => 0; + } + + // remount? + if (REMOUNT) { + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0; + } + + // truncate to new size + lfsr_file_truncate(&lfs, &file, TO) => 0; + if (TO < AND) { + memset(sim+TO, 0, AND-TO); + } + + // close + lfsr_file_close(&lfs, &file) => 0; + + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + } + + // check our file with stat + struct lfs_info info; + lfsr_stat(&lfs, "hello", &info) => 0; + assert(strcmp(info.name, "hello") == 0); + assert(info.type == LFS_TYPE_REG); + assert(info.size == TO); + + // and with dir read + lfsr_dir_t dir; + lfsr_dir_open(&lfs, &dir, "/") => 0; + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, ".") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "..") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "hello") == 0); + assert(info.type == LFS_TYPE_REG); + assert(info.size == TO); + lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT; + lfsr_dir_close(&lfs, &dir) => 0; + + // try reading our file + lfsr_file_open(&lfs, &file, "hello", LFS_O_RDONLY) => 0; + // is size correct? + lfsr_file_size(&lfs, &file) => TO; + // try reading + uint8_t rbuf[2*TO]; + memset(rbuf, 0xaa, 2*TO); + lfsr_file_read(&lfs, &file, rbuf, 2*TO) => TO; + assert(memcmp(rbuf, sim, TO) == 0); + lfsr_file_close(&lfs, &file) => 0; + + lfsr_unmount(&lfs) => 0; +''' + +# simple fruncate test +[cases.test_files_fruncate] +defines.FROM = ['0', 'CACHE_SIZE/2', '2*CACHE_SIZE'] +defines.TO = ['0', 'CACHE_SIZE/2', '2*CACHE_SIZE'] +defines.SYNC = [false, true] +defines.REMOUNT = [false, true] +reentrant = true +code = ''' + // format once per test + lfs_t lfs; + int err = lfsr_mount(&lfs, CFG); + if (err) { + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + } + // create a file, truncating in case of powerloss lfsr_file_t file; lfsr_file_open(&lfs, &file, "hello", @@ -1079,27 +1195,16 @@ code = ''' lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0; } - // truncate to intermediate size - lfsr_file_truncate(&lfs, &file, AND) => 0; - memset(&sim[AND], 0, lfs_max32(FROM,TO) - - lfs_min32(AND, lfs_max32(FROM,TO))); - - // sync? - if (SYNC) { - lfsr_file_sync(&lfs, &file) => 0; + // fruncate to new size + lfsr_file_fruncate(&lfs, &file, TO) => 0; + if (TO > FROM) { + memmove(sim+TO-FROM, sim, FROM); + memset(sim, 0, TO-FROM); + } else if (TO < FROM) { + memmove(sim, sim+FROM-TO, TO); + memset(sim+TO, 0, FROM-TO); } - // remount? - if (REMOUNT) { - lfsr_file_close(&lfs, &file) => 0; - lfsr_unmount(&lfs) => 0; - lfsr_mount(&lfs, CFG) => 0; - lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0; - } - - // truncate to new size - lfsr_file_truncate(&lfs, &file, TO) => 0; - // close lfsr_file_close(&lfs, &file) => 0; @@ -1146,9 +1251,128 @@ code = ''' lfsr_unmount(&lfs) => 0; ''' -# TODO -# [cases.test_files_fruncate] -# [cases.test_files_fruncate_2] +# one purpose of this test is to check that data is not hidden +# and then revealed by fruncate, that would be bad +[cases.test_files_fruncate_2] +defines.FROM = ['0', 'CACHE_SIZE/2', '2*CACHE_SIZE'] +defines.AND = ['0', 'CACHE_SIZE/2', '2*CACHE_SIZE'] +defines.TO = ['0', 'CACHE_SIZE/2', '2*CACHE_SIZE'] +defines.SYNC = [false, true] +defines.REMOUNT = [false, true] +reentrant = true +code = ''' + // format once per test + lfs_t lfs; + int err = lfsr_mount(&lfs, CFG); + if (err) { + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + } + + // create a file, truncating in case of powerloss + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "hello", + LFS_O_WRONLY | LFS_O_CREAT | LFS_O_TRUNC) => 0; + // simulate our file in ram + uint8_t sim[lfs_max32(FROM,lfs_max32(AND,TO))]; + memset(sim, 0, lfs_max32(FROM,lfs_max32(AND,TO))); + uint32_t prng = 42; + for (lfs_size_t i = 0; i < FROM; i++) { + sim[i] = 'a' + (TEST_PRNG(&prng) % 26); + } + lfsr_file_write(&lfs, &file, sim, FROM) => FROM; + + // sync? + if (SYNC) { + lfsr_file_sync(&lfs, &file) => 0; + } + + // remount? + if (REMOUNT) { + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0; + } + + // fruncate to intermediate size + lfsr_file_fruncate(&lfs, &file, AND) => 0; + if (AND > FROM) { + memmove(sim+AND-FROM, sim, FROM); + memset(sim, 0, AND-FROM); + } else if (AND < FROM) { + memmove(sim, sim+FROM-AND, AND); + memset(sim+AND, 0, FROM-AND); + } + + // sync? + if (SYNC) { + lfsr_file_sync(&lfs, &file) => 0; + } + + // remount? + if (REMOUNT) { + lfsr_file_close(&lfs, &file) => 0; + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + lfsr_file_open(&lfs, &file, "hello", LFS_O_WRONLY) => 0; + } + + // fruncate to new size + lfsr_file_fruncate(&lfs, &file, TO) => 0; + if (TO > AND) { + memmove(sim+TO-AND, sim, AND); + memset(sim, 0, TO-AND); + } else if (TO < AND) { + memmove(sim, sim+AND-TO, TO); + memset(sim+TO, 0, AND-TO); + } + + // close + lfsr_file_close(&lfs, &file) => 0; + + // remount? + if (REMOUNT) { + lfsr_unmount(&lfs) => 0; + lfsr_mount(&lfs, CFG) => 0; + } + + // check our file with stat + struct lfs_info info; + lfsr_stat(&lfs, "hello", &info) => 0; + assert(strcmp(info.name, "hello") == 0); + assert(info.type == LFS_TYPE_REG); + assert(info.size == TO); + + // and with dir read + lfsr_dir_t dir; + lfsr_dir_open(&lfs, &dir, "/") => 0; + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, ".") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "..") == 0); + assert(info.type == LFS_TYPE_DIR); + lfsr_dir_read(&lfs, &dir, &info) => 0; + assert(strcmp(info.name, "hello") == 0); + assert(info.type == LFS_TYPE_REG); + assert(info.size == TO); + lfsr_dir_read(&lfs, &dir, &info) => LFS_ERR_NOENT; + lfsr_dir_close(&lfs, &dir) => 0; + + // try reading our file + lfsr_file_open(&lfs, &file, "hello", LFS_O_RDONLY) => 0; + // is size correct? + lfsr_file_size(&lfs, &file) => TO; + // try reading + uint8_t rbuf[2*TO]; + memset(rbuf, 0xaa, 2*TO); + lfsr_file_read(&lfs, &file, rbuf, 2*TO) => TO; + assert(memcmp(rbuf, sim, TO) == 0); + lfsr_file_close(&lfs, &file) => 0; + + lfsr_unmount(&lfs) => 0; +''' # writing any data structure backwards always reveals issues [cases.test_files_reversed] @@ -1958,7 +2182,7 @@ code = ''' lfsr_unmount(&lfs) => 0; ''' -# heavy fuzz test with rw seeks, truncate, and TODO fruncate +# heavy fuzz test with rw seeks, truncate, and fruncate [cases.test_files_rwtf_fuzz] defines.N = 100 defines.SEED = 'range(100)' @@ -2013,8 +2237,8 @@ code = ''' } for (lfs_size_t i = 0; i < N; i++) { - // and if we are reading, writing, or truncating - uint8_t op = TEST_PRNG(&prng) % 3; + // and if we are reading, writing, truncating, or fruncating + uint8_t op = TEST_PRNG(&prng) % 4; // writing? if (op == 0) { @@ -2081,11 +2305,30 @@ code = ''' lfs_off_t size_ = TEST_PRNG(&prng) % SIZE; // update the sim - memset(&sim[size_], 0, size - lfs_min32(size_, size)); + if (size_ < size) { + memset(sim+size_, 0, size-size_); + } size = size_; // truncate the file lfsr_file_truncate(&lfs, &file, size_) => 0; + + } else if (op == 3) { + // choose a random new file size + lfs_off_t size_ = TEST_PRNG(&prng) % SIZE; + + // update the sim + if (size_ > size) { + memmove(sim+size_-size, sim, size); + memset(sim, 0, size_-size); + } else if (size_ < size) { + memmove(sim, sim+size-size_, size_); + memset(sim+size_, 0, size-size_); + } + size = size_; + + // truncate the file + lfsr_file_fruncate(&lfs, &file, size_) => 0; } } lfsr_file_close(&lfs, &file) => 0; @@ -2130,7 +2373,6 @@ code = ''' # TODO -# [cases.test_files_rwtf_fuzz] ? # [cases.test_files_push] ? # [cases.test_files_pop] ? # [cases.test_files_rwtfpp_fuzz] ?