diff --git a/lfs.c b/lfs.c index 47817247..aed2baf9 100644 --- a/lfs.c +++ b/lfs.c @@ -9681,6 +9681,27 @@ static int lfsr_ftree_carve(lfs_t *lfs, lfsr_ftree_t *ftree, // copies during file writes, but it is nice to prove this constraint is // possible in case we ever don't track temporary copies. + // TODO is this the best place for this? + // + // before we touch anything, if our file is a btree, we need to mark all + // other references btree as unerased + if (lfsr_ftree_isbtree(ftree)) { + for (lfsr_openedmdir_t *opened_ = lfs->opened[ + LFS_TYPE_REG-LFS_TYPE_REG]; + opened_; + opened_ = opened_->next) { + lfsr_ftree_t *ftree_ = (lfsr_ftree_t*)opened_; + if (ftree_ != ftree + && lfsr_ftree_isbtree(ftree_) + && lfsr_btree_cmp( + &ftree_->u.btree, + &ftree->u.btree) == 0) { + // mark as unerased + ftree_->u.btree.eoff = -1; + } + } + } + // always convert to bshrub/btree when this function is called if (!lfsr_ftree_isbshruborbtree(ftree)) { lfsr_attr_t attrs_[2]; @@ -10462,7 +10483,7 @@ static int lfsr_ftree_flush(lfs_t *lfs, lfsr_ftree_t *ftree, return 0; } -static int lfsr_ftree_sync(lfs_t *lfs, lfsr_ftree_t *ftree, bool unflushed, +static int lfsr_ftree_sync(lfs_t *lfs, lfsr_ftree_t *ftree, uint32_t flags, lfs_off_t buffer_pos, const uint8_t *buffer, lfs_size_t buffer_size) { // note because of small-file caching and our current write // strategy, we never actually end up with only a direct data @@ -10472,38 +10493,41 @@ static int lfsr_ftree_sync(lfs_t *lfs, lfsr_ftree_t *ftree, bool unflushed, LFS_ASSERT(!lfsr_ftree_isbsprout(ftree)); LFS_ASSERT(!lfsr_ftree_isbleaf(ftree)); // small files should start as zero, const prop should optimize this out - LFS_ASSERT(!unflushed || buffer_pos == 0); + LFS_ASSERT(!lfsr_f_isunflushed(flags) || buffer_pos == 0); // small files/ftree should be exclusive here - LFS_ASSERT(!unflushed || lfsr_ftree_size(ftree) == 0); + LFS_ASSERT(!lfsr_f_isunflushed(flags) || lfsr_ftree_size(ftree) == 0); // small files must be inlined entirely in our buffer - LFS_ASSERT(!unflushed + LFS_ASSERT(!lfsr_f_isunflushed(flags) || (buffer_size <= lfs->cfg->cache_size && buffer_size <= lfs->cfg->inline_size && buffer_size <= lfs->cfg->fragment_size)); - // commit our file's metadata - uint8_t buf[LFSR_BTREE_DSIZE]; - int err = lfsr_mdir_commit(lfs, &ftree->mdir, LFSR_ATTRS( - (unflushed && buffer_size == 0) - ? LFSR_ATTR(ftree->mdir.mid, - WIDE(RM(STRUCT)), 0, - NULL()) - : (unflushed) - ? LFSR_ATTR(ftree->mdir.mid, - WIDE(DATA), 0, - BUF(buffer, buffer_size)) - : (lfsr_ftree_isbshrub(ftree)) - ? LFSR_ATTR(ftree->mdir.mid, - WIDE(SHRUBTRUNK), 0, - SHRUBTRUNK(&ftree->u.bshrub)) - : LFSR_ATTR(ftree->mdir.mid, - WIDE(BTREE), 0, - FROMBTREE(&ftree->u.btree, buf)))); - if (err) { - return err; + // don't write to disk if disk is already in-sync + if (lfsr_f_isunsynced(flags)) { + // commit our file's metadata + uint8_t buf[LFSR_BTREE_DSIZE]; + int err = lfsr_mdir_commit(lfs, &ftree->mdir, LFSR_ATTRS( + (lfsr_f_isunflushed(flags) && buffer_size == 0) + ? LFSR_ATTR(ftree->mdir.mid, + WIDE(RM(STRUCT)), 0, + NULL()) + : (lfsr_f_isunflushed(flags)) + ? LFSR_ATTR(ftree->mdir.mid, + WIDE(DATA), 0, + BUF(buffer, buffer_size)) + : (lfsr_ftree_isbshrub(ftree)) + ? LFSR_ATTR(ftree->mdir.mid, + WIDE(SHRUBTRUNK), 0, + SHRUBTRUNK(&ftree->u.bshrub)) + : LFSR_ATTR(ftree->mdir.mid, + WIDE(BTREE), 0, + FROMBTREE(&ftree->u.btree, buf)))); + if (err) { + return err; + } } - // update other file handles + // but do update other file handles for (lfsr_openedmdir_t *opened = lfs->opened[ LFS_TYPE_REG-LFS_TYPE_REG]; opened; @@ -10514,9 +10538,12 @@ static int lfsr_ftree_sync(lfs_t *lfs, lfsr_ftree_t *ftree, bool unflushed, && &file_->ftree != ftree // don't update desynced file handles && !lfsr_o_isdesync(file_->flags)) { - if (unflushed) { + file_->flags &= ~LFS_F_UNSYNCED; + if (lfsr_f_isunflushed(flags)) { + file_->flags |= LFS_F_UNFLUSHED; file_->size = buffer_size; } else { + file_->flags &= ~LFS_F_UNFLUSHED; file_->size = lfsr_ftree_size(ftree); } file_->ftree.u = ftree->u; @@ -10771,7 +10798,8 @@ lfs_ssize_t lfsr_file_write(lfs_t *lfs, lfsr_file_t *file, // sync if requested if (lfsr_o_issync(file->flags)) { // sync - err = lfsr_ftree_sync(lfs, &ftree_, unflushed_, + err = lfsr_ftree_sync(lfs, &ftree_, + LFS_F_UNSYNCED | ((unflushed_) ? LFS_F_UNFLUSHED : 0), buffer_pos_, buffer_, buffer_size_); if (err) { goto failed; @@ -10895,13 +10923,6 @@ int lfsr_file_sync(lfs_t *lfs, lfsr_file_t *file) { return 0; } - // do nothing if we're already in sync - if (!lfsr_f_isunsynced(file->flags)) { - // but do clear desync flag - file->flags &= ~LFS_O_DESYNC; - return 0; - } - // first flush any data in our buffer, this is a noop if already // flushed // @@ -10923,7 +10944,7 @@ int lfsr_file_sync(lfs_t *lfs, lfsr_file_t *file) { lfsr_ftree_size(&file->ftree))); // commit our file's metadata - err = lfsr_ftree_sync(lfs, &file->ftree, lfsr_f_isunflushed(file->flags), + err = lfsr_ftree_sync(lfs, &file->ftree, file->flags, file->buffer_pos, file->buffer, file->buffer_size); if (err) { goto failed; @@ -11001,6 +11022,9 @@ int lfsr_file_truncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size) { lfs_alloc_ckpoint(lfs); // copy state so we can recover from errors + bool unflushed_ = lfsr_f_isunflushed(file->flags); + lfs_off_t buffer_pos_ = file->buffer_pos; + lfs_size_t buffer_size_ = file->buffer_size; lfsr_ftree_t ftree_ = file->ftree; // add to tracked mdirs lfsr_mdir_addopened(lfs, LFS_TYPE_REG, (lfsr_openedmdir_t*)&ftree_); @@ -11014,6 +11038,10 @@ int lfsr_file_truncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size) { // need to flush so our buffer is available to hold everything if (file->buffer_pos > 0 || file->buffer_size < lfs_min32(size, file->size)) { + // TODO use ftree flush to avoid double ftree tracking? + // TODO or move this before our tracking here? we MUST update + // the file to use its buffer + // // note that flush does not change the actual file data, so if // a read fails it's ok to fall back to our flushed state err = lfsr_file_flush(lfs, file); @@ -11029,21 +11057,21 @@ int lfsr_file_truncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size) { err = d; goto failed; } - file->buffer_pos = 0; - file->buffer_size = size; + buffer_pos_ = 0; + buffer_size_ = size; } // we may need to zero some of our buffer - if (size > file->buffer_size) { - memset(&file->buffer[file->buffer_size], + if (size > buffer_size_) { + memset(&file->buffer[buffer_size_], 0, - size - file->buffer_size); + size - buffer_size_); } // small files remain perpetually unflushed - file->flags |= LFS_F_UNFLUSHED; - file->buffer_pos = 0; - file->buffer_size = size; + unflushed_ = true; + buffer_pos_ = 0; + buffer_size_ = size; ftree_.u.size = LFSR_FTREE_NULL; // truncate our file normally @@ -11059,18 +11087,64 @@ int lfsr_file_truncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size) { } // truncate our buffer - file->buffer_pos = lfs_min32(file->buffer_pos, size); - file->buffer_size = lfs_min32( - file->buffer_size, - size - lfs_min32(file->buffer_pos, size)); + buffer_pos_ = lfs_min32(buffer_pos_, size); + buffer_size_ = lfs_min32( + buffer_size_, + size - lfs_min32(buffer_pos_, size)); + } + + bool unsynced_ = true; + // flush if requested + // + // this initially seems unreachable, but it's possible if we transition + // from a small file to a non-small file + if (lfsr_o_isflush(file->flags) || lfsr_o_issync(file->flags)) { + // keep small files unflushed + if (unflushed_ && !( + size <= lfs->cfg->cache_size + && size <= lfs->cfg->inline_size + && size <= lfs->cfg->fragment_size)) { + // flush + err = lfsr_ftree_flush(lfs, &ftree_, + buffer_pos_, file->buffer, buffer_size_); + if (err) { + goto failed; + } + unflushed_ = false; + } + } + // sync if requested + if (lfsr_o_issync(file->flags)) { + // sync + err = lfsr_ftree_sync(lfs, &ftree_, + LFS_F_UNSYNCED | ((unflushed_) ? LFS_F_UNFLUSHED : 0), + buffer_pos_, file->buffer, buffer_size_); + if (err) { + goto failed; + } + + // mark as in-sync + unsynced_ = false; + file->flags &= ~LFS_O_DESYNC; } // remove from tracked mdirs lfsr_mdir_removeopened(lfs, LFS_TYPE_REG, (lfsr_openedmdir_t*)&ftree_); // mark as unsynced and update our size - file->flags |= LFS_F_UNSYNCED; - file->ftree.u = ftree_.u; + if (unflushed_) { + file->flags |= LFS_F_UNFLUSHED; + } else { + file->flags &= ~LFS_F_UNFLUSHED; + } + if (unsynced_) { + file->flags |= LFS_F_UNSYNCED; + } else { + file->flags &= ~LFS_F_UNSYNCED; + } file->size = size; + file->ftree.u = ftree_.u; + file->buffer_pos = buffer_pos_; + file->buffer_size = buffer_size_; LFS_ASSERT(file->size == lfs_max32( file->buffer_pos + file->buffer_size, lfsr_ftree_size(&file->ftree))); @@ -11099,6 +11173,9 @@ int lfsr_file_fruncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size) { lfs_alloc_ckpoint(lfs); // copy state so we can recover from errors + bool unflushed_ = lfsr_f_isunflushed(file->flags); + lfs_off_t buffer_pos_ = file->buffer_pos; + lfs_size_t buffer_size_ = file->buffer_size; lfsr_ftree_t ftree_ = file->ftree; // add to tracked mdirs lfsr_mdir_addopened(lfs, LFS_TYPE_REG, (lfsr_openedmdir_t*)&ftree_); @@ -11112,6 +11189,10 @@ int lfsr_file_fruncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size) { // need to flush so our buffer is available to hold everything if (file->buffer_pos + file->buffer_size < file->size || file->buffer_size < lfs_min32(size, file->size)) { + // TODO use ftree flush to avoid double ftree tracking? + // TODO or move this before our tracking here? we MUST update + // the file to use its buffer + // // note that flush does not change the actual file data, so if // a read fails it's ok to fall back to our flushed state err = lfsr_file_flush(lfs, file); @@ -11128,30 +11209,30 @@ int lfsr_file_fruncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size) { err = d; goto failed; } - file->buffer_pos = 0; - file->buffer_size = size; + buffer_pos_ = 0; + buffer_size_ = size; } // we may need to move the data in our buffer - if (file->buffer_size > size) { + if (buffer_size_ > size) { memmove(file->buffer, - &file->buffer[file->buffer_size - size], - file->buffer_size); + &file->buffer[buffer_size_ - size], + buffer_size_); } // we may need to zero some of our buffer - if (size > file->buffer_size) { - memmove(&file->buffer[size - file->buffer_size], + if (size > buffer_size_) { + memmove(&file->buffer[size - buffer_size_], file->buffer, - file->buffer_size); + buffer_size_); memset(file->buffer, 0, - size - file->buffer_size); + size - buffer_size_); } // small files remain perpetually unflushed - file->flags |= LFS_F_UNFLUSHED; - file->buffer_pos = 0; - file->buffer_size = size; + unflushed_ = true; + buffer_pos_ = 0; + buffer_size_ = size; ftree_.u.size = LFSR_FTREE_NULL; // fruncate our file normally @@ -11169,23 +11250,69 @@ int lfsr_file_fruncate(lfs_t *lfs, lfsr_file_t *file, lfs_off_t size) { // fruncate our buffer memmove(file->buffer, &file->buffer[lfs_min32( - lfs_smax32(file->size - size - file->buffer_pos, 0), - file->buffer_size)], - file->buffer_size - lfs_min32( - lfs_smax32(file->size - size - file->buffer_pos, 0), - file->buffer_size)); - file->buffer_size -= lfs_min32( - lfs_smax32(file->size - size - file->buffer_pos, 0), - file->buffer_size); - file->buffer_pos -= lfs_smin32(file->size - size, file->buffer_pos); + lfs_smax32(file->size - size - buffer_pos_, 0), + buffer_size_)], + buffer_size_ - lfs_min32( + lfs_smax32(file->size - size - buffer_pos_, 0), + buffer_size_)); + buffer_size_ -= lfs_min32( + lfs_smax32(file->size - size - buffer_pos_, 0), + buffer_size_); + buffer_pos_ -= lfs_smin32(file->size - size, buffer_pos_); + } + + bool unsynced_ = true; + // flush if requested + // + // this initially seems unreachable, but it's possible if we transition + // from a small file to a non-small file + if (lfsr_o_isflush(file->flags) || lfsr_o_issync(file->flags)) { + // keep small files unflushed + if (unflushed_ && !( + size <= lfs->cfg->cache_size + && size <= lfs->cfg->inline_size + && size <= lfs->cfg->fragment_size)) { + // flush + err = lfsr_ftree_flush(lfs, &ftree_, + buffer_pos_, file->buffer, buffer_size_); + if (err) { + goto failed; + } + unflushed_ = false; + } + } + // sync if requested + if (lfsr_o_issync(file->flags)) { + // sync + err = lfsr_ftree_sync(lfs, &ftree_, + LFS_F_UNSYNCED | ((unflushed_) ? LFS_F_UNFLUSHED : 0), + buffer_pos_, file->buffer, buffer_size_); + if (err) { + goto failed; + } + + // mark as in-sync + unsynced_ = false; + file->flags &= ~LFS_O_DESYNC; } // remove from tracked mdirs lfsr_mdir_removeopened(lfs, LFS_TYPE_REG, (lfsr_openedmdir_t*)&ftree_); // mark as unsynced and update our size - file->flags |= LFS_F_UNSYNCED; - file->ftree.u = ftree_.u; + if (unflushed_) { + file->flags |= LFS_F_UNFLUSHED; + } else { + file->flags &= ~LFS_F_UNFLUSHED; + } + if (unsynced_) { + file->flags |= LFS_F_UNSYNCED; + } else { + file->flags &= ~LFS_F_UNSYNCED; + } file->size = size; + file->ftree.u = ftree_.u; + file->buffer_pos = buffer_pos_; + file->buffer_size = buffer_size_; LFS_ASSERT(file->size == lfs_max32( file->buffer_pos + file->buffer_size, lfsr_ftree_size(&file->ftree))); diff --git a/tests/test_fmulti.toml b/tests/test_fmulti.toml index cd180588..5ef98cf0 100644 --- a/tests/test_fmulti.toml +++ b/tests/test_fmulti.toml @@ -899,6 +899,325 @@ code = ''' lfsr_unmount(&lfs) => 0; ''' +# Test multiple rw files without fixed size +[cases.test_fmulti_rwrw_sparse_fuzz] +defines.RW = 4 +# 0 => no sync, readers not updated +# 1 => sync via lfsr_file_sync +# 2 => sync via LFS_O_SYNC +defines.SYNC = [0, 1, 2] +# 0 => no flush +# 1 => flush via lfsr_file_flush +# 2 => flush via LFS_O_FLUSH +defines.FLUSH = [0, 1, 2] +defines.SEED = 'range(10)' +defines.N = 40 +defines.SIZE = [ + 'CACHE_SIZE/2', + '2*CACHE_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '4*BLOCK_SIZE', +] +defines.CHUNK = '(SIZE+16-1) / 16' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + uint32_t prng = 42; + uint8_t between[RW][SIZE]; + lfs_size_t between_size[RW]; + for (lfs_size_t rw = 0; rw < RW; rw++) { + between_size[rw] = 0; + } + uint8_t after[SIZE]; + lfs_size_t after_size = 0; + + // write RW rdwrs in parallel + lfsr_file_t rdwrs[RW]; + for (lfs_size_t rw = 0; rw < RW; rw++) { + // open files + lfsr_file_open(&lfs, &rdwrs[rw], "jello", + LFS_O_RDWR + | LFS_O_CREAT + | ((FLUSH == 2) ? LFS_O_FLUSH : 0) + | ((SYNC == 2) ? LFS_O_SYNC : 0)) => 0; + } + for (lfs_size_t i = 0; i < SIZE; i += CHUNK) { + for (lfs_size_t rw = 0; rw < RW; rw++) { + // choose a random operation + uint8_t op = TEST_PRNG(&prng) % 2; + + // writing? + if (op == 0) { + // choose a random offset + lfs_off_t off = (between_size[rw] > 0) + ? TEST_PRNG(&prng) % between_size[rw] + : 0; + lfs_size_t size = lfs_min32(CHUNK, SIZE - off); + uint8_t wbuf[CHUNK]; + for (lfs_size_t j = 0; j < size; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + + // write + lfsr_file_seek(&lfs, &rdwrs[rw], off, LFS_SEEK_SET) => off; + lfsr_file_write(&lfs, &rdwrs[rw], wbuf, size) => size; + if (FLUSH == 1) { + lfsr_file_flush(&lfs, &rdwrs[rw]) => 0; + } + if (SYNC == 1) { + lfsr_file_sync(&lfs, &rdwrs[rw]) => 0; + } + + // update sim + if (off > between_size[rw]) { + memset(&between[rw][between_size[rw]], + 0, + off - between_size[rw]); + } + memcpy(&between[rw][off], wbuf, size); + between_size[rw] = lfs_max32(off + size, between_size[rw]); + + // reading? + } else if (op == 1) { + // choose a random offset + lfs_off_t off = (between_size[rw] > 0) + ? TEST_PRNG(&prng) % between_size[rw] + : 0; + lfs_size_t size = lfs_min32(CHUNK, between_size[rw] - off); + + // read + lfsr_file_seek(&lfs, &rdwrs[rw], off, LFS_SEEK_SET) => off; + uint8_t rbuf[CHUNK]; + lfsr_file_read(&lfs, &rdwrs[rw], rbuf, CHUNK) => size; + assert(memcmp(rbuf, &between[rw][off], size) == 0); + } + + // broadcast sim? + if (SYNC) { + memcpy(after, between[rw], SIZE); + after_size = between_size[rw]; + + for (lfs_size_t rw_ = 0; rw_ < RW; rw_++) { + memcpy(between[rw_], between[rw], SIZE); + between_size[rw_] = between_size[rw]; + } + } + } + } + for (lfs_size_t rw = 0; rw < RW; rw++) { + // close files + lfsr_file_close(&lfs, &rdwrs[rw]) => 0; + + // broadcast sim one last time? + memcpy(after, between[rw], SIZE); + after_size = between_size[rw]; + + for (lfs_size_t rw_ = 0; rw_ < RW; rw_++) { + memcpy(between[rw_], between[rw], SIZE); + between_size[rw_] = between_size[rw]; + } + } + + // check that file was written as expected + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jello", LFS_O_RDONLY) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => after_size; + assert(memcmp(rbuf, after, after_size) == 0); + lfsr_file_close(&lfs, &file) => 0; + + lfsr_unmount(&lfs) => 0; +''' + +# Test multiple rw files while also truncating/fruncating +[cases.test_fmulti_rwtfrwtf_sparse_fuzz] +defines.RW = 4 +# 0 => no sync, readers not updated +# 1 => sync via lfsr_file_sync +# 2 => sync via LFS_O_SYNC +defines.SYNC = [0, 1, 2] +# 0 => no flush +# 1 => flush via lfsr_file_flush +# 2 => flush via LFS_O_FLUSH +defines.FLUSH = [0, 1, 2] +defines.SEED = 'range(10)' +defines.N = 40 +defines.SIZE = [ + 'CACHE_SIZE/2', + '2*CACHE_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '4*BLOCK_SIZE', +] +defines.CHUNK = '(SIZE+16-1) / 16' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + uint32_t prng = 42; + uint8_t between[RW][SIZE]; + lfs_size_t between_size[RW]; + for (lfs_size_t rw = 0; rw < RW; rw++) { + between_size[rw] = 0; + } + uint8_t after[SIZE]; + lfs_size_t after_size = 0; + + // write RW rdwrs in parallel + lfsr_file_t rdwrs[RW]; + for (lfs_size_t rw = 0; rw < RW; rw++) { + // open files + lfsr_file_open(&lfs, &rdwrs[rw], "jello", + LFS_O_RDWR + | LFS_O_CREAT + | ((FLUSH == 2) ? LFS_O_FLUSH : 0) + | ((SYNC == 2) ? LFS_O_SYNC : 0)) => 0; + } + for (lfs_size_t i = 0; i < SIZE; i += CHUNK) { + for (lfs_size_t rw = 0; rw < RW; rw++) { + // choose a random operation + uint8_t op = TEST_PRNG(&prng) % 4; + + // writing? + if (op == 0) { + // choose a random offset + lfs_off_t off = (between_size[rw] > 0) + ? TEST_PRNG(&prng) % between_size[rw] + : 0; + lfs_size_t size = lfs_min32(CHUNK, SIZE - off); + uint8_t wbuf[CHUNK]; + for (lfs_size_t j = 0; j < size; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + + // write + lfsr_file_seek(&lfs, &rdwrs[rw], off, LFS_SEEK_SET) => off; + lfsr_file_write(&lfs, &rdwrs[rw], wbuf, size) => size; + if (FLUSH == 1) { + lfsr_file_flush(&lfs, &rdwrs[rw]) => 0; + } + if (SYNC == 1) { + lfsr_file_sync(&lfs, &rdwrs[rw]) => 0; + } + + // update sim + if (off > between_size[rw]) { + memset(&between[rw][between_size[rw]], + 0, + off - between_size[rw]); + } + memcpy(&between[rw][off], wbuf, size); + between_size[rw] = lfs_max32(off + size, between_size[rw]); + + // reading? + } else if (op == 1) { + // choose a random offset + lfs_off_t off = (between_size[rw] > 0) + ? TEST_PRNG(&prng) % between_size[rw] + : 0; + lfs_size_t size = lfs_min32(CHUNK, between_size[rw] - off); + + // read + lfsr_file_seek(&lfs, &rdwrs[rw], off, LFS_SEEK_SET) => off; + uint8_t rbuf[CHUNK]; + lfsr_file_read(&lfs, &rdwrs[rw], rbuf, CHUNK) => size; + assert(memcmp(rbuf, &between[rw][off], size) == 0); + + // truncating? + } else if (op == 2) { + // choose a random new file size + lfs_off_t size = TEST_PRNG(&prng) % SIZE; + + // truncate + lfsr_file_truncate(&lfs, &rdwrs[rw], size) => 0; + if (FLUSH == 1) { // (flush does nothing) + lfsr_file_flush(&lfs, &rdwrs[rw]) => 0; + } + if (SYNC == 1) { + lfsr_file_sync(&lfs, &rdwrs[rw]) => 0; + } + + // update the sim + if (size > between_size[rw]) { + memset(&between[rw][between_size[rw]], + 0, + size - between_size[rw]); + } + between_size[rw] = size; + + // fruncating? + } else if (op == 3) { + // choose a random new file size + lfs_off_t size = TEST_PRNG(&prng) % SIZE; + + // fruncate + lfsr_file_fruncate(&lfs, &rdwrs[rw], size) => 0; + if (FLUSH == 1) { // (flush does nothing) + lfsr_file_flush(&lfs, &rdwrs[rw]) => 0; + } + if (SYNC == 1) { + lfsr_file_sync(&lfs, &rdwrs[rw]) => 0; + } + + // update the sim + if (size > between_size[rw]) { + memmove(&between[rw][size - between_size[rw]], + between[rw], + between_size[rw]); + memset(between[rw], + 0, + size - between_size[rw]); + } else { + memmove(between[rw], + &between[rw][between_size[rw] - size], + size); + } + between_size[rw] = size; + } + + // broadcast sim? + if (SYNC) { + memcpy(after, between[rw], SIZE); + after_size = between_size[rw]; + + for (lfs_size_t rw_ = 0; rw_ < RW; rw_++) { + memcpy(between[rw_], between[rw], SIZE); + between_size[rw_] = between_size[rw]; + } + } + } + } + for (lfs_size_t rw = 0; rw < RW; rw++) { + // close files + lfsr_file_close(&lfs, &rdwrs[rw]) => 0; + + // broadcast sim one last time? + memcpy(after, between[rw], SIZE); + after_size = between_size[rw]; + + for (lfs_size_t rw_ = 0; rw_ < RW; rw_++) { + memcpy(between[rw_], between[rw], SIZE); + between_size[rw_] = between_size[rw]; + } + } + + // check that file was written as expected + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jello", LFS_O_RDONLY) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => after_size; + assert(memcmp(rbuf, after, after_size) == 0); + lfsr_file_close(&lfs, &file) => 0; + + lfsr_unmount(&lfs) => 0; +''' + # Desynced files make things interesting @@ -1289,7 +1608,7 @@ code = ''' ''' # Test multiple desynced rd/wrers -[cases.test_fmulti_rdrd] +[cases.test_fmulti_rwdrwd] defines.RW = 4 # 0 => no sync, readers not updated # 1 => sync via lfsr_file_sync @@ -1397,7 +1716,7 @@ code = ''' lfsr_unmount(&lfs) => 0; ''' -[cases.test_fmulti_rdrd_fuzz] +[cases.test_fmulti_rwdrwd_fuzz] defines.RW = 4 # 0 => no sync, readers not updated # 1 => sync via lfsr_file_sync @@ -1516,3 +1835,340 @@ code = ''' lfsr_unmount(&lfs) => 0; ''' + +# Test multiple rwd files without fixed size +[cases.test_fmulti_rwdrwd_sparse_fuzz] +defines.RW = 4 +# 0 => no flush +# 1 => flush via lfsr_file_flush +# 2 => flush via LFS_O_FLUSH +defines.FLUSH = [0, 1, 2] +defines.SEED = 'range(10)' +defines.N = 40 +defines.SIZE = [ + 'CACHE_SIZE/2', + '2*CACHE_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '4*BLOCK_SIZE', +] +defines.CHUNK = '(SIZE+16-1) / 16' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + uint32_t prng = 42; + uint8_t between[RW][SIZE]; + lfs_size_t between_size[RW]; + bool between_desync[RW]; + for (lfs_size_t rw = 0; rw < RW; rw++) { + between_size[rw] = 0; + between_desync[rw] = false; + } + uint8_t after[SIZE]; + lfs_size_t after_size = 0; + + // write RW rdwrs in parallel + lfsr_file_t rdwrs[RW]; + for (lfs_size_t rw = 0; rw < RW; rw++) { + // open files + lfsr_file_open(&lfs, &rdwrs[rw], "jello", + LFS_O_RDWR + | LFS_O_CREAT + | ((FLUSH == 2) ? LFS_O_FLUSH : 0)) => 0; + } + for (lfs_size_t i = 0; i < SIZE; i += CHUNK) { + for (lfs_size_t rw = 0; rw < RW; rw++) { + // choose a random operation + uint8_t op = TEST_PRNG(&prng) % 2; + // choose a random sync state + uint8_t sync = TEST_PRNG(&prng) % 3; + + // writing? + if (op == 0) { + // choose a random offset + lfs_off_t off = (between_size[rw] > 0) + ? TEST_PRNG(&prng) % between_size[rw] + : 0; + lfs_size_t size = lfs_min32(CHUNK, SIZE - off); + uint8_t wbuf[CHUNK]; + for (lfs_size_t j = 0; j < size; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + + // write + lfsr_file_seek(&lfs, &rdwrs[rw], off, LFS_SEEK_SET) => off; + lfsr_file_write(&lfs, &rdwrs[rw], wbuf, size) => size; + if (FLUSH == 1) { + lfsr_file_flush(&lfs, &rdwrs[rw]) => 0; + } + + // update sim + if (off > between_size[rw]) { + memset(&between[rw][between_size[rw]], + 0, + off - between_size[rw]); + } + memcpy(&between[rw][off], wbuf, size); + between_size[rw] = lfs_max32(off + size, between_size[rw]); + + // reading? + } else if (op == 1) { + // choose a random offset + lfs_off_t off = (between_size[rw] > 0) + ? TEST_PRNG(&prng) % between_size[rw] + : 0; + lfs_size_t size = lfs_min32(CHUNK, between_size[rw] - off); + + // read + lfsr_file_seek(&lfs, &rdwrs[rw], off, LFS_SEEK_SET) => off; + uint8_t rbuf[CHUNK]; + lfsr_file_read(&lfs, &rdwrs[rw], rbuf, CHUNK) => size; + assert(memcmp(rbuf, &between[rw][off], size) == 0); + } + + // desync? + if (sync == 0) { + lfsr_file_desync(&lfs, &rdwrs[rw]) => 0; + between_desync[rw] = true; + + // sync? + } else if (sync == 1) { + lfsr_file_sync(&lfs, &rdwrs[rw]) => 0; + between_desync[rw] = false; + + // otherwise no change + } + + // broadcast sim? + if (sync == 1) { + memcpy(after, between[rw], SIZE); + after_size = between_size[rw]; + + for (lfs_size_t rw_ = 0; rw_ < RW; rw_++) { + if (!between_desync[rw_]) { + memcpy(between[rw_], between[rw], SIZE); + between_size[rw_] = between_size[rw]; + } + } + } + } + } + for (lfs_size_t rw = 0; rw < RW; rw++) { + // close files + lfsr_file_close(&lfs, &rdwrs[rw]) => 0; + + // broadcast sim one last time? + if (!between_desync[rw]) { + memcpy(after, between[rw], SIZE); + after_size = between_size[rw]; + + for (lfs_size_t rw_ = 0; rw_ < RW; rw_++) { + if (!between_desync[rw_]) { + memcpy(between[rw_], between[rw], SIZE); + between_size[rw_] = between_size[rw]; + } + } + } + } + + // check that file was written as expected + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jello", LFS_O_RDONLY) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => after_size; + assert(memcmp(rbuf, after, after_size) == 0); + lfsr_file_close(&lfs, &file) => 0; + + lfsr_unmount(&lfs) => 0; +''' + +# Test multiple rwd files while also truncating/fruncating +[cases.test_fmulti_rwtfdrwtfd_sparse_fuzz] +defines.RW = 4 +# 0 => no flush +# 1 => flush via lfsr_file_flush +# 2 => flush via LFS_O_FLUSH +defines.FLUSH = [0, 1, 2] +defines.SEED = 'range(10)' +defines.N = 40 +defines.SIZE = [ + 'CACHE_SIZE/2', + '2*CACHE_SIZE', + 'BLOCK_SIZE/2', + 'BLOCK_SIZE', + '2*BLOCK_SIZE', + '4*BLOCK_SIZE', +] +defines.CHUNK = '(SIZE+16-1) / 16' +code = ''' + lfs_t lfs; + lfsr_format(&lfs, CFG) => 0; + lfsr_mount(&lfs, CFG) => 0; + + uint32_t prng = 42; + uint8_t between[RW][SIZE]; + lfs_size_t between_size[RW]; + bool between_desync[RW]; + for (lfs_size_t rw = 0; rw < RW; rw++) { + between_size[rw] = 0; + between_desync[rw] = false; + } + uint8_t after[SIZE]; + lfs_size_t after_size = 0; + + // write RW rdwrs in parallel + lfsr_file_t rdwrs[RW]; + for (lfs_size_t rw = 0; rw < RW; rw++) { + // open files + lfsr_file_open(&lfs, &rdwrs[rw], "jello", + LFS_O_RDWR + | LFS_O_CREAT + | ((FLUSH == 2) ? LFS_O_FLUSH : 0)) => 0; + } + for (lfs_size_t i = 0; i < SIZE; i += CHUNK) { + for (lfs_size_t rw = 0; rw < RW; rw++) { + // choose a random operation + uint8_t op = TEST_PRNG(&prng) % 4; + // choose a random sync state + uint8_t sync = TEST_PRNG(&prng) % 3; + + // writing? + if (op == 0) { + // choose a random offset + lfs_off_t off = (between_size[rw] > 0) + ? TEST_PRNG(&prng) % between_size[rw] + : 0; + lfs_size_t size = lfs_min32(CHUNK, SIZE - off); + uint8_t wbuf[CHUNK]; + for (lfs_size_t j = 0; j < size; j++) { + wbuf[j] = 'a' + (TEST_PRNG(&prng) % 26); + } + + // write + lfsr_file_seek(&lfs, &rdwrs[rw], off, LFS_SEEK_SET) => off; + lfsr_file_write(&lfs, &rdwrs[rw], wbuf, size) => size; + if (FLUSH == 1) { + lfsr_file_flush(&lfs, &rdwrs[rw]) => 0; + } + + // update sim + if (off > between_size[rw]) { + memset(&between[rw][between_size[rw]], + 0, + off - between_size[rw]); + } + memcpy(&between[rw][off], wbuf, size); + between_size[rw] = lfs_max32(off + size, between_size[rw]); + + // reading? + } else if (op == 1) { + // choose a random offset + lfs_off_t off = (between_size[rw] > 0) + ? TEST_PRNG(&prng) % between_size[rw] + : 0; + lfs_size_t size = lfs_min32(CHUNK, between_size[rw] - off); + + // read + lfsr_file_seek(&lfs, &rdwrs[rw], off, LFS_SEEK_SET) => off; + uint8_t rbuf[CHUNK]; + lfsr_file_read(&lfs, &rdwrs[rw], rbuf, CHUNK) => size; + assert(memcmp(rbuf, &between[rw][off], size) == 0); + + // truncating? + } else if (op == 2) { + // choose a random new file size + lfs_off_t size = TEST_PRNG(&prng) % SIZE; + + // truncate + lfsr_file_truncate(&lfs, &rdwrs[rw], size) => 0; + + // update the sim + if (size > between_size[rw]) { + memset(&between[rw][between_size[rw]], + 0, + size - between_size[rw]); + } + between_size[rw] = size; + + // fruncating? + } else if (op == 3) { + // choose a random new file size + lfs_off_t size = TEST_PRNG(&prng) % SIZE; + + // fruncate + lfsr_file_fruncate(&lfs, &rdwrs[rw], size) => 0; + + // update the sim + if (size > between_size[rw]) { + memmove(&between[rw][size - between_size[rw]], + between[rw], + between_size[rw]); + memset(between[rw], + 0, + size - between_size[rw]); + } else { + memmove(between[rw], + &between[rw][between_size[rw] - size], + size); + } + between_size[rw] = size; + } + + // desync? + if (sync == 0) { + lfsr_file_desync(&lfs, &rdwrs[rw]) => 0; + between_desync[rw] = true; + + // sync? + } else if (sync == 1) { + lfsr_file_sync(&lfs, &rdwrs[rw]) => 0; + between_desync[rw] = false; + + // otherwise no change + } + + // broadcast sim? + if (sync == 1) { + memcpy(after, between[rw], SIZE); + after_size = between_size[rw]; + + for (lfs_size_t rw_ = 0; rw_ < RW; rw_++) { + if (!between_desync[rw_]) { + memcpy(between[rw_], between[rw], SIZE); + between_size[rw_] = between_size[rw]; + } + } + } + } + } + for (lfs_size_t rw = 0; rw < RW; rw++) { + // close files + lfsr_file_close(&lfs, &rdwrs[rw]) => 0; + + // broadcast sim one last time? + if (!between_desync[rw]) { + memcpy(after, between[rw], SIZE); + after_size = between_size[rw]; + + for (lfs_size_t rw_ = 0; rw_ < RW; rw_++) { + if (!between_desync[rw_]) { + memcpy(between[rw_], between[rw], SIZE); + between_size[rw_] = between_size[rw]; + } + } + } + } + + // check that file was written as expected + lfsr_file_t file; + lfsr_file_open(&lfs, &file, "jello", LFS_O_RDONLY) => 0; + uint8_t rbuf[SIZE]; + lfsr_file_read(&lfs, &file, rbuf, SIZE) => after_size; + assert(memcmp(rbuf, after, after_size) == 0); + lfsr_file_close(&lfs, &file) => 0; + + lfsr_unmount(&lfs) => 0; +''' diff --git a/tests/test_fwrite.toml b/tests/test_fwrite.toml index 6afa2e85..3e3be918 100644 --- a/tests/test_fwrite.toml +++ b/tests/test_fwrite.toml @@ -2382,6 +2382,7 @@ code = ''' // truncate the file lfsr_file_truncate(&lfs, &file, size_) => 0; + // fruncating? } else if (op == 3) { // choose a random new file size lfs_off_t size_ = TEST_PRNG(&prng) % SIZE;