From 88180b6081402c8f0c37e4e6680ab31f286feba2 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Wed, 23 Jul 2025 15:32:35 -0500 Subject: [PATCH] bmap: Initial scaffolding for on-disk block map This is pretty exploratory work, so I'm going to try to be less thorough in commit messages until the dust settles. --- New tag for gbmapdelta: LFS3_TAG_GBMAPDELTA 0x0104 v--- ---1 ---- -1rr New tags for in-bmap block types: LFS3_TAG_BMRANGE 0x033u v--- --11 --11 uuuu LFS3_TAG_BMFREE 0x0330 v--- --11 --11 ---- LFS3_TAG_BMINFLIGHT 0x0331 v--- --11 --11 ---1 LFS3_TAG_BMINUSE 0x0332 v--- --11 --11 --1- LFS3_TAG_BMBAD 0x0333 v--- --11 --11 --11 LFS3_TAG_BMERASED 0x0334 v--- --11 --11 -1-- New gstate decoding for gbmap: .---+- -+- -+- -+- -. cursor: 1 leb128 <=5 bytes | cursor | known: 1 leb128 <=5 bytes +---+- -+- -+- -+- -+ block: 1 leb128 <=5 bytes | known | trunk: 1 leb128 <=4 bytes +---+- -+- -+- -+- -+ cksum: 1 le32 4 bytes | block | total: 23 bytes +---+- -+- -+- -+- -' | trunk | +---+- -+- -+- -+ | cksum | '---+---+---+---' New bmap node revdbg string: vvv---- -111111- -11---1- -11---1- (62 62 7e v0 bb~r) bmap node New mount/format/info flags (still unsure about these): LFS3_M_BMAPMODE 0x03000000 On-disk block map mode LFS3_M_BMAPNONE 0x00000000 Don't use the bmap LFS3_M_BMAPCACHE 0x01000000 Use the bmap to cache lookahead scans LFS3_M_BMAPSLOW 0x02000000 Use the slow bmap algorithm LFS3_M_BMAPFAST 0x03000000 Use the fast bmap algorithm New gbmap wcompat flag: LFS3_WCOMPAT_GBMAP 0x00002000 Global block-map in use --- lfs3.c | 326 +++++++++++++++++++++++++++++++++++++---- lfs3.h | 81 +++++++++- lfs3_util.h | 29 ++++ runners/bench_runner.h | 11 +- runners/test_runner.h | 11 +- scripts/dbgbmap.py | 54 +++++-- scripts/dbgbmapsvg.py | 54 +++++-- scripts/dbgbtree.py | 15 ++ scripts/dbgflags.py | 23 ++- scripts/dbglfs3.py | 164 +++++++++++++++++++-- scripts/dbgmtree.py | 15 ++ scripts/dbgrbyd.py | 15 ++ scripts/dbgtag.py | 15 ++ 13 files changed, 755 insertions(+), 58 deletions(-) diff --git a/lfs3.c b/lfs3.c index 5acb2cef..3109d631 100644 --- a/lfs3.c +++ b/lfs3.c @@ -1034,6 +1034,7 @@ enum lfs3_tag { // global-state tags LFS3_TAG_GDELTA = 0x0100, LFS3_TAG_GRMDELTA = 0x0100, + LFS3_TAG_GBMAPDELTA = 0x0104, // name tags LFS3_TAG_NAME = 0x0200, @@ -1060,6 +1061,12 @@ enum lfs3_tag { LFS3_TAG_MROOT = 0x0321, LFS3_TAG_MDIR = 0x0325, LFS3_TAG_MTREE = 0x032c, + LFS3_TAG_BMRANGE = 0x0330, + LFS3_TAG_BMFREE = 0x0330, + LFS3_TAG_BMINFLIGHT = 0x0331, + LFS3_TAG_BMINUSE = 0x0332, + LFS3_TAG_BMBAD = 0x0333, + LFS3_TAG_BMERASED = 0x0334, // user/sys attributes LFS3_TAG_ATTR = 0x0400, @@ -5133,8 +5140,8 @@ static lfs3_data_t lfs3_data_frombranch(const lfs3_rbyd_t *branch, #endif #ifndef LFS3_2BONLY -static int lfs3_data_readbranch(lfs3_t *lfs3, - lfs3_bid_t weight, lfs3_data_t *data, +static int lfs3_data_readbranch(lfs3_t *lfs3, lfs3_data_t *data, + lfs3_bid_t weight, lfs3_rbyd_t *branch) { // setting eoff to 0 here will trigger asserts if we try to append // without fetching first @@ -5198,7 +5205,7 @@ static int lfs3_data_fetchbranch(lfs3_t *lfs3, lfs3_data_t *data, lfs3_bid_t weight, lfs3_rbyd_t *branch) { // decode branch and fetch - int err = lfs3_data_readbranch(lfs3, weight, data, + int err = lfs3_data_readbranch(lfs3, data, weight, branch); if (err) { return err; @@ -5244,11 +5251,12 @@ static int lfs3_data_readbtree(lfs3_t *lfs3, lfs3_data_t *data, return err; } - err = lfs3_data_readbranch(lfs3, weight, data, &btree->r); + err = lfs3_data_readbranch(lfs3, data, weight, &btree->r); if (err) { return err; } + // make sure to zero btree leaf lfs3_btree_discardleaf(btree); return 0; } @@ -5450,7 +5458,7 @@ static int lfs3_btree_parent(lfs3_t *lfs3, const lfs3_btree_t *btree, // fetch the next branch lfs3_rbyd_t child__; - int err = lfs3_data_readbranch(lfs3, weight__, &data__, &child__); + int err = lfs3_data_readbranch(lfs3, &data__, weight__, &child__); if (err) { return err; } @@ -7402,10 +7410,80 @@ static inline bool lfs3_m_isckdatacksums(uint32_t flags) { } #endif +#ifdef LFS3_BMAP +static inline bool lfs3_m_isbmapnone(uint32_t flags) { + (void)flags; + #if defined(LFS3_YES_BMAPNONE) + return true; + #elif defined(LFS3_YES_BMAPFAST) \ + || defined(LFS3_YES_BMAPSLOW) \ + || defined(LFS3_YES_BMAPCACHE) \ + || defined(LFS3_YES_BMAPNONE) + return false; + #else + return (flags & LFS3_M_BMAPMODE) == LFS3_M_BMAPNONE; + #endif +} +#endif + +#ifdef LFS3_BMAP +static inline bool lfs3_m_isbmapcache(uint32_t flags) { + (void)flags; + #if defined(LFS3_YES_BMAPCACHE) + return true; + #elif defined(LFS3_YES_BMAPFAST) \ + || defined(LFS3_YES_BMAPSLOW) \ + || defined(LFS3_YES_BMAPCACHE) \ + || defined(LFS3_YES_BMAPNONE) + return false; + #else + return (flags & LFS3_M_BMAPMODE) == LFS3_M_BMAPCACHE; + #endif +} +#endif + +#ifdef LFS3_BMAP +static inline bool lfs3_m_isbmapslow(uint32_t flags) { + (void)flags; + #if defined(LFS3_YES_BMAPSLOW) + return true; + #elif defined(LFS3_YES_BMAPFAST) \ + || defined(LFS3_YES_BMAPSLOW) \ + || defined(LFS3_YES_BMAPCACHE) \ + || defined(LFS3_YES_BMAPNONE) + return false; + #else + return (flags & LFS3_M_BMAPMODE) == LFS3_M_BMAPSLOW; + #endif +} +#endif + +#ifdef LFS3_BMAP +static inline bool lfs3_m_isbmapfast(uint32_t flags) { + (void)flags; + #if defined(LFS3_YES_BMAPFAST) + return true; + #elif defined(LFS3_YES_BMAPFAST) \ + || defined(LFS3_YES_BMAPSLOW) \ + || defined(LFS3_YES_BMAPCACHE) \ + || defined(LFS3_YES_BMAPNONE) + return false; + #else + return (flags & LFS3_M_BMAPMODE) == LFS3_M_BMAPFAST; + #endif +} +#endif + // other internal flags #ifdef LFS3_REVDBG static inline bool lfs3_i_isinmtree(uint32_t flags) { - return flags & LFS3_i_INMTREE; + return (flags & LFS3_i_INMODE) == LFS3_i_INMTREE; +} +#endif + +#ifdef LFS3_REVDBG +static inline bool lfs3_i_isinbmap(uint32_t flags) { + return (flags & LFS3_i_INMODE) == LFS3_i_INBMAP; } #endif @@ -7716,11 +7794,12 @@ static int lfs3_fs_consumegdelta(lfs3_t *lfs3, const lfs3_mdir_t *mdir) { // on rbyd type, though it may be overwritten by the recycle counter if // it overlaps: // -// vvvv---- --1----1 -11-1--1 -11-1--- (68 69 21 v0 hi!.) mroot anchor -// vvvv---- -111111- -111--1- -11-11-1 (6d 72 7e v0 mr~.) mroot -// vvvv---- -111111- -11--1-- -11-11-1 (6d 64 7e v0 md~.) mdir -// vvvv---- -111111- -111-1-- -11---1- (62 74 7e v0 bt~.) file btree node -// vvvv---- -111111- -11-11-1 -11---1- (62 6d 7e v0 bm~.) mtree node +// vvvv---- --1----1 -11-1--1 -11-1--- (68 69 21 v0 hi!r) mroot anchor +// vvvv---- -111111- -111--1- -11-11-1 (6d 72 7e v0 mr~r) mroot +// vvvv---- -111111- -11--1-- -11-11-1 (6d 64 7e v0 md~r) mdir +// vvvv---- -111111- -111-1-- -11---1- (62 74 7e v0 bt~r) file btree node +// vvvv---- -111111- -11-11-1 -11---1- (62 6d 7e v0 bm~r) mtree node +// vvvv---- -111111- -11---1- -11---1- (62 62 7e v0 bb~r) bmap node // // needed in lfs3_rev_init @@ -7741,10 +7820,10 @@ static inline uint32_t lfs3_rev_init(lfs3_t *lfs3, const lfs3_mdir_t *mdir, if (lfs3_m_isrevdbg(lfs3->flags)) { // mroot? if (mdir->mid == -1 || lfs3_mdir_cmp(mdir, &lfs3->mroot) == 0) { - rev |= 0x007e726d; + rev |= 0x007e726d; // mr~r // mdir? } else { - rev |= 0x007e646d; + rev |= 0x007e646d; // md~r } } #endif @@ -7769,10 +7848,13 @@ static inline uint32_t lfs3_rev_btree(lfs3_t *lfs3) { if (lfs3_m_isrevdbg(lfs3->flags)) { // mtree? if (lfs3_i_isinmtree(lfs3->flags)) { - rev |= 0x007e6d62; + rev |= 0x007e6d62; // bm~r + // bmap? + } else if (lfs3_i_isinbmap(lfs3->flags)) { + rev |= 0x007e6262; // bb~r // file btree? } else { - rev |= 0x007e7462; + rev |= 0x007e7462; // bt~r } } #endif @@ -8055,7 +8137,18 @@ static int lfs3_mtree_commit(lfs3_t *lfs3, lfs3_btree_t *mtree, #ifdef LFS3_REVDBG lfs3->flags |= LFS3_i_INMTREE; #endif + int err = lfs3_btree_commit(lfs3, mtree, bid, rattrs, rattr_count); + if (err) { + goto failed; + } + + #ifdef LFS3_REVDBG + lfs3->flags &= ~LFS3_i_INMTREE; + #endif + return 0; + +failed:; #ifdef LFS3_REVDBG lfs3->flags &= ~LFS3_i_INMTREE; #endif @@ -10336,6 +10429,66 @@ eot:; + +/// Optional on-disk block map /// + +#if !defined(LFS3_RDONLY) && !defined(LFS3_2BONLY) && defined(LFS3_BMAP) +static lfs3_data_t lfs3_data_fromgbmap(const lfs3_t *lfs3, + uint8_t buffer[static LFS3_GBMAP_DSIZE]) { + // cursor should not exceed 31-bits + LFS3_ASSERT(lfs3->gbmap.cursor <= 0x7fffffff); + // known should not exceed 31-bits + LFS3_ASSERT(lfs3->gbmap.known <= 0x7fffffff); + + // make sure to zero so we don't leak any info + lfs3_memset(buffer, 0, LFS3_GBMAP_DSIZE); + + lfs3_ssize_t d = 0; + lfs3_ssize_t d_ = lfs3_toleb128(lfs3->gbmap.cursor, &buffer[d], 5); + if (d_ < 0) { + LFS3_UNREACHABLE(); + } + d += d_; + + d_ = lfs3_toleb128(lfs3->gbmap.known, &buffer[d], 5); + if (d_ < 0) { + LFS3_UNREACHABLE(); + } + d += d_; + + lfs3_data_t data = lfs3_data_frombranch(&lfs3->gbmap.b.r, &buffer[d]); + d += lfs3_data_size(data); + + return LFS3_DATA_BUF(buffer, lfs3_memlen(buffer, LFS3_GBMAP_DSIZE)); +} +#endif + +#if !defined(LFS3_RDONLY) && !defined(LFS3_2BONLY) && defined(LFS3_BMAP) +static int lfs3_data_readgbmap(lfs3_t *lfs3, lfs3_data_t *data) { + int err = lfs3_data_readleb128(lfs3, data, &lfs3->gbmap.cursor); + if (err) { + return err; + } + + err = lfs3_data_readleb128(lfs3, data, &lfs3->gbmap.known); + if (err) { + return err; + } + + err = lfs3_data_readbranch(lfs3, data, lfs3->block_count, + &lfs3->gbmap.b.r); + if (err) { + return err; + } + + // make sure to zero btree leaf + lfs3_btree_discardleaf(&lfs3->gbmap.b); + return 0; +} +#endif + + + /// Block allocator /// // checkpoint the allocator @@ -14319,7 +14472,8 @@ static int lfs3_init(lfs3_t *lfs3, uint32_t flags, | LFS3_IFDEF_CKPROGS(LFS3_M_CKPROGS, 0) | LFS3_IFDEF_CKFETCHES(LFS3_M_CKFETCHES, 0) | LFS3_IFDEF_CKMETAPARITY(LFS3_M_CKMETAPARITY, 0) - | LFS3_IFDEF_CKDATACKSUMS(LFS3_M_CKDATACKSUMS, 0))) == 0); + | LFS3_IFDEF_CKDATACKSUMS(LFS3_M_CKDATACKSUMS, 0) + | LFS3_IFDEF_BMAP(LFS3_M_BMAPMODE, 0))) == 0); // LFS3_M_REVDBG and LFS3_M_REVNOISE are incompatible #if defined(LFS3_REVNOISE) && defined(LFS3_REVDBG) LFS3_ASSERT(!lfs3_m_isrevdbg(flags) || !lfs3_m_isrevnoise(flags)); @@ -14457,6 +14611,19 @@ static int lfs3_init(lfs3_t *lfs3, uint32_t flags, lfs3_alloc_discard(lfs3); #endif + // setup treediff buffer + #if !defined(LFS3_RDONLY) && !defined(LFS3_2BONLY) && defined(LFS3_BMAP) + if (lfs3->cfg->treediff_buffer) { + lfs3->treediff.buffer = lfs3->cfg->treediff_buffer; + } else { + lfs3->treediff.buffer = lfs3_malloc(lfs3->cfg->treediff_size); + if (!lfs3->treediff.buffer) { + err = LFS3_ERR_NOMEM; + goto failed; + } + } + #endif + // check that the size limits are sane #ifndef LFS3_RDONLY LFS3_ASSERT(lfs3->cfg->name_limit <= LFS3_NAME_MAX); @@ -14629,6 +14796,12 @@ static int lfs3_deinit(lfs3_t *lfs3) { } #endif + #if !defined(LFS3_RDONLY) && !defined(LFS3_2BONLY) && defined(LFS3_BMAP) + if (!lfs3->cfg->treediff_buffer) { + lfs3_free(lfs3->treediff.buffer); + } + #endif + return 0; } @@ -14669,12 +14842,17 @@ static int lfs3_deinit(lfs3_t *lfs3) { #define LFS3_WCOMPAT_RDONLY 0x00000002 // Writing is disallowed #define LFS3_WCOMPAT_DIR 0x00000010 // Directory files in use #define LFS3_WCOMPAT_GCKSUM 0x00001000 // Global-checksum in use +#define LFS3_WCOMPAT_GBMAP 0x00002000 // Global block-map in use // internal #define LFS3_wcompat_OVERFLOW 0x80000000 // Can't represent all flags +// TODO this should really be calculated at runtime, we should allow no +// gbmap when bmap mode == LFS3_M_BMAPNONE even when compiling with +// LFS3_BMAP #define LFS3_WCOMPAT_COMPAT \ (LFS3_WCOMPAT_DIR \ - | LFS3_WCOMPAT_GCKSUM) + | LFS3_WCOMPAT_GCKSUM \ + | LFS3_IFDEF_BMAP(LFS3_WCOMPAT_GBMAP, 0)) #define LFS3_OCOMPAT_NONSTANDARD 0x00000001 // Non-standard filesystem format // internal @@ -15209,6 +15387,15 @@ int lfs3_mount(lfs3_t *lfs3, uint32_t flags, #ifdef LFS3_YES_CKDATA flags |= LFS3_M_CKDATA #endif + #if defined(LFS3_YES_BMAPFAST) + flags = (flags & ~LFS3_M_BMAPMODE) | LFS3_M_BMAPFAST; + #elif defined(LFS3_YES_BMAPSLOW) + flags = (flags & ~LFS3_M_BMAPMODE) | LFS3_M_BMAPSLOW; + #elif defined(LFS3_YES_BMAPCACHE) + flags = (flags & ~LFS3_M_BMAPMODE) | LFS3_M_BMAPCACHE; + #elif defined(LFS3_YES_BMAPNONE) + flags = (flags & ~LFS3_M_BMAPMODE) | LFS3_M_BMAPNONE; + #endif // unknown flags? LFS3_ASSERT((flags & ~( @@ -15226,7 +15413,8 @@ int lfs3_mount(lfs3_t *lfs3, uint32_t flags, | LFS3_IFDEF_RDONLY(0, LFS3_M_LOOKAHEAD) | LFS3_IFDEF_RDONLY(0, LFS3_M_COMPACT) | LFS3_M_CKMETA - | LFS3_M_CKDATA)) == 0); + | LFS3_M_CKDATA + | LFS3_IFDEF_BMAP(LFS3_M_BMAPMODE, 0))) == 0); // these flags require a writable filesystem LFS3_ASSERT(!lfs3_m_isrdonly(flags) || !lfs3_t_ismkconsistent(flags)); LFS3_ASSERT(!lfs3_m_isrdonly(flags) || !lfs3_t_islookahead(flags)); @@ -15243,7 +15431,8 @@ int lfs3_mount(lfs3_t *lfs3, uint32_t flags, | LFS3_IFDEF_CKPROGS(LFS3_M_CKPROGS, 0) | LFS3_IFDEF_CKFETCHES(LFS3_M_CKFETCHES, 0) | LFS3_IFDEF_CKMETAPARITY(LFS3_M_CKMETAPARITY, 0) - | LFS3_IFDEF_CKDATACKSUMS(LFS3_M_CKDATACKSUMS, 0)), + | LFS3_IFDEF_CKDATACKSUMS(LFS3_M_CKDATACKSUMS, 0) + | LFS3_IFDEF_BMAP(LFS3_M_BMAPMODE, 0)), cfg); if (err) { return err; @@ -15314,14 +15503,79 @@ int lfs3_unmount(lfs3_t *lfs3) { /// Format /// +#ifdef LFS3_BMAP +static int lfs3_formatbmap(lfs3_t *lfs3) { + #ifdef LFS3_REVDBG + lfs3->flags |= LFS3_i_INBMAP; + #endif + + // TODO should we try multiple blocks? + // assume we can write bmap to block 2 + lfs3->gbmap.cursor = 3; + lfs3->gbmap.known = lfs3->cfg->block_count; + lfs3->gbmap.b.r.blocks[0] = 2; + lfs3->gbmap.b.r.trunk = 0; + lfs3->gbmap.b.r.weight = 0; + lfs3->gbmap.b.r.eoff = 0; + lfs3->gbmap.b.r.cksum = 0; + + int err = lfs3_bd_erase(lfs3, lfs3->gbmap.b.r.blocks[0]); + if (err) { + goto failed; + } + + #if defined(LFS3_REVDBG) || defined(LFS3_REVNOISE) + // append a revision count? + err = lfs3_rbyd_appendrev(lfs3, &lfs3->gbmap.b.r, lfs3_rev_btree(lfs3)); + if (err) { + goto failed; + } + #endif + + err = lfs3_rbyd_commit(lfs3, &lfs3->gbmap.b.r, 0, LFS3_RATTRS( + // blocks 0..3 - in-use + LFS3_RATTR(LFS3_TAG_BMINUSE, +3), + // blocks 3..block_count - free + LFS3_RATTR(LFS3_TAG_BMFREE, +(lfs3->cfg->block_count - 3)))); + if (err) { + goto failed; + } + + #ifdef LFS3_REVDBG + lfs3->flags &= ~LFS3_i_INBMAP; + #endif + return 0; + +failed:; + #ifdef LFS3_REVDBG + lfs3->flags &= ~LFS3_i_INBMAP; + #endif + return err; +} +#endif + #ifndef LFS3_RDONLY static int lfs3_formatinited(lfs3_t *lfs3) { + int err; + // create an initial bmap + #ifdef LFS3_BMAP + err = lfs3_formatbmap(lfs3); + if (err) { + return err; + } + #endif + for (int i = 0; i < 2; i++) { // write superblock to both rbyds in the root mroot to hopefully // avoid mounting an older filesystem on disk - lfs3_rbyd_t rbyd = {.blocks[0]=i, .eoff=0, .trunk=0}; + lfs3_rbyd_t rbyd; + rbyd.blocks[0] = i; + rbyd.trunk = 0; + rbyd.weight = 0; + rbyd.eoff = 0; + rbyd.cksum = 0; - int err = lfs3_bd_erase(lfs3, rbyd.blocks[0]); + err = lfs3_bd_erase(lfs3, rbyd.blocks[0]); if (err) { return err; } @@ -15368,6 +15622,16 @@ static int lfs3_formatinited(lfs3_t *lfs3) { LFS3_RATTR_LEB128( LFS3_TAG_FILELIMIT, 0, lfs3->file_limit), + // TODO this is no good, switch to a builder pattern? + #ifdef LFS3_BMAP + (lfs3_m_isbmapfast(lfs3->flags) + || lfs3_m_isbmapslow(lfs3->flags) + || lfs3_m_isbmapcache(lfs3->flags)) + ? LFS3_RATTR_DATA(LFS3_TAG_GBMAPDELTA, 0, + (&((struct {lfs3_data_t d;}){ + lfs3_data_fromgbmap(lfs3, lfs3->gbmap_d)}).d)) + : LFS3_RATTR_NOOP(), + #endif LFS3_RATTR_NAME( LFS3_TAG_BOOKMARK, +1, 0, NULL, 0))); @@ -15391,7 +15655,7 @@ static int lfs3_formatinited(lfs3_t *lfs3) { } // sync on-disk state - int err = lfs3_bd_sync(lfs3); + err = lfs3_bd_sync(lfs3); if (err) { return err; } @@ -15427,6 +15691,15 @@ int lfs3_format(lfs3_t *lfs3, uint32_t flags, #ifdef LFS3_YES_CKDATA flags |= LFS3_F_CKDATA #endif + #if defined(LFS3_YES_BMAPFAST) + flags = (flags & ~LFS3_F_BMAPMODE) | LFS3_F_BMAPFAST; + #elif defined(LFS3_YES_BMAPSLOW) + flags = (flags & ~LFS3_F_BMAPMODE) | LFS3_F_BMAPSLOW; + #elif defined(LFS3_YES_BMAPCACHE) + flags = (flags & ~LFS3_F_BMAPMODE) | LFS3_F_BMAPCACHE; + #elif defined(LFS3_YES_BMAPNONE) + flags = (flags & ~LFS3_F_BMAPMODE) | LFS3_F_BMAPNONE; + #endif // unknown flags? LFS3_ASSERT((flags & ~( @@ -15438,7 +15711,8 @@ int lfs3_format(lfs3_t *lfs3, uint32_t flags, | LFS3_IFDEF_CKMETAPARITY(LFS3_F_CKMETAPARITY, 0) | LFS3_IFDEF_CKDATACKSUMS(LFS3_F_CKDATACKSUMS, 0) | LFS3_F_CKMETA - | LFS3_F_CKDATA)) == 0); + | LFS3_F_CKDATA + | LFS3_IFDEF_BMAP(LFS3_F_BMAPMODE, 0))) == 0); int err = lfs3_init(lfs3, flags & ( @@ -15448,7 +15722,8 @@ int lfs3_format(lfs3_t *lfs3, uint32_t flags, | LFS3_IFDEF_CKPROGS(LFS3_F_CKPROGS, 0) | LFS3_IFDEF_CKFETCHES(LFS3_F_CKFETCHES, 0) | LFS3_IFDEF_CKMETAPARITY(LFS3_F_CKMETAPARITY, 0) - | LFS3_IFDEF_CKDATACKSUMS(LFS3_F_CKDATACKSUMS, 0)), + | LFS3_IFDEF_CKDATACKSUMS(LFS3_F_CKDATACKSUMS, 0) + | LFS3_IFDEF_BMAP(LFS3_F_BMAPMODE, 0)), cfg); if (err) { return err; @@ -15515,7 +15790,8 @@ int lfs3_fs_stat(lfs3_t *lfs3, struct lfs3_fsinfo *fsinfo) { | LFS3_IFDEF_RDONLY(0, LFS3_I_LOOKAHEAD) | LFS3_IFDEF_RDONLY(0, LFS3_I_COMPACT) | LFS3_I_CKMETA - | LFS3_I_CKDATA); + | LFS3_I_CKDATA + | LFS3_IFDEF_BMAP(LFS3_I_BMAPMODE, 0)); // some flags we calculate on demand #ifndef LFS3_RDONLY fsinfo->flags |= (lfs3_grm_count(lfs3) > 0) ? LFS3_I_MKCONSISTENT : 0; diff --git a/lfs3.h b/lfs3.h index c40339ad..52a334dd 100644 --- a/lfs3.h +++ b/lfs3.h @@ -191,6 +191,15 @@ enum lfs3_type { #define LFS3_F_CKMETA 0x00001000 // Check metadata checksums #define LFS3_F_CKDATA 0x00002000 // Check metadata + data checksums + +#ifdef LFS3_BMAP +#define LFS3_F_BMAPMODE 0x03000000 // On-disk block map mode +#define LFS3_F_BMAPNONE 0x00000000 // Don't use the bmap +#define LFS3_F_BMAPCACHE \ + 0x01000000 // Use the bmap to cache lookahead scans +#define LFS3_F_BMAPSLOW 0x02000000 // Use the slow bmap algorithm +#define LFS3_F_BMAPFAST 0x03000000 // Use the fast bmap algorithm +#endif #endif // Filesystem mount flags @@ -237,6 +246,16 @@ enum lfs3_type { #define LFS3_M_CKMETA 0x00001000 // Check metadata checksums #define LFS3_M_CKDATA 0x00002000 // Check metadata + data checksums +#ifdef LFS3_BMAP +#define LFS3_M_BMAPMODE 0x03000000 // On-disk block map mode +#define LFS3_M_BMAPNONE 0x00000000 // Don't use the bmap +#define LFS3_M_BMAPCACHE \ + 0x01000000 // Use the bmap to cache lookahead scans +#define LFS3_M_BMAPSLOW 0x02000000 // Use the slow bmap algorithm +#define LFS3_M_BMAPFAST 0x03000000 // Use the fast bmap algorithm +#endif + + // Filesystem info flags #define LFS3_I_RDONLY 0x00000001 // Mounted read only #define LFS3_I_FLUSH 0x00000040 // Mounted with LFS3_M_FLUSH @@ -277,9 +296,20 @@ enum lfs3_type { #define LFS3_I_CKMETA 0x00001000 // Metadata checksums not checked recently #define LFS3_I_CKDATA 0x00002000 // Data checksums not checked recently +#ifdef LFS3_BMAP +#define LFS3_I_BMAPMODE 0x03000000 // On-disk block map mode +#define LFS3_I_BMAPNONE 0x00000000 // Mounted with LFS3_M_BMAPNONE +#define LFS3_I_BMAPCACHE \ + 0x01000000 // Mounted with LFS3_M_BMAPCACHE +#define LFS3_I_BMAPSLOW 0x02000000 // Mounted with LFS3_M_BMAPSLOW +#define LFS3_I_BMAPFAST 0x03000000 // Mounted with LFS3_M_BMAPFAST +#endif + // internally used flags, don't use these #ifdef LFS3_REVDBG -#define LFS3_i_INMTREE 0x08000000 // Committing to mtree +#define LFS3_i_INMODE 0x00030000 +#define LFS3_i_INMTREE 0x00010000 // Committing to mtree +#define LFS3_i_INBMAP 0x00020000 // Committing to bmap #endif @@ -427,13 +457,20 @@ struct lfs3_cfg { lfs3_size_t file_cache_size; // Size of the lookahead buffer in bytes. A larger lookahead buffer - // increases the number of blocks found during an allocation pass. The + // increases the number of blocks found during an allocation scan. The // lookahead buffer is stored as a compact bitmap, so each byte of RAM // can track 8 blocks. #ifndef LFS3_RDONLY lfs3_size_t lookahead_size; #endif + // Size of the treediff buffer in bytes. A larger treediff buffer speeds + // up tree diffing in BMAPSLOW and BMAPFAST modes. The treediff buffer + // also uses a compact bitmap, and sizes >block_count/8 have no effect. + #if !defined(LFS3_RDONLY) && defined(LFS3_BMAP) + lfs3_size_t treediff_size; + #endif + // Flags indicating what gc work to do during lfs3_gc calls. #ifdef LFS3_GC uint32_t gc_flags; @@ -482,6 +519,12 @@ struct lfs3_cfg { void *lookahead_buffer; #endif + // Optional statically allocated treediff buffer. Must be treediff_size. + // By default lfs3_malloc is used to allocate this buffer. + #if !defined(LFS3_RDONLY) && defined(LFS3_BMAP) + void *treediff_buffer; + #endif + // Optional upper limit on length of file names in bytes. No downside for // larger names except the size of the info struct which is controlled by // the LFS3_NAME_MAX define. Defaults to LFS3_NAME_MAX when zero. Stored in @@ -784,6 +827,8 @@ typedef struct lfs3_trv { lfs3_sblock_t blocks[2]; } lfs3_trv_t; +// littlefs global state + // grm encoding: // .- -+- -+- -+- -+- -. mids: 2 leb128s <=2x5 bytes // ' mids ' total: <=10 bytes @@ -797,6 +842,26 @@ typedef struct lfs3_grm { lfs3_smid_t queue[2]; } lfs3_grm_t; +// gbmap encoding: +// .---+- -+- -+- -+- -. cursor: 1 leb128 <=5 bytes +// | cursor | known: 1 leb128 <=5 bytes +// +---+- -+- -+- -+- -+ block: 1 leb128 <=5 bytes +// | known | trunk: 1 leb128 <=4 bytes +// +---+- -+- -+- -+- -+ cksum: 1 le32 4 bytes +// | block | total: 23 bytes +// +---+- -+- -+- -+- -' +// | trunk | +// +---+- -+- -+- -+ +// | cksum | +// '---+---+---+---' +#define LFS3_GBMAP_DSIZE (5+5+5+4+4) + +typedef struct lfs3_gbmap { + lfs3_block_t cursor; + lfs3_block_t known; + lfs3_btree_t b; +} lfs3_gbmap_t; + // The littlefs filesystem type typedef struct lfs3 { @@ -857,6 +922,12 @@ typedef struct lfs3 { } lookahead; #endif + #if !defined(LFS3_RDONLY) && !defined(LFS3_2BONLY) && defined(LFS3_BMAP) + struct lfs3_treediff { + uint8_t *buffer; + } treediff; + #endif + #if !defined(LFS3_RDONLY) && !defined(LFS3_2BONLY) const lfs3_data_t *graft; lfs3_ssize_t graft_count; @@ -877,6 +948,12 @@ typedef struct lfs3 { // TODO can we actually get rid of grm_d when LFS3_RDONLY? uint8_t grm_d[LFS3_GRM_DSIZE]; + #if !defined(LFS3_RDONLY) && !defined(LFS3_2BONLY) && defined(LFS3_BMAP) + lfs3_gbmap_t gbmap; + uint8_t gbmap_p[LFS3_GBMAP_DSIZE]; + uint8_t gbmap_d[LFS3_GBMAP_DSIZE]; + #endif + // optional incremental gc state #ifdef LFS3_GC struct { diff --git a/lfs3_util.h b/lfs3_util.h index 8ee727bc..36fdff95 100644 --- a/lfs3_util.h +++ b/lfs3_util.h @@ -47,6 +47,9 @@ #ifndef LFS3_GC #define LFS3_GC #endif +#ifndef LFS3_BMAP +#define LFS3_BMAP +#endif #endif // LFS3_YES_* variants imply the relevant LFS3_* macro @@ -80,6 +83,26 @@ #ifdef LFS3_YES_GC #define LFS3_GC #endif +#ifdef LFS3_YES_BMAP +#define LFS3_BMAP +#endif + +// TODO is this the best way to structure this? +// LFS3_BMAP mappings +// +// LFS3_YES_BMAP => LFS3_YES_BMAPFAST +#ifdef LFS3_YES_BMAP +#ifndef LFS3_YES_BMAPFAST +#define LFS3_YES_BMAPFAST +#endif +#endif +// LFS3_YES_BMAP* => LFS3_BMAP +#if defined(LFS3_YES_BMAPFAST) \ + || defined(LFS3_YES_BMAPSLOW) \ + || defined(LFS3_YES_BMAPCACHE) \ + || defined(LFS3_YES_BMAPNONE) +#define LFS3_BMAP +#endif // LFS3_NO_LOG disables all logging macros #ifdef LFS3_NO_LOG @@ -257,6 +280,12 @@ #define LFS3_IFDEF_GC(a, b) (b) #endif +#ifdef LFS3_BMAP +#define LFS3_IFDEF_BMAP(a, b) (a) +#else +#define LFS3_IFDEF_BMAP(a, b) (b) +#endif + // Some function attributes, no way around these diff --git a/runners/bench_runner.h b/runners/bench_runner.h index ebb048a6..47d73fbf 100644 --- a/runners/bench_runner.h +++ b/runners/bench_runner.h @@ -114,6 +114,7 @@ void bench_permutation(size_t i, uint32_t *buffer, size_t size); BENCH_DEFINE(PCACHE_SIZE, LFS3_MAX(16, PROG_SIZE) ) \ BENCH_DEFINE(FILE_CACHE_SIZE, 16 ) \ BENCH_DEFINE(LOOKAHEAD_SIZE, 16 ) \ + BENCH_DEFINE(TREEDIFF_SIZE, 16 ) \ BENCH_DEFINE(GC_FLAGS, 0 ) \ BENCH_DEFINE(GC_STEPS, 0 ) \ BENCH_DEFINE(GC_COMPACT_THRESH, 0 ) \ @@ -144,14 +145,22 @@ void bench_permutation(size_t i, uint32_t *buffer, size_t size); .pcache_size = PCACHE_SIZE, \ .file_cache_size = FILE_CACHE_SIZE, \ .lookahead_size = LOOKAHEAD_SIZE, \ + BENCH_BMAP_CFG \ BENCH_GC_CFG \ .gc_compact_thresh = GC_COMPACT_THRESH, \ .inline_size = INLINE_SIZE, \ .fragment_size = FRAGMENT_SIZE, \ .crystal_thresh = CRYSTAL_THRESH, +#ifdef LFS3_BMAP +#define BENCH_BMAP_CFG \ + .treediff_size = TREEDIFF_SIZE, +#else +#define BENCH_BMAP_CFG +#endif + #ifdef LFS3_GC -#define BENCH_GC_CFG \ +#define BENCH_GC_CFG \ .gc_flags = GC_FLAGS, \ .gc_steps = GC_STEPS, #else diff --git a/runners/test_runner.h b/runners/test_runner.h index 7223dfcf..c53014ec 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -105,6 +105,7 @@ void test_permutation(size_t i, uint32_t *buffer, size_t size); TEST_DEFINE(PCACHE_SIZE, LFS3_MAX(16, PROG_SIZE) ) \ TEST_DEFINE(FILE_CACHE_SIZE, 16 ) \ TEST_DEFINE(LOOKAHEAD_SIZE, 16 ) \ + TEST_DEFINE(TREEDIFF_SIZE, 16 ) \ TEST_DEFINE(GC_FLAGS, 0 ) \ TEST_DEFINE(GC_STEPS, 0 ) \ TEST_DEFINE(GC_COMPACT_THRESH, 0 ) \ @@ -135,14 +136,22 @@ void test_permutation(size_t i, uint32_t *buffer, size_t size); .pcache_size = PCACHE_SIZE, \ .file_cache_size = FILE_CACHE_SIZE, \ .lookahead_size = LOOKAHEAD_SIZE, \ + TEST_BMAP_CFG \ TEST_GC_CFG \ .gc_compact_thresh = GC_COMPACT_THRESH, \ .inline_size = INLINE_SIZE, \ .fragment_size = FRAGMENT_SIZE, \ .crystal_thresh = CRYSTAL_THRESH, +#ifdef LFS3_BMAP +#define TEST_BMAP_CFG \ + .treediff_size = TREEDIFF_SIZE, +#else +#define TEST_BMAP_CFG +#endif + #ifdef LFS3_GC -#define TEST_GC_CFG \ +#define TEST_GC_CFG \ .gc_flags = GC_FLAGS, \ .gc_steps = GC_STEPS, #else diff --git a/scripts/dbgbmap.py b/scripts/dbgbmap.py index d9ecf29b..a9a336ba 100755 --- a/scripts/dbgbmap.py +++ b/scripts/dbgbmap.py @@ -41,6 +41,7 @@ TAG_NAMELIMIT = 0x0039 # 0x0039 v--- ---- --11 1--1 TAG_FILELIMIT = 0x003a # 0x003a v--- ---- --11 1-1- TAG_GDELTA = 0x0100 ## 0x01tt v--- ---1 -ttt ttrr TAG_GRMDELTA = 0x0100 # 0x0100 v--- ---1 ---- ---- +TAG_GBMAPDELTA = 0x0104 # 0x0104 v--- ---1 ---- -1rr TAG_NAME = 0x0200 ## 0x02tt v--- --1- -ttt tttt TAG_BNAME = 0x0200 # 0x0200 v--- --1- ---- ---- TAG_REG = 0x0201 # 0x0201 v--- --1- ---- ---1 @@ -58,6 +59,12 @@ TAG_BTREE = 0x031c # 0x031c v--- --11 ---1 11rr TAG_MROOT = 0x0321 # 0x032r v--- --11 --1- --rr TAG_MDIR = 0x0325 # 0x0324 v--- --11 --1- -1rr TAG_MTREE = 0x032c # 0x032c v--- --11 --1- 11rr +TAG_BMRANGE = 0x0330 # 0x033u v--- --11 --11 uuuu +TAG_BMFREE = 0x0330 # 0x0330 v--- --11 --11 ---- +TAG_BMINFLIGHT = 0x0331 # 0x0331 v--- --11 --11 ---1 +TAG_BMINUSE = 0x0332 # 0x0332 v--- --11 --11 --1- +TAG_BMBAD = 0x0333 # 0x0333 v--- --11 --11 --11 +TAG_BMERASED = 0x0334 # 0x0334 v--- --11 --11 -1-- TAG_ATTR = 0x0400 ## 0x04aa v--- -1-a -aaa aaaa TAG_UATTR = 0x0400 # 0x04aa v--- -1-- -aaa aaaa TAG_SATTR = 0x0500 # 0x05aa v--- -1-1 -aaa aaaa @@ -310,6 +317,7 @@ def tagrepr(tag, weight=None, size=None, *, return '%s%s%s%s' % ( 'shrub' if tag & TAG_SHRUB else '', 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA + else 'gbmapdelta' if (tag & 0xfff) == TAG_GBMAPDELTA else 'gdelta 0x%02x' % (tag & 0xff), ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') @@ -339,6 +347,13 @@ def tagrepr(tag, weight=None, size=None, *, else 'mroot' if (tag & 0xfff) == TAG_MROOT else 'mdir' if (tag & 0xfff) == TAG_MDIR else 'mtree' if (tag & 0xfff) == TAG_MTREE + else 'bmfree' if (tag & 0xfff) == TAG_BMFREE + else 'bminflight' if (tag & 0xfff) == TAG_BMINFLIGHT + else 'bminuse' if (tag & 0xfff) == TAG_BMINUSE + else 'bmbad' if (tag & 0xfff) == TAG_BMBAD + else 'bmerased' if (tag & 0xfff) == TAG_BMERASED + else 'bmrange 0x%x' % (tag & 0xf) + if (tag & 0xff0) == TAG_BMRANGE else 'struct 0x%02x' % (tag & 0xff), ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') @@ -2585,8 +2600,9 @@ class Config: # lazy gstate object class Gstate: - def __init__(self, mtree): + def __init__(self, mtree, config): self.mtree = mtree + self.config = config # lookup a specific tag def lookup(self, tag=None, mask=None): @@ -2662,7 +2678,7 @@ class Gstate: tag = None mask = None - def __init__(self, mtree, tag, gdeltas): + def __init__(self, mtree, config, tag, gdeltas): # replace tag with what we find self.tag = tag # keep track of gdeltas for debugging @@ -2712,8 +2728,8 @@ class Gstate: class Gcksum(Gstate): tag = TAG_GCKSUMDELTA - def __init__(self, mtree, tag, gdeltas): - super().__init__(mtree, tag, gdeltas) + def __init__(self, mtree, config, tag, gdeltas): + super().__init__(mtree, config, tag, gdeltas) self.gcksum = fromle32(self.data) def __int__(self): @@ -2726,8 +2742,8 @@ class Gstate: class Grm(Gstate): tag = TAG_GRMDELTA - def __init__(self, mtree, tag, gdeltas): - super().__init__(mtree, tag, gdeltas) + def __init__(self, mtree, config, tag, gdeltas): + super().__init__(mtree, config, tag, gdeltas) queue = [] d = 0 for _ in range(2): @@ -2745,6 +2761,26 @@ class Gstate: def repr(self): return 'grm [%s]' % ', '.join(mid.repr() for mid in self.queue) + # the global block map + class Gbmap(Gstate): + tag = TAG_GBMAPDELTA + + def __init__(self, mtree, config, tag, gdeltas): + super().__init__(mtree, config, tag, gdeltas) + d = 0 + self.cursor, d_ = fromleb128(self.data, d); d += d_ + self.known, d_ = fromleb128(self.data, d); d += d_ + block, trunk, cksum, d_ = frombranch(self.data, d); d += d_ + self.btree = Btree.fetchck( + mtree.bd, block, trunk, + config.geometry.block_count, + cksum) + + def repr(self): + return 'gbmap %s k%s+%s' % ( + self.btree.addr(), + self.known, self.cursor) + # keep track of known gstate _known = [g for g in Gstate.__subclasses__() if g.tag is not None] @@ -2753,10 +2789,10 @@ class Gstate: # known config? for g in self._known: if (g.tag & ~(g.mask or 0)) == (tag & ~(g.mask or 0)): - return g(self.mtree, tag, gdeltas) + return g(self.mtree, self.config, tag, gdeltas) # otherwise return a marker class else: - return Unknown(self.mtree, tag, gdeltas) + return self.Unknown(self.mtree, self.config, tag, gdeltas) # create cached accessors for known gstate def _parser(g): @@ -2777,7 +2813,7 @@ class Lfs3: # create lazy config/gstate objects self.config = config or Config(self.mroot) - self.gstate = gstate or Gstate(self.mtree) + self.gstate = gstate or Gstate(self.mtree, self.config) # go ahead and fetch some expected fields self.version = self.config.version diff --git a/scripts/dbgbmapsvg.py b/scripts/dbgbmapsvg.py index c1a77a92..fb669012 100755 --- a/scripts/dbgbmapsvg.py +++ b/scripts/dbgbmapsvg.py @@ -39,6 +39,7 @@ TAG_NAMELIMIT = 0x0039 # 0x0039 v--- ---- --11 1--1 TAG_FILELIMIT = 0x003a # 0x003a v--- ---- --11 1-1- TAG_GDELTA = 0x0100 ## 0x01tt v--- ---1 -ttt ttrr TAG_GRMDELTA = 0x0100 # 0x0100 v--- ---1 ---- ---- +TAG_GBMAPDELTA = 0x0104 # 0x0104 v--- ---1 ---- -1rr TAG_NAME = 0x0200 ## 0x02tt v--- --1- -ttt tttt TAG_BNAME = 0x0200 # 0x0200 v--- --1- ---- ---- TAG_REG = 0x0201 # 0x0201 v--- --1- ---- ---1 @@ -56,6 +57,12 @@ TAG_BTREE = 0x031c # 0x031c v--- --11 ---1 11rr TAG_MROOT = 0x0321 # 0x032r v--- --11 --1- --rr TAG_MDIR = 0x0325 # 0x0324 v--- --11 --1- -1rr TAG_MTREE = 0x032c # 0x032c v--- --11 --1- 11rr +TAG_BMRANGE = 0x0330 # 0x033u v--- --11 --11 uuuu +TAG_BMFREE = 0x0330 # 0x0330 v--- --11 --11 ---- +TAG_BMINFLIGHT = 0x0331 # 0x0331 v--- --11 --11 ---1 +TAG_BMINUSE = 0x0332 # 0x0332 v--- --11 --11 --1- +TAG_BMBAD = 0x0333 # 0x0333 v--- --11 --11 --11 +TAG_BMERASED = 0x0334 # 0x0334 v--- --11 --11 -1-- TAG_ATTR = 0x0400 ## 0x04aa v--- -1-a -aaa aaaa TAG_UATTR = 0x0400 # 0x04aa v--- -1-- -aaa aaaa TAG_SATTR = 0x0500 # 0x05aa v--- -1-1 -aaa aaaa @@ -340,6 +347,7 @@ def tagrepr(tag, weight=None, size=None, *, return '%s%s%s%s' % ( 'shrub' if tag & TAG_SHRUB else '', 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA + else 'gbmapdelta' if (tag & 0xfff) == TAG_GBMAPDELTA else 'gdelta 0x%02x' % (tag & 0xff), ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') @@ -369,6 +377,13 @@ def tagrepr(tag, weight=None, size=None, *, else 'mroot' if (tag & 0xfff) == TAG_MROOT else 'mdir' if (tag & 0xfff) == TAG_MDIR else 'mtree' if (tag & 0xfff) == TAG_MTREE + else 'bmfree' if (tag & 0xfff) == TAG_BMFREE + else 'bminflight' if (tag & 0xfff) == TAG_BMINFLIGHT + else 'bminuse' if (tag & 0xfff) == TAG_BMINUSE + else 'bmbad' if (tag & 0xfff) == TAG_BMBAD + else 'bmerased' if (tag & 0xfff) == TAG_BMERASED + else 'bmrange 0x%x' % (tag & 0xf) + if (tag & 0xff0) == TAG_BMRANGE else 'struct 0x%02x' % (tag & 0xff), ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') @@ -2615,8 +2630,9 @@ class Config: # lazy gstate object class Gstate: - def __init__(self, mtree): + def __init__(self, mtree, config): self.mtree = mtree + self.config = config # lookup a specific tag def lookup(self, tag=None, mask=None): @@ -2692,7 +2708,7 @@ class Gstate: tag = None mask = None - def __init__(self, mtree, tag, gdeltas): + def __init__(self, mtree, config, tag, gdeltas): # replace tag with what we find self.tag = tag # keep track of gdeltas for debugging @@ -2742,8 +2758,8 @@ class Gstate: class Gcksum(Gstate): tag = TAG_GCKSUMDELTA - def __init__(self, mtree, tag, gdeltas): - super().__init__(mtree, tag, gdeltas) + def __init__(self, mtree, config, tag, gdeltas): + super().__init__(mtree, config, tag, gdeltas) self.gcksum = fromle32(self.data) def __int__(self): @@ -2756,8 +2772,8 @@ class Gstate: class Grm(Gstate): tag = TAG_GRMDELTA - def __init__(self, mtree, tag, gdeltas): - super().__init__(mtree, tag, gdeltas) + def __init__(self, mtree, config, tag, gdeltas): + super().__init__(mtree, config, tag, gdeltas) queue = [] d = 0 for _ in range(2): @@ -2775,6 +2791,26 @@ class Gstate: def repr(self): return 'grm [%s]' % ', '.join(mid.repr() for mid in self.queue) + # the global block map + class Gbmap(Gstate): + tag = TAG_GBMAPDELTA + + def __init__(self, mtree, config, tag, gdeltas): + super().__init__(mtree, config, tag, gdeltas) + d = 0 + self.cursor, d_ = fromleb128(self.data, d); d += d_ + self.known, d_ = fromleb128(self.data, d); d += d_ + block, trunk, cksum, d_ = frombranch(self.data, d); d += d_ + self.btree = Btree.fetchck( + mtree.bd, block, trunk, + config.geometry.block_count, + cksum) + + def repr(self): + return 'gbmap %s k%s+%s' % ( + self.btree.addr(), + self.known, self.cursor) + # keep track of known gstate _known = [g for g in Gstate.__subclasses__() if g.tag is not None] @@ -2783,10 +2819,10 @@ class Gstate: # known config? for g in self._known: if (g.tag & ~(g.mask or 0)) == (tag & ~(g.mask or 0)): - return g(self.mtree, tag, gdeltas) + return g(self.mtree, self.config, tag, gdeltas) # otherwise return a marker class else: - return Unknown(self.mtree, tag, gdeltas) + return self.Unknown(self.mtree, self.config, tag, gdeltas) # create cached accessors for known gstate def _parser(g): @@ -2807,7 +2843,7 @@ class Lfs3: # create lazy config/gstate objects self.config = config or Config(self.mroot) - self.gstate = gstate or Gstate(self.mtree) + self.gstate = gstate or Gstate(self.mtree, self.config) # go ahead and fetch some expected fields self.version = self.config.version diff --git a/scripts/dbgbtree.py b/scripts/dbgbtree.py index fb1bd4ad..bd36ef93 100755 --- a/scripts/dbgbtree.py +++ b/scripts/dbgbtree.py @@ -30,6 +30,7 @@ TAG_NAMELIMIT = 0x0039 # 0x0039 v--- ---- --11 1--1 TAG_FILELIMIT = 0x003a # 0x003a v--- ---- --11 1-1- TAG_GDELTA = 0x0100 ## 0x01tt v--- ---1 -ttt ttrr TAG_GRMDELTA = 0x0100 # 0x0100 v--- ---1 ---- ---- +TAG_GBMAPDELTA = 0x0104 # 0x0104 v--- ---1 ---- -1rr TAG_NAME = 0x0200 ## 0x02tt v--- --1- -ttt tttt TAG_BNAME = 0x0200 # 0x0200 v--- --1- ---- ---- TAG_REG = 0x0201 # 0x0201 v--- --1- ---- ---1 @@ -47,6 +48,12 @@ TAG_BTREE = 0x031c # 0x031c v--- --11 ---1 11rr TAG_MROOT = 0x0321 # 0x032r v--- --11 --1- --rr TAG_MDIR = 0x0325 # 0x0324 v--- --11 --1- -1rr TAG_MTREE = 0x032c # 0x032c v--- --11 --1- 11rr +TAG_BMRANGE = 0x0330 # 0x033u v--- --11 --11 uuuu +TAG_BMFREE = 0x0330 # 0x0330 v--- --11 --11 ---- +TAG_BMINFLIGHT = 0x0331 # 0x0331 v--- --11 --11 ---1 +TAG_BMINUSE = 0x0332 # 0x0332 v--- --11 --11 --1- +TAG_BMBAD = 0x0333 # 0x0333 v--- --11 --11 --11 +TAG_BMERASED = 0x0334 # 0x0334 v--- --11 --11 -1-- TAG_ATTR = 0x0400 ## 0x04aa v--- -1-a -aaa aaaa TAG_UATTR = 0x0400 # 0x04aa v--- -1-- -aaa aaaa TAG_SATTR = 0x0500 # 0x05aa v--- -1-1 -aaa aaaa @@ -218,6 +225,7 @@ def tagrepr(tag, weight=None, size=None, *, return '%s%s%s%s' % ( 'shrub' if tag & TAG_SHRUB else '', 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA + else 'gbmapdelta' if (tag & 0xfff) == TAG_GBMAPDELTA else 'gdelta 0x%02x' % (tag & 0xff), ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') @@ -247,6 +255,13 @@ def tagrepr(tag, weight=None, size=None, *, else 'mroot' if (tag & 0xfff) == TAG_MROOT else 'mdir' if (tag & 0xfff) == TAG_MDIR else 'mtree' if (tag & 0xfff) == TAG_MTREE + else 'bmfree' if (tag & 0xfff) == TAG_BMFREE + else 'bminflight' if (tag & 0xfff) == TAG_BMINFLIGHT + else 'bminuse' if (tag & 0xfff) == TAG_BMINUSE + else 'bmbad' if (tag & 0xfff) == TAG_BMBAD + else 'bmerased' if (tag & 0xfff) == TAG_BMERASED + else 'bmrange 0x%x' % (tag & 0xf) + if (tag & 0xff0) == TAG_BMRANGE else 'struct 0x%02x' % (tag & 0xff), ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') diff --git a/scripts/dbgflags.py b/scripts/dbgflags.py index 8fc7cfa2..1b844add 100755 --- a/scripts/dbgflags.py +++ b/scripts/dbgflags.py @@ -72,6 +72,12 @@ FLAGS = [ ('F_CKMETA', 0x00001000, "Check metadata checksums" ), ('F_CKDATA', 0x00002000, "Check metadata + data checksums" ), + ('F_BMAPMODE', 0x03000000, "On-disk block map mode" ), + ('^_BMAPNONE', 0x00000000, "Don't use the bmap" ), + ('^_BMAPCACHE', 0x01000000, "Use the bmap to cache lookahead scans" ), + ('^_BMAPSLOW', 0x02000000, "Use the slow bmap algorithm" ), + ('^_BMAPFAST', 0x03000000, "Use the fast bmap algorithm" ), + # Filesystem mount flags ('M_MODE', 1, "Mount's access mode" ), ('^_RDWR', 0, "Mount the filesystem as read and write" ), @@ -91,6 +97,12 @@ FLAGS = [ ('M_CKMETA', 0x00001000, "Check metadata checksums" ), ('M_CKDATA', 0x00002000, "Check metadata + data checksums" ), + ('M_BMAPMODE', 0x03000000, "On-disk block map mode" ), + ('^_BMAPNONE', 0x00000000, "Don't use the bmap" ), + ('^_BMAPCACHE', 0x01000000, "Use the bmap to cache lookahead scans" ), + ('^_BMAPSLOW', 0x02000000, "Use the slow bmap algorithm" ), + ('^_BMAPFAST', 0x03000000, "Use the fast bmap algorithm" ), + # GC flags ('GC_MKCONSISTENT',0x00000100, "Make the filesystem consistent" ), ('GC_LOOKAHEAD', 0x00000200, "Populate lookahead buffer" ), @@ -115,7 +127,15 @@ FLAGS = [ ('I_CKMETA', 0x00001000, "Metadata checksums not checked recently" ), ('I_CKDATA', 0x00002000, "Data checksums not checked recently" ), - ('i_INMTREE', 0x08000000, "Committing to mtree" ), + ('I_BMAPMODE', 0x03000000, "On-disk block map mode" ), + ('^_BMAPNONE', 0x00000000, "Mounted with LFS3_M_BMAPNONE" ), + ('^_BMAPCACHE', 0x01000000, "Mounted with LFS3_M_BMAPCACHE" ), + ('^_BMAPSLOW', 0x02000000, "Mounted with LFS3_M_BMAPSLOW" ), + ('^_BMAPFAST', 0x03000000, "Mounted with LFS3_M_BMAPFAST" ), + + ('i_INMTREE', 0x00030000, "Committing to mtree" ), + ('^_INMTREE', 0x00010000, "Committing to mtree" ), + ('^_INBMAP', 0x00020000, "Committing to bmap" ), # Traversal flags ('T_MODE', 1, "The traversal's access mode" ), @@ -181,6 +201,7 @@ FLAGS = [ ('WCOMPAT_RDONLY', 0x00000002, "Writing is disallowed" ), ('WCOMPAT_DIR', 0x00000010, "Directory file types in use" ), ('WCOMPAT_GCKSUM', 0x00001000, "Global-checksum in use" ), + ('WCOMPAT_GBMAP', 0x00002000, "Global block-map in use" ), ('wcompat_OVERFLOW', 0x80000000, "Can't represent all flags" ), diff --git a/scripts/dbglfs3.py b/scripts/dbglfs3.py index ffe05bcb..77cb2d45 100755 --- a/scripts/dbglfs3.py +++ b/scripts/dbglfs3.py @@ -31,6 +31,7 @@ TAG_NAMELIMIT = 0x0039 # 0x0039 v--- ---- --11 1--1 TAG_FILELIMIT = 0x003a # 0x003a v--- ---- --11 1-1- TAG_GDELTA = 0x0100 ## 0x01tt v--- ---1 -ttt ttrr TAG_GRMDELTA = 0x0100 # 0x0100 v--- ---1 ---- ---- +TAG_GBMAPDELTA = 0x0104 # 0x0104 v--- ---1 ---- -1rr TAG_NAME = 0x0200 ## 0x02tt v--- --1- -ttt tttt TAG_BNAME = 0x0200 # 0x0200 v--- --1- ---- ---- TAG_REG = 0x0201 # 0x0201 v--- --1- ---- ---1 @@ -48,6 +49,12 @@ TAG_BTREE = 0x031c # 0x031c v--- --11 ---1 11rr TAG_MROOT = 0x0321 # 0x032r v--- --11 --1- --rr TAG_MDIR = 0x0325 # 0x0324 v--- --11 --1- -1rr TAG_MTREE = 0x032c # 0x032c v--- --11 --1- 11rr +TAG_BMRANGE = 0x0330 # 0x033u v--- --11 --11 uuuu +TAG_BMFREE = 0x0330 # 0x0330 v--- --11 --11 ---- +TAG_BMINFLIGHT = 0x0331 # 0x0331 v--- --11 --11 ---1 +TAG_BMINUSE = 0x0332 # 0x0332 v--- --11 --11 --1- +TAG_BMBAD = 0x0333 # 0x0333 v--- --11 --11 --11 +TAG_BMERASED = 0x0334 # 0x0334 v--- --11 --11 -1-- TAG_ATTR = 0x0400 ## 0x04aa v--- -1-a -aaa aaaa TAG_UATTR = 0x0400 # 0x04aa v--- -1-- -aaa aaaa TAG_SATTR = 0x0500 # 0x05aa v--- -1-1 -aaa aaaa @@ -267,6 +274,7 @@ def tagrepr(tag, weight=None, size=None, *, return '%s%s%s%s' % ( 'shrub' if tag & TAG_SHRUB else '', 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA + else 'gbmapdelta' if (tag & 0xfff) == TAG_GBMAPDELTA else 'gdelta 0x%02x' % (tag & 0xff), ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') @@ -296,6 +304,13 @@ def tagrepr(tag, weight=None, size=None, *, else 'mroot' if (tag & 0xfff) == TAG_MROOT else 'mdir' if (tag & 0xfff) == TAG_MDIR else 'mtree' if (tag & 0xfff) == TAG_MTREE + else 'bmfree' if (tag & 0xfff) == TAG_BMFREE + else 'bminflight' if (tag & 0xfff) == TAG_BMINFLIGHT + else 'bminuse' if (tag & 0xfff) == TAG_BMINUSE + else 'bmbad' if (tag & 0xfff) == TAG_BMBAD + else 'bmerased' if (tag & 0xfff) == TAG_BMERASED + else 'bmrange 0x%x' % (tag & 0xf) + if (tag & 0xff0) == TAG_BMRANGE else 'struct 0x%02x' % (tag & 0xff), ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') @@ -2542,8 +2557,9 @@ class Config: # lazy gstate object class Gstate: - def __init__(self, mtree): + def __init__(self, mtree, config): self.mtree = mtree + self.config = config # lookup a specific tag def lookup(self, tag=None, mask=None): @@ -2619,7 +2635,7 @@ class Gstate: tag = None mask = None - def __init__(self, mtree, tag, gdeltas): + def __init__(self, mtree, config, tag, gdeltas): # replace tag with what we find self.tag = tag # keep track of gdeltas for debugging @@ -2669,8 +2685,8 @@ class Gstate: class Gcksum(Gstate): tag = TAG_GCKSUMDELTA - def __init__(self, mtree, tag, gdeltas): - super().__init__(mtree, tag, gdeltas) + def __init__(self, mtree, config, tag, gdeltas): + super().__init__(mtree, config, tag, gdeltas) self.gcksum = fromle32(self.data) def __int__(self): @@ -2683,8 +2699,8 @@ class Gstate: class Grm(Gstate): tag = TAG_GRMDELTA - def __init__(self, mtree, tag, gdeltas): - super().__init__(mtree, tag, gdeltas) + def __init__(self, mtree, config, tag, gdeltas): + super().__init__(mtree, config, tag, gdeltas) queue = [] d = 0 for _ in range(2): @@ -2702,6 +2718,26 @@ class Gstate: def repr(self): return 'grm [%s]' % ', '.join(mid.repr() for mid in self.queue) + # the global block map + class Gbmap(Gstate): + tag = TAG_GBMAPDELTA + + def __init__(self, mtree, config, tag, gdeltas): + super().__init__(mtree, config, tag, gdeltas) + d = 0 + self.cursor, d_ = fromleb128(self.data, d); d += d_ + self.known, d_ = fromleb128(self.data, d); d += d_ + block, trunk, cksum, d_ = frombranch(self.data, d); d += d_ + self.btree = Btree.fetchck( + mtree.bd, block, trunk, + config.geometry.block_count, + cksum) + + def repr(self): + return 'gbmap %s %s+%s' % ( + self.btree.addr(), + self.known, self.cursor) + # keep track of known gstate _known = [g for g in Gstate.__subclasses__() if g.tag is not None] @@ -2710,10 +2746,10 @@ class Gstate: # known config? for g in self._known: if (g.tag & ~(g.mask or 0)) == (tag & ~(g.mask or 0)): - return g(self.mtree, tag, gdeltas) + return g(self.mtree, self.config, tag, gdeltas) # otherwise return a marker class else: - return Unknown(self.mtree, tag, gdeltas) + return self.Unknown(self.mtree, self.config, tag, gdeltas) # create cached accessors for known gstate def _parser(g): @@ -2734,7 +2770,7 @@ class Lfs3: # create lazy config/gstate objects self.config = config or Config(self.mroot) - self.gstate = gstate or Gstate(self.mtree) + self.gstate = gstate or Gstate(self.mtree, self.config) # go ahead and fetch some expected fields self.version = self.config.version @@ -4040,6 +4076,7 @@ def dbg_config(lfs, *, color=False, w_width=2, **args): + # show config for i, config in enumerate(it.chain( lfs.config, lfs.attrs())): @@ -4078,6 +4115,103 @@ def dbg_gstate(lfs, *, color=False, w_width=2, **args): + + # print gstate structures + def dbg_gstruct(gstate): + # no tree? + if getattr(gstate, 'btree', None) is None: + return + + # precompute tree renderings + bt_width = 0 + if (args.get('tree') + or args.get('tree_rbyd') + or args.get('tree_btree')): + treeart = TreeArt.frombtree(gstate.btree, **args) + bt_width = treeart.width + + # dynamically size the id field + bw_width = mt.ceil(mt.log10(max(1, gstate.btree.weight)+1)) + + # only show the rbyd address on rbyd change + prbyd = None + # recursively print btree branches + def dbg_branch(d, bid, rbyd, rid, name): + nonlocal prbyd + + # show human-readable representation + for rattr in rbyd.rattrs(rid): + print('%12s %*s %s%*s %-*s %s' % ( + '%04x.%04x:' % (rbyd.block, rbyd.trunk) + if prbyd is None or rbyd != prbyd + else '', + 2*w_width+1, '', + treeart.repr( + (bid-(name.weight-1), d, rattr.tag), + color) + if args.get('tree') + or args.get('tree_rbyd') + or args.get('tree_btree') + else '', + 2*bw_width+1, '%d-%d' % (bid-(rattr.weight-1), bid) + if rattr.weight > 1 + else bid if rattr.weight > 0 + else '', + 21+2*bw_width+1, rattr.repr(), + next(xxd(rattr.data, 8), '') + if not args.get('raw') + and not args.get('no_truncate') + else '')) + prbyd = rbyd + + # show on-disk encoding of tags/data + if args.get('raw'): + for o, line in enumerate(xxd(rattr.tdata)): + print('%11s: %*s %*s%*s %s' % ( + '%04x' % (rattr.toff + o*16), + 2*w_width+1, '', + bt_width, '', + 2*bw_width+1, '', + line)) + if args.get('raw') or args.get('no_truncate'): + for o, line in enumerate(xxd(rattr.data)): + print('%11s: %*s %*s%*s %s' % ( + '%04x' % (rattr.off + o*16), + 2*w_width+1, '', + bt_width, '', + 2*bw_width+1, '', + line)) + + # traverse and print entries + ppath = [] + for bid, rbyd, path in gstate.btree.leaves( + path=True, + depth=args.get('depth')): + # print inner branches if requested + if args.get('inner'): + for d, (bid_, rbyd_, rid_, name_) in pathdelta( + path, ppath): + dbg_branch(d, bid_, rbyd_, rid_, name_) + ppath = path + + # corrupted? try to keep printing the tree + if not rbyd: + print('%s%11s: %*s %*s%s%s' % ( + '\x1b[31m' if color else '', + '%04x.%04x' % (rbyd.block, rbyd.trunk), + 2*w_width+1, '', + bt_width, '', + '(corrupted rbyd %s)' % rbyd.addr(), + '\x1b[m' if color else '')) + prbyd = None + continue + + for rid, name in rbyd.rids(): + bid_ = bid-(rbyd.weight-1) + rid + # show the leaf entry/branch + dbg_branch(len(path), bid_, rbyd, rid, name) + + # show gstate for i, gstate in enumerate(lfs.gstate): # some special situations worth reporting notes = [] @@ -4139,6 +4273,10 @@ def dbg_gstate(lfs, *, 2*w_width+1, '', line)) + # print gstate structures? + if args.get('gstructs'): + dbg_gstruct(gstate) + # show the littlefs file tree def dbg_files(lfs, paths, *, color=False, @@ -4317,6 +4455,7 @@ def dbg_files(lfs, paths, *, def dbg_branch(d, bid, rbyd, rid, name): nonlocal pmdir + # show human-readable representation for rattr in rbyd.rattrs(rid): print('%12s %*s %s%*s %-*s %s' % ( '%04x.%04x:' % (rbyd.block, rbyd.trunk) @@ -4538,7 +4677,8 @@ def main(disk, mroots=None, paths=None, *, # can show if requested show_config = args.get('config') show_gstate = (args.get('gstate') - or args.get('gdelta')) + or args.get('gdelta') + or args.get('gstructs')) show_files = (args.get('files') or args.get('structs') or args.get('attrs')) @@ -4725,6 +4865,10 @@ if __name__ == "__main__": action='store_true', help="Show relevant gdeltas used to build the gstate. " "Implies --gstate.") + parser.add_argument( + '--gstructs', + action='store_true', + help="Show gstate structures. Implies --gstate.") parser.add_argument( '--files', action='store_true', diff --git a/scripts/dbgmtree.py b/scripts/dbgmtree.py index 2843659d..56f84a69 100755 --- a/scripts/dbgmtree.py +++ b/scripts/dbgmtree.py @@ -30,6 +30,7 @@ TAG_NAMELIMIT = 0x0039 # 0x0039 v--- ---- --11 1--1 TAG_FILELIMIT = 0x003a # 0x003a v--- ---- --11 1-1- TAG_GDELTA = 0x0100 ## 0x01tt v--- ---1 -ttt ttrr TAG_GRMDELTA = 0x0100 # 0x0100 v--- ---1 ---- ---- +TAG_GBMAPDELTA = 0x0104 # 0x0104 v--- ---1 ---- -1rr TAG_NAME = 0x0200 ## 0x02tt v--- --1- -ttt tttt TAG_BNAME = 0x0200 # 0x0200 v--- --1- ---- ---- TAG_REG = 0x0201 # 0x0201 v--- --1- ---- ---1 @@ -47,6 +48,12 @@ TAG_BTREE = 0x031c # 0x031c v--- --11 ---1 11rr TAG_MROOT = 0x0321 # 0x032r v--- --11 --1- --rr TAG_MDIR = 0x0325 # 0x0324 v--- --11 --1- -1rr TAG_MTREE = 0x032c # 0x032c v--- --11 --1- 11rr +TAG_BMRANGE = 0x0330 # 0x033u v--- --11 --11 uuuu +TAG_BMFREE = 0x0330 # 0x0330 v--- --11 --11 ---- +TAG_BMINFLIGHT = 0x0331 # 0x0331 v--- --11 --11 ---1 +TAG_BMINUSE = 0x0332 # 0x0332 v--- --11 --11 --1- +TAG_BMBAD = 0x0333 # 0x0333 v--- --11 --11 --11 +TAG_BMERASED = 0x0334 # 0x0334 v--- --11 --11 -1-- TAG_ATTR = 0x0400 ## 0x04aa v--- -1-a -aaa aaaa TAG_UATTR = 0x0400 # 0x04aa v--- -1-- -aaa aaaa TAG_SATTR = 0x0500 # 0x05aa v--- -1-1 -aaa aaaa @@ -233,6 +240,7 @@ def tagrepr(tag, weight=None, size=None, *, return '%s%s%s%s' % ( 'shrub' if tag & TAG_SHRUB else '', 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA + else 'gbmapdelta' if (tag & 0xfff) == TAG_GBMAPDELTA else 'gdelta 0x%02x' % (tag & 0xff), ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') @@ -262,6 +270,13 @@ def tagrepr(tag, weight=None, size=None, *, else 'mroot' if (tag & 0xfff) == TAG_MROOT else 'mdir' if (tag & 0xfff) == TAG_MDIR else 'mtree' if (tag & 0xfff) == TAG_MTREE + else 'bmfree' if (tag & 0xfff) == TAG_BMFREE + else 'bminflight' if (tag & 0xfff) == TAG_BMINFLIGHT + else 'bminuse' if (tag & 0xfff) == TAG_BMINUSE + else 'bmbad' if (tag & 0xfff) == TAG_BMBAD + else 'bmerased' if (tag & 0xfff) == TAG_BMERASED + else 'bmrange 0x%x' % (tag & 0xf) + if (tag & 0xff0) == TAG_BMRANGE else 'struct 0x%02x' % (tag & 0xff), ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') diff --git a/scripts/dbgrbyd.py b/scripts/dbgrbyd.py index 05c70e83..864f8cb6 100755 --- a/scripts/dbgrbyd.py +++ b/scripts/dbgrbyd.py @@ -40,6 +40,7 @@ TAG_NAMELIMIT = 0x0039 # 0x0039 v--- ---- --11 1--1 TAG_FILELIMIT = 0x003a # 0x003a v--- ---- --11 1-1- TAG_GDELTA = 0x0100 ## 0x01tt v--- ---1 -ttt ttrr TAG_GRMDELTA = 0x0100 # 0x0100 v--- ---1 ---- ---- +TAG_GBMAPDELTA = 0x0104 # 0x0104 v--- ---1 ---- -1rr TAG_NAME = 0x0200 ## 0x02tt v--- --1- -ttt tttt TAG_BNAME = 0x0200 # 0x0200 v--- --1- ---- ---- TAG_REG = 0x0201 # 0x0201 v--- --1- ---- ---1 @@ -57,6 +58,12 @@ TAG_BTREE = 0x031c # 0x031c v--- --11 ---1 11rr TAG_MROOT = 0x0321 # 0x032r v--- --11 --1- --rr TAG_MDIR = 0x0325 # 0x0324 v--- --11 --1- -1rr TAG_MTREE = 0x032c # 0x032c v--- --11 --1- 11rr +TAG_BMRANGE = 0x0330 # 0x033u v--- --11 --11 uuuu +TAG_BMFREE = 0x0330 # 0x0330 v--- --11 --11 ---- +TAG_BMINFLIGHT = 0x0331 # 0x0331 v--- --11 --11 ---1 +TAG_BMINUSE = 0x0332 # 0x0332 v--- --11 --11 --1- +TAG_BMBAD = 0x0333 # 0x0333 v--- --11 --11 --11 +TAG_BMERASED = 0x0334 # 0x0334 v--- --11 --11 -1-- TAG_ATTR = 0x0400 ## 0x04aa v--- -1-a -aaa aaaa TAG_UATTR = 0x0400 # 0x04aa v--- -1-- -aaa aaaa TAG_SATTR = 0x0500 # 0x05aa v--- -1-1 -aaa aaaa @@ -221,6 +228,7 @@ def tagrepr(tag, weight=None, size=None, *, return '%s%s%s%s' % ( 'shrub' if tag & TAG_SHRUB else '', 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA + else 'gbmapdelta' if (tag & 0xfff) == TAG_GBMAPDELTA else 'gdelta 0x%02x' % (tag & 0xff), ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') @@ -250,6 +258,13 @@ def tagrepr(tag, weight=None, size=None, *, else 'mroot' if (tag & 0xfff) == TAG_MROOT else 'mdir' if (tag & 0xfff) == TAG_MDIR else 'mtree' if (tag & 0xfff) == TAG_MTREE + else 'bmfree' if (tag & 0xfff) == TAG_BMFREE + else 'bminflight' if (tag & 0xfff) == TAG_BMINFLIGHT + else 'bminuse' if (tag & 0xfff) == TAG_BMINUSE + else 'bmbad' if (tag & 0xfff) == TAG_BMBAD + else 'bmerased' if (tag & 0xfff) == TAG_BMERASED + else 'bmrange 0x%x' % (tag & 0xf) + if (tag & 0xff0) == TAG_BMRANGE else 'struct 0x%02x' % (tag & 0xff), ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') diff --git a/scripts/dbgtag.py b/scripts/dbgtag.py index e0322359..b1588ff0 100755 --- a/scripts/dbgtag.py +++ b/scripts/dbgtag.py @@ -23,6 +23,7 @@ TAG_NAMELIMIT = 0x0039 # 0x0039 v--- ---- --11 1--1 TAG_FILELIMIT = 0x003a # 0x003a v--- ---- --11 1-1- TAG_GDELTA = 0x0100 ## 0x01tt v--- ---1 -ttt ttrr TAG_GRMDELTA = 0x0100 # 0x0100 v--- ---1 ---- ---- +TAG_GBMAPDELTA = 0x0104 # 0x0104 v--- ---1 ---- -1rr TAG_NAME = 0x0200 ## 0x02tt v--- --1- -ttt tttt TAG_BNAME = 0x0200 # 0x0200 v--- --1- ---- ---- TAG_REG = 0x0201 # 0x0201 v--- --1- ---- ---1 @@ -40,6 +41,12 @@ TAG_BTREE = 0x031c # 0x031c v--- --11 ---1 11rr TAG_MROOT = 0x0321 # 0x032r v--- --11 --1- --rr TAG_MDIR = 0x0325 # 0x0324 v--- --11 --1- -1rr TAG_MTREE = 0x032c # 0x032c v--- --11 --1- 11rr +TAG_BMRANGE = 0x0330 # 0x033u v--- --11 --11 uuuu +TAG_BMFREE = 0x0330 # 0x0330 v--- --11 --11 ---- +TAG_BMINFLIGHT = 0x0331 # 0x0331 v--- --11 --11 ---1 +TAG_BMINUSE = 0x0332 # 0x0332 v--- --11 --11 --1- +TAG_BMBAD = 0x0333 # 0x0333 v--- --11 --11 --11 +TAG_BMERASED = 0x0334 # 0x0334 v--- --11 --11 -1-- TAG_ATTR = 0x0400 ## 0x04aa v--- -1-a -aaa aaaa TAG_UATTR = 0x0400 # 0x04aa v--- -1-- -aaa aaaa TAG_SATTR = 0x0500 # 0x05aa v--- -1-1 -aaa aaaa @@ -125,6 +132,7 @@ def tagrepr(tag, weight=None, size=None, *, return '%s%s%s%s' % ( 'shrub' if tag & TAG_SHRUB else '', 'grmdelta' if (tag & 0xfff) == TAG_GRMDELTA + else 'gbmapdelta' if (tag & 0xfff) == TAG_GBMAPDELTA else 'gdelta 0x%02x' % (tag & 0xff), ' w%d' % weight if weight else '', ' %s' % size if size is not None else '') @@ -154,6 +162,13 @@ def tagrepr(tag, weight=None, size=None, *, else 'mroot' if (tag & 0xfff) == TAG_MROOT else 'mdir' if (tag & 0xfff) == TAG_MDIR else 'mtree' if (tag & 0xfff) == TAG_MTREE + else 'bmfree' if (tag & 0xfff) == TAG_BMFREE + else 'bminflight' if (tag & 0xfff) == TAG_BMINFLIGHT + else 'bminuse' if (tag & 0xfff) == TAG_BMINUSE + else 'bmbad' if (tag & 0xfff) == TAG_BMBAD + else 'bmerased' if (tag & 0xfff) == TAG_BMERASED + else 'bmrange 0x%x' % (tag & 0xf) + if (tag & 0xff0) == TAG_BMRANGE else 'struct 0x%02x' % (tag & 0xff), ' w%d' % weight if weight else '', ' %s' % size if size is not None else '')