Github/littlefs
1
Fork 0
Code Actions Activity

Adopted new struct encoding scheme with redund tag bits

Browse Source 
Struct tags, in littlefs, generally encode pointers to different on-disk
data structures. At this point, they've gotten a bit complex, with the
btree struct, for example, containing 1. a block address, 2. the trunk
offset, 3. the weight of the trunk, and 4. a checksum.

Also some future plans:

1. Block redundancy will make it so these pointers may have a variable
   number of block addresses to contend with.

2. Different checksum types may make the checksum field itself variable
   length, at least on larger builds of littlefs.

   This may also happen if we support truncated checksums in littlefs
   for storage saving reasons.

Having two variable sized fields becomes a bit of a pain. We can use the
encoded tag size to figure out the size of one of these fields, but not
both.

The change here makes it so the tag size now determines the checksum
size, requiring the redundancy amount to go somewhere else. This makes
it so checksums can be variably sized, and the explicit redundancy
amount avoids the need to parse the leb128s fully to know how many
blocks we're expecting.

But where to put the redundancy amount?

This commit carves out 2-bits from the struct tag to store the amount of
redundancy to allow up to 3 blocks of redundancy:

  v0000011 0TTTTTrr
  ^--^---^-^----^-^- valid bit
     '---|-|----|-|- 3-bit mode (0x0 for structs)
         '-|----|-|- 4-bit suptype (0x3 for structs)
           '----|-|- 0 bit (reserved for leb128)
                '-|- 5-bit subtype
                  '- 2-bit redund

3 blocks may sound extremely limiting, but it's a common limit for
filesystems, 1. because you have to keep in mind each redundant block
adds that much more writing/reading overhead and 2. the fact
that 2^(2^n)-1 is always divisible by 3 makes >3 parity blocks much more
complicated mathematically.

Worst case, if we ever have >3 redundant blocks, we can create new
struct subtypes. Maybe adding extended struct types that prefix the
block addresses with a leb128 encoding the redundancy amount.

---

As a part of this, reorganized the on-disk btree and ecksum encodings to
put the checksum last.

Also split out the btree and inner btree branches as separate struct
types. The btree includes the weight, whereas the weight is implicit in
inner btree branches. This came about after realizing context-specific
prefixes are relatively easy to add thanks to the composability of our
parsers.

This led to some name collisions though:

- BRANCH   -> BNAME
- BOOKMARK -> DMARK
This commit is contained in:
Christopher Haster
2023-08-11 01:44:51 -05:00
parent d069fed3ed
commit 314c832588
5 changed files with 221 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+117 -84
View File
@@ -589,19 +589,20 @@ enum lfsr_tag_type {
LFSR_TAG_GRM = 0x0100, LFSR_TAG_GRM = 0x0100,
LFSR_TAG_NAME = 0x0200, LFSR_TAG_NAME = 0x0200,
LFSR_TAG_BRANCH = 0x0200, LFSR_TAG_BNAME = 0x0200,
LFSR_TAG_BOOKMARK = 0x0201, LFSR_TAG_DMARK = 0x0201,
LFSR_TAG_REG = 0x0202, LFSR_TAG_REG = 0x0202,
LFSR_TAG_DIR = 0x0203, LFSR_TAG_DIR = 0x0203,
LFSR_TAG_STRUCT = 0x0300, LFSR_TAG_STRUCT = 0x0300,
LFSR_TAG_INLINED = 0x0300, LFSR_TAG_INLINED = 0x0300,
LFSR_TAG_BLOCK = 0x0302, LFSR_TAG_BLOCK = 0x0308,
LFSR_TAG_BTREE = 0x0303, LFSR_TAG_BTREE = 0x030c,
LFSR_TAG_MROOT = 0x0304, LFSR_TAG_MDIR = 0x0311,
LFSR_TAG_MDIR = 0x0305, LFSR_TAG_MTREE = 0x0314,
LFSR_TAG_MTREE = 0x0306, LFSR_TAG_MROOT = 0x0318,
LFSR_TAG_DID = 0x0307, LFSR_TAG_BRANCH = 0x031c,
LFSR_TAG_DID = 0x0320,
LFSR_TAG_UATTR = 0x0400, LFSR_TAG_UATTR = 0x0400,
LFSR_TAG_SATTR = 0x0500, LFSR_TAG_SATTR = 0x0500,
@@ -1433,8 +1434,8 @@ typedef struct lfsr_attr {
// erased-state checksum on-disk encoding // erased-state checksum on-disk encoding
typedef struct lfsr_ecksum { typedef struct lfsr_ecksum {
uint32_t cksum;
lfs_size_t size; lfs_size_t size;
uint32_t cksum;
} lfsr_ecksum_t; } lfsr_ecksum_t;
// 1 leb128 + 1 crc32c => 9 bytes (worst case) // 1 leb128 + 1 crc32c => 9 bytes (worst case)
@@ -1445,26 +1446,26 @@ static lfs_ssize_t lfsr_ecksum_todisk(lfs_t *lfs, const lfsr_ecksum_t *ecksum,
(void)lfs; (void)lfs;
lfs_ssize_t d = 0; lfs_ssize_t d = 0;
lfs_tole32_(ecksum->cksum, &buffer[d]);
d += 4;
lfs_ssize_t d_ = lfs_toleb128(ecksum->size, &buffer[d], 5); lfs_ssize_t d_ = lfs_toleb128(ecksum->size, &buffer[d], 5);
if (d_ < 0) { if (d_ < 0) {
return d_; return d_;
} }
d += d_; d += d_;
lfs_tole32_(ecksum->cksum, &buffer[d]);
d += 4;
return d; return d;
} }
static int lfsr_data_readecksum(lfs_t *lfs, lfsr_data_t *data, static int lfsr_data_readecksum(lfs_t *lfs, lfsr_data_t *data,
lfsr_ecksum_t *ecksum) { lfsr_ecksum_t *ecksum) {
int err = lfsr_data_readle32(lfs, data, &ecksum->cksum); int err = lfsr_data_readleb128(lfs, data, (int32_t*)&ecksum->size);
if (err) { if (err) {
return err; return err;
} }
err = lfsr_data_readleb128(lfs, data, (int32_t*)&ecksum->size); err = lfsr_data_readle32(lfs, data, &ecksum->cksum);
if (err) { if (err) {
return err; return err;
} }
@@ -2793,7 +2794,7 @@ static int lfsr_rbyd_appendcksum(lfs_t *lfs, lfsr_rbyd_t *rbyd) {
// find the expected ecksum, don't bother avoiding a reread of the // find the expected ecksum, don't bother avoiding a reread of the
// perturb byte, as it should still be in our cache // perturb byte, as it should still be in our cache
lfsr_ecksum_t ecksum = {.cksum=0, .size=lfs->cfg->prog_size}; lfsr_ecksum_t ecksum = {.size=lfs->cfg->prog_size, .cksum=0};
err = lfsr_bd_cksum(lfs, rbyd->block, aligned_eoff, lfs->cfg->prog_size, err = lfsr_bd_cksum(lfs, rbyd->block, aligned_eoff, lfs->cfg->prog_size,
lfs->cfg->prog_size, lfs->cfg->prog_size,
&ecksum.cksum); &ecksum.cksum);
@@ -3439,7 +3440,7 @@ static int lfsr_rbyd_namelookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
// if we have no name or a vestigial name, treat this rid as always lt // if we have no name or a vestigial name, treat this rid as always lt
lfs_scmp_t cmp; lfs_scmp_t cmp;
if ((tag__ == LFSR_TAG_BRANCH && rid__-(weight__-1) == 0) if ((tag__ == LFSR_TAG_BNAME && rid__-(weight__-1) == 0)
|| lfsr_tag_suptype(tag__) != LFSR_TAG_NAME) { || lfsr_tag_suptype(tag__) != LFSR_TAG_NAME) {
cmp = LFS_CMP_LT; cmp = LFS_CMP_LT;
@@ -3522,8 +3523,60 @@ static inline lfs_size_t lfsr_btree_setinlined(lfs_size_t weight) {
// btree on-disk encoding // btree on-disk encoding
// 1 crc32c + 3 leb128 => 19 bytes (worst case) // 2 leb128 + 1 crc32c => 14 bytes (worst case)
#define LFSR_BTREE_DSIZE (4+5+5+5) #define LFSR_BRANCH_DSIZE (5+5+4)
static lfs_ssize_t lfsr_branch_todisk(lfs_t *lfs, const lfsr_rbyd_t *branch,
uint8_t buffer[static LFSR_BRANCH_DSIZE]) {
(void)lfs;
lfs_ssize_t d = 0;
lfs_ssize_t d_ = lfs_toleb128(branch->block, &buffer[d], 5);
if (d_ < 0) {
return d_;
}
d += d_;
d_ = lfs_toleb128(branch->trunk, &buffer[d], 5);
if (d_ < 0) {
return d_;
}
d += d_;
lfs_tole32_(branch->cksum, &buffer[d]);
d += 4;
return d;
}
static int lfsr_data_readbranch(lfs_t *lfs, lfsr_data_t *data,
lfs_size_t weight,
lfsr_rbyd_t *branch) {
// setting off to 0 here will trigger asserts if we try to append
// without fetching first
branch->eoff = 0;
branch->weight = weight;
int err = lfsr_data_readleb128(lfs, data, (int32_t*)&branch->block);
if (err) {
return err;
}
err = lfsr_data_readleb128(lfs, data, (int32_t*)&branch->trunk);
if (err) {
return err;
}
err = lfsr_data_readle32(lfs, data, &branch->cksum);
if (err) {
return err;
}
return 0;
}
// 3 leb128 + 1 crc32c => 19 bytes (worst case)
#define LFSR_BTREE_DSIZE (5+LFSR_BRANCH_DSIZE)
static lfs_ssize_t lfsr_btree_todisk(lfs_t *lfs, const lfsr_rbyd_t *btree, static lfs_ssize_t lfsr_btree_todisk(lfs_t *lfs, const lfsr_rbyd_t *btree,
uint8_t buffer[static LFSR_BTREE_DSIZE]) { uint8_t buffer[static LFSR_BTREE_DSIZE]) {
@@ -3532,22 +3585,13 @@ static lfs_ssize_t lfsr_btree_todisk(lfs_t *lfs, const lfsr_rbyd_t *btree,
LFS_ASSERT(!lfsr_btree_isinlined((const lfsr_btree_t*)btree)); LFS_ASSERT(!lfsr_btree_isinlined((const lfsr_btree_t*)btree));
lfs_ssize_t d = 0; lfs_ssize_t d = 0;
lfs_tole32_(btree->cksum, &buffer[d]);
d += 4;
lfs_ssize_t d_ = lfs_toleb128(btree->weight, &buffer[d], 5); lfs_ssize_t d_ = lfs_toleb128(btree->weight, &buffer[d], 5);
if (d_ < 0) { if (d_ < 0) {
return d_; return d_;
} }
d += d_; d += d_;
d_ = lfs_toleb128(btree->trunk, &buffer[d], 5); d_ = lfsr_branch_todisk(lfs, btree, &buffer[d]);
if (d_ < 0) {
return d_;
}
d += d_;
d_ = lfs_toleb128(btree->block, &buffer[d], 5);
if (d_ < 0) { if (d_ < 0) {
return d_; return d_;
} }
@@ -3556,12 +3600,13 @@ static lfs_ssize_t lfsr_btree_todisk(lfs_t *lfs, const lfsr_rbyd_t *btree,
return d; return d;
} }
static int lfsr_btree_inline(lfs_t *lfs, lfsr_btree_t *btree, static int lfsr_data_readbtreeinlined(lfs_t *lfs, lfsr_data_t *data,
lfsr_tag_t tag, lfs_size_t weight, lfsr_data_t data) { lfsr_tag_t tag, lfs_size_t weight,
lfsr_btree_t *btree) {
// mark as inlined // mark as inlined
btree->u.i.weight = lfsr_btree_setinlined(weight); btree->u.i.weight = lfsr_btree_setinlined(weight);
btree->u.i.tag = tag; btree->u.i.tag = tag;
lfs_ssize_t size = lfsr_data_read(lfs, &data, lfs_ssize_t size = lfsr_data_read(lfs, data,
btree->u.i.buffer, LFSR_BTREE_INLINESIZE); btree->u.i.buffer, LFSR_BTREE_INLINESIZE);
if (size < 0) { if (size < 0) {
return size; return size;
@@ -3576,22 +3621,13 @@ static int lfsr_data_readbtree(lfs_t *lfs, lfsr_data_t *data,
// without fetching first // without fetching first
btree->eoff = 0; btree->eoff = 0;
int err = lfsr_data_readle32(lfs, data, &btree->cksum); lfs_size_t weight;
int err = lfsr_data_readleb128(lfs, data, (int32_t*)&weight);
if (err) { if (err) {
return err; return err;
} }
err = lfsr_data_readleb128(lfs, data, (int32_t*)&btree->weight); err = lfsr_data_readbranch(lfs, data, weight, btree);
if (err) {
return err;
}
err = lfsr_data_readleb128(lfs, data, (int32_t*)&btree->trunk);
if (err) {
return err;
}
err = lfsr_data_readleb128(lfs, data, (int32_t*)&btree->block);
if (err) { if (err) {
return err; return err;
} }
@@ -3658,16 +3694,15 @@ static int lfsr_btree_lookupnext_(lfs_t *lfs,
} }
// found another branch // found another branch
if (tag__ == LFSR_TAG_BTREE) { if (tag__ == LFSR_TAG_BRANCH) {
// adjust rid with subtree's weight // adjust rid with subtree's weight
rid -= (rid__ - (weight__-1)); rid -= (rid__ - (weight__-1));
// fetch the next branch // fetch the next branch
err = lfsr_data_readbtree(lfs, &data__, &branch); err = lfsr_data_readbranch(lfs, &data__, weight__, &branch);
if (err) { if (err) {
return err; return err;
} }
LFS_ASSERT(branch.weight == weight__);
// found our bid // found our bid
} else { } else {
@@ -3779,7 +3814,7 @@ static int lfsr_btree_parent(lfs_t *lfs,
} }
// didn't find our child? // didn't find our child?
if (tag__ != LFSR_TAG_BTREE) { if (tag__ != LFSR_TAG_BRANCH) {
return LFS_ERR_NOENT; return LFS_ERR_NOENT;
} }
@@ -3788,11 +3823,10 @@ static int lfsr_btree_parent(lfs_t *lfs,
// fetch the next branch // fetch the next branch
lfsr_rbyd_t branch_; lfsr_rbyd_t branch_;
err = lfsr_data_readbtree(lfs, &data__, &branch_); err = lfsr_data_readbranch(lfs, &data__, weight__, &branch_);
if (err) { if (err) {
return err; return err;
} }
LFS_ASSERT(branch_.weight == weight__);
// found our child? // found our child?
if (branch_.block == child->block && branch_.trunk == child->trunk) { if (branch_.block == child->block && branch_.trunk == child->trunk) {
@@ -3818,7 +3852,7 @@ static int lfsr_btree_parent(lfs_t *lfs,
// array allocations // array allocations
#define LFSR_BTREE_SCRATCHATTRS ( \ #define LFSR_BTREE_SCRATCHATTRS ( \
4 \ 4 \
+ ((2*LFSR_BTREE_DSIZE) + sizeof(lfsr_attr_t)-1) \ + ((2*LFSR_BRANCH_DSIZE) + sizeof(lfsr_attr_t)-1) \
/ sizeof(lfsr_attr_t)) / sizeof(lfsr_attr_t))
// this macro creates an attr list with enough reserved space for // this macro creates an attr list with enough reserved space for
@@ -3890,7 +3924,7 @@ static int lfsr_btree_commit(lfs_t *lfs,
// cannibalize some attributes in our attr list to store // cannibalize some attributes in our attr list to store
// our branch // our branch
uint8_t *scratch_buf = (uint8_t*)&attrs[2]; uint8_t *scratch_buf = (uint8_t*)&attrs[2];
lfs_ssize_t scratch_dsize = lfsr_btree_todisk(lfs, rbyd, scratch_buf); lfs_ssize_t scratch_dsize = lfsr_branch_todisk(lfs, rbyd, scratch_buf);
if (scratch_dsize < 0) { if (scratch_dsize < 0) {
return scratch_dsize; return scratch_dsize;
} }
@@ -3905,7 +3939,7 @@ static int lfsr_btree_commit(lfs_t *lfs,
attr_count = 1; attr_count = 1;
} else { } else {
attrs[0] = LFSR_ATTR(pid, GROW(RM), +rbyd->weight-pweight, NULL); attrs[0] = LFSR_ATTR(pid, GROW(RM), +rbyd->weight-pweight, NULL);
attrs[1] = LFSR_ATTR(pid+rbyd->weight-pweight, BTREE, 0, attrs[1] = LFSR_ATTR(pid+rbyd->weight-pweight, BRANCH, 0,
BUF(scratch_buf, scratch_dsize)); BUF(scratch_buf, scratch_dsize));
attr_count = 2; attr_count = 2;
} }
@@ -4001,7 +4035,7 @@ static int lfsr_btree_commit(lfs_t *lfs,
// cannibalize some attributes in our attr list to store // cannibalize some attributes in our attr list to store
// our branch // our branch
scratch_buf = (uint8_t*)&attrs[2]; scratch_buf = (uint8_t*)&attrs[2];
scratch_dsize = lfsr_btree_todisk(lfs, rbyd, scratch_buf); scratch_dsize = lfsr_branch_todisk(lfs, rbyd, scratch_buf);
if (scratch_dsize < 0) { if (scratch_dsize < 0) {
return scratch_dsize; return scratch_dsize;
} }
@@ -4016,7 +4050,7 @@ static int lfsr_btree_commit(lfs_t *lfs,
attr_count = 1; attr_count = 1;
} else { } else {
attrs[0] = LFSR_ATTR(pid, GROW(RM), +rbyd->weight-pweight, NULL); attrs[0] = LFSR_ATTR(pid, GROW(RM), +rbyd->weight-pweight, NULL);
attrs[1] = LFSR_ATTR(pid+rbyd->weight-pweight, BTREE, 0, attrs[1] = LFSR_ATTR(pid+rbyd->weight-pweight, BRANCH, 0,
BUF(scratch_buf, scratch_dsize)); BUF(scratch_buf, scratch_dsize));
attr_count = 2; attr_count = 2;
} }
@@ -4109,13 +4143,13 @@ static int lfsr_btree_commit(lfs_t *lfs,
// cannibalize some attributes in our attr list to store // cannibalize some attributes in our attr list to store
// our branches // our branches
uint8_t *scratch1_buf = (uint8_t*)&attrs[4]; uint8_t *scratch1_buf = (uint8_t*)&attrs[4];
uint8_t *scratch2_buf = (uint8_t*)&attrs[4] + LFSR_BTREE_DSIZE; uint8_t *scratch2_buf = (uint8_t*)&attrs[4] + LFSR_BRANCH_DSIZE;
lfs_ssize_t scratch1_dsize = lfsr_btree_todisk(lfs, &rbyd_, lfs_ssize_t scratch1_dsize = lfsr_branch_todisk(lfs, &rbyd_,
scratch1_buf); scratch1_buf);
if (scratch1_dsize < 0) { if (scratch1_dsize < 0) {
return scratch1_dsize; return scratch1_dsize;
} }
lfs_ssize_t scratch2_dsize = lfsr_btree_todisk(lfs, &sibling, lfs_ssize_t scratch2_dsize = lfsr_branch_todisk(lfs, &sibling,
scratch2_buf); scratch2_buf);
if (scratch2_dsize < 0) { if (scratch2_dsize < 0) {
return scratch2_dsize; return scratch2_dsize;
@@ -4129,16 +4163,16 @@ static int lfsr_btree_commit(lfs_t *lfs,
} }
// prepare commit to parent, tail recursing upwards // prepare commit to parent, tail recursing upwards
attrs[0] = LFSR_ATTR(0, BTREE, +rbyd_.weight, attrs[0] = LFSR_ATTR(0, BRANCH, +rbyd_.weight,
BUF(scratch1_buf, scratch1_dsize)); BUF(scratch1_buf, scratch1_dsize));
attrs[1] = (lfsr_tag_suptype(stag) == LFSR_TAG_NAME attrs[1] = (lfsr_tag_suptype(stag) == LFSR_TAG_NAME
? LFSR_ATTR(rbyd_.weight, BRANCH, +sibling.weight, ? LFSR_ATTR(rbyd_.weight, BNAME, +sibling.weight,
DATA(sdata)) DATA(sdata))
: LFSR_ATTR_NOOP); : LFSR_ATTR_NOOP);
attrs[2] = (lfsr_tag_suptype(stag) == LFSR_TAG_NAME attrs[2] = (lfsr_tag_suptype(stag) == LFSR_TAG_NAME
? LFSR_ATTR(0+rbyd_.weight+sibling.weight-1, BTREE, 0, ? LFSR_ATTR(0+rbyd_.weight+sibling.weight-1, BRANCH, 0,
BUF(scratch2_buf, scratch2_dsize)) BUF(scratch2_buf, scratch2_dsize))
: LFSR_ATTR(0+rbyd_.weight, BTREE, +sibling.weight, : LFSR_ATTR(0+rbyd_.weight, BRANCH, +sibling.weight,
BUF(scratch2_buf, scratch2_dsize))); BUF(scratch2_buf, scratch2_dsize)));
attr_count = 3; attr_count = 3;
@@ -4146,19 +4180,19 @@ static int lfsr_btree_commit(lfs_t *lfs,
} else { } else {
// prepare commit to parent, tail recursing upwards // prepare commit to parent, tail recursing upwards
attrs[0] = LFSR_ATTR(pid, GROW(RM), +rbyd_.weight-pweight, NULL); attrs[0] = LFSR_ATTR(pid, GROW(RM), +rbyd_.weight-pweight, NULL);
attrs[1] = LFSR_ATTR(pid-(pweight-1)+rbyd_.weight-1, BTREE, 0, attrs[1] = LFSR_ATTR(pid-(pweight-1)+rbyd_.weight-1, BRANCH, 0,
BUF(scratch1_buf, scratch1_dsize)); BUF(scratch1_buf, scratch1_dsize));
attrs[2] = (lfsr_tag_suptype(stag) == LFSR_TAG_NAME attrs[2] = (lfsr_tag_suptype(stag) == LFSR_TAG_NAME
? LFSR_ATTR(pid-(pweight-1)+rbyd_.weight, ? LFSR_ATTR(pid-(pweight-1)+rbyd_.weight,
BRANCH, +sibling.weight, BNAME, +sibling.weight,
DATA(sdata)) DATA(sdata))
: LFSR_ATTR_NOOP); : LFSR_ATTR_NOOP);
attrs[3] = (lfsr_tag_suptype(stag) == LFSR_TAG_NAME attrs[3] = (lfsr_tag_suptype(stag) == LFSR_TAG_NAME
? LFSR_ATTR(pid-(pweight-1)+rbyd_.weight+sibling.weight-1, ? LFSR_ATTR(pid-(pweight-1)+rbyd_.weight+sibling.weight-1,
BTREE, 0, BRANCH, 0,
BUF(scratch2_buf, scratch2_dsize)) BUF(scratch2_buf, scratch2_dsize))
: LFSR_ATTR(pid-(pweight-1)+rbyd_.weight, : LFSR_ATTR(pid-(pweight-1)+rbyd_.weight,
BTREE, +sibling.weight, BRANCH, +sibling.weight,
BUF(scratch2_buf, scratch2_dsize))); BUF(scratch2_buf, scratch2_dsize)));
attr_count = 4; attr_count = 4;
} }
@@ -4232,15 +4266,14 @@ static int lfsr_btree_commit(lfs_t *lfs,
} }
// no sibling? can't merge // no sibling? can't merge
if (stag != LFSR_TAG_BTREE) { if (stag != LFSR_TAG_BRANCH) {
continue; continue;
} }
err = lfsr_data_readbtree(lfs, &sdata, &sibling); err = lfsr_data_readbranch(lfs, &sdata, sweight, &sibling);
if (err) { if (err) {
return err; return err;
} }
LFS_ASSERT(sibling.weight == sweight);
// estimate if our sibling will fit // estimate if our sibling will fit
// //
@@ -4324,7 +4357,7 @@ static int lfsr_btree_commit(lfs_t *lfs,
} }
err = lfsr_rbyd_append(lfs, &rbyd_, err = lfsr_rbyd_append(lfs, &rbyd_,
split_rid, LFSR_TAG_BRANCH, 0, split_data); split_rid, LFSR_TAG_BNAME, 0, split_data);
if (err) { if (err) {
LFS_ASSERT(err != LFS_ERR_RANGE); LFS_ASSERT(err != LFS_ERR_RANGE);
return err; return err;
@@ -4356,7 +4389,7 @@ static int lfsr_btree_commit(lfs_t *lfs,
// cannibalize some attributes in our attr list to store // cannibalize some attributes in our attr list to store
// our branch // our branch
uint8_t *scratch_buf = (uint8_t*)&attrs[3]; uint8_t *scratch_buf = (uint8_t*)&attrs[3];
lfs_ssize_t scratch_dsize = lfsr_btree_todisk(lfs, &rbyd_, lfs_ssize_t scratch_dsize = lfsr_branch_todisk(lfs, &rbyd_,
scratch_buf); scratch_buf);
if (scratch_dsize < 0) { if (scratch_dsize < 0) {
return scratch_dsize; return scratch_dsize;
@@ -4365,7 +4398,7 @@ static int lfsr_btree_commit(lfs_t *lfs,
// prepare commit to parent, tail recursing upwards // prepare commit to parent, tail recursing upwards
attrs[0] = LFSR_ATTR(sid, RM, -sweight, NULL); attrs[0] = LFSR_ATTR(sid, RM, -sweight, NULL);
attrs[1] = LFSR_ATTR(pid, GROW(RM), +rbyd_.weight-pweight, NULL); attrs[1] = LFSR_ATTR(pid, GROW(RM), +rbyd_.weight-pweight, NULL);
attrs[2] = LFSR_ATTR(pid+rbyd_.weight-pweight, BTREE, 0, attrs[2] = LFSR_ATTR(pid+rbyd_.weight-pweight, BRANCH, 0,
BUF(scratch_buf, scratch_dsize)); BUF(scratch_buf, scratch_dsize));
attr_count = 3; attr_count = 3;
} }
@@ -4657,7 +4690,7 @@ static int lfsr_btree_split(lfs_t *lfs, lfsr_btree_t *btree,
err = lfsr_rbyd_commit(lfs, &rbyd, LFSR_ATTRS( err = lfsr_rbyd_commit(lfs, &rbyd, LFSR_ATTRS(
LFSR_ATTR(0, TAG(tag1), +weight1, DATA(data1)), LFSR_ATTR(0, TAG(tag1), +weight1, DATA(data1)),
(lfsr_data_size(&name) > 0 (lfsr_data_size(&name) > 0
? LFSR_ATTR(weight1, BRANCH, +weight2, DATA(name)) ? LFSR_ATTR(weight1, BNAME, +weight2, DATA(name))
: LFSR_ATTR_NOOP), : LFSR_ATTR_NOOP),
(lfsr_data_size(&name) > 0 (lfsr_data_size(&name) > 0
? LFSR_ATTR(weight1+weight2-1, TAG(tag2), 0, DATA(data2)) ? LFSR_ATTR(weight1+weight2-1, TAG(tag2), 0, DATA(data2))
@@ -4690,7 +4723,7 @@ static int lfsr_btree_split(lfs_t *lfs, lfsr_btree_t *btree,
DATA(data1)), DATA(data1)),
(lfsr_data_size(&name) > 0 (lfsr_data_size(&name) > 0
? LFSR_ATTR(rid-(rweight-1)+weight1, ? LFSR_ATTR(rid-(rweight-1)+weight1,
BRANCH, +weight2, DATA(name)) BNAME, +weight2, DATA(name))
: LFSR_ATTR_NOOP), : LFSR_ATTR_NOOP),
(lfsr_data_size(&name) > 0 (lfsr_data_size(&name) > 0
? LFSR_ATTR(rid-(rweight-1)+weight1+weight2-1, ? LFSR_ATTR(rid-(rweight-1)+weight1+weight2-1,
@@ -4759,16 +4792,15 @@ static int lfsr_btree_namelookup(lfs_t *lfs, const lfsr_btree_t *btree,
} }
// found another branch // found another branch
if (tag__ == LFSR_TAG_BTREE) { if (tag__ == LFSR_TAG_BRANCH) {
// update our bid // update our bid
bid += rid__ - (weight__-1); bid += rid__ - (weight__-1);
// fetch the next branch // fetch the next branch
err = lfsr_data_readbtree(lfs, &data__, &branch); err = lfsr_data_readbranch(lfs, &data__, weight__, &branch);
if (err) { if (err) {
return err; return err;
} }
LFS_ASSERT(branch.weight == weight__);
// found our rid // found our rid
} else { } else {
@@ -4891,12 +4923,13 @@ static int lfsr_btree_traversal_next(lfs_t *lfs,
} }
// found another branch // found another branch
if (tag__ == LFSR_TAG_BTREE) { if (tag__ == LFSR_TAG_BRANCH) {
// adjust rid with subtree's weight // adjust rid with subtree's weight
traversal->rid -= (rid__ - (weight__-1)); traversal->rid -= (rid__ - (weight__-1));
// fetch the next branch // fetch the next branch
err = lfsr_data_readbtree(lfs, &data__, &traversal->branch); err = lfsr_data_readbranch(lfs, &data__, weight__,
&traversal->branch);
if (err) { if (err) {
return err; return err;
} }
@@ -6438,8 +6471,8 @@ static int lfsr_mtree_traversal_next(lfs_t *lfs,
if (err == LFS_ERR_NOENT) { if (err == LFS_ERR_NOENT) {
lfs->mtree = LFSR_BTREE_NULL; lfs->mtree = LFSR_BTREE_NULL;
} else if (tag == LFSR_TAG_MDIR) { } else if (tag == LFSR_TAG_MDIR) {
err = lfsr_btree_inline(lfs, &lfs->mtree, err = lfsr_data_readbtreeinlined(lfs, &data, LFSR_TAG_MDIR, 1,
LFSR_TAG_MDIR, 1, data); &lfs->mtree);
if (err) { if (err) {
return err; return err;
} }
@@ -6966,7 +6999,7 @@ static int lfsr_formatinited(lfs_t *lfs) {
LFSR_ATTR(-1, SUPERMAGIC, 0, BUF("littlefs", 8)), LFSR_ATTR(-1, SUPERMAGIC, 0, BUF("littlefs", 8)),
LFSR_ATTR(-1, SUPERCONFIG, 0, LFSR_ATTR(-1, SUPERCONFIG, 0,
BUF(superconfig_buf, superconfig_dsize)), BUF(superconfig_buf, superconfig_dsize)),
LFSR_ATTR(0, BOOKMARK, +1, NAME(0, NULL, 0)))); LFSR_ATTR(0, DMARK, +1, NAME(0, NULL, 0))));
if (err) { if (err) {
return err; return err;
} }
@@ -7236,7 +7269,7 @@ int lfsr_mkdir(lfs_t *lfs, const char *path) {
// lose power before writing the entry in our parent // lose power before writing the entry in our parent
// //
err = lfsr_mdir_commit(lfs, &mdir, LFSR_ATTRS( err = lfsr_mdir_commit(lfs, &mdir, LFSR_ATTRS(
LFSR_ATTR(mdir.mid.rid, BOOKMARK, +1, NAME(did, NULL, 0)), LFSR_ATTR(mdir.mid.rid, DMARK, +1, NAME(did, NULL, 0)),
LFSR_ATTR(-1, GRM, 0, GRM(&((lfsr_grm_t){{ LFSR_ATTR(-1, GRM, 0, GRM(&((lfsr_grm_t){{
mdir.mid, mdir.mid,
LFSR_MID(-1, -1)}}))))); LFSR_MID(-1, -1)}})))));
@@ -7329,7 +7362,7 @@ int lfsr_remove(lfs_t *lfs, const char *path) {
return err; return err;
} }
if (tag_ != LFSR_TAG_BOOKMARK) { if (tag_ != LFSR_TAG_DMARK) {
return LFS_ERR_NOTEMPTY; return LFS_ERR_NOTEMPTY;
} }
} }
@@ -7461,7 +7494,7 @@ int lfsr_rename(lfs_t *lfs, const char *old_path, const char *new_path) {
return err; return err;
} }
if (tag_ != LFSR_TAG_BOOKMARK) { if (tag_ != LFSR_TAG_DMARK) {
return LFS_ERR_NOTEMPTY; return LFS_ERR_NOTEMPTY;
} }
} }
scripts/dbgbtree.py
+22 -20
View File
@@ -14,18 +14,19 @@ TAG_SUPERCONFIG = 0x0004
TAG_GSTATE = 0x0100 TAG_GSTATE = 0x0100
TAG_GRM = 0x0100 TAG_GRM = 0x0100
TAG_NAME = 0x0200 TAG_NAME = 0x0200
TAG_BRANCH = 0x0200 TAG_BNAME = 0x0200
TAG_BOOKMARK = 0x0201 TAG_DMARK = 0x0201
TAG_REG = 0x0202 TAG_REG = 0x0202
TAG_DIR = 0x0203 TAG_DIR = 0x0203
TAG_STRUCT = 0x0300 TAG_STRUCT = 0x0300
TAG_INLINED = 0x0300 TAG_INLINED = 0x0300
TAG_BLOCK = 0x0302 TAG_BLOCK = 0x0308
TAG_BTREE = 0x0303 TAG_BTREE = 0x030c
TAG_MROOT = 0x0304 TAG_MDIR = 0x0311
TAG_MDIR = 0x0305 TAG_MTREE = 0x0314
TAG_MTREE = 0x0306 TAG_MROOT = 0x0318
TAG_DID = 0x0307 TAG_BRANCH = 0x031c
TAG_DID = 0x0320
TAG_UATTR = 0x0400 TAG_UATTR = 0x0400
TAG_SATTR = 0x0500 TAG_SATTR = 0x0500
TAG_ALT = 0x4000 TAG_ALT = 0x4000
@@ -97,12 +98,12 @@ def fromtag(data):
size, d_ = fromleb128(data[2+d:]) size, d_ = fromleb128(data[2+d:])
return tag>>15, tag&0x7fff, weight, size, 2+d+d_ return tag>>15, tag&0x7fff, weight, size, 2+d+d_
def frombtree(data): def frombranch(data):
cksum = fromle32(data) d = 0
w, d1 = fromleb128(data[4:]) block, d_ = fromleb128(data[d:]); d += d_
trunk, d2 = fromleb128(data[4+d1:]) trunk, d_ = fromleb128(data[d:]); d += d_
block, d3 = fromleb128(data[4+d1+d2:]) cksum = fromle32(data[d:]); d += 4
return w, trunk, block, cksum return block, trunk, cksum
def popc(x): def popc(x):
return bin(x).count('1') return bin(x).count('1')
@@ -138,8 +139,8 @@ def tagrepr(tag, w, size, off=None):
size) size)
elif (tag & 0xff00) == TAG_NAME: elif (tag & 0xff00) == TAG_NAME:
return '%s%s %d' % ( return '%s%s %d' % (
'branch' if tag == TAG_BRANCH 'bname' if tag == TAG_BNAME
else 'bookmark' if tag == TAG_BOOKMARK else 'dmark' if tag == TAG_DMARK
else 'reg' if tag == TAG_REG else 'reg' if tag == TAG_REG
else 'dir' if tag == TAG_DIR else 'dir' if tag == TAG_DIR
else 'name 0x%02x' % (tag & 0xff), else 'name 0x%02x' % (tag & 0xff),
@@ -150,9 +151,10 @@ def tagrepr(tag, w, size, off=None):
'inlined' if tag == TAG_INLINED 'inlined' if tag == TAG_INLINED
else 'block' if tag == TAG_BLOCK else 'block' if tag == TAG_BLOCK
else 'btree' if tag == TAG_BTREE else 'btree' if tag == TAG_BTREE
else 'mroot' if tag == TAG_MROOT
else 'mdir' if tag == TAG_MDIR else 'mdir' if tag == TAG_MDIR
else 'mtree' if tag == TAG_MTREE else 'mtree' if tag == TAG_MTREE
else 'mroot' if tag == TAG_MROOT
else 'branch' if tag == TAG_BRANCH
else 'did' if tag == TAG_DID else 'did' if tag == TAG_DID
else 'struct 0x%02x' % (tag & 0xff), else 'struct 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
@@ -542,7 +544,7 @@ def main(disk, roots=None, *,
rid_, w = rid__, w_ rid_, w = rid__, w_
# catch any branches # catch any branches
if tag == TAG_BTREE: if tag == TAG_BRANCH:
branch = (tag, j, d, data) branch = (tag, j, d, data)
tags.append((tag, j, d, data)) tags.append((tag, j, d, data))
@@ -554,7 +556,7 @@ def main(disk, roots=None, *,
if branch is not None and ( if branch is not None and (
not depth or depth_ < depth): not depth or depth_ < depth):
tag, j, d, data = branch tag, j, d, data = branch
w_, trunk, block, cksum = frombtree(data) block, trunk, cksum = frombranch(data)
rbyd = Rbyd.fetch(f, block_size, block, trunk) rbyd = Rbyd.fetch(f, block_size, block, trunk)
# corrupted? bail here so we can keep traversing the tree # corrupted? bail here so we can keep traversing the tree
@@ -648,7 +650,7 @@ def main(disk, roots=None, *,
)) ))
d_ += max(bdepths.get(d, 0), 1) d_ += max(bdepths.get(d, 0), 1)
leaf = (bid-(w-1), d, rid-(w-1), TAG_BTREE) leaf = (bid-(w-1), d, rid-(w-1), TAG_BRANCH)
# remap branches to leaves if we aren't showing inner branches # remap branches to leaves if we aren't showing inner branches
if not args.get('inner'): if not args.get('inner'):
scripts/dbglfs.py
+40 -32
View File
@@ -15,18 +15,19 @@ TAG_SUPERCONFIG = 0x0004
TAG_GSTATE = 0x0100 TAG_GSTATE = 0x0100
TAG_GRM = 0x0100 TAG_GRM = 0x0100
TAG_NAME = 0x0200 TAG_NAME = 0x0200
TAG_BRANCH = 0x0200 TAG_BNAME = 0x0200
TAG_BOOKMARK = 0x0201 TAG_DMARK = 0x0201
TAG_REG = 0x0202 TAG_REG = 0x0202
TAG_DIR = 0x0203 TAG_DIR = 0x0203
TAG_STRUCT = 0x0300 TAG_STRUCT = 0x0300
TAG_INLINED = 0x0300 TAG_INLINED = 0x0300
TAG_BLOCK = 0x0302 TAG_BLOCK = 0x0308
TAG_BTREE = 0x0303 TAG_BTREE = 0x030c
TAG_MROOT = 0x0304 TAG_MDIR = 0x0311
TAG_MDIR = 0x0305 TAG_MTREE = 0x0314
TAG_MTREE = 0x0306 TAG_MROOT = 0x0318
TAG_DID = 0x0307 TAG_BRANCH = 0x031c
TAG_DID = 0x0320
TAG_UATTR = 0x0400 TAG_UATTR = 0x0400
TAG_SATTR = 0x0500 TAG_SATTR = 0x0500
TAG_ALT = 0x4000 TAG_ALT = 0x4000
@@ -106,12 +107,18 @@ def frommdir(data):
d += d_ d += d_
return blocks return blocks
def frombranch(data):
d = 0
block, d_ = fromleb128(data[d:]); d += d_
trunk, d_ = fromleb128(data[d:]); d += d_
cksum = fromle32(data[d:]); d += 4
return block, trunk, cksum
def frombtree(data): def frombtree(data):
cksum = fromle32(data) d = 0
w, d1 = fromleb128(data[4:]) w, d_ = fromleb128(data[d:]); d += d_
trunk, d2 = fromleb128(data[4+d1:]) block, trunk, cksum = frombranch(data[d:])
block, d3 = fromleb128(data[4+d1+d2:]) return w, block, trunk, cksum
return w, trunk, block, cksum
def popc(x): def popc(x):
return bin(x).count('1') return bin(x).count('1')
@@ -147,8 +154,8 @@ def tagrepr(tag, w, size, off=None):
size) size)
elif (tag & 0xff00) == TAG_NAME: elif (tag & 0xff00) == TAG_NAME:
return '%s%s %d' % ( return '%s%s %d' % (
'branch' if tag == TAG_BRANCH 'bname' if tag == TAG_BNAME
else 'bookmark' if tag == TAG_BOOKMARK else 'dmark' if tag == TAG_DMARK
else 'reg' if tag == TAG_REG else 'reg' if tag == TAG_REG
else 'dir' if tag == TAG_DIR else 'dir' if tag == TAG_DIR
else 'name 0x%02x' % (tag & 0xff), else 'name 0x%02x' % (tag & 0xff),
@@ -159,9 +166,10 @@ def tagrepr(tag, w, size, off=None):
'inlined' if tag == TAG_INLINED 'inlined' if tag == TAG_INLINED
else 'block' if tag == TAG_BLOCK else 'block' if tag == TAG_BLOCK
else 'btree' if tag == TAG_BTREE else 'btree' if tag == TAG_BTREE
else 'mroot' if tag == TAG_MROOT
else 'mdir' if tag == TAG_MDIR else 'mdir' if tag == TAG_MDIR
else 'mtree' if tag == TAG_MTREE else 'mtree' if tag == TAG_MTREE
else 'mroot' if tag == TAG_MROOT
else 'branch' if tag == TAG_BRANCH
else 'did' if tag == TAG_DID else 'did' if tag == TAG_DID
else 'struct 0x%02x' % (tag & 0xff), else 'struct 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
@@ -428,7 +436,7 @@ class Rbyd:
rid_, w = rid__, w_ rid_, w = rid__, w_
# catch any branches # catch any branches
if tag == TAG_BTREE: if tag == TAG_BRANCH:
branch = (tag, j, d, data) branch = (tag, j, d, data)
tags.append((tag, j, d, data)) tags.append((tag, j, d, data))
@@ -440,7 +448,7 @@ class Rbyd:
if branch is not None and ( if branch is not None and (
not depth or depth_ < depth): not depth or depth_ < depth):
tag, j, d, data = branch tag, j, d, data = branch
w_, trunk, block, cksum = frombtree(data) block, trunk, cksum = frombranch(data)
rbyd = Rbyd.fetch(f, block_size, block, trunk) rbyd = Rbyd.fetch(f, block_size, block, trunk)
# corrupted? bail here so we can keep traversing the tree # corrupted? bail here so we can keep traversing the tree
@@ -457,7 +465,7 @@ class Rbyd:
# have mtree? # have mtree?
done, rid, tag, w, j, d, data, _ = self.lookup(-1, TAG_MTREE) done, rid, tag, w, j, d, data, _ = self.lookup(-1, TAG_MTREE)
if not done and rid == -1 and tag == TAG_MTREE: if not done and rid == -1 and tag == TAG_MTREE:
w, trunk, block, cksum = frombtree(data) w, block, trunk, cksum = frombtree(data)
mtree = Rbyd.fetch(f, block_size, block, trunk) mtree = Rbyd.fetch(f, block_size, block, trunk)
# corrupted? # corrupted?
if not mtree: if not mtree:
@@ -508,7 +516,7 @@ class Rbyd:
break break
# treat vestigial names as a catch-all # treat vestigial names as a catch-all
if ((tag == TAG_BRANCH and rid-(w-1) == 0) if ((tag == TAG_BNAME and rid-(w-1) == 0)
or (tag & 0xff00) != TAG_NAME): or (tag & 0xff00) != TAG_NAME):
did_ = 0 did_ = 0
name_ = b'' name_ = b''
@@ -540,11 +548,11 @@ class Rbyd:
done, rid_, tag_, w_, j, d, data, _ = rbyd.lookup(rid, TAG_STRUCT) done, rid_, tag_, w_, j, d, data, _ = rbyd.lookup(rid, TAG_STRUCT)
# found another branch # found another branch
if tag_ == TAG_BTREE: if tag_ == TAG_BRANCH:
# update our bid # update our bid
bid += rid - (w-1) bid += rid - (w-1)
w_, trunk, block, cksum = frombtree(data) block, trunk, cksum = frombranch(data)
rbyd = Rbyd.fetch(f, block_size, block, trunk) rbyd = Rbyd.fetch(f, block_size, block, trunk)
# found best match # found best match
@@ -556,7 +564,7 @@ class Rbyd:
# have mtree? # have mtree?
done, rid, tag, w, j, d, data, _ = self.lookup(-1, TAG_MTREE) done, rid, tag, w, j, d, data, _ = self.lookup(-1, TAG_MTREE)
if not done and rid == -1 and tag == TAG_MTREE: if not done and rid == -1 and tag == TAG_MTREE:
w, trunk, block, cksum = frombtree(data) w, block, trunk, cksum = frombtree(data)
mtree = Rbyd.fetch(f, block_size, block, trunk) mtree = Rbyd.fetch(f, block_size, block, trunk)
# corrupted? # corrupted?
if not mtree: if not mtree:
@@ -708,7 +716,7 @@ def grepr(tag, data):
return 'gstate 0x%02x %d' % (tag, len(data)) return 'gstate 0x%02x %d' % (tag, len(data))
def frepr(mdir, rid, tag): def frepr(mdir, rid, tag):
if tag == TAG_BOOKMARK: if tag == TAG_DMARK:
# read the did # read the did
did = '?' did = '?'
done, rid_, tag_, w_, j, d, data, _ = mdir.lookup(rid, tag) done, rid_, tag_, w_, j, d, data, _ = mdir.lookup(rid, tag)
@@ -787,7 +795,7 @@ def main(disk, mroots=None, *,
did, d = fromleb128(data) did, d = fromleb128(data)
dir_dids.append( dir_dids.append(
(did, data[d:], -1, mroot, rid, tag, w)) (did, data[d:], -1, mroot, rid, tag, w))
elif tag == TAG_BOOKMARK: elif tag == TAG_DMARK:
did, d = fromleb128(data) did, d = fromleb128(data)
bookmark_dids.append( bookmark_dids.append(
(did, data[d:], -1, mroot, rid, tag, w)) (did, data[d:], -1, mroot, rid, tag, w))
@@ -821,7 +829,7 @@ def main(disk, mroots=None, *,
did, d = fromleb128(data) did, d = fromleb128(data)
dir_dids.append(( dir_dids.append((
did, data[d:], 0, mdir, rid, tag, w)) did, data[d:], 0, mdir, rid, tag, w))
elif tag == TAG_BOOKMARK: elif tag == TAG_DMARK:
did, d = fromleb128(data) did, d = fromleb128(data)
bookmark_dids.append(( bookmark_dids.append((
did, data[d:], 0, mdir, rid, tag, w)) did, data[d:], 0, mdir, rid, tag, w))
@@ -830,7 +838,7 @@ def main(disk, mroots=None, *,
mtree = None mtree = None
done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MTREE) done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MTREE)
if not done and rid == -1 and tag == TAG_MTREE: if not done and rid == -1 and tag == TAG_MTREE:
w, trunk, block, cksum = frombtree(data) w, block, trunk, cksum = frombtree(data)
mtree = Rbyd.fetch(f, block_size, block, trunk) mtree = Rbyd.fetch(f, block_size, block, trunk)
mweight = w mweight = w
@@ -872,7 +880,7 @@ def main(disk, mroots=None, *,
did, d = fromleb128(data) did, d = fromleb128(data)
dir_dids.append(( dir_dids.append((
did, data[d:], mid, mdir_, rid, tag, w)) did, data[d:], mid, mdir_, rid, tag, w))
elif tag == TAG_BOOKMARK: elif tag == TAG_DMARK:
did, d = fromleb128(data) did, d = fromleb128(data)
bookmark_dids.append(( bookmark_dids.append((
did, data[d:], mid, mdir_, rid, tag, w)) did, data[d:], mid, mdir_, rid, tag, w))
@@ -1069,7 +1077,7 @@ def main(disk, mroots=None, *,
f, block_size, did): f, block_size, did):
if not args.get('all'): if not args.get('all'):
# skip bookmarks # skip bookmarks
if tag == TAG_BOOKMARK: if tag == TAG_DMARK:
continue continue
# skip grmed entries # skip grmed entries
if (max(mid, 0), rid) in gstate.grm: if (max(mid, 0), rid) in gstate.grm:
@@ -1104,7 +1112,7 @@ def main(disk, mroots=None, *,
if did_ not in grmed_bookmark_dids: if did_ not in grmed_bookmark_dids:
notes.append('missing bookmark') notes.append('missing bookmark')
# orphaned? # orphaned?
if tag == TAG_BOOKMARK: if tag == TAG_DMARK:
done, rid_, tag_, w_, j, d, data, _ = mdir.lookup( done, rid_, tag_, w_, j, d, data, _ = mdir.lookup(
rid, tag) rid, tag)
if not done and rid_ == rid and tag_ == tag: if not done and rid_ == rid and tag_ == tag:
@@ -1114,7 +1122,7 @@ def main(disk, mroots=None, *,
# print human readable dtree entry # print human readable dtree entry
print('%s%12s %*s %-*s %s%s%s' % ( print('%s%12s %*s %-*s %s%s%s' % (
'\x1b[90m' if color and (grmed or tag == TAG_BOOKMARK) '\x1b[90m' if color and (grmed or tag == TAG_DMARK)
else '', else '',
'{%s}:' % ','.join('%04x' % block '{%s}:' % ','.join('%04x' % block
for block in it.chain([mdir.block], for block in it.chain([mdir.block],
@@ -1132,7 +1140,7 @@ def main(disk, mroots=None, *,
', '.join(notes), ', '.join(notes),
'\x1b[m' if color and not grmed else '') '\x1b[m' if color and not grmed else '')
if notes else '', if notes else '',
'\x1b[m' if color and (grmed or tag == TAG_BOOKMARK) '\x1b[m' if color and (grmed or tag == TAG_DMARK)
else '')) else ''))
pmid = mid pmid = mid
scripts/dbgmtree.py
+29 -21
View File
@@ -14,18 +14,19 @@ TAG_SUPERCONFIG = 0x0004
TAG_GSTATE = 0x0100 TAG_GSTATE = 0x0100
TAG_GRM = 0x0100 TAG_GRM = 0x0100
TAG_NAME = 0x0200 TAG_NAME = 0x0200
TAG_BRANCH = 0x0200 TAG_BNAME = 0x0200
TAG_BOOKMARK = 0x0201 TAG_DMARK = 0x0201
TAG_REG = 0x0202 TAG_REG = 0x0202
TAG_DIR = 0x0203 TAG_DIR = 0x0203
TAG_STRUCT = 0x0300 TAG_STRUCT = 0x0300
TAG_INLINED = 0x0300 TAG_INLINED = 0x0300
TAG_BLOCK = 0x0302 TAG_BLOCK = 0x0308
TAG_BTREE = 0x0303 TAG_BTREE = 0x030c
TAG_MROOT = 0x0304 TAG_MDIR = 0x0311
TAG_MDIR = 0x0305 TAG_MTREE = 0x0314
TAG_MTREE = 0x0306 TAG_MROOT = 0x0318
TAG_DID = 0x0307 TAG_BRANCH = 0x031c
TAG_DID = 0x0320
TAG_UATTR = 0x0400 TAG_UATTR = 0x0400
TAG_SATTR = 0x0500 TAG_SATTR = 0x0500
TAG_ALT = 0x4000 TAG_ALT = 0x4000
@@ -105,12 +106,18 @@ def frommdir(data):
d += d_ d += d_
return blocks return blocks
def frombranch(data):
d = 0
block, d_ = fromleb128(data[d:]); d += d_
trunk, d_ = fromleb128(data[d:]); d += d_
cksum = fromle32(data[d:]); d += 4
return block, trunk, cksum
def frombtree(data): def frombtree(data):
cksum = fromle32(data) d = 0
w, d1 = fromleb128(data[4:]) w, d_ = fromleb128(data[d:]); d += d_
trunk, d2 = fromleb128(data[4+d1:]) block, trunk, cksum = frombranch(data[d:])
block, d3 = fromleb128(data[4+d1+d2:]) return w, block, trunk, cksum
return w, trunk, block, cksum
def popc(x): def popc(x):
return bin(x).count('1') return bin(x).count('1')
@@ -146,8 +153,8 @@ def tagrepr(tag, w, size, off=None):
size) size)
elif (tag & 0xff00) == TAG_NAME: elif (tag & 0xff00) == TAG_NAME:
return '%s%s %d' % ( return '%s%s %d' % (
'branch' if tag == TAG_BRANCH 'bname' if tag == TAG_BNAME
else 'bookmark' if tag == TAG_BOOKMARK else 'dmark' if tag == TAG_DMARK
else 'reg' if tag == TAG_REG else 'reg' if tag == TAG_REG
else 'dir' if tag == TAG_DIR else 'dir' if tag == TAG_DIR
else 'name 0x%02x' % (tag & 0xff), else 'name 0x%02x' % (tag & 0xff),
@@ -158,9 +165,10 @@ def tagrepr(tag, w, size, off=None):
'inlined' if tag == TAG_INLINED 'inlined' if tag == TAG_INLINED
else 'block' if tag == TAG_BLOCK else 'block' if tag == TAG_BLOCK
else 'btree' if tag == TAG_BTREE else 'btree' if tag == TAG_BTREE
else 'mroot' if tag == TAG_MROOT
else 'mdir' if tag == TAG_MDIR else 'mdir' if tag == TAG_MDIR
else 'mtree' if tag == TAG_MTREE else 'mtree' if tag == TAG_MTREE
else 'mroot' if tag == TAG_MROOT
else 'branch' if tag == TAG_BRANCH
else 'did' if tag == TAG_DID else 'did' if tag == TAG_DID
else 'struct 0x%02x' % (tag & 0xff), else 'struct 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',
@@ -519,7 +527,7 @@ class Rbyd:
rid_, w = rid__, w_ rid_, w = rid__, w_
# catch any branches # catch any branches
if tag == TAG_BTREE: if tag == TAG_BRANCH:
branch = (tag, j, d, data) branch = (tag, j, d, data)
tags.append((tag, j, d, data)) tags.append((tag, j, d, data))
@@ -531,7 +539,7 @@ class Rbyd:
if branch is not None and ( if branch is not None and (
not depth or depth_ < depth): not depth or depth_ < depth):
tag, j, d, data = branch tag, j, d, data = branch
w_, trunk, block, cksum = frombtree(data) block, trunk, cksum = frombranch(data)
rbyd = Rbyd.fetch(f, block_size, block, trunk) rbyd = Rbyd.fetch(f, block_size, block, trunk)
# corrupted? bail here so we can keep traversing the tree # corrupted? bail here so we can keep traversing the tree
@@ -626,7 +634,7 @@ class Rbyd:
)) ))
d_ += max(bdepths.get(d, 0), 1) d_ += max(bdepths.get(d, 0), 1)
leaf = (bid-(w-1), d, rid-(w-1), TAG_BTREE) leaf = (bid-(w-1), d, rid-(w-1), TAG_BRANCH)
# remap branches to leaves if we aren't showing inner branches # remap branches to leaves if we aren't showing inner branches
if not inner: if not inner:
@@ -805,7 +813,7 @@ def main(disk, mroots=None, *,
if not args.get('depth') or mdepth < args.get('depth'): if not args.get('depth') or mdepth < args.get('depth'):
done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MTREE) done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MTREE)
if not done and rid == -1 and tag == TAG_MTREE: if not done and rid == -1 and tag == TAG_MTREE:
w, trunk, block, cksum = frombtree(data) w, block, trunk, cksum = frombtree(data)
mtree = Rbyd.fetch(f, block_size, block, trunk) mtree = Rbyd.fetch(f, block_size, block, trunk)
mweight = w mweight = w
@@ -1474,7 +1482,7 @@ def main(disk, mroots=None, *,
if not args.get('depth') or mdepth < args.get('depth'): if not args.get('depth') or mdepth < args.get('depth'):
done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MTREE) done, rid, tag, w, j, d, data, _ = mroot.lookup(-1, TAG_MTREE)
if not done and rid == -1 and tag == TAG_MTREE: if not done and rid == -1 and tag == TAG_MTREE:
w, trunk, block, cksum = frombtree(data) w, block, trunk, cksum = frombtree(data)
mtree = Rbyd.fetch(f, block_size, block, trunk) mtree = Rbyd.fetch(f, block_size, block, trunk)
# traverse entries # traverse entries
scripts/dbgrbyd.py
+13 -12
View File
@@ -23,18 +23,19 @@ TAG_SUPERCONFIG = 0x0004
TAG_GSTATE = 0x0100 TAG_GSTATE = 0x0100
TAG_GRM = 0x0100 TAG_GRM = 0x0100
TAG_NAME = 0x0200 TAG_NAME = 0x0200
TAG_BRANCH = 0x0200 TAG_BNAME = 0x0200
TAG_BOOKMARK = 0x0201 TAG_DMARK = 0x0201
TAG_REG = 0x0202 TAG_REG = 0x0202
TAG_DIR = 0x0203 TAG_DIR = 0x0203
TAG_STRUCT = 0x0300 TAG_STRUCT = 0x0300
TAG_INLINED = 0x0300 TAG_INLINED = 0x0300
TAG_BLOCK = 0x0302 TAG_BLOCK = 0x0308
TAG_BTREE = 0x0303 TAG_BTREE = 0x030c
TAG_MROOT = 0x0304 TAG_MDIR = 0x0311
TAG_MDIR = 0x0305 TAG_MTREE = 0x0314
TAG_MTREE = 0x0306 TAG_MROOT = 0x0318
TAG_DID = 0x0307 TAG_BRANCH = 0x031c
TAG_DID = 0x0320
TAG_UATTR = 0x0400 TAG_UATTR = 0x0400
TAG_SATTR = 0x0500 TAG_SATTR = 0x0500
TAG_ALT = 0x4000 TAG_ALT = 0x4000
@@ -42,7 +43,6 @@ TAG_CKSUM = 0x2000
TAG_ECKSUM = 0x2100 TAG_ECKSUM = 0x2100
# parse some rbyd addr encodings # parse some rbyd addr encodings
# 0xa -> [0xa] # 0xa -> [0xa]
# 0xa.b -> ([0xa], b) # 0xa.b -> ([0xa], b)
@@ -140,8 +140,8 @@ def tagrepr(tag, w, size, off=None):
size) size)
elif (tag & 0xff00) == TAG_NAME: elif (tag & 0xff00) == TAG_NAME:
return '%s%s %d' % ( return '%s%s %d' % (
'branch' if tag == TAG_BRANCH 'bname' if tag == TAG_BNAME
else 'bookmark' if tag == TAG_BOOKMARK else 'dmark' if tag == TAG_DMARK
else 'reg' if tag == TAG_REG else 'reg' if tag == TAG_REG
else 'dir' if tag == TAG_DIR else 'dir' if tag == TAG_DIR
else 'name 0x%02x' % (tag & 0xff), else 'name 0x%02x' % (tag & 0xff),
@@ -152,9 +152,10 @@ def tagrepr(tag, w, size, off=None):
'inlined' if tag == TAG_INLINED 'inlined' if tag == TAG_INLINED
else 'block' if tag == TAG_BLOCK else 'block' if tag == TAG_BLOCK
else 'btree' if tag == TAG_BTREE else 'btree' if tag == TAG_BTREE
else 'mroot' if tag == TAG_MROOT
else 'mdir' if tag == TAG_MDIR else 'mdir' if tag == TAG_MDIR
else 'mtree' if tag == TAG_MTREE else 'mtree' if tag == TAG_MTREE
else 'mroot' if tag == TAG_MROOT
else 'branch' if tag == TAG_BRANCH
else 'did' if tag == TAG_DID else 'did' if tag == TAG_DID
else 'struct 0x%02x' % (tag & 0xff), else 'struct 0x%02x' % (tag & 0xff),
' w%d' % w if w else '', ' w%d' % w if w else '',