Github/littlefs
1
Fork 0
Code Actions Activity

Changed in-tree tags to store their weights

Browse Source 
Sorting weights instead of ids just had a number of benefits, suggesting
this is a better design:

- Calculating the id and delta of each rbyd trunk is surprisingly
  easier - id is now just lower+w-1, and no extra conditions are
  needed for unr tags, which just have a weight of zero.

- Removes ambiguity around which id unr tags should be assigned to,
  especially unrs that delete ids.

- No more +-1 weirdness when encoding/decoding tag ids - the weight
  can be written as-is and -1 ids are infered from their weight and
  position in the tree (lower+w-1 = 0+0-1 = -1).

- Weights compress better under leb128 encoding, since they are usually
  quite small.
This commit is contained in:
Christopher Haster
2023-04-05 14:23:35 -05:00
parent 355c7466f1
commit 0ccf283321
3 changed files with 337 additions and 300 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+55 -53
View File
@@ -611,23 +611,23 @@ static inline lfsr_tag_t lfsr_tag_key(lfsr_tag_t tag) {
return tag & ~0xf; return tag & ~0xf;
} }
static inline bool lfsr_tag_follow(lfsr_tag_t alt, lfs_ssize_t weight, static inline bool lfsr_tag_follow(lfsr_tag_t alt, lfs_size_t weight,
lfs_ssize_t lower, lfs_ssize_t upper, lfs_ssize_t lower, lfs_ssize_t upper,
lfsr_tag_t tag, lfs_ssize_t id) { lfsr_tag_t tag, lfs_ssize_t id) {
if (lfsr_tag_isgt(alt)) { if (lfsr_tag_isgt(alt)) {
return id > upper - weight - 1 return id > upper - (lfs_ssize_t)weight - 1
|| (id == upper - weight - 1 || (id == upper - (lfs_ssize_t)weight - 1
&& lfsr_tag_key(tag) > lfsr_tag_key(alt)); && lfsr_tag_key(tag) > lfsr_tag_key(alt));
} else { } else {
return id < lower + weight return id < lower + (lfs_ssize_t)weight
|| (id == lower + weight || (id == lower + (lfs_ssize_t)weight
&& lfsr_tag_key(tag) <= lfsr_tag_key(alt)); && lfsr_tag_key(tag) <= lfsr_tag_key(alt));
} }
} }
static inline bool lfsr_tag_follow2( static inline bool lfsr_tag_follow2(
lfsr_tag_t alt, lfs_ssize_t weight, lfsr_tag_t alt, lfs_size_t weight,
lfsr_tag_t alt2, lfs_ssize_t weight2, lfsr_tag_t alt2, lfs_size_t weight2,
lfs_ssize_t lower, lfs_ssize_t upper, lfs_ssize_t lower, lfs_ssize_t upper,
lfsr_tag_t tag, lfs_ssize_t id) { lfsr_tag_t tag, lfs_ssize_t id) {
if (lfsr_tag_isred(alt2) && lfsr_tag_isparallel(alt, alt2)) { if (lfsr_tag_isred(alt2) && lfsr_tag_isparallel(alt, alt2)) {
@@ -657,13 +657,13 @@ static inline bool lfsr_tag_prune2(
} }
} }
static inline void lfsr_tag_flip(lfsr_tag_t *alt, lfs_ssize_t *weight, static inline void lfsr_tag_flip(lfsr_tag_t *alt, lfs_size_t *weight,
lfs_ssize_t lower, lfs_ssize_t upper) { lfs_ssize_t lower, lfs_ssize_t upper) {
*alt = *alt ^ 0x4; *alt = *alt ^ 0x4;
*weight = (upper-lower) - *weight - 1; *weight = (upper-lower) - *weight - 1;
} }
static inline void lfsr_tag_flip2(lfsr_tag_t *alt, lfs_ssize_t *weight, static inline void lfsr_tag_flip2(lfsr_tag_t *alt, lfs_size_t *weight,
lfsr_tag_t alt2, lfs_size_t weight2, lfsr_tag_t alt2, lfs_size_t weight2,
lfs_ssize_t lower, lfs_ssize_t upper) { lfs_ssize_t lower, lfs_ssize_t upper) {
if (lfsr_tag_isred(alt2)) { if (lfsr_tag_isred(alt2)) {
@@ -674,7 +674,7 @@ static inline void lfsr_tag_flip2(lfsr_tag_t *alt, lfs_ssize_t *weight,
} }
static inline void lfsr_tag_trim( static inline void lfsr_tag_trim(
lfsr_tag_t alt, lfs_ssize_t weight, lfsr_tag_t alt, lfs_size_t weight,
lfs_ssize_t *lower_id, lfs_ssize_t *upper_id, lfs_ssize_t *lower_id, lfs_ssize_t *upper_id,
lfsr_tag_t *lower_tag, lfsr_tag_t *upper_tag) { lfsr_tag_t *lower_tag, lfsr_tag_t *upper_tag) {
if (lfsr_tag_isgt(alt)) { if (lfsr_tag_isgt(alt)) {
@@ -691,8 +691,8 @@ static inline void lfsr_tag_trim(
} }
static inline void lfsr_tag_trim2( static inline void lfsr_tag_trim2(
lfsr_tag_t alt, lfs_ssize_t weight, lfsr_tag_t alt, lfs_size_t weight,
lfsr_tag_t alt2, lfs_ssize_t weight2, lfsr_tag_t alt2, lfs_size_t weight2,
lfs_ssize_t *lower_id, lfs_ssize_t *upper_id, lfs_ssize_t *lower_id, lfs_ssize_t *upper_id,
lfsr_tag_t *lower_tag, lfsr_tag_t *upper_tag) { lfsr_tag_t *lower_tag, lfsr_tag_t *upper_tag) {
if (lfsr_tag_isred(alt2)) { if (lfsr_tag_isred(alt2)) {
@@ -1264,7 +1264,7 @@ static int lfsr_rbyd_alloc(lfs_t *lfs, lfsr_rbyd_t *rbyd, uint32_t rev) {
static lfs_ssize_t lfsr_rbyd_readtag(lfs_t *lfs, static lfs_ssize_t lfsr_rbyd_readtag(lfs_t *lfs,
const lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_size_t hint, const lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_size_t hint,
lfs_block_t block, lfs_off_t off, lfs_block_t block, lfs_off_t off,
lfsr_tag_t *tag, lfs_ssize_t *id, lfs_size_t *size, uint32_t *crc) { lfsr_tag_t *tag, lfs_size_t *weight, lfs_size_t *size, uint32_t *crc) {
// // needed to quiet an uninitialized warning, zeroing tag on error is // // needed to quiet an uninitialized warning, zeroing tag on error is
// // probably a good idea anyways // // probably a good idea anyways
// *tag = 0; // *tag = 0;
@@ -1303,15 +1303,15 @@ static lfs_ssize_t lfsr_rbyd_readtag(lfs_t *lfs,
uint16_t tag_ = lfs_fromle16_(&buffer[0]); uint16_t tag_ = lfs_fromle16_(&buffer[0]);
lfs_size_t id_; lfs_size_t weight_;
ssize_t delta = 2; ssize_t delta = 2;
lfs_ssize_t delta_ = lfs_fromleb128(&id_, &buffer[delta], 5); lfs_ssize_t delta_ = lfs_fromleb128(&weight_, &buffer[delta], 5);
if (delta_ < 0) { if (delta_ < 0) {
return delta_; return delta_;
} }
delta += delta_; delta += delta_;
if (id_ > 0x7fffffff) { if (weight_ > 0x7fffffff) {
return LFS_ERR_CORRUPT; return LFS_ERR_CORRUPT;
} }
@@ -1335,11 +1335,11 @@ static lfs_ssize_t lfsr_rbyd_readtag(lfs_t *lfs,
// - clear the valid bit from tag, we checked this earlier // - clear the valid bit from tag, we checked this earlier
// - adjust id so reserved id is -1, so we don't have mixed zero/one indexed // - adjust id so reserved id is -1, so we don't have mixed zero/one indexed
// //
if (!lfsr_tag_isalt(tag_)) { // if (!lfsr_tag_isalt(tag_)) {
id_ -= 1; // id_ -= 1;
} // }
*tag = tag_ & ~0x1; *tag = tag_ & ~0x1;
*id = id_; *weight = weight_;
*size = size_; *size = size_;
return delta; return delta;
@@ -1389,11 +1389,11 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd,
// scan tags, checking valid bits, crcs, etc // scan tags, checking valid bits, crcs, etc
while (off < limit) { while (off < limit) {
lfsr_tag_t tag; lfsr_tag_t tag;
lfs_ssize_t id; lfs_size_t w;
lfs_size_t size; lfs_size_t size;
lfs_ssize_t delta = lfsr_rbyd_readtag(lfs, lfs_ssize_t delta = lfsr_rbyd_readtag(lfs,
NULL, &lfs->rcache, limit-off, NULL, &lfs->rcache, limit-off,
block, off, &tag, &id, &size, &crc); block, off, &tag, &w, &size, &crc);
if (delta < 0) { if (delta < 0) {
if (delta == LFS_ERR_INVAL || delta == LFS_ERR_CORRUPT) { if (delta == LFS_ERR_INVAL || delta == LFS_ERR_CORRUPT) {
maybeerased = maybeerased && delta == LFS_ERR_INVAL; maybeerased = maybeerased && delta == LFS_ERR_INVAL;
@@ -1419,18 +1419,16 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd,
// on a bad crc // on a bad crc
if (lfsr_tag_isalt(tag)) { if (lfsr_tag_isalt(tag)) {
if (lfsr_tag_isgt(tag)) { if (lfsr_tag_isgt(tag)) {
upper += id; upper += w;
} else { } else {
lower += id; lower += w;
} }
} else if (lfsr_tag_istrunk(tag)) { } else if (lfsr_tag_istrunk(tag)) {
// note we need to include our id's weight in our weight // note we need to include our id's weight in our weight
// calculation, but only when our id is included in the tree // calculation, but only when our id is included in the tree
lfs_ssize_t delta = (lower+upper) - weight; lfs_ssize_t delta = (lower+upper+w) - weight;
if (!lfsr_tag_isrm(tag)) { weight = lower+upper+ w;
delta += id+1-lower;
}
// adjust any pending finds // adjust any pending finds
if (find && find->predicted_id >= (lfs_ssize_t)lower) { if (find && find->predicted_id >= (lfs_ssize_t)lower) {
@@ -1438,13 +1436,11 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd,
if (delta < 0 if (delta < 0
&& find->predicted_id < (lfs_ssize_t)lower + -delta) { && find->predicted_id < (lfs_ssize_t)lower + -delta) {
find->predicted_tag = 0; find->predicted_tag = 0;
find->predicted_id = lower-1; // TODO id-1? find->predicted_id = lower-1; // TODO is this correct for insertion behavior?
} else { } else {
find->predicted_id += delta; find->predicted_id += delta;
} }
} }
weight += delta;
} }
// we mostly just skip non-data tags here // we mostly just skip non-data tags here
@@ -1510,6 +1506,7 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd,
} }
// found match? // found match?
lfs_ssize_t id = lower+w-1;
if (cmp == LFS_CMP_EQ) { if (cmp == LFS_CMP_EQ) {
find->predicted_tag = tag; find->predicted_tag = tag;
find->predicted_id = id; find->predicted_id = id;
@@ -1614,7 +1611,7 @@ static int lfsr_rbyd_lookup(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
// descend down tree // descend down tree
while (true) { while (true) {
lfsr_tag_t alt; lfsr_tag_t alt;
lfs_ssize_t weight; lfs_size_t weight;
lfs_off_t jump; lfs_off_t jump;
lfs_ssize_t delta = lfsr_rbyd_readtag(lfs, lfs_ssize_t delta = lfsr_rbyd_readtag(lfs,
&lfs->pcache, &lfs->rcache, 0, &lfs->pcache, &lfs->rcache, 0,
@@ -1851,25 +1848,25 @@ static int lfsr_rbyd_progdata(lfs_t *lfs, lfsr_rbyd_t *rbyd_,
} }
static int lfsr_rbyd_progtag(lfs_t *lfs, lfsr_rbyd_t *rbyd_, static int lfsr_rbyd_progtag(lfs_t *lfs, lfsr_rbyd_t *rbyd_,
lfsr_tag_t tag, lfs_ssize_t id, lfs_size_t size, uint32_t *crc) { lfsr_tag_t tag, lfs_size_t weight, lfs_size_t size, uint32_t *crc) {
// check for underflow issues // check for underflow issues
LFS_ASSERT((lfs_size_t)(id+1) < 0x80000000); LFS_ASSERT(weight < 0x80000000);
LFS_ASSERT(size < 0x80000000); LFS_ASSERT(size < 0x80000000);
// make sure to include the parity of the current crc // make sure to include the parity of the current crc
tag |= lfs_popc(rbyd_->crc) & 1; tag |= lfs_popc(rbyd_->crc) & 1;
// change ids to on-disk representation // // change ids to on-disk representation
if (!lfsr_tag_isalt(tag)) { // if (!lfsr_tag_isalt(tag)) {
id += 1; // id += 1;
} // }
// compress into an le16 and pair of leb128s // compress into an le16 and pair of leb128s
uint8_t buf[LFSR_TAG_DSIZE]; uint8_t buf[LFSR_TAG_DSIZE];
lfs_tole16_(tag, &buf[0]); lfs_tole16_(tag, &buf[0]);
lfs_size_t delta = 2; lfs_size_t delta = 2;
ssize_t delta_ = lfs_toleb128(id, &buf[delta], 5); ssize_t delta_ = lfs_toleb128(weight, &buf[delta], 5);
if (delta_ < 0) { if (delta_ < 0) {
return delta_; return delta_;
} }
@@ -1891,7 +1888,7 @@ static int lfsr_rbyd_progtag(lfs_t *lfs, lfsr_rbyd_t *rbyd_,
static int lfsr_rbyd_p_flush(lfs_t *lfs, lfsr_rbyd_t *rbyd_, static int lfsr_rbyd_p_flush(lfs_t *lfs, lfsr_rbyd_t *rbyd_,
lfsr_tag_t p_alts[static 3], lfsr_tag_t p_alts[static 3],
lfs_ssize_t p_weights[static 3], lfs_size_t p_weights[static 3],
lfs_off_t p_jumps[static 3], lfs_off_t p_jumps[static 3],
unsigned count) { unsigned count) {
// write out some number of alt pointers in our queue // write out some number of alt pointers in our queue
@@ -1899,7 +1896,7 @@ static int lfsr_rbyd_p_flush(lfs_t *lfs, lfsr_rbyd_t *rbyd_,
if (p_alts[3-1-i]) { if (p_alts[3-1-i]) {
// change to a relative jump at the last minute // change to a relative jump at the last minute
lfsr_tag_t alt = p_alts[3-1-i]; lfsr_tag_t alt = p_alts[3-1-i];
lfs_ssize_t weight = p_weights[3-1-i]; lfs_size_t weight = p_weights[3-1-i];
lfs_off_t jump = rbyd_->off - p_jumps[3-1-i]; lfs_off_t jump = rbyd_->off - p_jumps[3-1-i];
int err = lfsr_rbyd_progtag(lfs, rbyd_, int err = lfsr_rbyd_progtag(lfs, rbyd_,
@@ -1915,7 +1912,7 @@ static int lfsr_rbyd_p_flush(lfs_t *lfs, lfsr_rbyd_t *rbyd_,
static inline int lfsr_rbyd_p_push(lfs_t *lfs, lfsr_rbyd_t *rbyd_, static inline int lfsr_rbyd_p_push(lfs_t *lfs, lfsr_rbyd_t *rbyd_,
lfsr_tag_t p_alts[static 3], lfsr_tag_t p_alts[static 3],
lfs_ssize_t p_weights[static 3], lfs_size_t p_weights[static 3],
lfs_off_t p_jumps[static 3], lfs_off_t p_jumps[static 3],
lfsr_tag_t alt, lfs_ssize_t weight, lfs_off_t jump) { lfsr_tag_t alt, lfs_ssize_t weight, lfs_off_t jump) {
int err = lfsr_rbyd_p_flush(lfs, rbyd_, p_alts, p_weights, p_jumps, 1); int err = lfsr_rbyd_p_flush(lfs, rbyd_, p_alts, p_weights, p_jumps, 1);
@@ -1935,10 +1932,10 @@ static inline int lfsr_rbyd_p_push(lfs_t *lfs, lfsr_rbyd_t *rbyd_,
static inline void lfsr_rbyd_p_pop( static inline void lfsr_rbyd_p_pop(
lfsr_tag_t p_alts[static 3], lfsr_tag_t p_alts[static 3],
lfs_ssize_t p_weights[static 3], lfs_size_t p_weights[static 3],
lfs_off_t p_jumps[static 3]) { lfs_off_t p_jumps[static 3]) {
memmove(p_alts, p_alts+1, 2*sizeof(lfsr_tag_t)); memmove(p_alts, p_alts+1, 2*sizeof(lfsr_tag_t));
memmove(p_weights, p_weights+1, 2*sizeof(lfs_ssize_t)); memmove(p_weights, p_weights+1, 2*sizeof(lfs_size_t));
memmove(p_jumps, p_jumps+1, 2*sizeof(lfs_off_t)); memmove(p_jumps, p_jumps+1, 2*sizeof(lfs_off_t));
p_alts[2] = 0; p_alts[2] = 0;
p_weights[2] = 0; p_weights[2] = 0;
@@ -1947,7 +1944,7 @@ static inline void lfsr_rbyd_p_pop(
static void lfsr_rbyd_p_red( static void lfsr_rbyd_p_red(
lfsr_tag_t p_alts[static 3], lfsr_tag_t p_alts[static 3],
lfs_ssize_t p_weights[static 3], lfs_size_t p_weights[static 3],
lfs_off_t p_jumps[static 3]) { lfs_off_t p_jumps[static 3]) {
// propagate a red edge upwards // propagate a red edge upwards
p_alts[0] = lfsr_tag_setblack(p_alts[0]); p_alts[0] = lfsr_tag_setblack(p_alts[0]);
@@ -1961,7 +1958,7 @@ static void lfsr_rbyd_p_red(
// no reorder needed // no reorder needed
} else if (lfsr_tag_isparallel(p_alts[0], p_alts[2])) { } else if (lfsr_tag_isparallel(p_alts[0], p_alts[2])) {
lfsr_tag_t alt_ = p_alts[1]; lfsr_tag_t alt_ = p_alts[1];
lfs_ssize_t weight_ = p_weights[1]; lfs_size_t weight_ = p_weights[1];
lfs_off_t jump_ = p_jumps[1]; lfs_off_t jump_ = p_jumps[1];
p_alts[1] = lfsr_tag_setred(p_alts[0]); p_alts[1] = lfsr_tag_setred(p_alts[0]);
p_weights[1] = p_weights[0]; p_weights[1] = p_weights[0];
@@ -1971,7 +1968,7 @@ static void lfsr_rbyd_p_red(
p_jumps[0] = jump_; p_jumps[0] = jump_;
} else if (lfsr_tag_isparallel(p_alts[0], p_alts[1])) { } else if (lfsr_tag_isparallel(p_alts[0], p_alts[1])) {
lfsr_tag_t alt_ = p_alts[2]; lfsr_tag_t alt_ = p_alts[2];
lfs_ssize_t weight_ = p_weights[2]; lfs_size_t weight_ = p_weights[2];
lfs_off_t jump_ = p_jumps[2]; lfs_off_t jump_ = p_jumps[2];
p_alts[2] = lfsr_tag_setred(p_alts[1]); p_alts[2] = lfsr_tag_setred(p_alts[1]);
p_weights[2] = p_weights[1]; p_weights[2] = p_weights[1];
@@ -2103,7 +2100,7 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
// queue of pending alts we can emulate rotations with // queue of pending alts we can emulate rotations with
lfsr_tag_t p_alts[3] = {0, 0, 0}; lfsr_tag_t p_alts[3] = {0, 0, 0};
lfs_ssize_t p_weights[3] = {0, 0, 0}; lfs_size_t p_weights[3] = {0, 0, 0};
lfs_off_t p_jumps[3] = {0, 0, 0}; lfs_off_t p_jumps[3] = {0, 0, 0};
lfs_off_t graft = 0; lfs_off_t graft = 0;
@@ -2111,7 +2108,7 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
while (true) { while (true) {
// read the alt pointer // read the alt pointer
lfsr_tag_t alt; lfsr_tag_t alt;
lfs_ssize_t weight; lfs_size_t weight;
lfs_off_t jump; lfs_off_t jump;
lfs_ssize_t d = lfsr_rbyd_readtag(lfs, lfs_ssize_t d = lfsr_rbyd_readtag(lfs,
&lfs->pcache, &lfs->rcache, 0, &lfs->pcache, &lfs->rcache, 0,
@@ -2213,7 +2210,7 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
lfs_swap32(&jump, &branch_); lfs_swap32(&jump, &branch_);
lfs_swap16(&p_alts[0], &alt); lfs_swap16(&p_alts[0], &alt);
lfs_sswap32(&p_weights[0], &weight); lfs_swap32(&p_weights[0], &weight);
lfs_swap32(&p_jumps[0], &jump); lfs_swap32(&p_jumps[0], &jump);
alt = lfsr_tag_setblack(alt); alt = lfsr_tag_setblack(alt);
@@ -2278,7 +2275,7 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
lower_id, upper_id, lower_id, upper_id,
tag_, id_)) { tag_, id_)) {
lfs_swap16(&p_alts[0], &alt); lfs_swap16(&p_alts[0], &alt);
lfs_sswap32(&p_weights[0], &weight); lfs_swap32(&p_weights[0], &weight);
lfs_swap32(&p_jumps[0], &jump); lfs_swap32(&p_jumps[0], &jump);
p_alts[0] = lfsr_tag_setred(p_alts[0]); p_alts[0] = lfsr_tag_setred(p_alts[0]);
alt = lfsr_tag_setblack(alt); alt = lfsr_tag_setblack(alt);
@@ -2383,10 +2380,12 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
// if removed make our tag unreachable // if removed make our tag unreachable
alt = LFSR_TAG_ALT(B, GT, 0); alt = LFSR_TAG_ALT(B, GT, 0);
weight = upper_id - lower_id - 1 + delta; weight = upper_id - lower_id - 1 + delta;
upper_id -= weight;
} else { } else {
// split less than // split less than
alt = LFSR_TAG_ALT(R, LE, tag_); alt = LFSR_TAG_ALT(R, LE, tag_);
weight = id_ - lower_id; weight = id_ - lower_id;
lower_id += weight;
} }
} else if (id_ > id } else if (id_ > id
@@ -2397,10 +2396,12 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
// if removed make our tag unreachable // if removed make our tag unreachable
alt = LFSR_TAG_ALT(B, GT, 0); alt = LFSR_TAG_ALT(B, GT, 0);
weight = upper_id - lower_id - 1 + delta; weight = upper_id - lower_id - 1 + delta;
upper_id -= weight;
} else { } else {
// split greater than // split greater than
alt = LFSR_TAG_ALT(R, GT, tag); alt = LFSR_TAG_ALT(R, GT, tag);
weight = upper_id - id - 1; weight = upper_id - id - 1;
upper_id -= weight;
} }
} }
@@ -2433,7 +2434,8 @@ leaf:;
// note we always need something after the alts! without something between // note we always need something after the alts! without something between
// alts we may not be able to find the trunk of our tree // alts we may not be able to find the trunk of our tree
err = lfsr_rbyd_progtag(lfs, rbyd, err = lfsr_rbyd_progtag(lfs, rbyd,
lfsr_tag_setnomk(tag), id + delta, lfsr_data_len(data), &rbyd->crc); lfsr_tag_setnomk(tag), upper_id - lower_id - 1 + delta,
lfsr_data_len(data), &rbyd->crc);
if (err) { if (err) {
rbyd->erased = false; rbyd->erased = false;
return err; return err;
@@ -2576,7 +2578,7 @@ static int lfsr_rbyd_commit(lfs_t *lfs, lfsr_rbyd_t *rbyd,
uint8_t fbuf[LFSR_FCRC_DSIZE]; uint8_t fbuf[LFSR_FCRC_DSIZE];
lfs_size_t fcrc_delta = lfsr_fcrc_todisk(&fcrc, fbuf); lfs_size_t fcrc_delta = lfsr_fcrc_todisk(&fcrc, fbuf);
err = lfsr_rbyd_progtag(lfs, &rbyd_, err = lfsr_rbyd_progtag(lfs, &rbyd_,
LFSR_TAG_FCRC, -1, fcrc_delta, &rbyd_.crc); LFSR_TAG_FCRC, 0, fcrc_delta, &rbyd_.crc);
if (err) { if (err) {
rbyd_.erased = false; rbyd_.erased = false;
return err; return err;
scripts/dbgbtree.py
+187 -176
View File
@@ -65,9 +65,9 @@ def fromleb128(data):
def fromtag(data): def fromtag(data):
tag = fromle16(data) tag = fromle16(data)
id, delta = fromleb128(data[2:]) weight, delta = fromleb128(data[2:])
size, delta_ = fromleb128(data[2+delta:]) size, delta_ = fromleb128(data[2+delta:])
return tag&1, tag&~1, id if tag&0x8 else id-1, size, 2+delta+delta_ return tag&1, tag&~1, weight, size, 2+delta+delta_
def popc(x): def popc(x):
return bin(x).count('1') return bin(x).count('1')
@@ -82,56 +82,56 @@ def xxd(data, width=16, crc=False):
b if b >= ' ' and b <= '~' else '.' b if b >= ' ' and b <= '~' else '.'
for b in map(chr, data[i:i+width]))) for b in map(chr, data[i:i+width])))
def tagrepr(tag, id, size, off=None): def tagrepr(tag, w, size, off=None):
if (tag & 0xfffe) == TAG_UNR: if (tag & 0xfffe) == TAG_UNR:
return 'unr id%d%s' % ( return 'unr%s%s' % (
id, ' w%d' % w if w else '',
' %d' % size if size else '') ' %d' % size if size else '')
elif (tag & 0xf00c) == TAG_NAME: elif (tag & 0xf00c) == TAG_NAME:
return '%s%s id%d %d' % ( return '%s%s%s %d' % (
'rm' if tag & 0x2 else '', 'rm' if tag & 0x2 else '',
'bname' if (tag & 0xfffe) == TAG_BNAME 'bname' if (tag & 0xfffe) == TAG_BNAME
else 'reg' if (tag & 0xfffe) == TAG_REG else 'reg' if (tag & 0xfffe) == TAG_REG
else 'dir' if (tag & 0xfffe) == TAG_DIR else 'dir' if (tag & 0xfffe) == TAG_DIR
else 'name 0x%02x' % ((tag & 0x0ff0) >> 4), else 'name 0x%02x' % ((tag & 0x0ff0) >> 4),
id, ' w%d' % w if w else '',
size) size)
elif (tag & 0xf00c) == TAG_STRUCT: elif (tag & 0xf00c) == TAG_STRUCT:
return '%s%s id%d %d' % ( return '%s%s%s %d' % (
'rm' if tag & 0x2 else '', 'rm' if tag & 0x2 else '',
'inlined' if (tag & 0xfffe) == TAG_INLINED 'inlined' if (tag & 0xfffe) == TAG_INLINED
else 'block' if (tag & 0xfffe) == TAG_BLOCK else 'block' if (tag & 0xfffe) == TAG_BLOCK
else 'branch' if (tag & 0xfffe) == TAG_BRANCH else 'branch' if (tag & 0xfffe) == TAG_BRANCH
else 'btree' if (tag & 0xfffe) == TAG_BTREE else 'btree' if (tag & 0xfffe) == TAG_BTREE
else 'struct 0x%02x' % ((tag & 0x0ff0) >> 4), else 'struct 0x%02x' % ((tag & 0x0ff0) >> 4),
id, ' w%d' % w if w else '',
size) size)
elif (tag & 0xf00c) == TAG_UATTR: elif (tag & 0xf00c) == TAG_UATTR:
return '%suattr 0x%02x%s%s' % ( return '%suattr 0x%02x%s%s' % (
'rm' if tag & 0x2 else '', 'rm' if tag & 0x2 else '',
(tag & 0x0ff0) >> 4, (tag & 0x0ff0) >> 4,
' id%d' % id if id != -1 else '', ' w%d' % w if w else '',
' %d' % size if not tag & 0x2 or size else '') ' %d' % size if not tag & 0x2 or size else '')
elif (tag & 0xf00e) == TAG_CRC: elif (tag & 0xf00e) == TAG_CRC:
return 'crc%x%s %d' % ( return 'crc%x%s %d' % (
1 if tag & 0x10 else 0, 1 if tag & 0x10 else 0,
' 0x%02x' % id if id != -1 else '', ' 0x%x' % w if w > 0 else '',
size) size)
elif (tag & 0xfffe) == TAG_FCRC: elif (tag & 0xfffe) == TAG_FCRC:
return 'fcrc%s %d' % ( return 'fcrc%s %d' % (
' 0x%02x' % id if id != -1 else '', ' 0x%x' % w if w > 0 else '',
size) size)
elif tag & 0x8: elif tag & 0x8:
return 'alt%s%s 0x%x w%d %s' % ( return 'alt%s%s 0x%x w%d %s' % (
'r' if tag & 0x2 else 'b', 'r' if tag & 0x2 else 'b',
'gt' if tag & 0x4 else 'le', 'gt' if tag & 0x4 else 'le',
tag & 0xfff0, tag & 0xfff0,
id, w,
'0x%x' % (0xffffffff & (off-size)) '0x%x' % (0xffffffff & (off-size))
if off is not None if off is not None
else '-%d' % off) else '-%d' % off)
else: else:
return '0x%04x id%d %d' % (tag, id, size) return '0x%04x w%d %d' % (tag, w, size)
class Rbyd: class Rbyd:
def __init__(self, block, limit, data, rev, off, trunk, weight): def __init__(self, block, limit, data, rev, off, trunk, weight):
@@ -161,7 +161,7 @@ class Rbyd:
weight_ = 0 weight_ = 0
wastrunk = False wastrunk = False
while j_ < limit: while j_ < limit:
v, tag, id, size, delta = fromtag(data[j_:]) v, tag, w, size, delta = fromtag(data[j_:])
if v != (popc(crc) & 1): if v != (popc(crc) & 1):
break break
crc = crc32c(data[j_:j_+delta], crc) crc = crc32c(data[j_:j_+delta], crc)
@@ -176,13 +176,11 @@ class Rbyd:
# keep track of weight # keep track of weight
if tag & 0x8: if tag & 0x8:
if tag & 0x4: if tag & 0x4:
upper_ += id upper_ += w
else: else:
lower_ += id lower_ += w
elif (tag & 0xc) == 0x0: elif (tag & 0xc) == 0x0:
weight_ = lower_+upper_ weight_ = lower_+upper_+w
if not tag & 0x2:
weight_ += id+1-lower_
# take care of crcs # take care of crcs
if not tag & 0x8: if not tag & 0x8:
@@ -301,9 +299,9 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
# corrupted? return a corrupted block once # corrupted? return a corrupted block once
if not rbyd: if not rbyd:
return (id > 0, id, 0, rbyd, -1, 0, return (id > 0, id, 0, rbyd, -1,
0, 0, b'', (0, 0, 0, b''),
0, 0, b'', (0, 0, 0, b''),
path) path)
while True: while True:
@@ -311,49 +309,51 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
(done, name_tag, rid_, w, (done, name_tag, rid_, w,
name_j, name_d, name) = rbyd.lookup(0, rid) name_j, name_d, name) = rbyd.lookup(0, rid)
if done: if done:
return (True, id, 0, rbyd, -1, 0, return (True, id, 0, rbyd, -1,
0, 0, b'', (0, 0, 0, b''),
0, 0, b'', (0, 0, 0, b''),
path) path)
if name_tag & 0xf00f == TAG_NAME: if name_tag & 0xf00f == TAG_NAME:
# then lookup struct # then lookup struct
(done, tag, _, _, (done, struct_tag, _, _,
struct_j, struct_d, struct_) = rbyd.lookup( struct_j, struct_d, struct_) = rbyd.lookup(
TAG_STRUCT, rid_) TAG_STRUCT, rid_)
if done: if done:
return (True, id, 0, rbyd, -1, 0, return (True, id, 0, rbyd, -1,
0, 0, b'', (0, 0, 0, b''),
0, 0, b'', (0, 0, 0, b''),
path) path)
else: else:
tag = name_tag tag = name_tag
struct_j, struct_d, struct_ = name_j, name_d, name struct_tag, struct_j, struct_d, struct_ = (
name_j, name_d, name = name_j, 0, b'' name_tag, name_j, name_d, name)
name_tag, name_j, name_d, name = 0, 0, 0, b''
path.append((id + (rid_-rid), w, rbyd, rid_, tag, path.append((id + (rid_-rid), w, rbyd, rid_,
name_j, name_d, name, (name_tag, name_j, name_d, name),
struct_j, struct_d, struct_)) (struct_tag, struct_j, struct_d, struct_)))
# is it another branch? continue down tree # is it another branch? continue down tree
if tag == TAG_BRANCH and (depth is None or depth_ < depth): if struct_tag == TAG_BRANCH and (
depth is None or depth_ < depth):
block, delta = fromleb128(struct_) block, delta = fromleb128(struct_)
limit, _ = fromleb128(struct_[delta:]) limit, _ = fromleb128(struct_[delta:])
rbyd = Rbyd.fetch(f, block_size, block, limit) rbyd = Rbyd.fetch(f, block_size, block, limit)
# corrupted? bail here so we can keep traversing the tree # corrupted? bail here so we can keep traversing the tree
if not rbyd: if not rbyd:
return (False, id + (rid_-rid), w, rbyd, -1, 0, return (False, id + (rid_-rid), w, rbyd, -1,
0, 0, b'', (0, 0, 0, b''),
0, 0, b'', (0, 0, 0, b''),
path) path)
rid -= (rid_-(w-1)) rid -= (rid_-(w-1))
depth_ += 1 depth_ += 1
else: else:
return (False, id + (rid_-rid), w, rbyd, rid_, tag, return (False, id + (rid_-rid), w, rbyd, rid_,
name_j, name_d, name, (name_tag, name_j, name_d, name),
struct_j, struct_d, struct_, (struct_tag, struct_j, struct_d, struct_),
path) path)
# if we're printing the tree, first find the max depth so we know how # if we're printing the tree, first find the max depth so we know how
@@ -363,9 +363,7 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
t_depth = 0 t_depth = 0
id = -1 id = -1
while True: while True:
(done, id, w, rbyd, rid, tag, (done, id, w, rbyd, rid, _, _,
name_j, name_d, name,
struct_j, struct_d, struct_,
path) = (lookup(id+1, depth=args.get('depth'))) path) = (lookup(id+1, depth=args.get('depth')))
if done: if done:
break break
@@ -375,23 +373,28 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
t_width = 2*t_depth+2 if t_depth > 0 else 0 t_width = 2*t_depth+2 if t_depth > 0 else 0
t_branches = [(0, trunk.weight)] t_branches = [(0, trunk.weight)]
def t_repr(id, w, d=None): def t_repr(id, w, leaf=True, depth=None):
branches_ = [] branches_ = []
for i in range(len(t_branches)): for i in range(len(t_branches)):
if d is not None and d == i-1: if depth is not None and depth == i-1:
branches_.append('+') branches_.append('+' if leaf else '|')
elif i+1 < len(t_branches): break
if (id-(w-1) == t_branches[i+1][0]
if i+1 < len(t_branches):
if (leaf
and id-(w-1) == t_branches[i+1][0]
and t_branches[i][0] == t_branches[i+1][0] and t_branches[i][0] == t_branches[i+1][0]
and (not args.get('inner') and (not args.get('inner')
or (i == 0 and d == 0))): or (i == 0 and depth == 0))):
branches_.append('+-') branches_.append('+-')
elif (id-(w-1) == t_branches[i+1][0] elif (leaf
and id-(w-1) == t_branches[i+1][0]
and t_branches[i][1] == t_branches[i+1][1] and t_branches[i][1] == t_branches[i+1][1]
and (not args.get('inner') or d == i)): and (not args.get('inner') or depth == i)):
branches_.append('\'-') branches_.append('\'-')
elif (id-(w-1) == t_branches[i+1][0] elif (leaf
and (not args.get('inner') or d == i)): and id-(w-1) == t_branches[i+1][0]
and (not args.get('inner') or depth == i)):
branches_.append('|-') branches_.append('|-')
elif (id-(w-1) >= t_branches[i][0] elif (id-(w-1) >= t_branches[i][0]
and id-(w-1) < t_branches[i][1] and id-(w-1) < t_branches[i][1]
@@ -400,14 +403,22 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
else: else:
branches_.append(' ') branches_.append(' ')
else: else:
if (id-(w-1) == t_branches[i][0] if (leaf
and id-(w-1) == t_branches[i][0]
and (not args.get('inner') or i == 0)): and (not args.get('inner') or i == 0)):
branches_.append('+-%s> ' % ('-'*2*(t_depth-i-1))) branches_.append('+-%s> ' % ('-'*2*(t_depth-i-1)))
elif id == t_branches[i][1]-1: elif (leaf
and id == t_branches[i][1]-1):
branches_.append('\'-%s> ' % ('-'*2*(t_depth-i-1))) branches_.append('\'-%s> ' % ('-'*2*(t_depth-i-1)))
elif (id >= t_branches[i][0] elif (leaf
and id >= t_branches[i][0]
and id-(w-1) < t_branches[i][1]): and id-(w-1) < t_branches[i][1]):
branches_.append('|-%s> ' % ('-'*2*(t_depth-i-1))) branches_.append('|-%s> ' % ('-'*2*(t_depth-i-1)))
elif (id >= t_branches[i][0]
and id < t_branches[i][1]-1):
branches_.append('|')
else:
branches_.append(' ')
return '%s%-*s%s' % ( return '%s%-*s%s' % (
'\x1b[90m' if color else '', '\x1b[90m' if color else '',
@@ -416,24 +427,123 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
# print header # print header
w_width = 2*m.ceil(m.log10(max(1, trunk.weight)+1))+1 w_width = 2*m.ceil(m.log10(max(1, trunk.weight)+1))+1
print('%-9s %*s%-*s %-8s %-22s %s' % ( print('%-9s %*s%-*s %-22s %s' % (
'block', 'block',
t_width, '', t_width, '',
w_width, 'ids', w_width, 'ids',
'name',
'tag', 'tag',
'data (truncated)' 'data (truncated)'
if not args.get('no_truncate') else '')) if not args.get('no_truncate') else ''))
# prbyd here means the last rendered rbyd, we update
# in show_entry to always print interleaved addresses
prbyd = None
def show_entry(id, w, rbyd, rid,
name_tag, name_j, name_d, name,
struct_tag, struct_j, struct_d, struct_,
depth=None):
nonlocal prbyd
# show human-readable representation
if name_tag:
print('%10s %s%*s %-22s %s' % (
'%04x.%04x:' % (rbyd.block, rbyd.limit)
if prbyd is None or rbyd != prbyd
else '',
t_repr(id, w, True, depth) if args.get('tree') else '',
w_width, '%d-%d' % (id-(w-1), id) if w > 1
else id if w > 0
else '',
tagrepr(name_tag, w, len(name), None),
''.join(
b if b >= ' ' and b <= '~' else '.'
for b in map(chr, name))))
prbyd = rbyd
print('%10s %s%*s %-22s %s' % (
'%04x.%04x:' % (rbyd.block, rbyd.limit)
if prbyd is None or rbyd != prbyd
else '',
t_repr(id, w, not name_tag, depth) if args.get('tree') else '',
w_width, '' if name_tag
else '%d-%d' % (id-(w-1), id) if w > 1
else id if w > 0
else '',
tagrepr(struct_tag,
w if not name_tag else 0,
len(struct_), None),
next(xxd(struct_, 8), '')
if not args.get('no_truncate') else ''))
prbyd = rbyd
# show in-device representation
if args.get('device'):
if name_tag:
print('%9s %*s%*s %-22s%s' % (
'',
t_width, '',
w_width, '',
'%04x %08x %07x' % (name_tag, w, len(name)),
' %s' % ' '.join(
'%08x' % struct.unpack('<I',
rbyd.data[name_j+name_d+i*4
: name_j+name_d + min(i*4+4,len(name))]
.ljust(4, b'\0'))
for i in range(min(m.ceil(len(name)/4), 3)))[:23]))
print('%9s %*s%*s %-22s%s' % (
'',
t_width, '',
w_width, '',
'%04x %08x %07x' % (
struct_tag, w if not name_tag else 0, len(struct_)),
' %s' % ' '.join(
'%08x' % struct.unpack('<I',
rbyd.data[struct_j+struct_d+i*4
: struct_j+struct_d + min(i*4+4,len(struct_))]
.ljust(4, b'\0'))
for i in range(min(m.ceil(len(struct_)/4), 3)))[:23]))
# show on-disk encoding of tags/data
if args.get('raw'):
for o, line in enumerate(xxd(
rbyd.data[name_j:name_j+name_d])):
print('%9s: %*s%*s %s' % (
'%04x' % (name_j + o*16),
t_width, '',
w_width, '',
line))
if args.get('raw'):
for o, line in enumerate(xxd(name)):
print('%9s: %*s%*s %s' % (
'%04x' % (name_j+name_d + o*16),
t_width, '',
w_width, '',
line))
if args.get('raw'):
for o, line in enumerate(xxd(
rbyd.data[struct_j:struct_j+struct_d])):
print('%9s: %*s%*s %s' % (
'%04x' % (struct_j + o*16),
t_width, '',
w_width, '',
line))
if args.get('raw') or args.get('no_truncate'):
for o, line in enumerate(xxd(struct_)):
print('%9s: %*s%*s %s' % (
'%04x' % (struct_j+struct_d + o*16),
t_width, '',
w_width, '',
line))
# traverse and print entries # traverse and print entries
id = -1 id = -1
prbyd = None prbyd = None
ppath = [] ppath = []
corrupted = False corrupted = False
while True: while True:
(done, id, w, rbyd, rid, tag, (done, id, w, rbyd, rid,
name_j, name_d, name, (name_tag, name_j, name_d, name),
struct_j, struct_d, struct_, (struct_tag, struct_j, struct_d, struct_),
path) = (lookup(id+1, depth=args.get('depth'))) path) = (lookup(id+1, depth=args.get('depth')))
if done: if done:
break break
@@ -446,9 +556,9 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
if x is None: if x is None:
break break
(id_, w_, rbyd_, rid_, tag_, (id_, w_, rbyd_, rid_,
name_j_, name_d_, name_, (name_tag_, name_j_, name_d_, name_),
struct_j_, struct_d_, struct__) = x (struct_tag_, struct_j_, struct_d_, struct__)) = x
t_branches.append((id_-(w_-1), id_+1)) t_branches.append((id_-(w_-1), id_+1))
if args.get('inner'): if args.get('inner'):
@@ -456,61 +566,11 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
continue continue
changed = True changed = True
# show human-readable representation # show the inner entry
print('%10s %s%*s %-8s %-22s %s' % ( show_entry(id_, w_, rbyd_, rid_,
'%04x.%04x:' % (rbyd_.block, rbyd_.limit) name_tag_, name_j_, name_d_, name_,
if prbyd is None or rbyd_ != prbyd struct_tag_, struct_j_, struct_d_, struct__,
else '', i)
t_repr(id_, w_, i) if args.get('tree') else '',
w_width, '%d-%d' % (id_-(w_-1), id_)
if w_ > 1 else id_
if w_ > 0 else '',
''.join(
b if b >= ' ' and b <= '~' else '.'
for b in map(chr, name_)),
tagrepr(tag_, rid_, len(struct__), None),
next(xxd(struct__, 8), '')
if not args.get('no_truncate') else ''))
# show in-device representation
if args.get('device'):
print('%9s %*s%*s %8s %-22s%s' % (
'',
t_width, '',
w_width, '',
'',
'%04x %08x %07x' % (
tag_, 0xffffffff & rid_, len(struct__)),
' %s' % ' '.join(
'%08x' % struct.unpack('<I',
rbyd_.data[struct_j_+struct_d_+i*4
: struct_j_+struct_d_
+ min(i*4+4,len(struct__))]
.ljust(4, b'\0'))
for i in range(
min(m.ceil(len(struct__)/4), 3)))[:23]))
# show on-disk encoding of tags/data
for j, d, data in [
(name_j_, name_d_, name_),
(struct_j_, struct_d_, struct__)]:
if args.get('raw'):
for o, line in enumerate(
xxd(rbyd_.data[j:j+d])):
print('%9s: %s' % (
'%04x' % (j + o*16),
line))
# show on-disk encoding of tags
if args.get('raw') or args.get('no_truncate'):
for o, line in enumerate(xxd(data)):
print('%9s: %s' % (
'%04x' % (j+d + o*16),
line))
# prbyd here means the last rendered rbyd, we update
# here to always print interleaved addresses
prbyd = rbyd_
# corrupted? try to keep printing the tree # corrupted? try to keep printing the tree
if not rbyd: if not rbyd:
@@ -529,65 +589,16 @@ def main(disk, block_size=None, trunk=0, limit=None, *,
# if we're not showing inner nodes, prefer names higher in the tree # if we're not showing inner nodes, prefer names higher in the tree
# since this avoids showing vestigial names # since this avoids showing vestigial names
if not args.get('inner'): if not args.get('inner'):
for (id_, w_, rbyd_, rid_, tag_, for id_, w_, rbyd_, rid_, name_, struct__ in reversed(path):
name_j_, name_d_, name_, name_tag_, name_j, name_d, name = name_
struct_j_, struct_d_, struct__) in reversed(path):
name_j, name_d, name = name_j_, name_d_, name_
if rid_-(w_-1) != 0: if rid_-(w_-1) != 0:
break break
# show human-readable representation # show the entry
print('%10s %s%*s %-8s %-22s %s' % ( show_entry(id, w, rbyd, rid,
'%04x.%04x:' % (rbyd.block, rbyd.limit) name_tag, name_j, name_d, name,
if prbyd is None or rbyd != prbyd struct_tag, struct_j, struct_d, struct_)
else '',
t_repr(id, w) if args.get('tree') else '',
w_width, '%d-%d' % (id-(w-1), id)
if w > 1 else id
if w > 0 else '',
''.join(
b if b >= ' ' and b <= '~' else '.'
for b in map(chr, name)),
tagrepr(tag, rid, len(struct_), None),
next(xxd(struct_, 8), '')
if not args.get('no_truncate') else ''))
# show in-device representation
if args.get('device'):
print('%9s %*s%*s %8s %-22s%s' % (
'',
t_width, '',
w_width, '',
'',
'%04x %08x %07x' % (
tag, 0xffffffff & rid, len(struct_)),
' %s' % ' '.join(
'%08x' % struct.unpack('<I',
rbyd.data[struct_j+struct_d+i*4
: struct_j+struct_d
+ min(i*4+4,len(struct_))]
.ljust(4, b'\0'))
for i in range(
min(m.ceil(len(struct_)/4), 3)))[:23]))
# show on-disk encoding of tags/data
for j, d, data in [
(name_j, name_d, name),
(struct_j, struct_d, struct_)]:
if args.get('raw'):
for o, line in enumerate(xxd(rbyd.data[j:j+d])):
print('%9s: %s' % (
'%04x' % (j + o*16),
line))
# show on-disk encoding of tags
if args.get('raw') or args.get('no_truncate'):
for o, line in enumerate(xxd(data)):
print('%9s: %s' % (
'%04x' % (j+d + o*16),
line))
prbyd = rbyd
ppath = path ppath = path
if args.get('error_on_corrupt') and corrupted: if args.get('error_on_corrupt') and corrupted:
scripts/dbgrbyd.py
+95 -71
View File
@@ -74,9 +74,9 @@ def fromleb128(data):
def fromtag(data): def fromtag(data):
tag = fromle16(data) tag = fromle16(data)
id, delta = fromleb128(data[2:]) weight, delta = fromleb128(data[2:])
size, delta_ = fromleb128(data[2+delta:]) size, delta_ = fromleb128(data[2+delta:])
return tag&1, tag&~1, id if tag&0x8 else id-1, size, 2+delta+delta_ return tag&1, tag&~1, weight, size, 2+delta+delta_
def popc(x): def popc(x):
return bin(x).count('1') return bin(x).count('1')
@@ -91,58 +91,58 @@ def xxd(data, width=16, crc=False):
b if b >= ' ' and b <= '~' else '.' b if b >= ' ' and b <= '~' else '.'
for b in map(chr, data[i:i+width]))) for b in map(chr, data[i:i+width])))
def tagrepr(tag, id, size, off=None): def tagrepr(tag, w, size, off=None):
if (tag & 0xfffe) == TAG_UNR: if (tag & 0xfffe) == TAG_UNR:
return 'unr id%d%s' % ( return 'unr%s%s' % (
id, ' w%d' % w if w else '',
' %d' % size if size else '') ' %d' % size if size else '')
elif (tag & 0xf00c) == TAG_NAME: elif (tag & 0xf00c) == TAG_NAME:
return '%s%s id%d %d' % ( return '%s%s%s %d' % (
'rm' if tag & 0x2 else '', 'rm' if tag & 0x2 else '',
'bname' if (tag & 0xfffe) == TAG_BNAME 'bname' if (tag & 0xfffe) == TAG_BNAME
else 'reg' if (tag & 0xfffe) == TAG_REG else 'reg' if (tag & 0xfffe) == TAG_REG
else 'dir' if (tag & 0xfffe) == TAG_DIR else 'dir' if (tag & 0xfffe) == TAG_DIR
else 'name 0x%02x' % ((tag & 0x0ff0) >> 4), else 'name 0x%02x' % ((tag & 0x0ff0) >> 4),
id, ' w%d' % w if w else '',
size) size)
elif (tag & 0xf00c) == TAG_STRUCT: elif (tag & 0xf00c) == TAG_STRUCT:
return '%s%s id%d %d' % ( return '%s%s%s %d' % (
'rm' if tag & 0x2 else '', 'rm' if tag & 0x2 else '',
'inlined' if (tag & 0xfffe) == TAG_INLINED 'inlined' if (tag & 0xfffe) == TAG_INLINED
else 'block' if (tag & 0xfffe) == TAG_BLOCK else 'block' if (tag & 0xfffe) == TAG_BLOCK
else 'branch' if (tag & 0xfffe) == TAG_BRANCH else 'branch' if (tag & 0xfffe) == TAG_BRANCH
else 'btree' if (tag & 0xfffe) == TAG_BTREE else 'btree' if (tag & 0xfffe) == TAG_BTREE
else 'struct 0x%02x' % ((tag & 0x0ff0) >> 4), else 'struct 0x%02x' % ((tag & 0x0ff0) >> 4),
id, ' w%d' % w if w else '',
size) size)
elif (tag & 0xf00c) == TAG_UATTR: elif (tag & 0xf00c) == TAG_UATTR:
return '%suattr 0x%02x%s%s' % ( return '%suattr 0x%02x%s%s' % (
'rm' if tag & 0x2 else '', 'rm' if tag & 0x2 else '',
(tag & 0x0ff0) >> 4, (tag & 0x0ff0) >> 4,
' id%d' % id if id != -1 else '', ' w%d' % w if w else '',
' %d' % size if not tag & 0x2 or size else '') ' %d' % size if not tag & 0x2 or size else '')
elif (tag & 0xf00e) == TAG_CRC: elif (tag & 0xf00e) == TAG_CRC:
return 'crc%x%s %d' % ( return 'crc%x%s %d' % (
1 if tag & 0x10 else 0, 1 if tag & 0x10 else 0,
' 0x%02x' % id if id != -1 else '', ' 0x%x' % w if w > 0 else '',
size) size)
elif (tag & 0xfffe) == TAG_FCRC: elif (tag & 0xfffe) == TAG_FCRC:
return 'fcrc%s %d' % ( return 'fcrc%s %d' % (
' 0x%02x' % id if id != -1 else '', ' 0x%x' % w if w > 0 else '',
size) size)
elif tag & 0x8: elif tag & 0x8:
return 'alt%s%s 0x%x w%d %s' % ( return 'alt%s%s 0x%x w%d %s' % (
'r' if tag & 0x2 else 'b', 'r' if tag & 0x2 else 'b',
'gt' if tag & 0x4 else 'le', 'gt' if tag & 0x4 else 'le',
tag & 0xfff0, tag & 0xfff0,
id, w,
'0x%x' % (0xffffffff & (off-size)) '0x%x' % (0xffffffff & (off-size))
if off is not None if off is not None
else '-%d' % off) else '-%d' % off)
else: else:
return '0x%04x id%d %d' % (tag, id, size) return '0x%04x w%d %d' % (tag, w, size)
def show_log(block_size, data, rev, off, *, def show_log(block_size, data, rev, off, weight, *,
color=False, color=False,
**args): **args):
crc = crc32c(data[0:4]) crc = crc32c(data[0:4])
@@ -153,7 +153,7 @@ def show_log(block_size, data, rev, off, *,
j_ = 4 j_ = 4
while j_ < (block_size if args.get('all') else off): while j_ < (block_size if args.get('all') else off):
j = j_ j = j_
v, tag, id, size, delta = fromtag(data[j_:]) v, tag, w, size, delta = fromtag(data[j_:])
j_ += delta j_ += delta
if not tag & 0x8: if not tag & 0x8:
j_ += size j_ += size
@@ -250,7 +250,7 @@ def show_log(block_size, data, rev, off, *,
j_ = 4 j_ = 4
while j_ < (block_size if args.get('all') else off): while j_ < (block_size if args.get('all') else off):
j = j_ j = j_
v, tag, id, size, delta = fromtag(data[j_:]) v, tag, w, size, delta = fromtag(data[j_:])
j_ += delta j_ += delta
if not tag & 0x8: if not tag & 0x8:
j_ += size j_ += size
@@ -263,14 +263,13 @@ def show_log(block_size, data, rev, off, *,
# keep track of weight # keep track of weight
if tag & 0x8: if tag & 0x8:
if tag & 0x4: if tag & 0x4:
upper_ += id upper_ += w
else: else:
lower_ += id lower_ += w
elif (tag & 0xc) == 0x0: elif (tag & 0xc) == 0x0:
delta = (lower_+upper_) - weight_ delta = (lower_+upper_+w) - weight_
if not tag & 0x2: weight_ = lower_+upper_+w
delta += id+1-lower_ id = lower_+w-1
weight_ += delta
# note we ignore out-of-bounds here for debugging # note we ignore out-of-bounds here for debugging
if delta > 0: if delta > 0:
@@ -310,7 +309,7 @@ def show_log(block_size, data, rev, off, *,
weights = weights_ weights = weights_
lifetimes = lifetimes_ lifetimes = lifetimes_
if not tag & 0x2: if not tag & 0x2 and id >= 0:
# attach tag to lifetime # attach tag to lifetime
i, id_ = index(weights, id) i, id_ = index(weights, id)
if i < len(weights): if i < len(weights):
@@ -366,9 +365,11 @@ def show_log(block_size, data, rev, off, *,
lifetime_width - sum(len(r) for r in reprs), '') lifetime_width - sum(len(r) for r in reprs), '')
# print header # print header
print('%-8s %*s%-22s %s' % ( w_width = 2*m.ceil(m.log10(max(1, weight)+1))+1
print('%-8s %*s%-*s %-22s %s' % (
'off', 'off',
lifetime_width, '', lifetime_width, '',
w_width, 'ids',
'tag', 'tag',
'data (truncated)' 'data (truncated)'
if not args.get('no_truncate') else '')) if not args.get('no_truncate') else ''))
@@ -378,18 +379,35 @@ def show_log(block_size, data, rev, off, *,
print('%8s: %s' % ('%04x' % 0, next(xxd(data[0:4])))) print('%8s: %s' % ('%04x' % 0, next(xxd(data[0:4]))))
# print tags # print tags
lower_, upper_ = 0, 0
wastrunk = False
j_ = 4 j_ = 4
while j_ < (block_size if args.get('all') else off): while j_ < (block_size if args.get('all') else off):
notes = [] notes = []
j = j_ j = j_
v, tag, id, size, delta = fromtag(data[j_:]) v, tag, w, size, delta = fromtag(data[j_:])
if v != (popc(crc) & 1): if v != (popc(crc) & 1):
notes.append('v!=%x' % (popc(crc) & 1)) notes.append('v!=%x' % (popc(crc) & 1))
tag &= ~1 tag &= ~1
crc = crc32c(data[j_:j_+delta], crc) crc = crc32c(data[j_:j_+delta], crc)
j_ += delta j_ += delta
# find trunk
if not wastrunk and (tag & 0xc) != 0x4:
lower_, upper_ = 0, 0
wastrunk = not not tag & 0x8
# calculate id from alt weights
if tag & 0x8:
if tag & 0x4:
upper_ += w
else:
lower_ += w
elif (tag & 0xc) == 0x0:
weight_ = lower_+upper_+w
id = lower_+w-1
if not tag & 0x8: if not tag & 0x8:
if (tag & 0xf00f) != TAG_CRC: if (tag & 0xf00f) != TAG_CRC:
crc = crc32c(data[j_:j_+size], crc) crc = crc32c(data[j_:j_+size], crc)
@@ -401,14 +419,17 @@ def show_log(block_size, data, rev, off, *,
j_ += size j_ += size
# show human-readable tag representation # show human-readable tag representation
print('%s%08x:%s %s%s%-57s%s%s' % ( print('%s%08x:%s %*s%s%*s %-57s%s%s' % (
'\x1b[90m' if color and j >= off else '', '\x1b[90m' if color and j >= off else '',
j, j,
'\x1b[m' if color and j >= off else '', '\x1b[m' if color and j >= off else '',
lifetimerepr(j) if args.get('lifetimes') else '', lifetime_width, lifetimerepr(j) if args.get('lifetimes') else '',
'\x1b[90m' if color and j >= off else '', '\x1b[90m' if color and j >= off else '',
w_width, '' if (tag & 0xc) != 0x0
else '%d-%d' % (id-(w-1), id) if w > 1
else id,
'%-22s%s' % ( '%-22s%s' % (
tagrepr(tag, id, size, j), tagrepr(tag, w, size, j),
' %s' % next(xxd( ' %s' % next(xxd(
data[j+delta:j+delta+min(size, 8)], 8), '') data[j+delta:j+delta+min(size, 8)], 8), '')
if not args.get('no_truncate') if not args.get('no_truncate')
@@ -419,24 +440,16 @@ def show_log(block_size, data, rev, off, *,
if args.get('jumps') if args.get('jumps')
else '')) else ''))
if args.get('raw'):
# show on-disk encoding of tags
for o, line in enumerate(xxd(data[j:j+delta])):
print('%s%8s: %s%s' % (
'\x1b[90m' if color and j >= off else '',
'%04x' % (j + o*16),
line,
'\x1b[m' if color and j >= off else ''))
# show in-device representation, including some extra # show in-device representation, including some extra
# crc/parity info # crc/parity info
if args.get('device'): if args.get('device'):
print('%s%8s %*s%-47s %08x %x%s' % ( print('%s%8s %*s%*s %-47s %08x %x%s' % (
'\x1b[90m' if color and j >= off else '', '\x1b[90m' if color and j >= off else '',
'', '',
lifetime_width, '', lifetime_width, '',
w_width, '',
'%-22s%s' % ( '%-22s%s' % (
'%04x %08x %07x' % (tag, 0xffffffff & id, size), '%04x %08x %07x' % (tag, w, size),
' %s' % ' '.join( ' %s' % ' '.join(
'%08x' % struct.unpack('<I', '%08x' % struct.unpack('<I',
data[j+delta+i*4:j+delta+min(i*4+4,size)] data[j+delta+i*4:j+delta+min(i*4+4,size)]
@@ -448,13 +461,24 @@ def show_log(block_size, data, rev, off, *,
popc(crc) & 1, popc(crc) & 1,
'\x1b[m' if color and j >= off else '')) '\x1b[m' if color and j >= off else ''))
if not tag & 0x8: # show on-disk encoding of tags
# show on-disk encoding of data if args.get('raw'):
if args.get('raw') or args.get('no_truncate'): for o, line in enumerate(xxd(data[j:j+delta])):
print('%s%8s: %*s%*s %s%s' % (
'\x1b[90m' if color and j >= off else '',
'%04x' % (j + o*16),
lifetime_width, '',
w_width, '',
line,
'\x1b[m' if color and j >= off else ''))
if args.get('raw') or args.get('no_truncate'):
if not tag & 0x8:
for o, line in enumerate(xxd(data[j+delta:j+delta+size])): for o, line in enumerate(xxd(data[j+delta:j+delta+size])):
print('%s%8s: %s%s' % ( print('%s%8s: %*s%*s %s%s' % (
'\x1b[90m' if color and j >= off else '', '\x1b[90m' if color and j >= off else '',
'%04x' % (j+delta + o*16), '%04x' % (j+delta + o*16),
lifetime_width, '',
w_width, '',
line, line,
'\x1b[m' if color and j >= off else '')) '\x1b[m' if color and j >= off else ''))
@@ -475,16 +499,16 @@ def show_tree(block_size, data, rev, trunk, weight, *,
# descend down tree # descend down tree
j = trunk j = trunk
while True: while True:
_, alt, weight_, jump, delta = fromtag(data[j:]) _, alt, w, jump, delta = fromtag(data[j:])
# found an alt? # found an alt?
if alt & 0x8: if alt & 0x8:
# follow? # follow?
if ((id, tag & ~0xf) > (upper-weight_-1, alt & ~0xf) if ((id, tag & ~0xf) > (upper-w-1, alt & ~0xf)
if alt & 0x4 if alt & 0x4
else ((id, tag & ~0xf) <= (lower+weight_, alt & ~0xf))): else ((id, tag & ~0xf) <= (lower+w, alt & ~0xf))):
lower += upper-lower-1-weight_ if alt & 0x4 else 0 lower += upper-lower-1-w if alt & 0x4 else 0
upper -= upper-lower-1-weight_ if not alt & 0x4 else 0 upper -= upper-lower-1-w if not alt & 0x4 else 0
j = j - jump j = j - jump
if args.get('tree'): if args.get('tree'):
@@ -499,8 +523,8 @@ def show_tree(block_size, data, rev, trunk, weight, *,
path.append((j+jump, j, 'b')) path.append((j+jump, j, 'b'))
# stay on path # stay on path
else: else:
lower += weight_ if not alt & 0x4 else 0 lower += w if not alt & 0x4 else 0
upper -= weight_ if alt & 0x4 else 0 upper -= w if alt & 0x4 else 0
j = j + delta j = j + delta
if args.get('tree'): if args.get('tree'):
@@ -638,14 +662,14 @@ def show_tree(block_size, data, rev, trunk, weight, *,
break break
# show human-readable tag representation # show human-readable tag representation
print('%08x:%s %*s %-57s' % ( print('%08x:%s %-57s' % (
j, j,
treerepr(j) if args.get('tree') else '', treerepr(j) if args.get('tree') else '',
w_width, '%d-%d' % (id-(w-1), id) '%*s %-22s%s' % (
if w > 1 else id w_width, '%d-%d' % (id-(w-1), id)
if w > 0 else '', if w > 1 else id
'%-22s%s' % ( if w > 0 else '',
tagrepr(tag, id, size, j), tagrepr(tag, w, size, j),
' %s' % next(xxd( ' %s' % next(xxd(
data[j+delta:j+delta+min(size, 8)], 8), '') data[j+delta:j+delta+min(size, 8)], 8), '')
if not args.get('no_truncate') if not args.get('no_truncate')
@@ -667,19 +691,21 @@ def show_tree(block_size, data, rev, trunk, weight, *,
if not args.get('no_truncate') if not args.get('no_truncate')
and not tag & 0x8 else ''))) and not tag & 0x8 else '')))
# show on-disk encoding of tags
if args.get('raw'): if args.get('raw'):
# show on-disk encoding of tags
for o, line in enumerate(xxd(data[j:j+delta])): for o, line in enumerate(xxd(data[j:j+delta])):
print('%8s: %s' % ( print('%8s: %*s%*s %s' % (
'%04x' % (j + o*16), '%04x' % (j + o*16),
tree_width, '',
w_width, '',
line)) line))
if args.get('raw') or args.get('no_truncate'):
if not tag & 0x8: if not tag & 0x8:
# show on-disk encoding of data
if args.get('raw') or args.get('no_truncate'):
for o, line in enumerate(xxd(data[j+delta:j+delta+size])): for o, line in enumerate(xxd(data[j+delta:j+delta+size])):
print('%8s: %s' % ( print('%8s: %*s%*s %s' % (
'%04x' % (j+delta + o*16), '%04x' % (j+delta + o*16),
tree_width, '',
w_width, '',
line)) line))
@@ -733,7 +759,7 @@ def main(disk, block_size=None, block1=0, block2=None, *,
weight_ = 0 weight_ = 0
wastrunk = False wastrunk = False
while j_ < len(data): while j_ < len(data):
v, tag, id, size, delta = fromtag(data[j_:]) v, tag, w, size, delta = fromtag(data[j_:])
if v != (popc(crc) & 1): if v != (popc(crc) & 1):
break break
crc = crc32c(data[j_:j_+delta], crc) crc = crc32c(data[j_:j_+delta], crc)
@@ -748,13 +774,11 @@ def main(disk, block_size=None, block1=0, block2=None, *,
# keep track of weight # keep track of weight
if tag & 0x8: if tag & 0x8:
if tag & 0x4: if tag & 0x4:
upper_ += id upper_ += w
else: else:
lower_ += id lower_ += w
elif (tag & 0xc) == 0x0: elif (tag & 0xc) == 0x0:
weight_ = lower_+upper_ weight_ = lower_+upper_+w
if not tag & 0x2:
weight_ += id+1-lower_
# take care of crcs # take care of crcs
if not tag & 0x8: if not tag & 0x8:
@@ -801,7 +825,7 @@ def main(disk, block_size=None, block1=0, block2=None, *,
if len(blocks) > 1 else '')) if len(blocks) > 1 else ''))
if args.get('log'): if args.get('log'):
show_log(block_size, data, rev, off, show_log(block_size, data, rev, off, weight,
color=color, color=color,
**args) **args)
else: else: