Github/littlefs
1
Fork 0
Code Actions Activity

Moved bits around in tag encoding to allow leb128 custom attributes

Browse Source 
Yet another tag encoding, but hopefully narrowing in on a good long term
design. This change trades a subtype bit for the ability to extend
subtypes indefinitely via leb128 in the future.

The immediate benefit is ~unlimited custom attributes, though I'm not
sure how to make this configurable yet. Extended custom attributes may
have a significant impact on alt tag sizes, so it may be worth
defaulting to only 8-bit custom attributes still.

Tag encoding:

   vmmmtttt 0TTTTTTT 0wwwwwww 0sssssss
   ^--^---^--------^--------^--------^- valid bit
      '---|--------|--------|--------|- 3-bit mode
          '--------|--------|--------|- 4-bit suptype
                   '--------|--------|- leb128 subtype
                            '--------|- leb128 weight
                                     '- leb128 size/jump

This limits subtypes to 7-bits, but this seems very reasonable at the
moment.

This also seems to limit custom attributes to 7-bits, but we can use two
separate suptypes to bring this back up to 8-bits. I was planning to do
this anyways to have separate "user-attributes" and "system-attributes",
so this actually fits in really well.
This commit is contained in:
Christopher Haster
2023-06-16 01:41:42 -05:00
parent 4ff7c1f771
commit 2113d877d6
4 changed files with 404 additions and 405 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+124 -112
View File
@@ -584,124 +584,128 @@ static int lfsr_bd_erase(lfs_t *lfs, lfs_block_t block) {
// 16-bit metadata tags
enum lfsr_tag_type {
LFSR_TAG_UNR = 0x0002,
LFSR_TAG_MKUNR = 0x0006, // in-device only
LFSR_TAG_UNR = 0x1000,
LFSR_TAG_MKUNR = 0x3000, // in-device only
LFSR_TAG_MAGIC = 0x0030,
LFSR_TAG_CONFIG = 0x0040,
LFSR_TAG_MROOT = 0x0110,
LFSR_TAG_SUPERMAGIC = 0x0003,
LFSR_TAG_SUPERCONFIG = 0x0004,
LFSR_TAG_MROOT = 0x0304,
LFSR_TAG_NAME = 0x1000,
LFSR_TAG_BRANCH = 0x1000,
LFSR_TAG_MKBRANCH = 0x1004, // in-device only
LFSR_TAG_REG = 0x1010,
LFSR_TAG_MKREG = 0x1014, // in-device only
LFSR_TAG_DIR = 0x1020,
LFSR_TAG_MKDIR = 0x1024, // in-device only
LFSR_TAG_NAME = 0x0100,
LFSR_TAG_BRANCH = 0x0100,
LFSR_TAG_MKBRANCH = 0x2100, // in-device only
LFSR_TAG_REG = 0x0101,
LFSR_TAG_MKREG = 0x2101, // in-device only
LFSR_TAG_DIR = 0x0102,
LFSR_TAG_MKDIR = 0x2102, // in-device only
LFSR_TAG_STRUCT = 0x3000,
LFSR_TAG_INLINED = 0x3000,
LFSR_TAG_MKINLINED = 0x3004, // test only?
LFSR_TAG_BLOCK = 0x3100,
LFSR_TAG_MDIR = 0x3200,
LFSR_TAG_RMMDIR = 0x3202,
LFSR_TAG_BTREE = 0x3300,
LFSR_TAG_MKBTREE = 0x3304, // in-device only
LFSR_TAG_RMBTREE = 0x3302,
LFSR_TAG_STRUCT = 0x0300,
LFSR_TAG_INLINED = 0x0300,
LFSR_TAG_MKINLINED = 0x2300, // test only?
LFSR_TAG_BLOCK = 0x0302,
LFSR_TAG_BTREE = 0x0303,
LFSR_TAG_MKBTREE = 0x2303, // in-device only
LFSR_TAG_RMBTREE = 0x1303,
LFSR_TAG_MDIR = 0x0305,
LFSR_TAG_RMMDIR = 0x1305,
LFSR_TAG_UATTR = 0x4000,
LFSR_TAG_MKUATTR = 0x4004, // in-device only
LFSR_TAG_RMUATTR = 0x4002,
LFSR_TAG_UATTR = 0x0400,
LFSR_TAG_MKUATTR = 0x2400, // in-device only
LFSR_TAG_RMUATTR = 0x1400,
LFSR_TAG_ALT = 0x0008,
LFSR_TAG_ALTBLE = 0x0008,
LFSR_TAG_ALTRLE = 0x000a,
LFSR_TAG_ALTBGT = 0x000c,
LFSR_TAG_ALTRGT = 0x000e,
LFSR_TAG_ALT = 0x4000,
LFSR_TAG_ALTBLE = 0x4000,
LFSR_TAG_ALTRLE = 0x5000,
LFSR_TAG_ALTBGT = 0x6000,
LFSR_TAG_ALTRGT = 0x7000,
LFSR_TAG_CRC = 0x0004,
LFSR_TAG_FCRC = 0x1004,
LFSR_TAG_CRC = 0x2000,
LFSR_TAG_FCRC = 0x2100,
// in-device only
LFSR_TAG_GROW = 0xf000,
LFSR_TAG_SHRINK = 0xf010,
LFSR_TAG_FROM = 0xf020,
LFSR_TAG_GROW = 0x0f00,
LFSR_TAG_SHRINK = 0x0f01,
LFSR_TAG_FROM = 0x0f02,
};
#define LFSR_TAG_ALT_(color, dir, key) \
(LFSR_TAG_ALT \
| ((0x1 & (lfsr_tag_t)(color)) << 1) \
| ((0x1 & (lfsr_tag_t)(dir)) << 2) \
| ((0xfff0 & (lfsr_tag_t)(key))))
| ((0x1 & (lfsr_tag_t)(color)) << 13) \
| ((0x1 & (lfsr_tag_t)(dir)) << 14) \
| (0x0fff & (lfsr_tag_t)(key)))
#define LFSR_TAG_ALT(color, dir, key) \
(LFSR_TAG_ALT##color##dir \
| ((0xfff0 & (lfsr_tag_t)(key))))
| (0x0fff & (lfsr_tag_t)(key)))
#define LFSR_TAG_UATTR(attr) \
(LFSR_TAG_UATTR \
| ((0xff & (lfsr_tag_t)(attr)) << 4))
| (0xff & (lfsr_tag_t)(attr)))
#define LFSR_TAG_MKUATTR(attr) \
(LFSR_TAG_MKUATTR \
| ((0xff & (lfsr_tag_t)(attr)) << 4))
| (0xff & (lfsr_tag_t)(attr)))
#define LFSR_TAG_RMUATTR(attr) \
(LFSR_TAG_RMUATTR \
| ((0xff & (lfsr_tag_t)(attr)) << 4))
| (0xff & (lfsr_tag_t)(attr)))
// tag type operations
static inline lfsr_tag_t lfsr_tag_suptype(lfsr_tag_t tag) {
return tag & 0xf00f;
return tag & 0xff00;
}
static inline uint8_t lfsr_tag_subtype(lfsr_tag_t tag) {
return (tag & 0x0ff0) >> 4;
return tag & 0x00ff;
}
static inline bool lfsr_tag_isvalid(lfsr_tag_t tag) {
return !(tag & 0x8000);
}
static inline lfsr_tag_t lfsr_tag_setvalid(lfsr_tag_t tag) {
return tag & ~0x8000;
}
static inline lfsr_tag_t lfsr_tag_setinvalid(lfsr_tag_t tag) {
return tag | 0x8000;
}
static inline bool lfsr_tag_ismk(lfsr_tag_t tag) {
return tag & 0x4;
return tag & 0x2000;
}
static inline lfsr_tag_t lfsr_tag_setmk(lfsr_tag_t tag) {
return tag | 0x4;
return tag | 0x2000;
}
static inline lfsr_tag_t lfsr_tag_setnomk(lfsr_tag_t tag) {
return tag & ~0x4;
return tag & ~0x2000;
}
static inline bool lfsr_tag_isrm(lfsr_tag_t tag) {
return tag & 0x2;
return tag & 0x1000;
}
static inline lfsr_tag_t lfsr_tag_setrm(lfsr_tag_t tag) {
return tag | 0x2;
return tag | 0x1000;
}
static inline bool lfsr_tag_istrunk(lfsr_tag_t tag) {
return (tag & 0xc) != 0x4;
return (tag & 0x6000) != 0x2000;
}
static inline bool lfsr_tag_isalt(lfsr_tag_t tag) {
return tag & 0x8;
return tag & 0x4000;
}
static inline lfsr_tag_t lfsr_tag_next(lfsr_tag_t tag) {
return tag + 0x10;
return tag + 0x1;
}
// lfsr_rbyd_append specific flags
static inline bool lfsr_tag_isfound(lfsr_tag_t tag) {
return tag & 0x1;
}
static inline lfsr_tag_t lfsr_tag_setfound(lfsr_tag_t tag) {
return tag | 0x1;
}
static inline bool lfsr_tag_isupper(lfsr_tag_t tag) {
return tag & 0x4;
return tag & 0x2000;
}
static inline bool lfsr_tag_islower(lfsr_tag_t tag) {
@@ -709,48 +713,48 @@ static inline bool lfsr_tag_islower(lfsr_tag_t tag) {
}
static inline lfsr_tag_t lfsr_tag_setupper(lfsr_tag_t tag) {
return tag | 0x4;
return tag | 0x2000;
}
static inline bool lfsr_tag_hasdiverged(lfsr_tag_t tag) {
return tag & 0x8;
return tag & 0x4000;
}
static inline lfsr_tag_t lfsr_tag_setdiverged(lfsr_tag_t tag) {
return tag | 0x8;
return tag | 0x4000;
}
// alt operations
static inline bool lfsr_tag_isblack(lfsr_tag_t tag) {
return !(tag & 0x2);
return !(tag & 0x1000);
}
static inline bool lfsr_tag_isred(lfsr_tag_t tag) {
return tag & 0x2;
return tag & 0x1000;
}
static inline lfsr_tag_t lfsr_tag_setblack(lfsr_tag_t tag) {
return tag & ~0x2;
return tag & ~0x1000;
}
static inline lfsr_tag_t lfsr_tag_setred(lfsr_tag_t tag) {
return tag | 0x2;
return tag | 0x1000;
}
static inline bool lfsr_tag_isle(lfsr_tag_t tag) {
return !(tag & 0x4);
return !(tag & 0x2000);
}
static inline bool lfsr_tag_isgt(lfsr_tag_t tag) {
return tag & 0x4;
return tag & 0x2000;
}
static inline lfsr_tag_t lfsr_tag_isparallel(lfsr_tag_t a, lfsr_tag_t b) {
return (a & 0x4) == (b & 0x4);
return (a & 0x2000) == (b & 0x2000);
}
static inline lfsr_tag_t lfsr_tag_key(lfsr_tag_t tag) {
return tag & ~0xf;
return tag & 0x0fff;
}
static inline bool lfsr_tag_follow(lfsr_tag_t alt, lfs_size_t weight,
@@ -795,13 +799,13 @@ static inline bool lfsr_tag_prune2(
alt, weight,
alt2, weight2,
lower_id, upper_id,
upper_id-1, upper_tag-0x10);
upper_id-1, upper_tag-0x1);
}
}
static inline void lfsr_tag_flip(lfsr_tag_t *alt, lfs_size_t *weight,
lfs_ssize_t lower, lfs_ssize_t upper) {
*alt = *alt ^ 0x4;
*alt = *alt ^ 0x2000;
*weight = (upper-lower) - *weight - 1;
}
@@ -822,12 +826,12 @@ static inline void lfsr_tag_trim(
if (lfsr_tag_isgt(alt)) {
*upper_id -= weight;
if (upper_tag) {
*upper_tag = alt + 0x10;
*upper_tag = alt + 0x1;
}
} else {
*lower_id += weight;
if (lower_tag) {
*lower_tag = alt + 0x10;
*lower_tag = alt + 0x1;
}
}
}
@@ -846,9 +850,10 @@ static inline void lfsr_tag_trim2(
// support for encoding/decoding tags on disk
// each piece of metadata in an rbyd tree is prefixed with a 3-piece tag:
// each piece of metadata in an rbyd tree is prefixed with a 4-piece tag:
//
// - 16-bit type => 2 byte le16
// - 8-bit suptype => 1 byte
// - 8-bit subtype => 1 byte
// - 32-bit id/weight => 5 byte leb128 (worst case)
// - 32-bit size/jump => 5 byte leb128 (worst case)
// => 12 bytes total
@@ -870,7 +875,7 @@ static lfs_ssize_t lfsr_bd_readtag(lfs_t *lfs,
if (tsize < 2) {
return LFS_ERR_CORRUPT;
}
uint16_t tag = lfs_fromle16_(&buf[0]);
uint16_t tag = ((lfsr_tag_t)buf[0] << 8) | ((lfsr_tag_t)buf[1] << 0);
ssize_t d = 2;
if (csum_) {
@@ -881,7 +886,7 @@ static lfs_ssize_t lfsr_bd_readtag(lfs_t *lfs,
// note we need to do this before leb128 decoding as we may not have
// valid leb128 if we're erased, but we shouldn't treat a truncated
// leb128 here as corruption
if ((tag & 1) != (lfs_popc(*csum_) & 1)) {
if ((tag >> 15) != (lfs_popc(*csum_) & 1)) {
return LFS_ERR_INVAL;
}
}
@@ -917,7 +922,7 @@ static lfs_ssize_t lfsr_bd_readtag(lfs_t *lfs,
// - clear the valid bit from tag, we checked this earlier
// - adjust id so reserved id is -1, so we don't have mixed zero/one indexed
//
*tag_ = tag & ~0x1;
*tag_ = tag & 0x7fff;
*weight_ = weight;
*size_ = size;
return d;
@@ -932,11 +937,12 @@ static lfs_ssize_t lfsr_bd_progtag(lfs_t *lfs,
LFS_ASSERT(size < 0x80000000);
// make sure to include the parity of the current crc
tag |= lfs_popc(*csum_) & 1;
tag |= (lfs_popc(*csum_) & 1) << 15;
// encode into an le16 and pair of leb128s
// encode into a be16 and pair of leb128s
uint8_t buf[LFSR_TAG_DSIZE];
lfs_tole16_(tag, &buf[0]);
buf[0] = (uint8_t)(tag >> 8);
buf[1] = (uint8_t)(tag >> 0);
lfs_size_t d = 2;
ssize_t d_ = lfs_toleb128(weight, &buf[d], 5);
@@ -1814,7 +1820,7 @@ static int lfsr_rbyd_lookupnext(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
// make sure we never look up zero tags, the way we create
// unreachable tags has a hole here
tag = lfs_max16(tag, 0x10);
tag = lfs_max16(tag, 0x1);
// no trunk yet?
if (!branch) {
@@ -2063,8 +2069,8 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
id_ = id + 1;
other_id_ = id + 1;
// also note these tags MUST NOT be zero, due to unreachable tag holes
tag_ = 0x10;
other_tag_ = lfsr_tag_setupper(0x10);
tag_ = 0x1;
other_tag_ = 0x1;
} else if (lfsr_tag_ismk(tag) && delta < 0) {
LFS_ASSERT(id < (lfs_ssize_t)rbyd->weight);
@@ -2074,23 +2080,26 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
id_ = id - lfs_smax32(-delta, 0);
other_id_ = id;
// also note these tags MUST NOT be zero, due to unreachable tag holes
tag_ = 0x10;
other_tag_ = lfsr_tag_setupper(0x10);
tag_ = 0x1;
other_tag_ = 0x1;
} else if (lfsr_tag_isrm(tag)) {
LFS_ASSERT(id < (lfs_ssize_t)rbyd->weight);
id_ = id - lfs_smax32(-delta, 0);
other_id_ = id;
tag_ = lfsr_tag_key(tag);
other_tag_ = lfsr_tag_setupper(lfsr_tag_key(tag) + 0x10);
other_tag_ = lfsr_tag_key(tag) + 0x1;
} else {
LFS_ASSERT(id < (lfs_ssize_t)rbyd->weight);
id_ = id - lfs_smax32(-delta, 0);
other_id_ = id;
tag_ = lfsr_tag_key(tag);
other_tag_ = lfsr_tag_setupper(lfsr_tag_key(tag));
other_tag_ = lfsr_tag_key(tag);
}
// mark as invalid until found
tag_ = lfsr_tag_setinvalid(tag_);
other_tag_ = lfsr_tag_setinvalid(lfsr_tag_setupper(other_tag_));
// keep track of bounds as we descend down the tree
//
@@ -2348,15 +2357,15 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
// update the found tag/id
//
// note we:
// - preserve diverged bit (0x8)
// - preserve is upper tag (0x4)
// - set found tag (0x1)
tag_ = lfsr_tag_setfound(alt | (tag_ & 0xc));
// - clear valid bit (0x8000)
// - preserve diverged bit (0x4000)
// - preserve isupper tag (0x2000)
tag_ = lfsr_tag_setvalid(alt | (tag_ & 0x6000));
id_ = upper_id-1;
// done?
if (!lfsr_tag_hasdiverged(tag_)
|| lfsr_tag_isfound(other_tag_)) {
|| lfsr_tag_isvalid(other_tag_)) {
break;
}
}
@@ -2377,9 +2386,9 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
LFS_ASSERT(lfsr_tag_isblack(p_alts[0]));
// if we diverged, merge the bounds
LFS_ASSERT(lfsr_tag_isfound(tag_));
LFS_ASSERT(lfsr_tag_isvalid(tag_));
LFS_ASSERT(!lfsr_tag_hasdiverged(tag_)
|| lfsr_tag_isfound(other_tag_));
|| lfsr_tag_isvalid(other_tag_));
if (lfsr_tag_hasdiverged(tag_) && lfsr_tag_islower(tag_)) {
// finished on lower path
tag_ = other_tag_;
@@ -2666,7 +2675,8 @@ static int lfsr_rbyd_commit(lfs_t *lfs, lfsr_rbyd_t *rbyd,
// get around this catch-22 we just always write a fully-expanded leb128
// encoding
uint8_t buf[2+1+5+4];
lfs_tole16_(LFSR_TAG_CRC | (lfs_popc(rbyd_.crc) & 1), &buf[0]);
buf[0] = (LFSR_TAG_CRC >> 8) | ((lfs_popc(rbyd_.crc) & 1) << 7);
buf[1] = 0;
buf[2] = 0;
lfs_off_t padding = aligned - (rbyd_.off + 2+1+5);
@@ -2681,9 +2691,9 @@ static int lfsr_rbyd_commit(lfs_t *lfs, lfsr_rbyd_t *rbyd,
// commit if this happens, note parity(crc(m)) == parity(m) with crc32c,
// so we can really change any bit to make this happen, we've reserved a bit
// in crc tags just for this purpose
if ((lfs_popc(rbyd_.crc) & 1) == (perturb & 1)) {
buf[0] ^= 0x10;
rbyd_.crc ^= 0x847609b4; // note crc(a ^ b) == crc(a) ^ crc(b)
if ((lfs_popc(rbyd_.crc) & 1) == (perturb >> 7)) {
buf[1] ^= 0x01;
rbyd_.crc ^= 0x68032cc8; // note crc(a ^ b) == crc(a) ^ crc(b)
}
lfs_tole32_(rbyd_.crc, &buf[2+1+5]);
@@ -5416,13 +5426,15 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
// copy magic/config from current mroot
lfsr_data_t magic;
err = lfsr_mdir_lookup(lfs, &mchildroot, -1, LFSR_TAG_MAGIC, &magic);
err = lfsr_mdir_lookup(lfs, &mchildroot,
-1, LFSR_TAG_SUPERMAGIC, &magic);
if (err) {
return err;
}
lfsr_data_t config;
err = lfsr_mdir_lookup(lfs, &mchildroot, -1, LFSR_TAG_CONFIG, &config);
err = lfsr_mdir_lookup(lfs, &mchildroot,
-1, LFSR_TAG_SUPERCONFIG, &config);
if (err) {
return err;
}
@@ -5453,8 +5465,8 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
// compact into new mparentroot
err = lfsr_rbyd_commit(lfs, &mparentroot_.rbyd, LFSR_ATTRS(
LFSR_ATTR_DATA(-1, MAGIC, 0, magic),
LFSR_ATTR_DATA(-1, CONFIG, 0, config),
LFSR_ATTR_DATA(-1, SUPERMAGIC, 0, magic),
LFSR_ATTR_DATA(-1, SUPERCONFIG, 0, config),
LFSR_ATTR(-1, MROOT, 0, buf, d)));
if (err) {
return err;
@@ -6706,10 +6718,10 @@ static int lfsr_mdir_commit(lfs_t *lfs, lfsr_mdir_t *mdir, lfs_ssize_t *rid,
// - 32-bit file_limit => 5 byte leb128 (worst case)
// => 30 bytes total
//
#define LFSR_CONFIG_DSIZE (1+1+1+1+5+5+1+5+5+5)
#define LFSR_SUPERCONFIG_DSIZE (1+1+1+1+5+5+1+5+5+5)
static lfs_ssize_t lfsr_superconfig_todisk(lfs_t *lfs,
uint8_t buffer[static LFSR_CONFIG_DSIZE]) {
uint8_t buffer[static LFSR_SUPERCONFIG_DSIZE]) {
// TODO most of these should also be in the lfs_config/lfs_t structs
// note we take a shortcut for for single-byte leb128s, but these
@@ -6806,7 +6818,7 @@ static int lfsr_mountinited(lfs_t *lfs) {
if (mdir->mid == -1) {
// has magic string?
lfsr_data_t data;
err = lfsr_mdir_lookup(lfs, mdir, -1, LFSR_TAG_MAGIC, &data);
err = lfsr_mdir_lookup(lfs, mdir, -1, LFSR_TAG_SUPERMAGIC, &data);
if (err && err != LFS_ERR_NOENT) {
return err;
}
@@ -6830,7 +6842,7 @@ static int lfsr_mountinited(lfs_t *lfs) {
}
// lookup the superconfig
err = lfsr_mdir_lookup(lfs, mdir, -1, LFSR_TAG_CONFIG, &data);
err = lfsr_mdir_lookup(lfs, mdir, -1, LFSR_TAG_SUPERCONFIG, &data);
if (err && err != LFS_ERR_NOENT) {
return err;
}
@@ -7045,7 +7057,7 @@ static int lfsr_mountinited(lfs_t *lfs) {
}
static int lfsr_formatinited(lfs_t *lfs) {
uint8_t buf[LFSR_CONFIG_DSIZE];
uint8_t buf[LFSR_SUPERCONFIG_DSIZE];
lfs_ssize_t d = lfsr_superconfig_todisk(lfs, buf);
if (d < 0) {
return d;
@@ -7062,8 +7074,8 @@ static int lfsr_formatinited(lfs_t *lfs) {
}
err = lfsr_rbyd_commit(lfs, &rbyd, LFSR_ATTRS(
LFSR_ATTR(-1, MAGIC, 0, "littlefs", 8),
LFSR_ATTR(-1, CONFIG, 0, buf, d)));
LFSR_ATTR(-1, SUPERMAGIC, 0, "littlefs", 8),
LFSR_ATTR(-1, SUPERCONFIG, 0, buf, d)));
if (err) {
return err;
}
scripts/dbgbtree.py
+83 -87
View File
@@ -8,23 +8,24 @@ import os
import struct
TAG_UNR = 0x0002
TAG_MAGIC = 0x0030
TAG_CONFIG = 0x0040
TAG_MROOT = 0x0110
TAG_NAME = 0x1000
TAG_BRANCH = 0x1000
TAG_REG = 0x1010
TAG_DIR = 0x1020
TAG_STRUCT = 0x3000
TAG_INLINED = 0x3000
TAG_BLOCK = 0x3100
TAG_MDIR = 0x3200
TAG_BTREE = 0x3300
TAG_UATTR = 0x4000
TAG_ALT = 0x0008
TAG_CRC = 0x0004
TAG_FCRC = 0x1004
TAG_UNR = 0x1000
TAG_SUPERMAGIC = 0x0003
TAG_SUPERCONFIG = 0x0004
TAG_MROOT = 0x0304
TAG_NAME = 0x0100
TAG_BRANCH = 0x0100
TAG_REG = 0x0101
TAG_DIR = 0x0102
TAG_STRUCT = 0x0300
TAG_INLINED = 0x0300
TAG_BLOCK = 0x0302
TAG_BTREE = 0x0303
TAG_MDIR = 0x0305
TAG_UATTR = 0x0400
TAG_ALT = 0x4000
TAG_CRC = 0x2000
TAG_FCRC = 0x2100
# parse some rbyd addr encodings
# 0xa -> [0xa]
@@ -70,9 +71,6 @@ def crc32c(data, crc=0):
crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78)
return 0xffffffff ^ crc
def fromle16(data):
return struct.unpack('<H', data[0:2].ljust(2, b'\0'))[0]
def fromle32(data):
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0]
@@ -86,10 +84,10 @@ def fromleb128(data):
return word, len(data)
def fromtag(data):
tag = fromle16(data)
tag = (data[0] << 8) | data[1]
weight, d = fromleb128(data[2:])
size, d_ = fromleb128(data[2+d:])
return tag&1, tag&~1, weight, size, 2+d+d_
return tag>>15, tag&0x7fff, weight, size, 2+d+d_
def frombtree(data):
w, d1 = fromleb128(data)
@@ -112,64 +110,60 @@ def xxd(data, width=16, crc=False):
for b in map(chr, data[i:i+width])))
def tagrepr(tag, w, size, off=None):
if (tag & 0xfffe) == TAG_UNR:
if (tag & 0x7fff) == TAG_UNR:
return 'unr%s%s' % (
' w%d' % w if w else '',
' %d' % size if size else '')
elif (tag & 0xfffc) == TAG_MAGIC:
return '%smagic%s %d' % (
'rm' if tag & 0x2 else '',
elif (tag & 0x6fff) == TAG_SUPERMAGIC:
return '%ssupermagic%s%s' % (
'rm' if tag & 0x1000 else '',
' w%d' % w if w else '',
size)
elif (tag & 0xfffc) == TAG_CONFIG:
return '%sconfig%s %d' % (
'rm' if tag & 0x2 else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6fff) == TAG_SUPERCONFIG:
return '%ssuperconfig%s%s' % (
'rm' if tag & 0x1000 else '',
' w%d' % w if w else '',
size)
elif (tag & 0xfffc) == TAG_MROOT:
return '%smroot%s %d' % (
'rm' if tag & 0x2 else '',
' w%d' % w if w else '',
size)
elif (tag & 0xf00c) == TAG_NAME:
return '%s%s%s %d' % (
'rm' if tag & 0x2 else '',
'bname' if (tag & 0xfffe) == TAG_BRANCH
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_NAME:
return '%s%s%s%s' % (
'rm' if tag & 0x1000 else '',
'branch' if (tag & 0xfffe) == TAG_BRANCH
else 'reg' if (tag & 0xfffe) == TAG_REG
else 'dir' if (tag & 0xfffe) == TAG_DIR
else 'name 0x%02x' % ((tag & 0x0ff0) >> 4),
' w%d' % w if w else '',
size)
elif (tag & 0xf00c) == TAG_STRUCT:
return '%s%s%s %d' % (
'rm' if tag & 0x2 else '',
'inlined' if (tag & 0xfffe) == TAG_INLINED
else 'block' if (tag & 0xfffe) == TAG_BLOCK
else 'mdir' if (tag & 0xfffe) == TAG_MDIR
else 'btree' if (tag & 0xfffe) == TAG_BTREE
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_STRUCT:
return '%s%s%s%s' % (
'rm' if tag & 0x1000 else '',
'inlined' if (tag & 0x6fff) == TAG_INLINED
else 'block' if (tag & 0x6fff) == TAG_BLOCK
else 'btree' if (tag & 0x6fff) == TAG_BTREE
else 'mdir' if (tag & 0x6fff) == TAG_MROOT
else 'mdir' if (tag & 0x6fff) == TAG_MDIR
else 'struct 0x%02x' % ((tag & 0x0ff0) >> 4),
' w%d' % w if w else '',
size)
elif (tag & 0xf00c) == TAG_UATTR:
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_UATTR:
return '%suattr 0x%02x%s%s' % (
'rm' if tag & 0x2 else '',
(tag & 0x0ff0) >> 4,
'rm' if tag & 0x1000 else '',
tag & 0xff,
' w%d' % w if w else '',
' %d' % size if not tag & 0x2 or size else '')
elif (tag & 0xf00e) == TAG_CRC:
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x7f00) == TAG_CRC:
return 'crc%x%s %d' % (
1 if tag & 0x10 else 0,
1 if tag & 0x1 else 0,
' 0x%x' % w if w > 0 else '',
size)
elif (tag & 0xfffe) == TAG_FCRC:
elif (tag & 0x7fff) == TAG_FCRC:
return 'fcrc%s %d' % (
' 0x%x' % w if w > 0 else '',
size)
elif tag & 0x8:
elif tag & 0x4000:
return 'alt%s%s 0x%x w%d %s' % (
'r' if tag & 0x2 else 'b',
'gt' if tag & 0x4 else 'le',
tag & 0xfff0,
'r' if tag & 0x1000 else 'b',
'gt' if tag & 0x2000 else 'le',
tag & 0x0fff,
w,
'0x%x' % (0xffffffff & (off-size))
if off is not None
@@ -177,6 +171,7 @@ def tagrepr(tag, w, size, off=None):
else:
return '0x%04x w%d %d' % (tag, w, size)
# this type is used for tree representations
TBranch = co.namedtuple('TBranch', 'a, b, d, c')
@@ -253,12 +248,12 @@ class Rbyd:
break
crc_ = crc32c(data[j_:j_+d], crc_)
j_ += d
if not tag & 0x8 and j_ + size > len(data):
if not tag & 0x4000 and j_ + size > len(data):
break
# take care of crcs
if not tag & 0x8:
if (tag & 0xf00f) != TAG_CRC:
if not tag & 0x4000:
if (tag & 0x7f00) != TAG_CRC:
crc_ = crc32c(data[j_:j_+size], crc_)
# found a crc?
else:
@@ -272,7 +267,7 @@ class Rbyd:
weight = weight_
# evaluate trunks
if (tag & 0xc) != 0x4 and (
if (tag & 0x6000) != 0x2000 and (
not trunk or trunk >= j_-d or wastrunk):
# new trunk?
if not wastrunk:
@@ -281,8 +276,8 @@ class Rbyd:
wastrunk = True
# keep track of weight
if tag & 0x8:
if tag & 0x4:
if tag & 0x4000:
if tag & 0x2000:
upper_ += w
else:
lower_ += w
@@ -293,7 +288,7 @@ class Rbyd:
if trunk and j_ + size > trunk:
trunkoff = j_ + size
if not tag & 0x8:
if not tag & 0x4000:
j_ += size
return cls(block, data, rev, off, trunk_, weight)
@@ -312,19 +307,20 @@ class Rbyd:
_, alt, weight_, jump, d = fromtag(self.data[j:])
# found an alt?
if alt & 0x8:
if alt & 0x4000:
# follow?
if ((id, tag & ~0xf) > (upper-weight_-1, alt & ~0xf)
if alt & 0x4
else ((id, tag & ~0xf) <= (lower+weight_, alt & ~0xf))):
lower += upper-lower-1-weight_ if alt & 0x4 else 0
upper -= upper-lower-1-weight_ if not alt & 0x4 else 0
if ((id, tag & 0xfff) > (upper-weight_-1, alt & 0xfff)
if alt & 0x2000
else ((id, tag & 0xfff)
<= (lower+weight_, alt & 0xfff))):
lower += upper-lower-1-weight_ if alt & 0x2000 else 0
upper -= upper-lower-1-weight_ if not alt & 0x2000 else 0
j = j - jump
# figure out which color
if alt & 0x2:
if alt & 0x1000:
_, nalt, _, _, _ = fromtag(self.data[j+jump+d:])
if nalt & 0x2:
if nalt & 0x1000:
path.append((j+jump, j, True, 'y'))
else:
path.append((j+jump, j, True, 'r'))
@@ -333,14 +329,14 @@ class Rbyd:
# stay on path
else:
lower += weight_ if not alt & 0x4 else 0
upper -= weight_ if alt & 0x4 else 0
lower += weight_ if not alt & 0x2000 else 0
upper -= weight_ if alt & 0x2000 else 0
j = j + d
# figure out which color
if alt & 0x2:
if alt & 0x1000:
_, nalt, _, _, _ = fromtag(self.data[j:])
if nalt & 0x2:
if nalt & 0x1000:
path.append((j-d, j, False, 'y'))
else:
path.append((j-d, j, False, 'r'))
@@ -353,7 +349,7 @@ class Rbyd:
tag_ = alt
w_ = id_-lower
done = (id_, tag_) < (id, tag) or tag_ & 2
done = (id_, tag_) < (id, tag) or tag_ & 0x1000
return done, id_, tag_, w_, j, d, self.data[j+d:j+d+jump], path
@@ -371,7 +367,7 @@ class Rbyd:
id = -1
while True:
done, id, tag, w, j, d, data, _ = self.lookup(id, tag+0x10)
done, id, tag, w, j, d, data, _ = self.lookup(id, tag+0x1)
if done:
break
@@ -384,7 +380,7 @@ class Rbyd:
id, tag = -1, 0
while True:
done, id, tag, w, j, d, data, path = self.lookup(id, tag+0x10)
done, id, tag, w, j, d, data, path = self.lookup(id, tag+0x1)
# found end of tree?
if done:
break
@@ -519,7 +515,7 @@ def main(disk, roots=None, *,
w = 0
for i in it.count():
done, rid__, tag, w_, j, d, data, _ = rbyd.lookup(
rid_, tag+0x10)
rid_, tag+0x1)
if done or (i != 0 and rid__ != rid_):
break
@@ -698,7 +694,7 @@ def main(disk, roots=None, *,
name = None
for bid_, w_, rbyd_, rid_, tags_ in reversed(path):
for tag_, j_, d_, data_ in tags_:
if tag_ & 0xf00f == TAG_NAME:
if tag_ & 0x7f00 == TAG_NAME:
name = (tag_, j_, d_, data_)
if rid_-(w_-1) != 0:
@@ -826,7 +822,7 @@ def main(disk, roots=None, *,
''.join(
b if b >= ' ' and b <= '~' else '.'
for b in map(chr, data))
if tag & 0xf00f == TAG_NAME
if tag & 0x7f00 == TAG_NAME
else next(xxd(data, 8), '')
if not args.get('no_truncate')
else ''))
@@ -911,7 +907,7 @@ def main(disk, roots=None, *,
name = None
for bid_, w_, rbyd_, rid_, tags_ in reversed(path):
for tag_, j_, d_, data_ in tags_:
if tag_ & 0xf00f == TAG_NAME:
if tag_ & 0x7f00 == TAG_NAME:
name = (tag_, j_, d_, data_)
if rid_-(w_-1) != 0:
@@ -920,7 +916,7 @@ def main(disk, roots=None, *,
if name is not None:
tags = [name] + [(tag, j, d, data)
for tag, j, d, data in tags
if tag & 0xf00f != TAG_NAME]
if tag & 0x7f00 != TAG_NAME]
# show the branch
dbg_branch(bid, w, rbyd, rid, tags, len(path)-1)
scripts/dbgmtree.py
+98 -99
View File
@@ -8,23 +8,24 @@ import os
import struct
TAG_UNR = 0x0002
TAG_MAGIC = 0x0030
TAG_CONFIG = 0x0040
TAG_MROOT = 0x0110
TAG_NAME = 0x1000
TAG_BRANCH = 0x1000
TAG_REG = 0x1010
TAG_DIR = 0x1020
TAG_STRUCT = 0x3000
TAG_INLINED = 0x3000
TAG_BLOCK = 0x3100
TAG_MDIR = 0x3200
TAG_BTREE = 0x3300
TAG_UATTR = 0x4000
TAG_ALT = 0x0008
TAG_CRC = 0x0004
TAG_FCRC = 0x1004
TAG_UNR = 0x1000
TAG_SUPERMAGIC = 0x0003
TAG_SUPERCONFIG = 0x0004
TAG_MROOT = 0x0304
TAG_NAME = 0x0100
TAG_BRANCH = 0x0100
TAG_REG = 0x0101
TAG_DIR = 0x0102
TAG_STRUCT = 0x0300
TAG_INLINED = 0x0300
TAG_BLOCK = 0x0302
TAG_BTREE = 0x0303
TAG_MDIR = 0x0305
TAG_UATTR = 0x0400
TAG_ALT = 0x4000
TAG_CRC = 0x2000
TAG_FCRC = 0x2100
# parse some rbyd addr encodings
# 0xa -> [0xa]
@@ -70,9 +71,6 @@ def crc32c(data, crc=0):
crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78)
return 0xffffffff ^ crc
def fromle16(data):
return struct.unpack('<H', data[0:2].ljust(2, b'\0'))[0]
def fromle32(data):
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0]
@@ -86,10 +84,10 @@ def fromleb128(data):
return word, len(data)
def fromtag(data):
tag = fromle16(data)
tag = (data[0] << 8) | data[1]
weight, d = fromleb128(data[2:])
size, d_ = fromleb128(data[2+d:])
return tag&1, tag&~1, weight, size, 2+d+d_
return tag>>15, tag&0x7fff, weight, size, 2+d+d_
def frommdir(data):
blocks = []
@@ -121,64 +119,60 @@ def xxd(data, width=16, crc=False):
for b in map(chr, data[i:i+width])))
def tagrepr(tag, w, size, off=None):
if (tag & 0xfffe) == TAG_UNR:
if (tag & 0x7fff) == TAG_UNR:
return 'unr%s%s' % (
' w%d' % w if w else '',
' %d' % size if size else '')
elif (tag & 0xfffc) == TAG_MAGIC:
return '%smagic%s %d' % (
'rm' if tag & 0x2 else '',
elif (tag & 0x6fff) == TAG_SUPERMAGIC:
return '%ssupermagic%s%s' % (
'rm' if tag & 0x1000 else '',
' w%d' % w if w else '',
size)
elif (tag & 0xfffc) == TAG_CONFIG:
return '%sconfig%s %d' % (
'rm' if tag & 0x2 else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6fff) == TAG_SUPERCONFIG:
return '%ssuperconfig%s%s' % (
'rm' if tag & 0x1000 else '',
' w%d' % w if w else '',
size)
elif (tag & 0xfffc) == TAG_MROOT:
return '%smroot%s %d' % (
'rm' if tag & 0x2 else '',
' w%d' % w if w else '',
size)
elif (tag & 0xf00c) == TAG_NAME:
return '%s%s%s %d' % (
'rm' if tag & 0x2 else '',
'bname' if (tag & 0xfffe) == TAG_BRANCH
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_NAME:
return '%s%s%s%s' % (
'rm' if tag & 0x1000 else '',
'branch' if (tag & 0xfffe) == TAG_BRANCH
else 'reg' if (tag & 0xfffe) == TAG_REG
else 'dir' if (tag & 0xfffe) == TAG_DIR
else 'name 0x%02x' % ((tag & 0x0ff0) >> 4),
' w%d' % w if w else '',
size)
elif (tag & 0xf00c) == TAG_STRUCT:
return '%s%s%s %d' % (
'rm' if tag & 0x2 else '',
'inlined' if (tag & 0xfffe) == TAG_INLINED
else 'block' if (tag & 0xfffe) == TAG_BLOCK
else 'mdir' if (tag & 0xfffe) == TAG_MDIR
else 'btree' if (tag & 0xfffe) == TAG_BTREE
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_STRUCT:
return '%s%s%s%s' % (
'rm' if tag & 0x1000 else '',
'inlined' if (tag & 0x6fff) == TAG_INLINED
else 'block' if (tag & 0x6fff) == TAG_BLOCK
else 'btree' if (tag & 0x6fff) == TAG_BTREE
else 'mdir' if (tag & 0x6fff) == TAG_MROOT
else 'mdir' if (tag & 0x6fff) == TAG_MDIR
else 'struct 0x%02x' % ((tag & 0x0ff0) >> 4),
' w%d' % w if w else '',
size)
elif (tag & 0xf00c) == TAG_UATTR:
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_UATTR:
return '%suattr 0x%02x%s%s' % (
'rm' if tag & 0x2 else '',
(tag & 0x0ff0) >> 4,
'rm' if tag & 0x1000 else '',
tag & 0xff,
' w%d' % w if w else '',
' %d' % size if not tag & 0x2 or size else '')
elif (tag & 0xf00e) == TAG_CRC:
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x7f00) == TAG_CRC:
return 'crc%x%s %d' % (
1 if tag & 0x10 else 0,
1 if tag & 0x1 else 0,
' 0x%x' % w if w > 0 else '',
size)
elif (tag & 0xfffe) == TAG_FCRC:
elif (tag & 0x7fff) == TAG_FCRC:
return 'fcrc%s %d' % (
' 0x%x' % w if w > 0 else '',
size)
elif tag & 0x8:
elif tag & 0x4000:
return 'alt%s%s 0x%x w%d %s' % (
'r' if tag & 0x2 else 'b',
'gt' if tag & 0x4 else 'le',
tag & 0xfff0,
'r' if tag & 0x1000 else 'b',
'gt' if tag & 0x2000 else 'le',
tag & 0x0fff,
w,
'0x%x' % (0xffffffff & (off-size))
if off is not None
@@ -186,6 +180,7 @@ def tagrepr(tag, w, size, off=None):
else:
return '0x%04x w%d %d' % (tag, w, size)
# this type is used for tree representations
TBranch = co.namedtuple('TBranch', 'a, b, d, c')
@@ -262,12 +257,12 @@ class Rbyd:
break
crc_ = crc32c(data[j_:j_+d], crc_)
j_ += d
if not tag & 0x8 and j_ + size > len(data):
if not tag & 0x4000 and j_ + size > len(data):
break
# take care of crcs
if not tag & 0x8:
if (tag & 0xf00f) != TAG_CRC:
if not tag & 0x4000:
if (tag & 0x7f00) != TAG_CRC:
crc_ = crc32c(data[j_:j_+size], crc_)
# found a crc?
else:
@@ -281,7 +276,7 @@ class Rbyd:
weight = weight_
# evaluate trunks
if (tag & 0xc) != 0x4 and (
if (tag & 0x6000) != 0x2000 and (
not trunk or trunk >= j_-d or wastrunk):
# new trunk?
if not wastrunk:
@@ -290,8 +285,8 @@ class Rbyd:
wastrunk = True
# keep track of weight
if tag & 0x8:
if tag & 0x4:
if tag & 0x4000:
if tag & 0x2000:
upper_ += w
else:
lower_ += w
@@ -302,7 +297,7 @@ class Rbyd:
if trunk and j_ + size > trunk:
trunkoff = j_ + size
if not tag & 0x8:
if not tag & 0x4000:
j_ += size
return cls(block, data, rev, off, trunk_, weight)
@@ -321,19 +316,20 @@ class Rbyd:
_, alt, weight_, jump, d = fromtag(self.data[j:])
# found an alt?
if alt & 0x8:
if alt & 0x4000:
# follow?
if ((id, tag & ~0xf) > (upper-weight_-1, alt & ~0xf)
if alt & 0x4
else ((id, tag & ~0xf) <= (lower+weight_, alt & ~0xf))):
lower += upper-lower-1-weight_ if alt & 0x4 else 0
upper -= upper-lower-1-weight_ if not alt & 0x4 else 0
if ((id, tag & 0xfff) > (upper-weight_-1, alt & 0xfff)
if alt & 0x2000
else ((id, tag & 0xfff)
<= (lower+weight_, alt & 0xfff))):
lower += upper-lower-1-weight_ if alt & 0x2000 else 0
upper -= upper-lower-1-weight_ if not alt & 0x2000 else 0
j = j - jump
# figure out which color
if alt & 0x2:
if alt & 0x1000:
_, nalt, _, _, _ = fromtag(self.data[j+jump+d:])
if nalt & 0x2:
if nalt & 0x1000:
path.append((j+jump, j, True, 'y'))
else:
path.append((j+jump, j, True, 'r'))
@@ -342,14 +338,14 @@ class Rbyd:
# stay on path
else:
lower += weight_ if not alt & 0x4 else 0
upper -= weight_ if alt & 0x4 else 0
lower += weight_ if not alt & 0x2000 else 0
upper -= weight_ if alt & 0x2000 else 0
j = j + d
# figure out which color
if alt & 0x2:
if alt & 0x1000:
_, nalt, _, _, _ = fromtag(self.data[j:])
if nalt & 0x2:
if nalt & 0x1000:
path.append((j-d, j, False, 'y'))
else:
path.append((j-d, j, False, 'r'))
@@ -362,7 +358,7 @@ class Rbyd:
tag_ = alt
w_ = id_-lower
done = (id_, tag_) < (id, tag) or tag_ & 2
done = (id_, tag_) < (id, tag) or tag_ & 0x1000
return done, id_, tag_, w_, j, d, self.data[j+d:j+d+jump], path
@@ -380,7 +376,7 @@ class Rbyd:
id = -1
while True:
done, id, tag, w, j, d, data, _ = self.lookup(id, tag+0x10)
done, id, tag, w, j, d, data, _ = self.lookup(id, tag+0x1)
if done:
break
@@ -393,7 +389,7 @@ class Rbyd:
id, tag = -1, 0
while True:
done, id, tag, w, j, d, data, path = self.lookup(id, tag+0x10)
done, id, tag, w, j, d, data, path = self.lookup(id, tag+0x1)
# found end of tree?
if done:
break
@@ -497,7 +493,7 @@ class Rbyd:
w = 0
for i in it.count():
done, rid__, tag, w_, j, d, data, _ = rbyd.lookup(
rid_, tag+0x10)
rid_, tag+0x1)
if done or (i != 0 and rid__ != rid_):
break
@@ -679,7 +675,7 @@ class Rbyd:
name = None
for bid_, w_, rbyd_, rid_, tags_ in reversed(path):
for tag_, j_, d_, data_ in tags_:
if tag_ & 0xf00f == TAG_NAME:
if tag_ & 0x7f00 == TAG_NAME:
name = (tag_, j_, d_, data_)
if rid_-(w_-1) != 0:
@@ -850,7 +846,7 @@ def main(disk, mroots=None, *,
if root:
r_rid, r_tag = root.a
else:
_, r_rid, r_tag, _, _, _, _, _ = mroot_.lookup(-1, 0x10)
_, r_rid, r_tag, _, _, _, _, _ = mroot_.lookup(-1, 0x1)
tree.add(TBranch(
a=(-1, d-1, 0, -1, TAG_MROOT),
b=(-1, d, 0, r_rid, r_tag),
@@ -895,7 +891,7 @@ def main(disk, mroots=None, *,
if root:
r_rid, r_tag = root.a
else:
_, r_rid, r_tag, _, _, _, _, _ = mdir.lookup(-1, 0x10)
_, r_rid, r_tag, _, _, _, _, _ = mdir.lookup(-1, 0x1)
tree.add(TBranch(
a=(-1, d, 0, -1, TAG_MDIR),
b=(0, 0, 0, r_rid, r_tag),
@@ -1015,7 +1011,7 @@ def main(disk, mroots=None, *,
r_rid, r_tag = root.a
else:
_, r_rid, r_tag, _, _, _, _, _ = (
mdir_.lookup(-1, 0x10))
mdir_.lookup(-1, 0x1))
tree.add(TBranch(
a=branch.b,
b=(mid-(w-1), len(path), 0, r_rid, r_tag),
@@ -1097,7 +1093,7 @@ def main(disk, mroots=None, *,
# connect branch to our first tag
if d > 0:
done, rid, tag, w, j, _, data, _ = mroot_.lookup(-1, 0x10)
done, rid, tag, w, j, _, data, _ = mroot_.lookup(-1, 0x1)
if not done:
tree.add(TBranch(
a=(-1, d-1, 0, -1, TAG_MROOT),
@@ -1122,7 +1118,7 @@ def main(disk, mroots=None, *,
# create a branch to our mdir if there is one
if mdir:
# connect branch to our first tag
done, rid, tag, w, j, _, data, _ = mdir.lookup(-1, 0x10)
done, rid, tag, w, j, _, data, _ = mdir.lookup(-1, 0x1)
if not done:
tree.add(TBranch(
a=(-1, d, 0, -1, TAG_MDIR),
@@ -1192,7 +1188,7 @@ def main(disk, mroots=None, *,
# find the first entry in the mdir, map branches
# to this entry
done, rid, tag, _, j, d, data, _ = (
mdir_.lookup(-1, 0x10))
mdir_.lookup(-1, 0x1))
tree_ = set()
for branch in tree:
@@ -1293,8 +1289,9 @@ def main(disk, mroots=None, *,
# show in-device representation
if args.get('device'):
print('%11s %*s %s' % (
print('%11s %*s%*s %s' % (
'',
t_width, '',
w_width, '',
'%-22s%s' % (
'%04x %08x %07x' % (tag, w, len(data)),
@@ -1306,20 +1303,22 @@ def main(disk, mroots=None, *,
min(m.ceil(len(data)/4),
3)))[:23]
if not args.get('no_truncate')
and not tag & 0x8 else '')))
and not tag & 0x4000 else '')))
# show on-disk encoding of tags
if args.get('raw'):
for o, line in enumerate(xxd(mdir.data[j:j+d])):
print('%11s: %*s %s' % (
print('%11s: %*s%*s %s' % (
'%04x' % (j + o*16),
t_width, '',
w_width, '',
line))
if args.get('raw') or args.get('no_truncate'):
if not tag & 0x8:
if not tag & 0x4000:
for o, line in enumerate(xxd(data)):
print('%11s: %*s %s' % (
print('%11s: %*s%*s %s' % (
'%04x' % (j+d + o*16),
t_width, '',
w_width, '',
line))
@@ -1346,7 +1345,7 @@ def main(disk, mroots=None, *,
''.join(
b if b >= ' ' and b <= '~' else '.'
for b in map(chr, data))
if tag & 0xf00f == TAG_NAME
if tag & 0x7f00 == TAG_NAME
else next(xxd(data, 8), '')
if not args.get('no_truncate')
else ''))
@@ -1376,7 +1375,7 @@ def main(disk, mroots=None, *,
line))
# note we don't render name tags with no_truncate
if args.get('raw') or (
args.get('no_truncate') and tag & 0xf00f != TAG_NAME):
args.get('no_truncate') and tag & 0x7f00 != TAG_NAME):
for o, line in enumerate(xxd(data)):
print('%11s: %*s%*s %s' % (
'%04x' % (j+d + o*16),
@@ -1507,7 +1506,7 @@ def main(disk, mroots=None, *,
name = None
for mid_, w_, rbyd_, rid_, tags_ in reversed(path):
for tag_, j_, d_, data_ in tags_:
if tag_ & 0xf00f == TAG_NAME:
if tag_ & 0x7f00 == TAG_NAME:
name = (tag_, j_, d_, data_)
if rid_-(w_-1) != 0:
@@ -1516,7 +1515,7 @@ def main(disk, mroots=None, *,
if name is not None:
tags = [name] + [(tag, j, d, data)
for tag, j, d, data in tags
if tag & 0xf00f != TAG_NAME]
if tag & 0x7f00 != TAG_NAME]
# find mdir in the tags
mdir__ = None
scripts/dbgrbyd.py
+99 -107
View File
@@ -17,23 +17,23 @@ COLORS = [
]
TAG_UNR = 0x0002
TAG_MAGIC = 0x0030
TAG_CONFIG = 0x0040
TAG_MROOT = 0x0110
TAG_NAME = 0x1000
TAG_BRANCH = 0x1000
TAG_REG = 0x1010
TAG_DIR = 0x1020
TAG_STRUCT = 0x3000
TAG_INLINED = 0x3000
TAG_BLOCK = 0x3100
TAG_MDIR = 0x3200
TAG_BTREE = 0x3300
TAG_UATTR = 0x4000
TAG_ALT = 0x0008
TAG_CRC = 0x0004
TAG_FCRC = 0x1004
TAG_UNR = 0x1000
TAG_SUPERMAGIC = 0x0003
TAG_SUPERCONFIG = 0x0004
TAG_MROOT = 0x0304
TAG_NAME = 0x0100
TAG_BRANCH = 0x0100
TAG_REG = 0x0101
TAG_DIR = 0x0102
TAG_STRUCT = 0x0300
TAG_INLINED = 0x0300
TAG_BLOCK = 0x0302
TAG_BTREE = 0x0303
TAG_MDIR = 0x0305
TAG_UATTR = 0x0400
TAG_ALT = 0x4000
TAG_CRC = 0x2000
TAG_FCRC = 0x2100
# parse some rbyd addr encodings
# 0xa -> [0xa]
@@ -79,9 +79,6 @@ def crc32c(data, crc=0):
crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78)
return 0xffffffff ^ crc
def fromle16(data):
return struct.unpack('<H', data[0:2].ljust(2, b'\0'))[0]
def fromle32(data):
return struct.unpack('<I', data[0:4].ljust(4, b'\0'))[0]
@@ -95,10 +92,10 @@ def fromleb128(data):
return word, len(data)
def fromtag(data):
tag = fromle16(data)
tag = (data[0] << 8) | data[1]
weight, d = fromleb128(data[2:])
size, d_ = fromleb128(data[2+d:])
return tag&1, tag&~1, weight, size, 2+d+d_
return tag>>15, tag&0x7fff, weight, size, 2+d+d_
def popc(x):
return bin(x).count('1')
@@ -114,64 +111,60 @@ def xxd(data, width=16, crc=False):
for b in map(chr, data[i:i+width])))
def tagrepr(tag, w, size, off=None):
if (tag & 0xfffe) == TAG_UNR:
if (tag & 0x7fff) == TAG_UNR:
return 'unr%s%s' % (
' w%d' % w if w else '',
' %d' % size if size else '')
elif (tag & 0xfffc) == TAG_MAGIC:
return '%smagic%s %d' % (
'rm' if tag & 0x2 else '',
elif (tag & 0x6fff) == TAG_SUPERMAGIC:
return '%ssupermagic%s%s' % (
'rm' if tag & 0x1000 else '',
' w%d' % w if w else '',
size)
elif (tag & 0xfffc) == TAG_CONFIG:
return '%sconfig%s %d' % (
'rm' if tag & 0x2 else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6fff) == TAG_SUPERCONFIG:
return '%ssuperconfig%s%s' % (
'rm' if tag & 0x1000 else '',
' w%d' % w if w else '',
size)
elif (tag & 0xfffc) == TAG_MROOT:
return '%smroot%s %d' % (
'rm' if tag & 0x2 else '',
' w%d' % w if w else '',
size)
elif (tag & 0xf00c) == TAG_NAME:
return '%s%s%s %d' % (
'rm' if tag & 0x2 else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_NAME:
return '%s%s%s%s' % (
'rm' if tag & 0x1000 else '',
'branch' if (tag & 0xfffe) == TAG_BRANCH
else 'reg' if (tag & 0xfffe) == TAG_REG
else 'dir' if (tag & 0xfffe) == TAG_DIR
else 'name 0x%02x' % ((tag & 0x0ff0) >> 4),
' w%d' % w if w else '',
size)
elif (tag & 0xf00c) == TAG_STRUCT:
return '%s%s%s %d' % (
'rm' if tag & 0x2 else '',
'inlined' if (tag & 0xfffe) == TAG_INLINED
else 'block' if (tag & 0xfffe) == TAG_BLOCK
else 'mdir' if (tag & 0xfffe) == TAG_MDIR
else 'btree' if (tag & 0xfffe) == TAG_BTREE
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_STRUCT:
return '%s%s%s%s' % (
'rm' if tag & 0x1000 else '',
'inlined' if (tag & 0x6fff) == TAG_INLINED
else 'block' if (tag & 0x6fff) == TAG_BLOCK
else 'btree' if (tag & 0x6fff) == TAG_BTREE
else 'mdir' if (tag & 0x6fff) == TAG_MROOT
else 'mdir' if (tag & 0x6fff) == TAG_MDIR
else 'struct 0x%02x' % ((tag & 0x0ff0) >> 4),
' w%d' % w if w else '',
size)
elif (tag & 0xf00c) == TAG_UATTR:
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_UATTR:
return '%suattr 0x%02x%s%s' % (
'rm' if tag & 0x2 else '',
(tag & 0x0ff0) >> 4,
'rm' if tag & 0x1000 else '',
tag & 0xff,
' w%d' % w if w else '',
' %d' % size if not tag & 0x2 or size else '')
elif (tag & 0xf00e) == TAG_CRC:
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x7f00) == TAG_CRC:
return 'crc%x%s %d' % (
1 if tag & 0x10 else 0,
1 if tag & 0x1 else 0,
' 0x%x' % w if w > 0 else '',
size)
elif (tag & 0xfffe) == TAG_FCRC:
elif (tag & 0x7fff) == TAG_FCRC:
return 'fcrc%s %d' % (
' 0x%x' % w if w > 0 else '',
size)
elif tag & 0x8:
elif tag & 0x4000:
return 'alt%s%s 0x%x w%d %s' % (
'r' if tag & 0x2 else 'b',
'gt' if tag & 0x4 else 'le',
tag & 0xfff0,
'r' if tag & 0x1000 else 'b',
'gt' if tag & 0x2000 else 'le',
tag & 0x0fff,
w,
'0x%x' % (0xffffffff & (off-size))
if off is not None
@@ -192,14 +185,14 @@ def dbg_log(data, block_size, rev, off, weight, *,
j = j_
v, tag, w, size, d = fromtag(data[j_:])
j_ += d
if not tag & 0x8:
if not tag & 0x4000:
j_ += size
if tag & 0x8:
if tag & 0x4000:
# figure out which alt color
if tag & 0x2:
if tag & 0x1000:
_, ntag, _, _, _ = fromtag(data[j_:])
if ntag & 0x2:
if ntag & 0x1000:
jumps.append((j, j-size, 0, 'y'))
else:
jumps.append((j, j-size, 0, 'r'))
@@ -289,21 +282,21 @@ def dbg_log(data, block_size, rev, off, weight, *,
j = j_
v, tag, w, size, d = fromtag(data[j_:])
j_ += d
if not tag & 0x8:
if not tag & 0x4000:
j_ += size
# find trunk
if not wastrunk and (tag & 0xc) != 0x4:
if not wastrunk and (tag & 0x6000) != 0x2000:
lower_, upper_ = 0, 0
wastrunk = not not tag & 0x8
wastrunk = not not tag & 0x4000
# keep track of weight
if tag & 0x8:
if tag & 0x4:
if tag & 0x4000:
if tag & 0x2000:
upper_ += w
else:
lower_ += w
elif (tag & 0xc) == 0x0:
elif (tag & 0x6000) == 0x2000:
delta = (lower_+upper_+w) - weight_
weight_ = lower_+upper_+w
id = lower_+w-1
@@ -346,7 +339,7 @@ def dbg_log(data, block_size, rev, off, weight, *,
weights = weights_
lifetimes = lifetimes_
if not tag & 0x2 and id >= 0:
if not tag & 0x1000 and id >= 0:
# attach tag to lifetime
i, id_ = index(weights, id)
if i < len(weights):
@@ -431,27 +424,26 @@ def dbg_log(data, block_size, rev, off, weight, *,
v, tag, w, size, d = fromtag(data[j_:])
if v != (popc(crc) & 1):
notes.append('v!=%x' % (popc(crc) & 1))
tag &= ~1
crc = crc32c(data[j_:j_+d], crc)
j_ += d
# find trunk
if not wastrunk and (tag & 0xc) != 0x4:
if not wastrunk and (tag & 0x6000) != 0x2000:
lower_, upper_ = 0, 0
wastrunk = not not tag & 0x8
wastrunk = not not tag & 0x4000
# calculate id from alt weights
if tag & 0x8:
if tag & 0x4:
if tag & 0x4000:
if tag & 0x2000:
upper_ += w
else:
lower_ += w
elif (tag & 0xc) == 0x0:
elif (tag & 0x6000) == 0x0:
weight_ = lower_+upper_+w
id = lower_+w-1
if not tag & 0x8:
if (tag & 0xf00f) != TAG_CRC:
if not tag & 0x4000:
if (tag & 0x7f00) != TAG_CRC:
crc = crc32c(data[j_:j_+size], crc)
# found a crc?
else:
@@ -467,7 +459,7 @@ def dbg_log(data, block_size, rev, off, weight, *,
'\x1b[m' if color and j >= off else '',
lifetime_width, lifetimerepr(j) if args.get('lifetimes') else '',
'\x1b[90m' if color and j >= off else '',
w_width, '' if (tag & 0xc) != 0x0
w_width, '' if (tag & 0x6000) != 0x0
else '%d-%d' % (id-(w-1), id) if w > 1
else id,
'%-22s%s' % (
@@ -475,7 +467,7 @@ def dbg_log(data, block_size, rev, off, weight, *,
' %s' % next(xxd(
data[j+d:j+d+min(size, 8)], 8), '')
if not args.get('no_truncate')
and not tag & 0x8 else ''),
and not tag & 0x4000 else ''),
'\x1b[m' if color and j >= off else '',
' (%s)' % ', '.join(notes) if notes
else ' %s' % jumprepr(j)
@@ -497,7 +489,7 @@ def dbg_log(data, block_size, rev, off, weight, *,
data[j+d+i*4:j+d+min(i*4+4,size)])
for i in range(min(m.ceil(size/4), 3)))[:23]
if not args.get('no_truncate')
and not tag & 0x8 else ''),
and not tag & 0x4000 else ''),
crc,
popc(crc) & 1,
'\x1b[m' if color and j >= off else ''))
@@ -513,7 +505,7 @@ def dbg_log(data, block_size, rev, off, weight, *,
line,
'\x1b[m' if color and j >= off else ''))
if args.get('raw') or args.get('no_truncate'):
if not tag & 0x8:
if not tag & 0x4000:
for o, line in enumerate(xxd(data[j+d:j+d+size])):
print('%s%8s: %*s%*s %s%s' % (
'\x1b[90m' if color and j >= off else '',
@@ -543,19 +535,19 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
_, alt, w, jump, d = fromtag(data[j:])
# found an alt?
if alt & 0x8:
if alt & 0x4000:
# follow?
if ((id, tag & ~0xf) > (upper-w-1, alt & ~0xf)
if alt & 0x4
else ((id, tag & ~0xf) <= (lower+w, alt & ~0xf))):
lower += upper-lower-1-w if alt & 0x4 else 0
upper -= upper-lower-1-w if not alt & 0x4 else 0
if ((id, tag & 0xfff) > (upper-w-1, alt & 0xfff)
if alt & 0x2000
else ((id, tag & 0xfff) <= (lower+w, alt & 0xfff))):
lower += upper-lower-1-w if alt & 0x2000 else 0
upper -= upper-lower-1-w if not alt & 0x2000 else 0
j = j - jump
# figure out which color
if alt & 0x2:
if alt & 0x1000:
_, nalt, _, _, _ = fromtag(data[j+jump+d:])
if nalt & 0x2:
if nalt & 0x1000:
path.append((j+jump, j, True, 'y'))
else:
path.append((j+jump, j, True, 'r'))
@@ -564,14 +556,14 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
# stay on path
else:
lower += w if not alt & 0x4 else 0
upper -= w if alt & 0x4 else 0
lower += w if not alt & 0x2000 else 0
upper -= w if alt & 0x2000 else 0
j = j + d
# figure out which color
if alt & 0x2:
if alt & 0x1000:
_, nalt, _, _, _ = fromtag(data[j:])
if nalt & 0x2:
if nalt & 0x1000:
path.append((j-d, j, False, 'y'))
else:
path.append((j-d, j, False, 'r'))
@@ -584,7 +576,7 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
tag_ = alt
w_ = id_-lower
done = (id_, tag_) < (id, tag) or tag_ & 2
done = (id_, tag_) < (id, tag) or tag_ & 0x1000
return done, id_, tag_, w_, j, d, jump, path
@@ -596,7 +588,7 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
id, tag = -1, 0
while True:
done, id, tag, w, j, d, size, path = lookup(id, tag+0x10)
done, id, tag, w, j, d, size, path = lookup(id, tag+0x1)
# found end of tree?
if done:
break
@@ -742,7 +734,7 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
id, tag = -1, 0
while True:
done, id, tag, w, j, d, size, path = lookup(id, tag+0x10)
done, id, tag, w, j, d, size, path = lookup(id, tag+0x1)
# found end of tree?
if done:
break
@@ -759,7 +751,7 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
' %s' % next(xxd(
data[j+d:j+d+min(size, 8)], 8), '')
if not args.get('no_truncate')
and not tag & 0x8 else '')))
and not tag & 0x4000 else '')))
# show in-device representation
if args.get('device'):
@@ -774,7 +766,7 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
data[j+d+i*4:j+d+min(i*4+4,size)])
for i in range(min(m.ceil(size/4), 3)))[:23]
if not args.get('no_truncate')
and not tag & 0x8 else '')))
and not tag & 0x4000 else '')))
# show on-disk encoding of tags
if args.get('raw'):
@@ -785,7 +777,7 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
w_width, '',
line))
if args.get('raw') or args.get('no_truncate'):
if not tag & 0x8:
if not tag & 0x4000:
for o, line in enumerate(xxd(data[j+d:j+d+size])):
print('%8s: %*s%*s %s' % (
'%04x' % (j+d + o*16),
@@ -853,12 +845,12 @@ def main(disk, blocks=None, *,
break
crc_ = crc32c(data[j_:j_+d], crc_)
j_ += d
if not tag & 0x8 and j_ + size > len(data):
if not tag & 0x4000 and j_ + size > len(data):
break
# take care of crcs
if not tag & 0x8:
if (tag & 0xf00f) != TAG_CRC:
if not tag & 0x4000:
if (tag & 0x7f00) != TAG_CRC:
crc_ = crc32c(data[j_:j_+size], crc_)
# found a crc?
else:
@@ -872,7 +864,7 @@ def main(disk, blocks=None, *,
weight = weight_
# evaluate trunks
if (tag & 0xc) != 0x4 and (
if (tag & 0x6000) != 0x2000 and (
not trunk or trunk >= j_-d or wastrunk):
# new trunk?
if not wastrunk:
@@ -881,8 +873,8 @@ def main(disk, blocks=None, *,
wastrunk = True
# keep track of weight
if tag & 0x8:
if tag & 0x4:
if tag & 0x4000:
if tag & 0x2000:
upper_ += w
else:
lower_ += w
@@ -893,7 +885,7 @@ def main(disk, blocks=None, *,
if trunk and j_ + size > trunk:
trunkoff = j_ + size
if not tag & 0x8:
if not tag & 0x4000:
j_ += size
return rev, off, trunk_, weight