Github/littlefs
1
Fork 0
Code Actions Activity

Allowed "alta" (altbgt 0) to terminate rbyd trunks, dropped rm bit

Browse Source 
This replaces unr with null on disk, though note both the rm bit and unr
are used in-device still, they just don't get written to disk.

This removes the need for the rm bit on disk. Since we no longer need to
figure out what's been removed during fetch, we can save this bit for both
internal and future on-disk use.

Special handling of alta allows us to avoid emitting an unr tag (now null) if
the current trunk is truly unreachable. This is minor now, but important
for a theoretical rbyd rebalance operation (planned), which brings the
rbyd overhead down from ~3x to ~2x.

These changes give us two ways to terminate trunks without a tag:

1. With an alta, if the current trunk is unreachable:

     altbgt 0x403 w0 0x7b
     altbgt 0x402 w0 0x29
     alta w0 0x4

2. With a null, if the current trunk is reachable, either for
   code convenience or because emitting an alta is impossible (an empty
   rbyd for example):

     altbgt 0x403 w0 0x7b
     altbgt 0x402 w0 0x29
     altbgt 0x401 w0 0x4
     null
This commit is contained in:
Christopher Haster
2023-06-17 18:11:45 -05:00
parent aa559d30b0
commit 7180b70c9c
4 changed files with 275 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+74 -61
View File
@@ -584,7 +584,8 @@ static int lfsr_bd_erase(lfs_t *lfs, lfs_block_t block) {
// 16-bit metadata tags // 16-bit metadata tags
enum lfsr_tag_type { enum lfsr_tag_type {
LFSR_TAG_UNR = 0x1000, LFSR_TAG_NULL = 0x0000,
LFSR_TAG_UNR = 0x1000, // in-device only
LFSR_TAG_MKUNR = 0x3000, // in-device only LFSR_TAG_MKUNR = 0x3000, // in-device only
LFSR_TAG_SUPERMAGIC = 0x0003, LFSR_TAG_SUPERMAGIC = 0x0003,
@@ -618,6 +619,7 @@ enum lfsr_tag_type {
LFSR_TAG_ALTRLE = 0x5000, LFSR_TAG_ALTRLE = 0x5000,
LFSR_TAG_ALTBGT = 0x6000, LFSR_TAG_ALTBGT = 0x6000,
LFSR_TAG_ALTRGT = 0x7000, LFSR_TAG_ALTRGT = 0x7000,
LFSR_TAG_ALTA = 0x6000,
LFSR_TAG_CRC = 0x2000, LFSR_TAG_CRC = 0x2000,
LFSR_TAG_FCRC = 0x2100, LFSR_TAG_FCRC = 0x2100,
@@ -691,12 +693,16 @@ static inline lfsr_tag_t lfsr_tag_setrm(lfsr_tag_t tag) {
return tag | 0x1000; return tag | 0x1000;
} }
static inline bool lfsr_tag_istrunk(lfsr_tag_t tag) { static inline bool lfsr_tag_isalt(lfsr_tag_t tag) {
return tag & 0x4000;
}
static inline bool lfsr_tag_istrunkstart(lfsr_tag_t tag) {
return (tag & 0x6000) != 0x2000; return (tag & 0x6000) != 0x2000;
} }
static inline bool lfsr_tag_isalt(lfsr_tag_t tag) { static inline bool lfsr_tag_istrunkend(lfsr_tag_t tag) {
return tag & 0x4000; return !lfsr_tag_isalt(tag) || tag == LFSR_TAG_ALTA;
} }
static inline lfsr_tag_t lfsr_tag_next(lfsr_tag_t tag) { static inline lfsr_tag_t lfsr_tag_next(lfsr_tag_t tag) {
@@ -704,24 +710,33 @@ static inline lfsr_tag_t lfsr_tag_next(lfsr_tag_t tag) {
} }
// lfsr_rbyd_append specific flags // lfsr_rbyd_append specific flags
static inline bool lfsr_tag_isupper(lfsr_tag_t tag) { static inline bool lfsr_tag_hasdiverged(lfsr_tag_t tag) {
return tag & 0x2000; return tag & 0x2000;
} }
static inline bool lfsr_tag_islower(lfsr_tag_t tag) { static inline bool lfsr_tag_isdivergedupper(lfsr_tag_t tag) {
return !lfsr_tag_isupper(tag); return lfsr_tag_hasdiverged(tag) && (tag & 0x1000);
} }
static inline lfsr_tag_t lfsr_tag_setupper(lfsr_tag_t tag) { static inline bool lfsr_tag_isdivergedlower(lfsr_tag_t tag) {
return tag | 0x2000; return lfsr_tag_hasdiverged(tag) && !(tag & 0x1000);
}
static inline bool lfsr_tag_hasdiverged(lfsr_tag_t tag) {
return tag & 0x4000;
} }
static inline lfsr_tag_t lfsr_tag_setdiverged(lfsr_tag_t tag) { static inline lfsr_tag_t lfsr_tag_setdiverged(lfsr_tag_t tag) {
return tag | 0x4000; return tag | 0x2000;
}
static inline lfsr_tag_t lfsr_tag_setdivergedvalid(
lfsr_tag_t tag, lfsr_tag_t tag_) {
return (tag & 0x3000) | tag_;
}
static inline lfsr_tag_t lfsr_tag_setdivergedupper(lfsr_tag_t tag) {
return tag | 0x3000;
}
static inline lfsr_tag_t lfsr_tag_cleardiverged(lfsr_tag_t tag) {
return tag & ~0x3000;
} }
// alt operations // alt operations
@@ -1586,9 +1601,8 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd,
lfs_off_t off = sizeof(uint32_t); lfs_off_t off = sizeof(uint32_t);
lfs_off_t trunk_ = 0; lfs_off_t trunk_ = 0;
bool wastrunk = false; bool wastrunk = false;
lfs_size_t lower = 0;
lfs_size_t upper = 0;
lfs_size_t weight = 0; lfs_size_t weight = 0;
lfs_size_t weight_ = 0;
// assume unerased until proven otherwise // assume unerased until proven otherwise
lfsr_fcrc_t fcrc; lfsr_fcrc_t fcrc;
@@ -1687,35 +1701,30 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd,
} }
// found a trunk of a tree? // found a trunk of a tree?
if (lfsr_tag_istrunk(tag) && (!trunk || trunk >= off-d || wastrunk)) { if (lfsr_tag_istrunkstart(tag)
&& (!trunk || trunk >= off-d || wastrunk)) {
// start of trunk?
if (!wastrunk) { if (!wastrunk) {
wastrunk = true;
// save trunk entry point // save trunk entry point
trunk_ = off-d; trunk_ = off-d;
// reset weights // reset weight
lower = 0; weight_ = 0;
upper = 0;
wastrunk = true;
} }
// derive the new weight of the tree from alt pointers // derive weight of the tree from alt pointers
// //
// NOTE we can't check for overflow/underflow here because we // NOTE we can't check for overflow/underflow here because we
// may be overeagerly parsing an invalid commit, it's ok for // may be overeagerly parsing an invalid commit, it's ok for
// this to overflow/underflow as long as we throw it out later // this to overflow/underflow as long as we throw it out later
// on a bad crc // on a bad crc
if (lfsr_tag_isalt(tag)) { weight_ += w;
if (lfsr_tag_isgt(tag)) {
upper += w;
} else {
lower += w;
}
} else { // end of trunk?
// update current weight if (lfsr_tag_istrunkend(tag)) {
weight = lower+upper+w;
// any non-alt terminates the current trunk
wastrunk = false; wastrunk = false;
// update current weight
weight = weight_;
} }
} }
@@ -1801,9 +1810,10 @@ static int lfsr_rbyd_lookupnext(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
lfsr_tag_t tag__ = alt; lfsr_tag_t tag__ = alt;
// not what we're looking for? // not what we're looking for?
if (id__ < id if (!tag__
|| (id__ == id && lfsr_tag_key(tag__) < lfsr_tag_key(tag)) || id__ < id
|| lfsr_tag_isrm(tag__)) { || (id__ == id && lfsr_tag_key(tag__)
< lfsr_tag_key(tag))) {
return LFS_ERR_NOENT; return LFS_ERR_NOENT;
} }
@@ -2043,7 +2053,7 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
} }
// mark as invalid until found // mark as invalid until found
tag_ = lfsr_tag_setinvalid(tag_); tag_ = lfsr_tag_setinvalid(tag_);
other_tag_ = lfsr_tag_setinvalid(lfsr_tag_setupper(other_tag_)); other_tag_ = lfsr_tag_setinvalid(lfsr_tag_setdivergedupper(other_tag_));
// keep track of bounds as we descend down the tree // keep track of bounds as we descend down the tree
// //
@@ -2284,7 +2294,7 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
// prune inner alts if our tags diverged // prune inner alts if our tags diverged
if (lfsr_tag_hasdiverged(tag_) if (lfsr_tag_hasdiverged(tag_)
&& lfsr_tag_isupper(tag_) != lfsr_tag_isgt(alt)) { && lfsr_tag_isdivergedupper(tag_) != lfsr_tag_isgt(alt)) {
continue; continue;
} }
@@ -2301,10 +2311,9 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
// update the found tag/id // update the found tag/id
// //
// note we: // note we:
// - clear valid bit (0x8000) // - clear valid bit, marking the tag as found
// - preserve diverged bit (0x4000) // - preserve diverged state
// - preserve isupper tag (0x2000) tag_ = lfsr_tag_setdivergedvalid(tag_, alt);
tag_ = lfsr_tag_setvalid(alt | (tag_ & 0x6000));
id_ = upper_id-1; id_ = upper_id-1;
// done? // done?
@@ -2333,14 +2342,15 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
LFS_ASSERT(lfsr_tag_isvalid(tag_)); LFS_ASSERT(lfsr_tag_isvalid(tag_));
LFS_ASSERT(!lfsr_tag_hasdiverged(tag_) LFS_ASSERT(!lfsr_tag_hasdiverged(tag_)
|| lfsr_tag_isvalid(other_tag_)); || lfsr_tag_isvalid(other_tag_));
if (lfsr_tag_hasdiverged(tag_) && lfsr_tag_islower(tag_)) { if (lfsr_tag_isdivergedlower(tag_)) {
// finished on lower path // finished on lower path
tag_ = other_tag_; tag_ = lfsr_tag_cleardiverged(other_tag_);
id_ = other_id_; id_ = other_id_;
branch = other_branch; branch = other_branch;
upper_id = other_upper_id; upper_id = other_upper_id;
} else if (lfsr_tag_hasdiverged(tag_) && lfsr_tag_isupper(tag_)) { } else if (lfsr_tag_isdivergedupper(tag_)) {
// finished on upper path // finished on upper path
tag_ = lfsr_tag_cleardiverged(tag_);
lower_id = other_lower_id; lower_id = other_lower_id;
} }
@@ -2350,17 +2360,14 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
// always finds the next biggest tag // always finds the next biggest tag
lfsr_tag_t alt = 0; lfsr_tag_t alt = 0;
lfs_size_t weight = 0; lfs_size_t weight = 0;
if (lfsr_tag_isrm(tag_)) { // note if tag_ is null, we found a removed tag that we should just prune
// found an old removed tag, no split needed, just prune the if (tag_ && (id_ < id-lfs_smax32(-delta, 0)
// removed tag
} else if (id_ < id-lfs_smax32(-delta, 0)
|| (id_ == id-lfs_smax32(-delta, 0) || (id_ == id-lfs_smax32(-delta, 0)
&& ((lfsr_tag_ismk(tag) && delta > 0) && ((lfsr_tag_ismk(tag) && delta > 0)
|| lfsr_tag_key(tag_) < lfsr_tag_key(tag)))) { || lfsr_tag_key(tag_) < lfsr_tag_key(tag))))) {
if (lfsr_tag_isrm(tag)) { if (lfsr_tag_isrm(tag)) {
// if removed make our tag unreachable // if removed, make our tag unreachable
alt = LFSR_TAG_ALT(B, GT, 0); alt = LFSR_TAG_ALTA;
weight = upper_id - lower_id - 1 + delta; weight = upper_id - lower_id - 1 + delta;
upper_id -= weight; upper_id -= weight;
} else { } else {
@@ -2370,13 +2377,13 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
lower_id += weight; lower_id += weight;
} }
} else if (id_ > id } else if (tag_ && (id_ > id
|| (id_ == id || (id_ == id
&& ((lfsr_tag_ismk(tag) && delta > 0) && ((lfsr_tag_ismk(tag) && delta > 0)
|| lfsr_tag_key(tag_) > lfsr_tag_key(tag)))) { || lfsr_tag_key(tag_) > lfsr_tag_key(tag))))) {
if (lfsr_tag_isrm(tag)) { if (lfsr_tag_isrm(tag)) {
// if removed make our tag unreachable // if removed, make our tag unreachable
alt = LFSR_TAG_ALT(B, GT, 0); alt = LFSR_TAG_ALTA;
weight = upper_id - lower_id - 1 + delta; weight = upper_id - lower_id - 1 + delta;
upper_id -= weight; upper_id -= weight;
} else { } else {
@@ -2385,6 +2392,12 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
weight = upper_id - id - 1; weight = upper_id - id - 1;
upper_id -= weight; upper_id -= weight;
} }
} else {
if (lfsr_tag_isrm(tag)) {
// if removed, replace our tag with a null tag
tag = LFSR_TAG_NULL;
}
} }
if (alt) { if (alt) {
@@ -2409,10 +2422,8 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
} }
leaf:; leaf:;
if (!lfsr_tag_isrm(tag)) {
// write the actual tag // write the actual tag
//
// note we always need something after the alts! without something between
// alts we may not be able to find the trunk of our tree
lfs_ssize_t d = lfsr_bd_progtag(lfs, rbyd->block, rbyd->off, lfs_ssize_t d = lfsr_bd_progtag(lfs, rbyd->block, rbyd->off,
lfsr_tag_setnomk(tag), upper_id - lower_id - 1 + delta, lfsr_tag_setnomk(tag), upper_id - lower_id - 1 + delta,
lfsr_data_size(data), lfsr_data_size(data),
@@ -2424,11 +2435,13 @@ leaf:;
rbyd->off += d; rbyd->off += d;
// don't forget the data! // don't forget the data!
err = lfsr_bd_progdata(lfs, rbyd->block, rbyd->off, data, &rbyd->crc); err = lfsr_bd_progdata(lfs, rbyd->block, rbyd->off, data,
&rbyd->crc);
if (err) { if (err) {
goto failed; goto failed;
} }
rbyd->off += lfsr_data_size(data); rbyd->off += lfsr_data_size(data);
}
return 0; return 0;
scripts/dbgbtree.py
+49 -48
View File
@@ -8,7 +8,7 @@ import os
import struct import struct
TAG_UNR = 0x1000 TAG_NULL = 0x0000
TAG_SUPERMAGIC = 0x0003 TAG_SUPERMAGIC = 0x0003
TAG_SUPERCONFIG = 0x0004 TAG_SUPERCONFIG = 0x0004
TAG_MROOT = 0x0304 TAG_MROOT = 0x0304
@@ -23,6 +23,7 @@ TAG_BTREE = 0x0303
TAG_MDIR = 0x0305 TAG_MDIR = 0x0305
TAG_UATTR = 0x0400 TAG_UATTR = 0x0400
TAG_ALT = 0x4000 TAG_ALT = 0x4000
TAG_ALTA = 0x6000
TAG_CRC = 0x2000 TAG_CRC = 0x2000
TAG_FCRC = 0x2100 TAG_FCRC = 0x2100
@@ -110,55 +111,56 @@ def xxd(data, width=16, crc=False):
for b in map(chr, data[i:i+width]))) for b in map(chr, data[i:i+width])))
def tagrepr(tag, w, size, off=None): def tagrepr(tag, w, size, off=None):
if (tag & 0x7fff) == TAG_UNR: if tag == TAG_NULL:
return 'unr%s%s' % ( return 'null%s%s' % (
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if size else '') ' %d' % size if size else '')
elif (tag & 0x6fff) == TAG_SUPERMAGIC: elif tag == TAG_SUPERMAGIC:
return '%ssupermagic%s%s' % ( return 'supermagic%s %d' % (
'rm' if tag & 0x1000 else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '') size)
elif (tag & 0x6fff) == TAG_SUPERCONFIG: elif tag == TAG_SUPERCONFIG:
return '%ssuperconfig%s%s' % ( return 'superconfig%s %d' % (
'rm' if tag & 0x1000 else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '') size)
elif (tag & 0x6f00) == TAG_NAME: elif (tag & 0xff00) == TAG_NAME:
return '%s%s%s%s' % ( return '%s%s %d' % (
'rm' if tag & 0x1000 else '', 'branch' if tag == TAG_BRANCH
'branch' if (tag & 0xfffe) == TAG_BRANCH else 'reg' if tag == TAG_REG
else 'reg' if (tag & 0xfffe) == TAG_REG else 'dir' if tag == TAG_DIR
else 'dir' if (tag & 0xfffe) == TAG_DIR else 'name 0x%02x' % (tag & 0xff),
else 'name 0x%02x' % ((tag & 0x0ff0) >> 4),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '') size)
elif (tag & 0x6f00) == TAG_STRUCT: elif (tag & 0xff00) == TAG_STRUCT:
return '%s%s%s%s' % ( return '%s%s %d' % (
'rm' if tag & 0x1000 else '', 'inlined' if tag == TAG_INLINED
'inlined' if (tag & 0x6fff) == TAG_INLINED else 'block' if tag == TAG_BLOCK
else 'block' if (tag & 0x6fff) == TAG_BLOCK else 'btree' if tag == TAG_BTREE
else 'btree' if (tag & 0x6fff) == TAG_BTREE else 'mdir' if tag == TAG_MROOT
else 'mdir' if (tag & 0x6fff) == TAG_MROOT else 'mdir' if tag == TAG_MDIR
else 'mdir' if (tag & 0x6fff) == TAG_MDIR else 'struct 0x%02x' % (tag & 0xff),
else 'struct 0x%02x' % ((tag & 0x0ff0) >> 4),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '') size)
elif (tag & 0x6f00) == TAG_UATTR: elif (tag & 0xff00) == TAG_UATTR:
return '%suattr 0x%02x%s%s' % ( return 'uattr 0x%02x%s %d' % (
'rm' if tag & 0x1000 else '',
tag & 0xff, tag & 0xff,
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '') size)
elif (tag & 0x7f00) == TAG_CRC: elif (tag & 0xff00) == TAG_CRC:
return 'crc%x%s %d' % ( return 'crc%x%s %d' % (
1 if tag & 0x1 else 0, 1 if tag & 0x1 else 0,
' 0x%x' % w if w > 0 else '', ' 0x%x' % w if w > 0 else '',
size) size)
elif (tag & 0x7fff) == TAG_FCRC: elif tag == TAG_FCRC:
return 'fcrc%s %d' % ( return 'fcrc%s %d' % (
' 0x%x' % w if w > 0 else '', ' 0x%x' % w if w > 0 else '',
size) size)
elif tag == TAG_ALTA:
return 'alta w%d %s' % (
w,
'0x%x' % (0xffffffff & (off-size))
if off is not None
else '-%d' % off)
elif tag & 0x4000: elif tag & 0x4000:
return 'alt%s%s 0x%x w%d %s' % ( return 'alt%s%s 0x%x w%d %s' % (
'r' if tag & 0x1000 else 'b', 'r' if tag & 0x1000 else 'b',
@@ -238,8 +240,8 @@ class Rbyd:
trunk_ = 0 trunk_ = 0
trunk__ = 0 trunk__ = 0
weight = 0 weight = 0
lower_, upper_ = 0, 0
weight_ = 0 weight_ = 0
weight__ = 0
wastrunk = False wastrunk = False
trunkoff = None trunkoff = None
while j_ < len(data) and (not trunk or off <= trunk): while j_ < len(data) and (not trunk or off <= trunk):
@@ -253,7 +255,7 @@ class Rbyd:
# take care of crcs # take care of crcs
if not tag & 0x4000: if not tag & 0x4000:
if (tag & 0x7f00) != TAG_CRC: if (tag & 0xff00) != TAG_CRC:
crc_ = crc32c(data[j_:j_+size], crc_) crc_ = crc32c(data[j_:j_+size], crc_)
# found a crc? # found a crc?
else: else:
@@ -267,23 +269,22 @@ class Rbyd:
weight = weight_ weight = weight_
# evaluate trunks # evaluate trunks
if (tag & 0x6000) != 0x2000 and ( if (tag & 0xe000) != 0x2000 and (
not trunk or trunk >= j_-d or wastrunk): not trunk or trunk >= j_-d or wastrunk):
# new trunk? # new trunk?
if not wastrunk: if not wastrunk:
trunk__ = j_-d
lower_, upper_ = 0, 0
wastrunk = True wastrunk = True
trunk__ = j_-d
weight__ = 0
# keep track of weight # keep track of weight
if tag & 0x4000: weight__ += w
if tag & 0x2000:
upper_ += w # end of trunk?
else: if not tag & 0x4000 or tag == TAG_ALTA:
lower_ += w
else:
weight_ = lower_+upper_+w
wastrunk = False wastrunk = False
# update weight
weight_ = weight__
# keep track of off for best matching trunk # keep track of off for best matching trunk
if trunk and j_ + size > trunk: if trunk and j_ + size > trunk:
trunkoff = j_ + size trunkoff = j_ + size
@@ -349,7 +350,7 @@ class Rbyd:
tag_ = alt tag_ = alt
w_ = id_-lower w_ = id_-lower
done = (id_, tag_) < (id, tag) or tag_ & 0x1000 done = not tag_ or (id_, tag_) < (id, tag)
return done, id_, tag_, w_, j, d, self.data[j+d:j+d+jump], path return done, id_, tag_, w_, j, d, self.data[j+d:j+d+jump], path
scripts/dbgmtree.py
+49 -48
View File
@@ -8,7 +8,7 @@ import os
import struct import struct
TAG_UNR = 0x1000 TAG_NULL = 0x0000
TAG_SUPERMAGIC = 0x0003 TAG_SUPERMAGIC = 0x0003
TAG_SUPERCONFIG = 0x0004 TAG_SUPERCONFIG = 0x0004
TAG_MROOT = 0x0304 TAG_MROOT = 0x0304
@@ -23,6 +23,7 @@ TAG_BTREE = 0x0303
TAG_MDIR = 0x0305 TAG_MDIR = 0x0305
TAG_UATTR = 0x0400 TAG_UATTR = 0x0400
TAG_ALT = 0x4000 TAG_ALT = 0x4000
TAG_ALTA = 0x6000
TAG_CRC = 0x2000 TAG_CRC = 0x2000
TAG_FCRC = 0x2100 TAG_FCRC = 0x2100
@@ -119,55 +120,56 @@ def xxd(data, width=16, crc=False):
for b in map(chr, data[i:i+width]))) for b in map(chr, data[i:i+width])))
def tagrepr(tag, w, size, off=None): def tagrepr(tag, w, size, off=None):
if (tag & 0x7fff) == TAG_UNR: if tag == TAG_NULL:
return 'unr%s%s' % ( return 'null%s%s' % (
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if size else '') ' %d' % size if size else '')
elif (tag & 0x6fff) == TAG_SUPERMAGIC: elif tag == TAG_SUPERMAGIC:
return '%ssupermagic%s%s' % ( return 'supermagic%s %d' % (
'rm' if tag & 0x1000 else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '') size)
elif (tag & 0x6fff) == TAG_SUPERCONFIG: elif tag == TAG_SUPERCONFIG:
return '%ssuperconfig%s%s' % ( return 'superconfig%s %d' % (
'rm' if tag & 0x1000 else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '') size)
elif (tag & 0x6f00) == TAG_NAME: elif (tag & 0xff00) == TAG_NAME:
return '%s%s%s%s' % ( return '%s%s %d' % (
'rm' if tag & 0x1000 else '', 'branch' if tag == TAG_BRANCH
'branch' if (tag & 0xfffe) == TAG_BRANCH else 'reg' if tag == TAG_REG
else 'reg' if (tag & 0xfffe) == TAG_REG else 'dir' if tag == TAG_DIR
else 'dir' if (tag & 0xfffe) == TAG_DIR else 'name 0x%02x' % (tag & 0xff),
else 'name 0x%02x' % ((tag & 0x0ff0) >> 4),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '') size)
elif (tag & 0x6f00) == TAG_STRUCT: elif (tag & 0xff00) == TAG_STRUCT:
return '%s%s%s%s' % ( return '%s%s %d' % (
'rm' if tag & 0x1000 else '', 'inlined' if tag == TAG_INLINED
'inlined' if (tag & 0x6fff) == TAG_INLINED else 'block' if tag == TAG_BLOCK
else 'block' if (tag & 0x6fff) == TAG_BLOCK else 'btree' if tag == TAG_BTREE
else 'btree' if (tag & 0x6fff) == TAG_BTREE else 'mdir' if tag == TAG_MROOT
else 'mdir' if (tag & 0x6fff) == TAG_MROOT else 'mdir' if tag == TAG_MDIR
else 'mdir' if (tag & 0x6fff) == TAG_MDIR else 'struct 0x%02x' % (tag & 0xff),
else 'struct 0x%02x' % ((tag & 0x0ff0) >> 4),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '') size)
elif (tag & 0x6f00) == TAG_UATTR: elif (tag & 0xff00) == TAG_UATTR:
return '%suattr 0x%02x%s%s' % ( return 'uattr 0x%02x%s %d' % (
'rm' if tag & 0x1000 else '',
tag & 0xff, tag & 0xff,
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '') size)
elif (tag & 0x7f00) == TAG_CRC: elif (tag & 0xff00) == TAG_CRC:
return 'crc%x%s %d' % ( return 'crc%x%s %d' % (
1 if tag & 0x1 else 0, 1 if tag & 0x1 else 0,
' 0x%x' % w if w > 0 else '', ' 0x%x' % w if w > 0 else '',
size) size)
elif (tag & 0x7fff) == TAG_FCRC: elif tag == TAG_FCRC:
return 'fcrc%s %d' % ( return 'fcrc%s %d' % (
' 0x%x' % w if w > 0 else '', ' 0x%x' % w if w > 0 else '',
size) size)
elif tag == TAG_ALTA:
return 'alta w%d %s' % (
w,
'0x%x' % (0xffffffff & (off-size))
if off is not None
else '-%d' % off)
elif tag & 0x4000: elif tag & 0x4000:
return 'alt%s%s 0x%x w%d %s' % ( return 'alt%s%s 0x%x w%d %s' % (
'r' if tag & 0x1000 else 'b', 'r' if tag & 0x1000 else 'b',
@@ -247,8 +249,8 @@ class Rbyd:
trunk_ = 0 trunk_ = 0
trunk__ = 0 trunk__ = 0
weight = 0 weight = 0
lower_, upper_ = 0, 0
weight_ = 0 weight_ = 0
weight__ = 0
wastrunk = False wastrunk = False
trunkoff = None trunkoff = None
while j_ < len(data) and (not trunk or off <= trunk): while j_ < len(data) and (not trunk or off <= trunk):
@@ -262,7 +264,7 @@ class Rbyd:
# take care of crcs # take care of crcs
if not tag & 0x4000: if not tag & 0x4000:
if (tag & 0x7f00) != TAG_CRC: if (tag & 0xff00) != TAG_CRC:
crc_ = crc32c(data[j_:j_+size], crc_) crc_ = crc32c(data[j_:j_+size], crc_)
# found a crc? # found a crc?
else: else:
@@ -276,23 +278,22 @@ class Rbyd:
weight = weight_ weight = weight_
# evaluate trunks # evaluate trunks
if (tag & 0x6000) != 0x2000 and ( if (tag & 0xe000) != 0x2000 and (
not trunk or trunk >= j_-d or wastrunk): not trunk or trunk >= j_-d or wastrunk):
# new trunk? # new trunk?
if not wastrunk: if not wastrunk:
trunk__ = j_-d
lower_, upper_ = 0, 0
wastrunk = True wastrunk = True
trunk__ = j_-d
weight__ = 0
# keep track of weight # keep track of weight
if tag & 0x4000: weight__ += w
if tag & 0x2000:
upper_ += w # end of trunk?
else: if not tag & 0x4000 or tag == TAG_ALTA:
lower_ += w
else:
weight_ = lower_+upper_+w
wastrunk = False wastrunk = False
# update weight
weight_ = weight__
# keep track of off for best matching trunk # keep track of off for best matching trunk
if trunk and j_ + size > trunk: if trunk and j_ + size > trunk:
trunkoff = j_ + size trunkoff = j_ + size
@@ -358,7 +359,7 @@ class Rbyd:
tag_ = alt tag_ = alt
w_ = id_-lower w_ = id_-lower
done = (id_, tag_) < (id, tag) or tag_ & 0x1000 done = not tag_ or (id_, tag_) < (id, tag)
return done, id_, tag_, w_, j, d, self.data[j+d:j+d+jump], path return done, id_, tag_, w_, j, d, self.data[j+d:j+d+jump], path
scripts/dbgrbyd.py
+88 -78
View File
@@ -17,7 +17,7 @@ COLORS = [
] ]
TAG_UNR = 0x1000 TAG_NULL = 0x0000
TAG_SUPERMAGIC = 0x0003 TAG_SUPERMAGIC = 0x0003
TAG_SUPERCONFIG = 0x0004 TAG_SUPERCONFIG = 0x0004
TAG_MROOT = 0x0304 TAG_MROOT = 0x0304
@@ -32,9 +32,11 @@ TAG_BTREE = 0x0303
TAG_MDIR = 0x0305 TAG_MDIR = 0x0305
TAG_UATTR = 0x0400 TAG_UATTR = 0x0400
TAG_ALT = 0x4000 TAG_ALT = 0x4000
TAG_ALTA = 0x6000
TAG_CRC = 0x2000 TAG_CRC = 0x2000
TAG_FCRC = 0x2100 TAG_FCRC = 0x2100
# parse some rbyd addr encodings # parse some rbyd addr encodings
# 0xa -> [0xa] # 0xa -> [0xa]
# 0xa.b -> ([0xa], b) # 0xa.b -> ([0xa], b)
@@ -111,55 +113,56 @@ def xxd(data, width=16, crc=False):
for b in map(chr, data[i:i+width]))) for b in map(chr, data[i:i+width])))
def tagrepr(tag, w, size, off=None): def tagrepr(tag, w, size, off=None):
if (tag & 0x7fff) == TAG_UNR: if tag == TAG_NULL:
return 'unr%s%s' % ( return 'null%s%s' % (
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if size else '') ' %d' % size if size else '')
elif (tag & 0x6fff) == TAG_SUPERMAGIC: elif tag == TAG_SUPERMAGIC:
return '%ssupermagic%s%s' % ( return 'supermagic%s %d' % (
'rm' if tag & 0x1000 else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '') size)
elif (tag & 0x6fff) == TAG_SUPERCONFIG: elif tag == TAG_SUPERCONFIG:
return '%ssuperconfig%s%s' % ( return 'superconfig%s %d' % (
'rm' if tag & 0x1000 else '',
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '') size)
elif (tag & 0x6f00) == TAG_NAME: elif (tag & 0xff00) == TAG_NAME:
return '%s%s%s%s' % ( return '%s%s %d' % (
'rm' if tag & 0x1000 else '', 'branch' if tag == TAG_BRANCH
'branch' if (tag & 0xfffe) == TAG_BRANCH else 'reg' if tag == TAG_REG
else 'reg' if (tag & 0xfffe) == TAG_REG else 'dir' if tag == TAG_DIR
else 'dir' if (tag & 0xfffe) == TAG_DIR else 'name 0x%02x' % (tag & 0xff),
else 'name 0x%02x' % ((tag & 0x0ff0) >> 4),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '') size)
elif (tag & 0x6f00) == TAG_STRUCT: elif (tag & 0xff00) == TAG_STRUCT:
return '%s%s%s%s' % ( return '%s%s %d' % (
'rm' if tag & 0x1000 else '', 'inlined' if tag == TAG_INLINED
'inlined' if (tag & 0x6fff) == TAG_INLINED else 'block' if tag == TAG_BLOCK
else 'block' if (tag & 0x6fff) == TAG_BLOCK else 'btree' if tag == TAG_BTREE
else 'btree' if (tag & 0x6fff) == TAG_BTREE else 'mdir' if tag == TAG_MROOT
else 'mdir' if (tag & 0x6fff) == TAG_MROOT else 'mdir' if tag == TAG_MDIR
else 'mdir' if (tag & 0x6fff) == TAG_MDIR else 'struct 0x%02x' % (tag & 0xff),
else 'struct 0x%02x' % ((tag & 0x0ff0) >> 4),
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '') size)
elif (tag & 0x6f00) == TAG_UATTR: elif (tag & 0xff00) == TAG_UATTR:
return '%suattr 0x%02x%s%s' % ( return 'uattr 0x%02x%s %d' % (
'rm' if tag & 0x1000 else '',
tag & 0xff, tag & 0xff,
' w%d' % w if w else '', ' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '') size)
elif (tag & 0x7f00) == TAG_CRC: elif (tag & 0xff00) == TAG_CRC:
return 'crc%x%s %d' % ( return 'crc%x%s %d' % (
1 if tag & 0x1 else 0, 1 if tag & 0x1 else 0,
' 0x%x' % w if w > 0 else '', ' 0x%x' % w if w > 0 else '',
size) size)
elif (tag & 0x7fff) == TAG_FCRC: elif tag == TAG_FCRC:
return 'fcrc%s %d' % ( return 'fcrc%s %d' % (
' 0x%x' % w if w > 0 else '', ' 0x%x' % w if w > 0 else '',
size) size)
elif tag == TAG_ALTA:
return 'alta w%d %s' % (
w,
'0x%x' % (0xffffffff & (off-size))
if off is not None
else '-%d' % off)
elif tag & 0x4000: elif tag & 0x4000:
return 'alt%s%s 0x%x w%d %s' % ( return 'alt%s%s 0x%x w%d %s' % (
'r' if tag & 0x1000 else 'b', 'r' if tag & 0x1000 else 'b',
@@ -172,6 +175,7 @@ def tagrepr(tag, w, size, off=None):
else: else:
return '0x%04x w%d %d' % (tag, w, size) return '0x%04x w%d %d' % (tag, w, size)
def dbg_log(data, block_size, rev, off, weight, *, def dbg_log(data, block_size, rev, off, weight, *,
color=False, color=False,
**args): **args):
@@ -285,22 +289,26 @@ def dbg_log(data, block_size, rev, off, weight, *,
if not tag & 0x4000: if not tag & 0x4000:
j_ += size j_ += size
# find trunk # evaluate trunks
if not wastrunk and (tag & 0x6000) != 0x2000: if (tag & 0xe000) != 0x2000:
if not wastrunk:
wastrunk = True
lower_, upper_ = 0, 0 lower_, upper_ = 0, 0
wastrunk = not not tag & 0x4000
# keep track of weight if (tag & 0xe000) == 0x4000:
if tag & 0x4000:
if tag & 0x2000:
upper_ += w
else:
lower_ += w lower_ += w
elif (tag & 0x6000) == 0x2000: else:
delta = (lower_+upper_+w) - weight_ upper_ += w
weight_ = lower_+upper_+w
id = lower_+w-1
if not tag & 0x4000 or tag == TAG_ALTA:
wastrunk = False
# derive the current tag's id from alt weights
delta = (lower_+upper_) - weight_
weight_ = lower_+upper_
id = lower_ + w-1
if ((tag & 0xe000) != 0x2000
and (not tag & 0x4000 or tag == TAG_ALTA)):
# note we ignore out-of-bounds here for debugging # note we ignore out-of-bounds here for debugging
if delta > 0: if delta > 0:
# grow lifetimes # grow lifetimes
@@ -339,7 +347,7 @@ def dbg_log(data, block_size, rev, off, weight, *,
weights = weights_ weights = weights_
lifetimes = lifetimes_ lifetimes = lifetimes_
if not tag & 0x1000 and id >= 0: if id >= 0:
# attach tag to lifetime # attach tag to lifetime
i, id_ = index(weights, id) i, id_ = index(weights, id)
if i < len(weights): if i < len(weights):
@@ -427,23 +435,9 @@ def dbg_log(data, block_size, rev, off, weight, *,
crc = crc32c(data[j_:j_+d], crc) crc = crc32c(data[j_:j_+d], crc)
j_ += d j_ += d
# find trunk # take care of crcs
if not wastrunk and (tag & 0x6000) != 0x2000:
lower_, upper_ = 0, 0
wastrunk = not not tag & 0x4000
# calculate id from alt weights
if tag & 0x4000:
if tag & 0x2000:
upper_ += w
else:
lower_ += w
elif (tag & 0x6000) == 0x0:
weight_ = lower_+upper_+w
id = lower_+w-1
if not tag & 0x4000: if not tag & 0x4000:
if (tag & 0x7f00) != TAG_CRC: if (tag & 0xff00) != TAG_CRC:
crc = crc32c(data[j_:j_+size], crc) crc = crc32c(data[j_:j_+size], crc)
# found a crc? # found a crc?
else: else:
@@ -452,6 +446,22 @@ def dbg_log(data, block_size, rev, off, weight, *,
notes.append('crc!=%08x' % crc) notes.append('crc!=%08x' % crc)
j_ += size j_ += size
# evaluate trunks
if (tag & 0xe000) != 0x2000:
if not wastrunk:
wastrunk = True
lower_, upper_ = 0, 0
if (tag & 0xe000) == 0x4000:
lower_ += w
else:
upper_ += w
if not tag & 0x4000 or tag == TAG_ALTA:
wastrunk = False
# derive the current tag's id from alt weights
id = lower_ + w-1
# show human-readable tag representation # show human-readable tag representation
print('%s%08x:%s %*s%s%*s %-57s%s%s' % ( print('%s%08x:%s %*s%s%*s %-57s%s%s' % (
'\x1b[90m' if color and j >= off else '', '\x1b[90m' if color and j >= off else '',
@@ -459,7 +469,8 @@ def dbg_log(data, block_size, rev, off, weight, *,
'\x1b[m' if color and j >= off else '', '\x1b[m' if color and j >= off else '',
lifetime_width, lifetimerepr(j) if args.get('lifetimes') else '', lifetime_width, lifetimerepr(j) if args.get('lifetimes') else '',
'\x1b[90m' if color and j >= off else '', '\x1b[90m' if color and j >= off else '',
w_width, '' if (tag & 0x6000) != 0x0 w_width, '-' if tag == TAG_ALTA
else '' if (tag & 0xe000) != 0x0000
else '%d-%d' % (id-(w-1), id) if w > 1 else '%d-%d' % (id-(w-1), id) if w > 1
else id, else id,
'%-22s%s' % ( '%-22s%s' % (
@@ -576,7 +587,7 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
tag_ = alt tag_ = alt
w_ = id_-lower w_ = id_-lower
done = (id_, tag_) < (id, tag) or tag_ & 0x1000 done = not tag_ or (id_, tag_) < (id, tag)
return done, id_, tag_, w_, j, d, jump, path return done, id_, tag_, w_, j, d, jump, path
@@ -835,8 +846,8 @@ def main(disk, blocks=None, *,
trunk_ = 0 trunk_ = 0
trunk__ = 0 trunk__ = 0
weight = 0 weight = 0
lower_, upper_ = 0, 0
weight_ = 0 weight_ = 0
weight__ = 0
wastrunk = False wastrunk = False
trunkoff = None trunkoff = None
while j_ < len(data) and (not trunk or off <= trunk): while j_ < len(data) and (not trunk or off <= trunk):
@@ -850,7 +861,7 @@ def main(disk, blocks=None, *,
# take care of crcs # take care of crcs
if not tag & 0x4000: if not tag & 0x4000:
if (tag & 0x7f00) != TAG_CRC: if (tag & 0xff00) != TAG_CRC:
crc_ = crc32c(data[j_:j_+size], crc_) crc_ = crc32c(data[j_:j_+size], crc_)
# found a crc? # found a crc?
else: else:
@@ -864,23 +875,22 @@ def main(disk, blocks=None, *,
weight = weight_ weight = weight_
# evaluate trunks # evaluate trunks
if (tag & 0x6000) != 0x2000 and ( if (tag & 0xe000) != 0x2000 and (
not trunk or trunk >= j_-d or wastrunk): not trunk or trunk >= j_-d or wastrunk):
# new trunk? # new trunk?
if not wastrunk: if not wastrunk:
trunk__ = j_-d
lower_, upper_ = 0, 0
wastrunk = True wastrunk = True
trunk__ = j_-d
weight__ = 0
# keep track of weight # keep track of weight
if tag & 0x4000: weight__ += w
if tag & 0x2000:
upper_ += w # end of trunk?
else: if not tag & 0x4000 or tag == TAG_ALTA:
lower_ += w
else:
weight_ = lower_+upper_+w
wastrunk = False wastrunk = False
# update weight
weight_ = weight__
# keep track of off for best matching trunk # keep track of off for best matching trunk
if trunk and j_ + size > trunk: if trunk and j_ + size > trunk:
trunkoff = j_ + size trunkoff = j_ + size