Github/littlefs
1
Fork 0
Code Actions Activity

Allowed "alta" (altbgt 0) to terminate rbyd trunks, dropped rm bit

Browse Source 
This replaces unr with null on disk, though note both the rm bit and unr
are used in-device still, they just don't get written to disk.

This removes the need for the rm bit on disk. Since we no longer need to
figure out what's been removed during fetch, we can save this bit for both
internal and future on-disk use.

Special handling of alta allows us to avoid emitting an unr tag (now null) if
the current trunk is truly unreachable. This is minor now, but important
for a theoretical rbyd rebalance operation (planned), which brings the
rbyd overhead down from ~3x to ~2x.

These changes give us two ways to terminate trunks without a tag:

1. With an alta, if the current trunk is unreachable:

     altbgt 0x403 w0 0x7b
     altbgt 0x402 w0 0x29
     alta w0 0x4

2. With a null, if the current trunk is reachable, either for
   code convenience or because emitting an alta is impossible (an empty
   rbyd for example):

     altbgt 0x403 w0 0x7b
     altbgt 0x402 w0 0x29
     altbgt 0x401 w0 0x4
     null
This commit is contained in:
Christopher Haster
2023-06-17 18:11:45 -05:00
parent aa559d30b0
commit 7180b70c9c
4 changed files with 275 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lfs.c
+88 -75
View File
@@ -584,7 +584,8 @@ static int lfsr_bd_erase(lfs_t *lfs, lfs_block_t block) {
// 16-bit metadata tags
enum lfsr_tag_type {
LFSR_TAG_UNR = 0x1000,
LFSR_TAG_NULL = 0x0000,
LFSR_TAG_UNR = 0x1000, // in-device only
LFSR_TAG_MKUNR = 0x3000, // in-device only
LFSR_TAG_SUPERMAGIC = 0x0003,
@@ -618,6 +619,7 @@ enum lfsr_tag_type {
LFSR_TAG_ALTRLE = 0x5000,
LFSR_TAG_ALTBGT = 0x6000,
LFSR_TAG_ALTRGT = 0x7000,
LFSR_TAG_ALTA = 0x6000,
LFSR_TAG_CRC = 0x2000,
LFSR_TAG_FCRC = 0x2100,
@@ -691,12 +693,16 @@ static inline lfsr_tag_t lfsr_tag_setrm(lfsr_tag_t tag) {
return tag | 0x1000;
}
static inline bool lfsr_tag_istrunk(lfsr_tag_t tag) {
static inline bool lfsr_tag_isalt(lfsr_tag_t tag) {
return tag & 0x4000;
}
static inline bool lfsr_tag_istrunkstart(lfsr_tag_t tag) {
return (tag & 0x6000) != 0x2000;
}
static inline bool lfsr_tag_isalt(lfsr_tag_t tag) {
return tag & 0x4000;
static inline bool lfsr_tag_istrunkend(lfsr_tag_t tag) {
return !lfsr_tag_isalt(tag) || tag == LFSR_TAG_ALTA;
}
static inline lfsr_tag_t lfsr_tag_next(lfsr_tag_t tag) {
@@ -704,24 +710,33 @@ static inline lfsr_tag_t lfsr_tag_next(lfsr_tag_t tag) {
}
// lfsr_rbyd_append specific flags
static inline bool lfsr_tag_isupper(lfsr_tag_t tag) {
static inline bool lfsr_tag_hasdiverged(lfsr_tag_t tag) {
return tag & 0x2000;
}
static inline bool lfsr_tag_islower(lfsr_tag_t tag) {
return !lfsr_tag_isupper(tag);
static inline bool lfsr_tag_isdivergedupper(lfsr_tag_t tag) {
return lfsr_tag_hasdiverged(tag) && (tag & 0x1000);
}
static inline lfsr_tag_t lfsr_tag_setupper(lfsr_tag_t tag) {
return tag | 0x2000;
}
static inline bool lfsr_tag_hasdiverged(lfsr_tag_t tag) {
return tag & 0x4000;
static inline bool lfsr_tag_isdivergedlower(lfsr_tag_t tag) {
return lfsr_tag_hasdiverged(tag) && !(tag & 0x1000);
}
static inline lfsr_tag_t lfsr_tag_setdiverged(lfsr_tag_t tag) {
return tag | 0x4000;
return tag | 0x2000;
}
static inline lfsr_tag_t lfsr_tag_setdivergedvalid(
lfsr_tag_t tag, lfsr_tag_t tag_) {
return (tag & 0x3000) | tag_;
}
static inline lfsr_tag_t lfsr_tag_setdivergedupper(lfsr_tag_t tag) {
return tag | 0x3000;
}
static inline lfsr_tag_t lfsr_tag_cleardiverged(lfsr_tag_t tag) {
return tag & ~0x3000;
}
// alt operations
@@ -1586,9 +1601,8 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd,
lfs_off_t off = sizeof(uint32_t);
lfs_off_t trunk_ = 0;
bool wastrunk = false;
lfs_size_t lower = 0;
lfs_size_t upper = 0;
lfs_size_t weight = 0;
lfs_size_t weight_ = 0;
// assume unerased until proven otherwise
lfsr_fcrc_t fcrc;
@@ -1687,35 +1701,30 @@ static int lfsr_rbyd_fetch(lfs_t *lfs, lfsr_rbyd_t *rbyd,
}
// found a trunk of a tree?
if (lfsr_tag_istrunk(tag) && (!trunk || trunk >= off-d || wastrunk)) {
if (lfsr_tag_istrunkstart(tag)
&& (!trunk || trunk >= off-d || wastrunk)) {
// start of trunk?
if (!wastrunk) {
wastrunk = true;
// save trunk entry point
trunk_ = off-d;
// reset weights
lower = 0;
upper = 0;
wastrunk = true;
// reset weight
weight_ = 0;
}
// derive the new weight of the tree from alt pointers
// derive weight of the tree from alt pointers
//
// NOTE we can't check for overflow/underflow here because we
// may be overeagerly parsing an invalid commit, it's ok for
// this to overflow/underflow as long as we throw it out later
// on a bad crc
if (lfsr_tag_isalt(tag)) {
if (lfsr_tag_isgt(tag)) {
upper += w;
} else {
lower += w;
}
weight_ += w;
} else {
// update current weight
weight = lower+upper+w;
// any non-alt terminates the current trunk
// end of trunk?
if (lfsr_tag_istrunkend(tag)) {
wastrunk = false;
// update current weight
weight = weight_;
}
}
@@ -1801,9 +1810,10 @@ static int lfsr_rbyd_lookupnext(lfs_t *lfs, const lfsr_rbyd_t *rbyd,
lfsr_tag_t tag__ = alt;
// not what we're looking for?
if (id__ < id
|| (id__ == id && lfsr_tag_key(tag__) < lfsr_tag_key(tag))
|| lfsr_tag_isrm(tag__)) {
if (!tag__
|| id__ < id
|| (id__ == id && lfsr_tag_key(tag__)
< lfsr_tag_key(tag))) {
return LFS_ERR_NOENT;
}
@@ -2043,7 +2053,7 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
}
// mark as invalid until found
tag_ = lfsr_tag_setinvalid(tag_);
other_tag_ = lfsr_tag_setinvalid(lfsr_tag_setupper(other_tag_));
other_tag_ = lfsr_tag_setinvalid(lfsr_tag_setdivergedupper(other_tag_));
// keep track of bounds as we descend down the tree
//
@@ -2284,7 +2294,7 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
// prune inner alts if our tags diverged
if (lfsr_tag_hasdiverged(tag_)
&& lfsr_tag_isupper(tag_) != lfsr_tag_isgt(alt)) {
&& lfsr_tag_isdivergedupper(tag_) != lfsr_tag_isgt(alt)) {
continue;
}
@@ -2301,10 +2311,9 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
// update the found tag/id
//
// note we:
// - clear valid bit (0x8000)
// - preserve diverged bit (0x4000)
// - preserve isupper tag (0x2000)
tag_ = lfsr_tag_setvalid(alt | (tag_ & 0x6000));
// - clear valid bit, marking the tag as found
// - preserve diverged state
tag_ = lfsr_tag_setdivergedvalid(tag_, alt);
id_ = upper_id-1;
// done?
@@ -2333,14 +2342,15 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
LFS_ASSERT(lfsr_tag_isvalid(tag_));
LFS_ASSERT(!lfsr_tag_hasdiverged(tag_)
|| lfsr_tag_isvalid(other_tag_));
if (lfsr_tag_hasdiverged(tag_) && lfsr_tag_islower(tag_)) {
if (lfsr_tag_isdivergedlower(tag_)) {
// finished on lower path
tag_ = other_tag_;
tag_ = lfsr_tag_cleardiverged(other_tag_);
id_ = other_id_;
branch = other_branch;
upper_id = other_upper_id;
} else if (lfsr_tag_hasdiverged(tag_) && lfsr_tag_isupper(tag_)) {
} else if (lfsr_tag_isdivergedupper(tag_)) {
// finished on upper path
tag_ = lfsr_tag_cleardiverged(tag_);
lower_id = other_lower_id;
}
@@ -2350,17 +2360,14 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
// always finds the next biggest tag
lfsr_tag_t alt = 0;
lfs_size_t weight = 0;
if (lfsr_tag_isrm(tag_)) {
// found an old removed tag, no split needed, just prune the
// removed tag
} else if (id_ < id-lfs_smax32(-delta, 0)
// note if tag_ is null, we found a removed tag that we should just prune
if (tag_ && (id_ < id-lfs_smax32(-delta, 0)
|| (id_ == id-lfs_smax32(-delta, 0)
&& ((lfsr_tag_ismk(tag) && delta > 0)
|| lfsr_tag_key(tag_) < lfsr_tag_key(tag)))) {
|| lfsr_tag_key(tag_) < lfsr_tag_key(tag))))) {
if (lfsr_tag_isrm(tag)) {
// if removed make our tag unreachable
alt = LFSR_TAG_ALT(B, GT, 0);
// if removed, make our tag unreachable
alt = LFSR_TAG_ALTA;
weight = upper_id - lower_id - 1 + delta;
upper_id -= weight;
} else {
@@ -2370,13 +2377,13 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
lower_id += weight;
}
} else if (id_ > id
} else if (tag_ && (id_ > id
|| (id_ == id
&& ((lfsr_tag_ismk(tag) && delta > 0)
|| lfsr_tag_key(tag_) > lfsr_tag_key(tag)))) {
|| lfsr_tag_key(tag_) > lfsr_tag_key(tag))))) {
if (lfsr_tag_isrm(tag)) {
// if removed make our tag unreachable
alt = LFSR_TAG_ALT(B, GT, 0);
// if removed, make our tag unreachable
alt = LFSR_TAG_ALTA;
weight = upper_id - lower_id - 1 + delta;
upper_id -= weight;
} else {
@@ -2385,6 +2392,12 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
weight = upper_id - id - 1;
upper_id -= weight;
}
} else {
if (lfsr_tag_isrm(tag)) {
// if removed, replace our tag with a null tag
tag = LFSR_TAG_NULL;
}
}
if (alt) {
@@ -2409,26 +2422,26 @@ static int lfsr_rbyd_append(lfs_t *lfs, lfsr_rbyd_t *rbyd,
}
leaf:;
// write the actual tag
//
// note we always need something after the alts! without something between
// alts we may not be able to find the trunk of our tree
lfs_ssize_t d = lfsr_bd_progtag(lfs, rbyd->block, rbyd->off,
lfsr_tag_setnomk(tag), upper_id - lower_id - 1 + delta,
lfsr_data_size(data),
&rbyd->crc);
if (d < 0) {
err = d;
goto failed;
}
rbyd->off += d;
if (!lfsr_tag_isrm(tag)) {
// write the actual tag
lfs_ssize_t d = lfsr_bd_progtag(lfs, rbyd->block, rbyd->off,
lfsr_tag_setnomk(tag), upper_id - lower_id - 1 + delta,
lfsr_data_size(data),
&rbyd->crc);
if (d < 0) {
err = d;
goto failed;
}
rbyd->off += d;
// don't forget the data!
err = lfsr_bd_progdata(lfs, rbyd->block, rbyd->off, data, &rbyd->crc);
if (err) {
goto failed;
// don't forget the data!
err = lfsr_bd_progdata(lfs, rbyd->block, rbyd->off, data,
&rbyd->crc);
if (err) {
goto failed;
}
rbyd->off += lfsr_data_size(data);
}
rbyd->off += lfsr_data_size(data);
return 0;
scripts/dbgbtree.py
+49 -48
View File
@@ -8,7 +8,7 @@ import os
import struct
TAG_UNR = 0x1000
TAG_NULL = 0x0000
TAG_SUPERMAGIC = 0x0003
TAG_SUPERCONFIG = 0x0004
TAG_MROOT = 0x0304
@@ -23,6 +23,7 @@ TAG_BTREE = 0x0303
TAG_MDIR = 0x0305
TAG_UATTR = 0x0400
TAG_ALT = 0x4000
TAG_ALTA = 0x6000
TAG_CRC = 0x2000
TAG_FCRC = 0x2100
@@ -110,55 +111,56 @@ def xxd(data, width=16, crc=False):
for b in map(chr, data[i:i+width])))
def tagrepr(tag, w, size, off=None):
if (tag & 0x7fff) == TAG_UNR:
return 'unr%s%s' % (
if tag == TAG_NULL:
return 'null%s%s' % (
' w%d' % w if w else '',
' %d' % size if size else '')
elif (tag & 0x6fff) == TAG_SUPERMAGIC:
return '%ssupermagic%s%s' % (
'rm' if tag & 0x1000 else '',
elif tag == TAG_SUPERMAGIC:
return 'supermagic%s %d' % (
' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6fff) == TAG_SUPERCONFIG:
return '%ssuperconfig%s%s' % (
'rm' if tag & 0x1000 else '',
size)
elif tag == TAG_SUPERCONFIG:
return 'superconfig%s %d' % (
' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_NAME:
return '%s%s%s%s' % (
'rm' if tag & 0x1000 else '',
'branch' if (tag & 0xfffe) == TAG_BRANCH
else 'reg' if (tag & 0xfffe) == TAG_REG
else 'dir' if (tag & 0xfffe) == TAG_DIR
else 'name 0x%02x' % ((tag & 0x0ff0) >> 4),
size)
elif (tag & 0xff00) == TAG_NAME:
return '%s%s %d' % (
'branch' if tag == TAG_BRANCH
else 'reg' if tag == TAG_REG
else 'dir' if tag == TAG_DIR
else 'name 0x%02x' % (tag & 0xff),
' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_STRUCT:
return '%s%s%s%s' % (
'rm' if tag & 0x1000 else '',
'inlined' if (tag & 0x6fff) == TAG_INLINED
else 'block' if (tag & 0x6fff) == TAG_BLOCK
else 'btree' if (tag & 0x6fff) == TAG_BTREE
else 'mdir' if (tag & 0x6fff) == TAG_MROOT
else 'mdir' if (tag & 0x6fff) == TAG_MDIR
else 'struct 0x%02x' % ((tag & 0x0ff0) >> 4),
size)
elif (tag & 0xff00) == TAG_STRUCT:
return '%s%s %d' % (
'inlined' if tag == TAG_INLINED
else 'block' if tag == TAG_BLOCK
else 'btree' if tag == TAG_BTREE
else 'mdir' if tag == TAG_MROOT
else 'mdir' if tag == TAG_MDIR
else 'struct 0x%02x' % (tag & 0xff),
' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_UATTR:
return '%suattr 0x%02x%s%s' % (
'rm' if tag & 0x1000 else '',
size)
elif (tag & 0xff00) == TAG_UATTR:
return 'uattr 0x%02x%s %d' % (
tag & 0xff,
' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x7f00) == TAG_CRC:
size)
elif (tag & 0xff00) == TAG_CRC:
return 'crc%x%s %d' % (
1 if tag & 0x1 else 0,
' 0x%x' % w if w > 0 else '',
size)
elif (tag & 0x7fff) == TAG_FCRC:
elif tag == TAG_FCRC:
return 'fcrc%s %d' % (
' 0x%x' % w if w > 0 else '',
size)
elif tag == TAG_ALTA:
return 'alta w%d %s' % (
w,
'0x%x' % (0xffffffff & (off-size))
if off is not None
else '-%d' % off)
elif tag & 0x4000:
return 'alt%s%s 0x%x w%d %s' % (
'r' if tag & 0x1000 else 'b',
@@ -238,8 +240,8 @@ class Rbyd:
trunk_ = 0
trunk__ = 0
weight = 0
lower_, upper_ = 0, 0
weight_ = 0
weight__ = 0
wastrunk = False
trunkoff = None
while j_ < len(data) and (not trunk or off <= trunk):
@@ -253,7 +255,7 @@ class Rbyd:
# take care of crcs
if not tag & 0x4000:
if (tag & 0x7f00) != TAG_CRC:
if (tag & 0xff00) != TAG_CRC:
crc_ = crc32c(data[j_:j_+size], crc_)
# found a crc?
else:
@@ -267,23 +269,22 @@ class Rbyd:
weight = weight_
# evaluate trunks
if (tag & 0x6000) != 0x2000 and (
if (tag & 0xe000) != 0x2000 and (
not trunk or trunk >= j_-d or wastrunk):
# new trunk?
if not wastrunk:
trunk__ = j_-d
lower_, upper_ = 0, 0
wastrunk = True
trunk__ = j_-d
weight__ = 0
# keep track of weight
if tag & 0x4000:
if tag & 0x2000:
upper_ += w
else:
lower_ += w
else:
weight_ = lower_+upper_+w
weight__ += w
# end of trunk?
if not tag & 0x4000 or tag == TAG_ALTA:
wastrunk = False
# update weight
weight_ = weight__
# keep track of off for best matching trunk
if trunk and j_ + size > trunk:
trunkoff = j_ + size
@@ -349,7 +350,7 @@ class Rbyd:
tag_ = alt
w_ = id_-lower
done = (id_, tag_) < (id, tag) or tag_ & 0x1000
done = not tag_ or (id_, tag_) < (id, tag)
return done, id_, tag_, w_, j, d, self.data[j+d:j+d+jump], path
scripts/dbgmtree.py
+49 -48
View File
@@ -8,7 +8,7 @@ import os
import struct
TAG_UNR = 0x1000
TAG_NULL = 0x0000
TAG_SUPERMAGIC = 0x0003
TAG_SUPERCONFIG = 0x0004
TAG_MROOT = 0x0304
@@ -23,6 +23,7 @@ TAG_BTREE = 0x0303
TAG_MDIR = 0x0305
TAG_UATTR = 0x0400
TAG_ALT = 0x4000
TAG_ALTA = 0x6000
TAG_CRC = 0x2000
TAG_FCRC = 0x2100
@@ -119,55 +120,56 @@ def xxd(data, width=16, crc=False):
for b in map(chr, data[i:i+width])))
def tagrepr(tag, w, size, off=None):
if (tag & 0x7fff) == TAG_UNR:
return 'unr%s%s' % (
if tag == TAG_NULL:
return 'null%s%s' % (
' w%d' % w if w else '',
' %d' % size if size else '')
elif (tag & 0x6fff) == TAG_SUPERMAGIC:
return '%ssupermagic%s%s' % (
'rm' if tag & 0x1000 else '',
elif tag == TAG_SUPERMAGIC:
return 'supermagic%s %d' % (
' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6fff) == TAG_SUPERCONFIG:
return '%ssuperconfig%s%s' % (
'rm' if tag & 0x1000 else '',
size)
elif tag == TAG_SUPERCONFIG:
return 'superconfig%s %d' % (
' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_NAME:
return '%s%s%s%s' % (
'rm' if tag & 0x1000 else '',
'branch' if (tag & 0xfffe) == TAG_BRANCH
else 'reg' if (tag & 0xfffe) == TAG_REG
else 'dir' if (tag & 0xfffe) == TAG_DIR
else 'name 0x%02x' % ((tag & 0x0ff0) >> 4),
size)
elif (tag & 0xff00) == TAG_NAME:
return '%s%s %d' % (
'branch' if tag == TAG_BRANCH
else 'reg' if tag == TAG_REG
else 'dir' if tag == TAG_DIR
else 'name 0x%02x' % (tag & 0xff),
' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_STRUCT:
return '%s%s%s%s' % (
'rm' if tag & 0x1000 else '',
'inlined' if (tag & 0x6fff) == TAG_INLINED
else 'block' if (tag & 0x6fff) == TAG_BLOCK
else 'btree' if (tag & 0x6fff) == TAG_BTREE
else 'mdir' if (tag & 0x6fff) == TAG_MROOT
else 'mdir' if (tag & 0x6fff) == TAG_MDIR
else 'struct 0x%02x' % ((tag & 0x0ff0) >> 4),
size)
elif (tag & 0xff00) == TAG_STRUCT:
return '%s%s %d' % (
'inlined' if tag == TAG_INLINED
else 'block' if tag == TAG_BLOCK
else 'btree' if tag == TAG_BTREE
else 'mdir' if tag == TAG_MROOT
else 'mdir' if tag == TAG_MDIR
else 'struct 0x%02x' % (tag & 0xff),
' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_UATTR:
return '%suattr 0x%02x%s%s' % (
'rm' if tag & 0x1000 else '',
size)
elif (tag & 0xff00) == TAG_UATTR:
return 'uattr 0x%02x%s %d' % (
tag & 0xff,
' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x7f00) == TAG_CRC:
size)
elif (tag & 0xff00) == TAG_CRC:
return 'crc%x%s %d' % (
1 if tag & 0x1 else 0,
' 0x%x' % w if w > 0 else '',
size)
elif (tag & 0x7fff) == TAG_FCRC:
elif tag == TAG_FCRC:
return 'fcrc%s %d' % (
' 0x%x' % w if w > 0 else '',
size)
elif tag == TAG_ALTA:
return 'alta w%d %s' % (
w,
'0x%x' % (0xffffffff & (off-size))
if off is not None
else '-%d' % off)
elif tag & 0x4000:
return 'alt%s%s 0x%x w%d %s' % (
'r' if tag & 0x1000 else 'b',
@@ -247,8 +249,8 @@ class Rbyd:
trunk_ = 0
trunk__ = 0
weight = 0
lower_, upper_ = 0, 0
weight_ = 0
weight__ = 0
wastrunk = False
trunkoff = None
while j_ < len(data) and (not trunk or off <= trunk):
@@ -262,7 +264,7 @@ class Rbyd:
# take care of crcs
if not tag & 0x4000:
if (tag & 0x7f00) != TAG_CRC:
if (tag & 0xff00) != TAG_CRC:
crc_ = crc32c(data[j_:j_+size], crc_)
# found a crc?
else:
@@ -276,23 +278,22 @@ class Rbyd:
weight = weight_
# evaluate trunks
if (tag & 0x6000) != 0x2000 and (
if (tag & 0xe000) != 0x2000 and (
not trunk or trunk >= j_-d or wastrunk):
# new trunk?
if not wastrunk:
trunk__ = j_-d
lower_, upper_ = 0, 0
wastrunk = True
trunk__ = j_-d
weight__ = 0
# keep track of weight
if tag & 0x4000:
if tag & 0x2000:
upper_ += w
else:
lower_ += w
else:
weight_ = lower_+upper_+w
weight__ += w
# end of trunk?
if not tag & 0x4000 or tag == TAG_ALTA:
wastrunk = False
# update weight
weight_ = weight__
# keep track of off for best matching trunk
if trunk and j_ + size > trunk:
trunkoff = j_ + size
@@ -358,7 +359,7 @@ class Rbyd:
tag_ = alt
w_ = id_-lower
done = (id_, tag_) < (id, tag) or tag_ & 0x1000
done = not tag_ or (id_, tag_) < (id, tag)
return done, id_, tag_, w_, j, d, self.data[j+d:j+d+jump], path
scripts/dbgrbyd.py
+89 -79
View File
@@ -17,7 +17,7 @@ COLORS = [
]
TAG_UNR = 0x1000
TAG_NULL = 0x0000
TAG_SUPERMAGIC = 0x0003
TAG_SUPERCONFIG = 0x0004
TAG_MROOT = 0x0304
@@ -32,9 +32,11 @@ TAG_BTREE = 0x0303
TAG_MDIR = 0x0305
TAG_UATTR = 0x0400
TAG_ALT = 0x4000
TAG_ALTA = 0x6000
TAG_CRC = 0x2000
TAG_FCRC = 0x2100
# parse some rbyd addr encodings
# 0xa -> [0xa]
# 0xa.b -> ([0xa], b)
@@ -111,55 +113,56 @@ def xxd(data, width=16, crc=False):
for b in map(chr, data[i:i+width])))
def tagrepr(tag, w, size, off=None):
if (tag & 0x7fff) == TAG_UNR:
return 'unr%s%s' % (
if tag == TAG_NULL:
return 'null%s%s' % (
' w%d' % w if w else '',
' %d' % size if size else '')
elif (tag & 0x6fff) == TAG_SUPERMAGIC:
return '%ssupermagic%s%s' % (
'rm' if tag & 0x1000 else '',
elif tag == TAG_SUPERMAGIC:
return 'supermagic%s %d' % (
' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6fff) == TAG_SUPERCONFIG:
return '%ssuperconfig%s%s' % (
'rm' if tag & 0x1000 else '',
size)
elif tag == TAG_SUPERCONFIG:
return 'superconfig%s %d' % (
' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_NAME:
return '%s%s%s%s' % (
'rm' if tag & 0x1000 else '',
'branch' if (tag & 0xfffe) == TAG_BRANCH
else 'reg' if (tag & 0xfffe) == TAG_REG
else 'dir' if (tag & 0xfffe) == TAG_DIR
else 'name 0x%02x' % ((tag & 0x0ff0) >> 4),
size)
elif (tag & 0xff00) == TAG_NAME:
return '%s%s %d' % (
'branch' if tag == TAG_BRANCH
else 'reg' if tag == TAG_REG
else 'dir' if tag == TAG_DIR
else 'name 0x%02x' % (tag & 0xff),
' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_STRUCT:
return '%s%s%s%s' % (
'rm' if tag & 0x1000 else '',
'inlined' if (tag & 0x6fff) == TAG_INLINED
else 'block' if (tag & 0x6fff) == TAG_BLOCK
else 'btree' if (tag & 0x6fff) == TAG_BTREE
else 'mdir' if (tag & 0x6fff) == TAG_MROOT
else 'mdir' if (tag & 0x6fff) == TAG_MDIR
else 'struct 0x%02x' % ((tag & 0x0ff0) >> 4),
size)
elif (tag & 0xff00) == TAG_STRUCT:
return '%s%s %d' % (
'inlined' if tag == TAG_INLINED
else 'block' if tag == TAG_BLOCK
else 'btree' if tag == TAG_BTREE
else 'mdir' if tag == TAG_MROOT
else 'mdir' if tag == TAG_MDIR
else 'struct 0x%02x' % (tag & 0xff),
' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x6f00) == TAG_UATTR:
return '%suattr 0x%02x%s%s' % (
'rm' if tag & 0x1000 else '',
size)
elif (tag & 0xff00) == TAG_UATTR:
return 'uattr 0x%02x%s %d' % (
tag & 0xff,
' w%d' % w if w else '',
' %d' % size if not tag & 0x1000 or size else '')
elif (tag & 0x7f00) == TAG_CRC:
size)
elif (tag & 0xff00) == TAG_CRC:
return 'crc%x%s %d' % (
1 if tag & 0x1 else 0,
' 0x%x' % w if w > 0 else '',
size)
elif (tag & 0x7fff) == TAG_FCRC:
elif tag == TAG_FCRC:
return 'fcrc%s %d' % (
' 0x%x' % w if w > 0 else '',
size)
elif tag == TAG_ALTA:
return 'alta w%d %s' % (
w,
'0x%x' % (0xffffffff & (off-size))
if off is not None
else '-%d' % off)
elif tag & 0x4000:
return 'alt%s%s 0x%x w%d %s' % (
'r' if tag & 0x1000 else 'b',
@@ -172,6 +175,7 @@ def tagrepr(tag, w, size, off=None):
else:
return '0x%04x w%d %d' % (tag, w, size)
def dbg_log(data, block_size, rev, off, weight, *,
color=False,
**args):
@@ -285,22 +289,26 @@ def dbg_log(data, block_size, rev, off, weight, *,
if not tag & 0x4000:
j_ += size
# find trunk
if not wastrunk and (tag & 0x6000) != 0x2000:
lower_, upper_ = 0, 0
wastrunk = not not tag & 0x4000
# evaluate trunks
if (tag & 0xe000) != 0x2000:
if not wastrunk:
wastrunk = True
lower_, upper_ = 0, 0
# keep track of weight
if tag & 0x4000:
if tag & 0x2000:
upper_ += w
else:
if (tag & 0xe000) == 0x4000:
lower_ += w
elif (tag & 0x6000) == 0x2000:
delta = (lower_+upper_+w) - weight_
weight_ = lower_+upper_+w
id = lower_+w-1
else:
upper_ += w
if not tag & 0x4000 or tag == TAG_ALTA:
wastrunk = False
# derive the current tag's id from alt weights
delta = (lower_+upper_) - weight_
weight_ = lower_+upper_
id = lower_ + w-1
if ((tag & 0xe000) != 0x2000
and (not tag & 0x4000 or tag == TAG_ALTA)):
# note we ignore out-of-bounds here for debugging
if delta > 0:
# grow lifetimes
@@ -339,7 +347,7 @@ def dbg_log(data, block_size, rev, off, weight, *,
weights = weights_
lifetimes = lifetimes_
if not tag & 0x1000 and id >= 0:
if id >= 0:
# attach tag to lifetime
i, id_ = index(weights, id)
if i < len(weights):
@@ -427,23 +435,9 @@ def dbg_log(data, block_size, rev, off, weight, *,
crc = crc32c(data[j_:j_+d], crc)
j_ += d
# find trunk
if not wastrunk and (tag & 0x6000) != 0x2000:
lower_, upper_ = 0, 0
wastrunk = not not tag & 0x4000
# calculate id from alt weights
if tag & 0x4000:
if tag & 0x2000:
upper_ += w
else:
lower_ += w
elif (tag & 0x6000) == 0x0:
weight_ = lower_+upper_+w
id = lower_+w-1
# take care of crcs
if not tag & 0x4000:
if (tag & 0x7f00) != TAG_CRC:
if (tag & 0xff00) != TAG_CRC:
crc = crc32c(data[j_:j_+size], crc)
# found a crc?
else:
@@ -452,6 +446,22 @@ def dbg_log(data, block_size, rev, off, weight, *,
notes.append('crc!=%08x' % crc)
j_ += size
# evaluate trunks
if (tag & 0xe000) != 0x2000:
if not wastrunk:
wastrunk = True
lower_, upper_ = 0, 0
if (tag & 0xe000) == 0x4000:
lower_ += w
else:
upper_ += w
if not tag & 0x4000 or tag == TAG_ALTA:
wastrunk = False
# derive the current tag's id from alt weights
id = lower_ + w-1
# show human-readable tag representation
print('%s%08x:%s %*s%s%*s %-57s%s%s' % (
'\x1b[90m' if color and j >= off else '',
@@ -459,7 +469,8 @@ def dbg_log(data, block_size, rev, off, weight, *,
'\x1b[m' if color and j >= off else '',
lifetime_width, lifetimerepr(j) if args.get('lifetimes') else '',
'\x1b[90m' if color and j >= off else '',
w_width, '' if (tag & 0x6000) != 0x0
w_width, '-' if tag == TAG_ALTA
else '' if (tag & 0xe000) != 0x0000
else '%d-%d' % (id-(w-1), id) if w > 1
else id,
'%-22s%s' % (
@@ -576,7 +587,7 @@ def dbg_tree(data, block_size, rev, trunk, weight, *,
tag_ = alt
w_ = id_-lower
done = (id_, tag_) < (id, tag) or tag_ & 0x1000
done = not tag_ or (id_, tag_) < (id, tag)
return done, id_, tag_, w_, j, d, jump, path
@@ -835,8 +846,8 @@ def main(disk, blocks=None, *,
trunk_ = 0
trunk__ = 0
weight = 0
lower_, upper_ = 0, 0
weight_ = 0
weight__ = 0
wastrunk = False
trunkoff = None
while j_ < len(data) and (not trunk or off <= trunk):
@@ -850,7 +861,7 @@ def main(disk, blocks=None, *,
# take care of crcs
if not tag & 0x4000:
if (tag & 0x7f00) != TAG_CRC:
if (tag & 0xff00) != TAG_CRC:
crc_ = crc32c(data[j_:j_+size], crc_)
# found a crc?
else:
@@ -864,23 +875,22 @@ def main(disk, blocks=None, *,
weight = weight_
# evaluate trunks
if (tag & 0x6000) != 0x2000 and (
if (tag & 0xe000) != 0x2000 and (
not trunk or trunk >= j_-d or wastrunk):
# new trunk?
if not wastrunk:
trunk__ = j_-d
lower_, upper_ = 0, 0
wastrunk = True
trunk__ = j_-d
weight__ = 0
# keep track of weight
if tag & 0x4000:
if tag & 0x2000:
upper_ += w
else:
lower_ += w
else:
weight_ = lower_+upper_+w
weight__ += w
# end of trunk?
if not tag & 0x4000 or tag == TAG_ALTA:
wastrunk = False
# update weight
weight_ = weight__
# keep track of off for best matching trunk
if trunk and j_ + size > trunk:
trunkoff = j_ + size