Github/littlefs
1
Fork 0
Code Actions Activity

Cleaned up tag encoding, now with clear chunk field

Browse Source 
Before, the tag format's type field was limited to 9-bits. This sounds
like a lot, but this field needed to encode up to 256 user-specified
types. This limited the flexibility of the encoded types. As time went
on, more bits in the type field were repurposed for various things,
leaving a rather fragile type field.

Here we make the jump to full 11-bit type fields. This comes at the cost
of a smaller length field, however the use of the length field was
always going to come with a RAM limitation. Rather than putting pressure
on RAM for inline files, the new type field lets us encode a chunk
number, splitting up inline files into multiple updatable units. This
actually pushes the theoretical inline max from 8KiB to 256KiB! (Note
that we only allow a single 1KiB chunk for now, chunky inline files
is just a theoretical future improvement).

Here is the new 32-bit tag format, note that there are multiple levels
of types which break down into more info:

[----            32             ----]
[1|--  11   --|--  10  --|--  10  --]
 ^.     ^     .     ^          ^- entry length
 |.     |     .     \------------ file id chunk info
 |.     \-----.------------------ type info (type3)
 \.-----------.------------------ valid bit
  [-3-|-- 8 --]
    ^     ^- chunk info
    \------- type info (type1)

Additionally, I've split the CREATE tag into separate SPLICE and NAME
tags. This simplified the new compact logic a bit. For now, littlefs
still follows the rule that a NAME tag precedes any other tags related
to a file, but this can change in the future.
This commit is contained in:
Christopher Haster
2018-12-29 07:53:12 -06:00
parent a548ce68c1
commit b989b4a89f
6 changed files with 426 additions and 460 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/corrupt.py
+1 -1
View File
@@ -19,7 +19,7 @@ def corrupt(block):
break
tag ^= ntag
size = (tag & 0x1fff) if (tag & 0x1fff) != 0x1fff else 0
size = (tag & 0x3ff) if (tag & 0x3ff) != 0x3ff else 0
file.seek(size, os.SEEK_CUR)
# lob off last 3 bytes
tests/debug.py
+30 -24
View File
@@ -4,23 +4,29 @@ import struct
import binascii
TYPES = {
(0x1ff, 0x011): 'create reg',
(0x1ff, 0x010): 'create dir',
(0x1ff, 0x001): 'superblock',
(0x1ff, 0x020): 'delete',
(0x1f0, 0x0e0): 'globals',
(0x1ff, 0x080): 'tail soft',
(0x1ff, 0x081): 'tail hard',
(0x1f0, 0x0a0): 'crc',
(0x1ff, 0x040): 'struct dir',
(0x1ff, 0x041): 'struct inline',
(0x1ff, 0x042): 'struct ctz',
(0x100, 0x100): 'attr',
(0x700, 0x400): 'splice',
(0x7ff, 0x401): 'create',
(0x7ff, 0x4ff): 'delete',
(0x700, 0x000): 'name',
(0x7ff, 0x001): 'name reg',
(0x7ff, 0x002): 'name dir',
(0x7ff, 0x0ff): 'name superblock',
(0x700, 0x200): 'struct',
(0x7ff, 0x200): 'struct dir',
(0x7ff, 0x202): 'struct ctz',
(0x7ff, 0x201): 'struct inline',
(0x700, 0x300): 'userattr',
(0x700, 0x600): 'tail',
(0x7ff, 0x600): 'tail soft',
(0x7ff, 0x601): 'tail hard',
(0x700, 0x700): 'gstate',
(0x7ff, 0x7ff): 'gstate move',
(0x700, 0x500): 'crc',
}
def typeof(type):
for prefix in range(9):
mask = 0x1ff & ~((1 << prefix)-1)
for prefix in range(12):
mask = 0x7ff & ~((1 << prefix)-1)
if (mask, type & mask) in TYPES:
return TYPES[mask, type & mask] + (
' %0*x' % (prefix/4, type & ((1 << prefix)-1))
@@ -59,7 +65,7 @@ def main(*blocks):
print "--- %s ---" % ', '.join(v for _,v in sorted(versions, reverse=True))
# go through each tag, print useful information
print "%-4s %-8s %-14s %3s %3s %s" % (
print "%-4s %-8s %-14s %3s %4s %s" % (
'off', 'tag', 'type', 'id', 'len', 'dump')
tag = 0xffffffff
@@ -75,26 +81,26 @@ def main(*blocks):
tag ^= ntag
off += 4
type = (tag & 0x7fc00000) >> 22
id = (tag & 0x003fe000) >> 13
size = (tag & 0x00001fff) >> 0
iscrc = (type & 0x1f0) == 0x0f0
type = (tag & 0x7ff00000) >> 20
id = (tag & 0x000ffc00) >> 10
size = (tag & 0x000003ff) >> 0
iscrc = (type & 0x700) == 0x500
data = file.read(size if size != 0x1fff else 0)
data = file.read(size if size != 0x3ff else 0)
if iscrc:
crc = binascii.crc32(data[:4], crc)
else:
crc = binascii.crc32(data, crc)
print '%04x: %08x %-14s %3s %3s %-23s %-8s' % (
print '%04x: %08x %-15s %3s %4s %-23s %-8s' % (
off, tag,
typeof(type) + (' bad!' if iscrc and ~crc else ''),
id if id != 0x1ff else '.',
size if size != 0x1fff else 'x',
id if id != 0x3ff else '.',
size if size != 0x3ff else 'x',
' '.join('%02x' % ord(c) for c in data[:8]),
''.join(c if c >= ' ' and c <= '~' else '.' for c in data[:8]))
off += size if size != 0x1fff else 0
off += size if size != 0x3ff else 0
if iscrc:
crc = 0
tag ^= (type & 1) << 31
tests/test_alloc.sh
+3 -3
View File
@@ -253,7 +253,7 @@ tests/test.py << TEST
// find out max file size
lfs_mkdir(&lfs, "exhaustiondir") => 0;
for (int i = 0; i < 9; i++) {
for (int i = 0; i < 10; i++) {
sprintf((char*)buffer, "dirwithanexhaustivelylongnameforpadding%d", i);
lfs_mkdir(&lfs, (char*)buffer) => 0;
}
@@ -275,7 +275,7 @@ tests/test.py << TEST
lfs_remove(&lfs, "exhaustion") => 0;
lfs_remove(&lfs, "exhaustiondir") => 0;
for (int i = 0; i < 9; i++) {
for (int i = 0; i < 10; i++) {
sprintf((char*)buffer, "dirwithanexhaustivelylongnameforpadding%d", i);
lfs_remove(&lfs, (char*)buffer) => 0;
}
@@ -287,7 +287,7 @@ tests/test.py << TEST
}
lfs_file_sync(&lfs, &file[0]) => 0;
for (int i = 0; i < 9; i++) {
for (int i = 0; i < 10; i++) {
sprintf((char*)buffer, "dirwithanexhaustivelylongnameforpadding%d", i);
lfs_mkdir(&lfs, (char*)buffer) => 0;
}
tests/test_format.sh
+1 -2
View File
@@ -12,8 +12,7 @@ TEST
echo "--- Basic mounting ---"
tests/test.py << TEST
lfs_format(&lfs, &cfg) => 0;
TEST
tests/test.py << TEST
lfs_mount(&lfs, &cfg) => 0;
lfs_unmount(&lfs) => 0;
TEST