WIP Added allocation randomization for dynamic wear-leveling

tests/corrupt.py

@@ -3,37 +3,41 @@
 import struct
 import sys
 import os
+import argparse
 
-def main(*paths):
-    # find most recent block
-    file = None
-    rev = None
-    for path in paths:
-        try:
-            nfile = open(path, 'r+b')
-            nrev, = struct.unpack('<I', nfile.read(4))
+def corrupt(block):
+    with open(block, 'r+b') as file:
+        # skip rev
+        file.read(4)
 
-            assert rev != nrev
-            if not file or ((rev - nrev) & 0x80000000):
-                file = nfile
-                rev = nrev
-        except IOError:
-            pass
+        # go to last commit
+        tag = 0
+        while True:
+            try:
+                ntag, = struct.unpack('<I', file.read(4))
+            except struct.error:
+                break
 
-    # go to last commit
-    tag = 0
-    while True:
-        try:
-            ntag, = struct.unpack('<I', file.read(4))
-        except struct.error:
-            break
+            tag ^= ntag
+            file.seek(tag & 0xfff, os.SEEK_CUR)
 
-        tag ^= ntag
-        file.seek(tag & 0xfff, os.SEEK_CUR)
+        # lob off last 3 bytes
+        file.seek(-((tag & 0xfff) + 3), os.SEEK_CUR)
+        file.truncate()
 
-    # lob off last 3 bytes
-    file.seek(-((tag & 0xfff) + 3), os.SEEK_CUR)
-    file.truncate()
+def main(args):
+    if args.n or not args.blocks:
+        with open('blocks/.history', 'rb') as file:
+            for i in range(int(args.n or 1)):
+                last, = struct.unpack('<I', file.read(4))
+                args.blocks.append('blocks/%x' % last)
+
+    for block in args.blocks:
+        print 'corrupting %s' % block
+        corrupt(block)
 
 if __name__ == "__main__":
-    main(*sys.argv[1:])
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-n')
+    parser.add_argument('blocks', nargs='*')
+    main(parser.parse_args())

tests/debug.py

@@ -0,0 +1,98 @@
+#!/usr/bin/env python2
+
+import struct
+import binascii
+
+TYPES = {
+    (0x1ff, 0x001): 'reg',
+    (0x1ff, 0x002): 'dir',
+    (0x1ff, 0x011): 'superblock',
+    (0x1ff, 0x012): 'root',
+    (0x1ff, 0x030): 'delete',
+    (0x1f0, 0x080): 'globals',
+    (0x1ff, 0x0c0): 'tail soft',
+    (0x1ff, 0x0c1): 'tail hard',
+    (0x1ff, 0x0f0): 'crc',
+    (0x1ff, 0x040): 'struct dir',
+    (0x1ff, 0x041): 'struct inline',
+    (0x1ff, 0x042): 'struct ctz',
+    (0x100, 0x100): 'attr',
+}
+
+def typeof(type):
+    for prefix in range(9):
+        mask = 0x1ff & ~((1 << prefix)-1)
+        if (mask, type & mask) in TYPES:
+            return TYPES[mask, type & mask] + (
+                ' [%0*x]' % (prefix/4, type & ((1 << prefix)-1))
+                if prefix else '')
+    else:
+        return '[%02x]' % type
+
+def main(*blocks):
+    # find most recent block
+    file = None
+    rev = None
+    crc = None
+    versions = []
+
+    for block in blocks:
+        try:
+            nfile = open(block, 'rb')
+            ndata = nfile.read(4)
+            ncrc = binascii.crc32(ndata)
+            nrev, = struct.unpack('<I', ndata)
+
+            assert rev != nrev
+            if not file or ((rev - nrev) & 0x80000000):
+                file = nfile
+                rev = nrev
+                crc = ncrc
+
+            versions.append((nrev, '%s (rev %d)' % (block, nrev)))
+        except IOError:
+            pass
+
+    print "--- %s ---" % ', '.join(v for _,v in sorted(versions, reverse=True))
+
+    # go through each tag, print useful information
+    print "%-4s  %-8s  %-14s  %3s  %3s  %s" % (
+        'off', 'tag', 'type', 'id', 'len', 'dump')
+
+    tag = 0
+    off = 4
+    while True:
+        try:
+            data = file.read(4)
+            crc = binascii.crc32(data, crc)
+            ntag, = struct.unpack('<I', data)
+        except struct.error:
+            break
+
+        tag ^= ntag
+        off += 4
+
+        type = (tag & 0x7fc00000) >> 22
+        id   = (tag & 0x003ff000) >> 12
+        size = (tag & 0x00000fff) >> 0
+
+        data = file.read(size)
+        if type == 0x0f0:
+            crc = binascii.crc32(data[:4], crc)
+        else:
+            crc = binascii.crc32(data, crc)
+
+        print '%04x: %08x  %-14s  %3s  %3d  %-23s  %-8s' % (
+            off, tag,
+            typeof(type) + (' bad!' if type == 0x0f0 and ~crc else ''),
+            id if id != 0x3ff else '.', size,
+            ' '.join('%02x' % ord(c) for c in data[:8]),
+            ''.join(c if c >= ' ' and c <= '~' else '.' for c in data[:8]))
+
+        off += tag & 0xfff
+        if type == 0x0f0:
+            crc = 0
+
+if __name__ == "__main__":
+    import sys
+    main(*sys.argv[1:])

tests/stats.py

@@ -7,7 +7,7 @@ import os
 import re
 
 def main():
-    with open('blocks/config') as file:
+    with open('blocks/.config') as file:
         s = struct.unpack('<LLLL', file.read())
         print 'read_size: %d' % s[0]
         print 'prog_size: %d' % s[1]
@@ -18,7 +18,7 @@ def main():
         os.path.getsize(os.path.join('blocks', f))
         for f in os.listdir('blocks') if re.match('\d+', f))
 
-    with open('blocks/stats') as file:
+    with open('blocks/.stats') as file:
         s = struct.unpack('<QQQ', file.read())
         print 'read_count: %d' % s[0]
         print 'prog_count: %d' % s[1]

tests/test_corrupt.sh

@@ -71,24 +71,25 @@ echo "--- Sanity check ---"
 rm -rf blocks
 lfs_mktree
 lfs_chktree
+BLOCKS="$(ls blocks | grep -vw '[01]')"
 
 echo "--- Block corruption ---"
-for i in {2..33}
+for b in $BLOCKS
 do 
     rm -rf blocks
     mkdir blocks
-    ln -s /dev/zero blocks/$(printf '%x' $i)
+    ln -s /dev/zero blocks/$b
     lfs_mktree
     lfs_chktree
 done
 
 echo "--- Block persistance ---"
-for i in {2..33}
+for b in $BLOCKS
 do 
     rm -rf blocks
     mkdir blocks
     lfs_mktree
-    chmod a-w blocks/$(printf '%x' $i) || true
+    chmod a-w blocks/$b
     lfs_mktree
     lfs_chktree
 done
@@ -96,7 +97,7 @@ done
 echo "--- Big region corruption ---"
 rm -rf blocks
 mkdir blocks
-for i in {2..255}
+for i in {2..512}
 do
     ln -s /dev/zero blocks/$(printf '%x' $i)
 done
@@ -106,7 +107,7 @@ lfs_chktree
 echo "--- Alternating corruption ---"
 rm -rf blocks
 mkdir blocks
-for i in {2..511..2}
+for i in {2..1024..2}
 do
     ln -s /dev/zero blocks/$(printf '%x' $i)
 done

tests/test_move.sh

@@ -59,7 +59,7 @@ tests/test.py << TEST
     lfs_rename(&lfs, "b/hello", "c/hello") => 0;
     lfs_unmount(&lfs) => 0;
 TEST
-tests/corrupt.py blocks/{4,5}
+tests/corrupt.py -n 1
 tests/test.py << TEST
     lfs_mount(&lfs, &cfg) => 0;
     lfs_dir_open(&lfs, &dir[0], "b") => 0;
@@ -86,8 +86,7 @@ tests/test.py << TEST
     lfs_rename(&lfs, "c/hello", "d/hello") => 0;
     lfs_unmount(&lfs) => 0;
 TEST
-tests/corrupt.py blocks/{6,7}
-tests/corrupt.py blocks/{8,9}
+tests/corrupt.py -n 2
 tests/test.py << TEST
     lfs_mount(&lfs, &cfg) => 0;
     lfs_dir_open(&lfs, &dir[0], "c") => 0;
@@ -166,7 +165,7 @@ tests/test.py << TEST
     lfs_rename(&lfs, "b/hi", "c/hi") => 0;
     lfs_unmount(&lfs) => 0;
 TEST
-tests/corrupt.py blocks/{4,5}
+tests/corrupt.py -n 1
 tests/test.py << TEST
     lfs_mount(&lfs, &cfg) => 0;
     lfs_dir_open(&lfs, &dir[0], "b") => 0;
@@ -193,8 +192,7 @@ tests/test.py << TEST
     lfs_rename(&lfs, "c/hi", "d/hi") => 0;
     lfs_unmount(&lfs) => 0;
 TEST
-tests/corrupt.py blocks/{6,7}
-tests/corrupt.py blocks/{8,9}
+tests/corrupt.py -n 2
 tests/test.py << TEST
     lfs_mount(&lfs, &cfg) => 0;
     lfs_dir_open(&lfs, &dir[0], "c") => 0;

tests/test_orphan.sh

@@ -17,7 +17,7 @@ tests/test.py << TEST
 TEST
 # corrupt most recent commit, this should be the update to the previous
 # linked-list entry and should orphan the child
-tests/corrupt.py blocks/{6,7}
+tests/corrupt.py
 tests/test.py << TEST
     lfs_mount(&lfs, &cfg) => 0;