From 3820be180d566a3dff38aa1172b81910325d18c7 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Mon, 7 Apr 2025 15:33:56 -0500 Subject: [PATCH] scripts: Adopted crc32c lib when available Jumping from a simple Python implementation to the fully hardware accelerated crc32c library basically deletes any crc32c related bottlenecks: crc32c.py disk (1MiB) w/ crc32c lib: 0m0.027s crc32c.py disk (1MiB) w/o crc32c lib: 0m0.844s This uses the same try-import trick we use for inotify_simple, so we get the speed improvement without losing portability. --- In dbgbmap.py: dbgbmap.py w/ crc32c lib: 0m0.273s dbgbmap.py w/o crc32c lib: 0m0.697s dbgbmap.py w/ crc32c lib --no-ckdata: 0m0.269s dbgbmap.py w/o crc32c lib --no-ckdata: 0m0.490s dbgbmap.old.py: 0m0.231s The bulk of the runtime is still in Rbyd.fetch, but this is now dominated by leb128 decoding, which makes sense. We do ~twice as many fetches in the new dbgbmap.py in order to calculate the gcksum (which we then ignore...). --- scripts/crc32c.py | 20 ++++++++++++++------ scripts/dbgblock.py | 20 ++++++++++++++------ scripts/dbgbmap.py | 20 ++++++++++++++------ scripts/dbgbmapd3.py | 20 ++++++++++++++------ scripts/dbgbtree.py | 20 ++++++++++++++------ scripts/dbgcat.py | 20 ++++++++++++++------ scripts/dbglfs.py | 20 ++++++++++++++------ scripts/dbgmtree.py | 20 ++++++++++++++------ scripts/dbgrbyd.py | 20 ++++++++++++++------ 9 files changed, 126 insertions(+), 54 deletions(-) diff --git a/scripts/crc32c.py b/scripts/crc32c.py index 65cee3a9..56d8cb1e 100755 --- a/scripts/crc32c.py +++ b/scripts/crc32c.py @@ -9,6 +9,11 @@ import os import struct import sys +try: + import crc32c as crc32c_lib +except ModuleNotFoundError: + crc32c_lib = None + def openio(path, mode='r', buffering=-1): # allow '-' for stdin/stdout @@ -22,12 +27,15 @@ def openio(path, mode='r', buffering=-1): return open(path, mode, buffering) def crc32c(data, crc=0): - crc ^= 0xffffffff - for b in data: - crc ^= b - for j in range(8): - crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78) - return 0xffffffff ^ crc + if crc32c_lib is not None: + return crc32c_lib.crc32c(data, crc) + else: + crc ^= 0xffffffff + for b in data: + crc ^= b + for j in range(8): + crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78) + return 0xffffffff ^ crc def main(paths, **args): diff --git a/scripts/dbgblock.py b/scripts/dbgblock.py index 5d609258..20048715 100755 --- a/scripts/dbgblock.py +++ b/scripts/dbgblock.py @@ -7,6 +7,11 @@ if __name__ == "__main__": import itertools as it import os +try: + import crc32c as crc32c_lib +except ModuleNotFoundError: + crc32c_lib = None + # some ways of block geometry representations # 512 -> 512 @@ -79,12 +84,15 @@ def xxd(data, width=16): for b in map(chr, data[i:i+width]))) def crc32c(data, crc=0): - crc ^= 0xffffffff - for b in data: - crc ^= b - for j in range(8): - crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78) - return 0xffffffff ^ crc + if crc32c_lib is not None: + return crc32c_lib.crc32c(data, crc) + else: + crc ^= 0xffffffff + for b in data: + crc ^= b + for j in range(8): + crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78) + return 0xffffffff ^ crc def main(disk, blocks=None, *, block_size=None, diff --git a/scripts/dbgbmap.py b/scripts/dbgbmap.py index c11e55da..2d9f39c6 100755 --- a/scripts/dbgbmap.py +++ b/scripts/dbgbmap.py @@ -23,6 +23,11 @@ try: except ModuleNotFoundError: inotify_simple = None +try: + import crc32c as crc32c_lib +except ModuleNotFoundError: + crc32c_lib = None + TAG_NULL = 0x0000 ## 0x0000 v--- ---- ---- ---- TAG_CONFIG = 0x0000 ## 0x00tt v--- ---- -ttt tttt @@ -161,12 +166,15 @@ def rbydaddr(s): return tuple(addr) def crc32c(data, crc=0): - crc ^= 0xffffffff - for b in data: - crc ^= b - for j in range(8): - crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78) - return 0xffffffff ^ crc + if crc32c_lib is not None: + return crc32c_lib.crc32c(data, crc) + else: + crc ^= 0xffffffff + for b in data: + crc ^= b + for j in range(8): + crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78) + return 0xffffffff ^ crc def pmul(a, b): r = 0 diff --git a/scripts/dbgbmapd3.py b/scripts/dbgbmapd3.py index 85a3d569..5781ab0c 100755 --- a/scripts/dbgbmapd3.py +++ b/scripts/dbgbmapd3.py @@ -20,6 +20,11 @@ import re import shlex import struct +try: + import crc32c as crc32c_lib +except ModuleNotFoundError: + crc32c_lib = None + TAG_NULL = 0x0000 ## 0x0000 v--- ---- ---- ---- TAG_CONFIG = 0x0000 ## 0x00tt v--- ---- -ttt tttt @@ -190,12 +195,15 @@ def rbydaddr(s): return tuple(addr) def crc32c(data, crc=0): - crc ^= 0xffffffff - for b in data: - crc ^= b - for j in range(8): - crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78) - return 0xffffffff ^ crc + if crc32c_lib is not None: + return crc32c_lib.crc32c(data, crc) + else: + crc ^= 0xffffffff + for b in data: + crc ^= b + for j in range(8): + crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78) + return 0xffffffff ^ crc def pmul(a, b): r = 0 diff --git a/scripts/dbgbtree.py b/scripts/dbgbtree.py index 33d6b7a4..931076e3 100755 --- a/scripts/dbgbtree.py +++ b/scripts/dbgbtree.py @@ -12,6 +12,11 @@ import math as mt import os import struct +try: + import crc32c as crc32c_lib +except ModuleNotFoundError: + crc32c_lib = None + TAG_NULL = 0x0000 ## 0x0000 v--- ---- ---- ---- TAG_CONFIG = 0x0000 ## 0x00tt v--- ---- -ttt tttt @@ -117,12 +122,15 @@ def rbydaddr(s): return tuple(addr) def crc32c(data, crc=0): - crc ^= 0xffffffff - for b in data: - crc ^= b - for j in range(8): - crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78) - return 0xffffffff ^ crc + if crc32c_lib is not None: + return crc32c_lib.crc32c(data, crc) + else: + crc ^= 0xffffffff + for b in data: + crc ^= b + for j in range(8): + crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78) + return 0xffffffff ^ crc def popc(x): return bin(x).count('1') diff --git a/scripts/dbgcat.py b/scripts/dbgcat.py index 6b8deb59..133a0707 100755 --- a/scripts/dbgcat.py +++ b/scripts/dbgcat.py @@ -7,6 +7,11 @@ if __name__ == "__main__": import itertools as it import os +try: + import crc32c as crc32c_lib +except ModuleNotFoundError: + crc32c_lib = None + # some ways of block geometry representations # 512 -> 512 @@ -79,12 +84,15 @@ def xxd(data, width=16): for b in map(chr, data[i:i+width]))) def crc32c(data, crc=0): - crc ^= 0xffffffff - for b in data: - crc ^= b - for j in range(8): - crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78) - return 0xffffffff ^ crc + if crc32c_lib is not None: + return crc32c_lib.crc32c(data, crc) + else: + crc ^= 0xffffffff + for b in data: + crc ^= b + for j in range(8): + crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78) + return 0xffffffff ^ crc def main(disk, blocks=None, *, block_size=None, diff --git a/scripts/dbglfs.py b/scripts/dbglfs.py index 51a07281..ca85fd81 100755 --- a/scripts/dbglfs.py +++ b/scripts/dbglfs.py @@ -13,6 +13,11 @@ import os import struct import sys +try: + import crc32c as crc32c_lib +except ModuleNotFoundError: + crc32c_lib = None + TAG_NULL = 0x0000 ## 0x0000 v--- ---- ---- ---- TAG_CONFIG = 0x0000 ## 0x00tt v--- ---- -ttt tttt @@ -118,12 +123,15 @@ def rbydaddr(s): return tuple(addr) def crc32c(data, crc=0): - crc ^= 0xffffffff - for b in data: - crc ^= b - for j in range(8): - crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78) - return 0xffffffff ^ crc + if crc32c_lib is not None: + return crc32c_lib.crc32c(data, crc) + else: + crc ^= 0xffffffff + for b in data: + crc ^= b + for j in range(8): + crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78) + return 0xffffffff ^ crc def pmul(a, b): r = 0 diff --git a/scripts/dbgmtree.py b/scripts/dbgmtree.py index 34d42aa2..09cebb89 100755 --- a/scripts/dbgmtree.py +++ b/scripts/dbgmtree.py @@ -12,6 +12,11 @@ import math as mt import os import struct +try: + import crc32c as crc32c_lib +except ModuleNotFoundError: + crc32c_lib = None + TAG_NULL = 0x0000 ## 0x0000 v--- ---- ---- ---- TAG_CONFIG = 0x0000 ## 0x00tt v--- ---- -ttt tttt @@ -117,12 +122,15 @@ def rbydaddr(s): return tuple(addr) def crc32c(data, crc=0): - crc ^= 0xffffffff - for b in data: - crc ^= b - for j in range(8): - crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78) - return 0xffffffff ^ crc + if crc32c_lib is not None: + return crc32c_lib.crc32c(data, crc) + else: + crc ^= 0xffffffff + for b in data: + crc ^= b + for j in range(8): + crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78) + return 0xffffffff ^ crc def popc(x): return bin(x).count('1') diff --git a/scripts/dbgrbyd.py b/scripts/dbgrbyd.py index 2fbb2b40..6b10dc40 100755 --- a/scripts/dbgrbyd.py +++ b/scripts/dbgrbyd.py @@ -12,6 +12,11 @@ import math as mt import os import struct +try: + import crc32c as crc32c_lib +except ModuleNotFoundError: + crc32c_lib = None + COLORS = [ '34', # blue @@ -127,12 +132,15 @@ def rbydaddr(s): return tuple(addr) def crc32c(data, crc=0): - crc ^= 0xffffffff - for b in data: - crc ^= b - for j in range(8): - crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78) - return 0xffffffff ^ crc + if crc32c_lib is not None: + return crc32c_lib.crc32c(data, crc) + else: + crc ^= 0xffffffff + for b in data: + crc ^= b + for j in range(8): + crc = (crc >> 1) ^ ((crc & 1) * 0x82f63b78) + return 0xffffffff ^ crc def popc(x): return bin(x).count('1')