diff --git a/.gitignore b/.gitignore index 8f1b90e6..640a47c3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ *.a.c *.gcno *.gcda +*.perf # Testing things blocks/ diff --git a/Makefile b/Makefile index 1c6a4584..16957fd1 100644 --- a/Makefile +++ b/Makefile @@ -19,44 +19,52 @@ TARGET ?= $(BUILDDIR)lfs.a endif -CC ?= gcc -AR ?= ar -SIZE ?= size -CTAGS ?= ctags -NM ?= nm -OBJDUMP ?= objdump -LCOV ?= lcov +CC ?= gcc +AR ?= ar +SIZE ?= size +CTAGS ?= ctags +NM ?= nm +OBJDUMP ?= objdump +VALGRIND ?= valgrind +GDB ?= gdb +PERF ?= perf -SRC ?= $(filter-out $(wildcard *.*.c),$(wildcard *.c)) -OBJ := $(SRC:%.c=$(BUILDDIR)%.o) -DEP := $(SRC:%.c=$(BUILDDIR)%.d) -ASM := $(SRC:%.c=$(BUILDDIR)%.s) -CI := $(SRC:%.c=$(BUILDDIR)%.ci) +SRC ?= $(filter-out $(wildcard *.*.c),$(wildcard *.c)) +OBJ := $(SRC:%.c=$(BUILDDIR)%.o) +DEP := $(SRC:%.c=$(BUILDDIR)%.d) +ASM := $(SRC:%.c=$(BUILDDIR)%.s) +CI := $(SRC:%.c=$(BUILDDIR)%.ci) GCDA := $(SRC:%.c=$(BUILDDIR)%.t.a.gcda) TESTS ?= $(wildcard tests/*.toml) TEST_SRC ?= $(SRC) \ $(filter-out $(wildcard bd/*.*.c),$(wildcard bd/*.c)) \ runners/test_runner.c -TEST_TC := $(TESTS:%.toml=$(BUILDDIR)%.t.c) $(TEST_SRC:%.c=$(BUILDDIR)%.t.c) -TEST_TAC := $(TEST_TC:%.t.c=%.t.a.c) -TEST_OBJ := $(TEST_TAC:%.t.a.c=%.t.a.o) -TEST_DEP := $(TEST_TAC:%.t.a.c=%.t.a.d) -TEST_CI := $(TEST_TAC:%.t.a.c=%.t.a.ci) +TEST_RUNNER ?= $(BUILDDIR)runners/test_runner +TEST_TC := $(TESTS:%.toml=$(BUILDDIR)%.t.c) \ + $(TEST_SRC:%.c=$(BUILDDIR)%.t.c) +TEST_TAC := $(TEST_TC:%.t.c=%.t.a.c) +TEST_OBJ := $(TEST_TAC:%.t.a.c=%.t.a.o) +TEST_DEP := $(TEST_TAC:%.t.a.c=%.t.a.d) +TEST_CI := $(TEST_TAC:%.t.a.c=%.t.a.ci) TEST_GCNO := $(TEST_TAC:%.t.a.c=%.t.a.gcno) TEST_GCDA := $(TEST_TAC:%.t.a.c=%.t.a.gcda) +TEST_PERF := $(TEST_RUNNER:%=%.perf) BENCHES ?= $(wildcard benches/*.toml) BENCH_SRC ?= $(SRC) \ $(filter-out $(wildcard bd/*.*.c),$(wildcard bd/*.c)) \ runners/bench_runner.c -BENCH_BC := $(BENCHES:%.toml=$(BUILDDIR)%.b.c) $(BENCH_SRC:%.c=$(BUILDDIR)%.b.c) -BENCH_BAC := $(BENCH_BC:%.b.c=%.b.a.c) -BENCH_OBJ := $(BENCH_BAC:%.b.a.c=%.b.a.o) -BENCH_DEP := $(BENCH_BAC:%.b.a.c=%.b.a.d) -BENCH_CI := $(BENCH_BAC:%.b.a.c=%.b.a.ci) +BENCH_RUNNER ?= $(BUILDDIR)runners/bench_runner +BENCH_BC := $(BENCHES:%.toml=$(BUILDDIR)%.b.c) \ + $(BENCH_SRC:%.c=$(BUILDDIR)%.b.c) +BENCH_BAC := $(BENCH_BC:%.b.c=%.b.a.c) +BENCH_OBJ := $(BENCH_BAC:%.b.a.c=%.b.a.o) +BENCH_DEP := $(BENCH_BAC:%.b.a.c=%.b.a.d) +BENCH_CI := $(BENCH_BAC:%.b.a.c=%.b.a.ci) BENCH_GCNO := $(BENCH_BAC:%.b.a.c=%.b.a.gcno) BENCH_GCDA := $(BENCH_BAC:%.b.a.c=%.b.a.gcda) +BENCH_PERF := $(BENCH_RUNNER:%=%.perf) ifdef DEBUG override CFLAGS += -O0 @@ -71,40 +79,67 @@ override CFLAGS += -I. override CFLAGS += -std=c99 -Wall -pedantic override CFLAGS += -Wextra -Wshadow -Wjump-misses-init -Wundef override CFLAGS += -ftrack-macro-expansion=0 +ifdef YES_COVERAGE +override CFLAGS += --coverage +endif +ifdef YES_PERF +override CFLAGS += -fno-omit-frame-pointer +endif -override TESTFLAGS += -b -override BENCHFLAGS += -b -# forward -j flag -override TESTFLAGS += $(filter -j%,$(MAKEFLAGS)) -override BENCHFLAGS += $(filter -j%,$(MAKEFLAGS)) ifdef VERBOSE -override CODEFLAGS += -v -override DATAFLAGS += -v -override STACKFLAGS += -v -override STRUCTFLAGS += -v -override COVERAGEFLAGS += -v -override TESTFLAGS += -v -override TESTCFLAGS += -v -override BENCHFLAGS += -v -override BENCHCFLAGS += -v -endif -ifdef EXEC -override TESTFLAGS += --exec="$(EXEC)" -override BENCHFLAGS += --exec="$(EXEC)" -endif -ifdef BUILDDIR -override CODEFLAGS += --build-dir="$(BUILDDIR:/=)" -override DATAFLAGS += --build-dir="$(BUILDDIR:/=)" -override STACKFLAGS += --build-dir="$(BUILDDIR:/=)" -override STRUCTFLAGS += --build-dir="$(BUILDDIR:/=)" -override COVERAGEFLAGS += --build-dir="$(BUILDDIR:/=)" +override CODEFLAGS += -v +override DATAFLAGS += -v +override STACKFLAGS += -v +override STRUCTFLAGS += -v +override COVERAGEFLAGS += -v +override PERFFLAGS += -v endif ifneq ($(NM),nm) override CODEFLAGS += --nm-tool="$(NM)" override DATAFLAGS += --nm-tool="$(NM)" endif ifneq ($(OBJDUMP),objdump) +override CODEFLAGS += --objdump-tool="$(OBJDUMP)" +override DATAFLAGS += --objdump-tool="$(OBJDUMP)" override STRUCTFLAGS += --objdump-tool="$(OBJDUMP)" +override PERFFLAGS += --objdump-tool="$(OBJDUMP)" +endif +ifneq ($(PERF),perf) +override PERFFLAGS += --perf-tool="$(PERF)" +endif + +override TESTFLAGS += -b +override BENCHFLAGS += -b +# forward -j flag +override TESTFLAGS += $(filter -j%,$(MAKEFLAGS)) +override BENCHFLAGS += $(filter -j%,$(MAKEFLAGS)) +ifdef YES_PERF +override TESTFLAGS += --perf=$(TEST_PERF) +endif +ifndef NO_PERF +override BENCHFLAGS += --perf=$(BENCH_PERF) +endif +ifdef VERBOSE +override TESTFLAGS += -v +override TESTCFLAGS += -v +override BENCHFLAGS += -v +override BENCHCFLAGS += -v +endif +ifdef EXEC +override TESTFLAGS += --exec="$(EXEC)" +override BENCHFLAGS += --exec="$(EXEC)" +endif +ifneq ($(GDB),gdb) +override TESTFLAGS += --gdb-tool="$(GDB)" +override BENCHFLAGS += --gdb-tool="$(GDB)" +endif +ifneq ($(VALGRIND),valgrind) +override TESTFLAGS += --valgrind-tool="$(VALGRIND)" +override BENCHFLAGS += --valgrind-tool="$(VALGRIND)" +endif +ifneq ($(PERF),perf) +override TESTFLAGS += --perf-tool="$(PERF)" +override BENCHFLAGS += --perf-tool="$(PERF)" endif @@ -124,28 +159,50 @@ tags: $(CTAGS) --totals --c-types=+p $(shell find -H -name '*.h') $(SRC) .PHONY: test-runner build-test +ifndef NO_COVERAGE test-runner build-test: override CFLAGS+=--coverage -test-runner build-test: $(BUILDDIR)runners/test_runner +endif +ifdef YES_PERF +bench-runner build-bench: override CFLAGS+=-fno-omit-frame-pointer +endif +test-runner build-test: $(TEST_RUNNER) +ifndef NO_COVERAGE rm -f $(TEST_GCDA) +endif +ifdef YES_PERF + rm -f $(TEST_PERF) +endif .PHONY: test test: test-runner - ./scripts/test.py $(BUILDDIR)runners/test_runner $(TESTFLAGS) + ./scripts/test.py $(TEST_RUNNER) $(TESTFLAGS) .PHONY: test-list test-list: test-runner - ./scripts/test.py $(BUILDDIR)runners/test_runner $(TESTFLAGS) -l + ./scripts/test.py $(TEST_RUNNER) $(TESTFLAGS) -l .PHONY: bench-runner build-bench -bench-runner build-bench: $(BUILDDIR)runners/bench_runner +ifdef YES_COVERAGE +bench-runner build-bench: override CFLAGS+=--coverage +endif +ifndef NO_PERF +bench-runner build-bench: override CFLAGS+=-fno-omit-frame-pointer +endif +bench-runner build-bench: $(BENCH_RUNNER) +ifdef YES_COVERAGE + rm -f $(BENCH_GCDA) +endif +ifndef NO_PERF + rm -f $(BENCH_PERF) +endif .PHONY: bench bench: bench-runner - ./scripts/bench.py $(BUILDDIR)runners/bench_runner $(BENCHFLAGS) + ./scripts/bench.py $(BENCH_RUNNER) $(BENCHFLAGS) .PHONY: bench-list bench-list: bench-runner - ./scripts/bench.py $(BUILDDIR)runners/bench_runner $(BENCHFLAGS) -l + ./scripts/bench.py $(BENCH_RUNNER) $(BENCHFLAGS) -l .PHONY: code code: $(OBJ) @@ -165,7 +222,17 @@ struct: $(OBJ) .PHONY: coverage coverage: $(GCDA) - ./scripts/coverage.py $^ -slines -sbranches $(COVERAGEFLAGS) + $(strip ./scripts/coverage.py \ + $^ $(patsubst %,-F%,$(SRC)) \ + -slines -sbranches \ + $(COVERAGEFLAGS)) + +.PHONY: perf +perf: $(BENCH_PERF) + $(strip ./scripts/perf.py \ + $^ $(patsubst %,-F%,$(SRC)) \ + -scycles \ + $(PERFFLAGS)) .PHONY: summary sizes summary sizes: $(BUILDDIR)lfs.csv @@ -203,7 +270,10 @@ $(BUILDDIR)lfs.struct.csv: $(OBJ) ./scripts/struct_.py $^ -q $(CODEFLAGS) -o $@ $(BUILDDIR)lfs.coverage.csv: $(GCDA) - ./scripts/coverage.py $^ -q $(COVERAGEFLAGS) -o $@ + ./scripts/coverage.py $^ $(patsubst %,-F%,$(SRC)) -q $(COVERAGEFLAGS) -o $@ + +$(BUILDDIR)lfs.perf.csv: $(BENCH_PERF) + ./scripts/perf.py $^ $(patsubst %,-F%,$(SRC)) -q $(PERFFLAGS) -o $@ $(BUILDDIR)lfs.csv: \ $(BUILDDIR)lfs.code.csv \ @@ -255,13 +325,13 @@ clean: $(BUILDDIR)lfs.data.csv \ $(BUILDDIR)lfs.stack.csv \ $(BUILDDIR)lfs.struct.csv \ - $(BUILDDIR)lfs.coverage.csv) - rm -f $(BUILDDIR)runners/test_runner - rm -f $(BUILDDIR)runners/bench_runner + $(BUILDDIR)lfs.coverage.csv \ + $(BUILDDIR)lfs.perf.csv) rm -f $(OBJ) rm -f $(DEP) rm -f $(ASM) rm -f $(CI) + rm -f $(TEST_RUNNER) rm -f $(TEST_TC) rm -f $(TEST_TAC) rm -f $(TEST_OBJ) @@ -269,6 +339,8 @@ clean: rm -f $(TEST_CI) rm -f $(TEST_GCNO) rm -f $(TEST_GCDA) + rm -f $(TEST_PERF) + rm -f $(BENCH_RUNNER) rm -f $(BENCH_BC) rm -f $(BENCH_BAC) rm -f $(BENCH_OBJ) @@ -276,3 +348,4 @@ clean: rm -f $(BENCH_CI) rm -f $(BENCH_GCNO) rm -f $(BENCH_GCDA) + rm -f $(BENCH_PERF) diff --git a/scripts/bench.py b/scripts/bench.py index e401d7cc..61db83e4 100755 --- a/scripts/bench.py +++ b/scripts/bench.py @@ -27,9 +27,13 @@ import time import toml -RUNNER_PATH = 'runners/bench_runner' +RUNNER_PATH = './runners/bench_runner' HEADER_PATH = 'runners/bench_runner.h' +GDB_TOOL = ['gdb'] +VALGRIND_TOOL = ['valgrind'] +PERF_SCRIPT = ['./scripts/perf.py'] + def openio(path, mode='r', buffering=-1, nb=False): if path == '-': @@ -502,12 +506,25 @@ def find_runner(runner, **args): # run under valgrind? if args.get('valgrind'): - cmd[:0] = filter(None, [ - 'valgrind', + cmd[:0] = args['valgrind_tool'] + [ '--leak-check=full', '--track-origins=yes', '--error-exitcode=4', - '-q']) + '-q'] + + # run under perf? + if args.get('perf'): + cmd[:0] = args['perf_script'] + list(filter(None, [ + '-R', + '--perf-freq=%s' % args['perf_freq'] + if args.get('perf_freq') else None, + '--perf-period=%s' % args['perf_period'] + if args.get('perf_period') else None, + '--perf-events=%s' % args['perf_events'] + if args.get('perf_events') else None, + '--perf-tool=%s' % args['perf_tool'] + if args.get('perf_tool') else None, + '-o%s' % args['perf']])) # other context if args.get('geometry'): @@ -789,9 +806,9 @@ def run_stage(name, runner_, ids, output_, **args): try: line = mpty.readline() except OSError as e: - if e.errno == errno.EIO: - break - raise + if e.errno != errno.EIO: + raise + break if not line: break last_stdout.append(line) @@ -1126,24 +1143,24 @@ def run(runner, bench_ids=[], **args): cmd = runner_ + [failure.id] if args.get('gdb_main'): - cmd[:0] = ['gdb', + cmd[:0] = args['gdb_tool'] + [ '-ex', 'break main', '-ex', 'run', '--args'] elif args.get('gdb_case'): path, lineno = find_path(runner_, failure.id, **args) - cmd[:0] = ['gdb', + cmd[:0] = args['gdb_tool'] + [ '-ex', 'break %s:%d' % (path, lineno), '-ex', 'run', '--args'] elif failure.assert_ is not None: - cmd[:0] = ['gdb', + cmd[:0] = args['gdb_tool'] + [ '-ex', 'run', '-ex', 'frame function raise', '-ex', 'up 2', '--args'] else: - cmd[:0] = ['gdb', + cmd[:0] = args['gdb_tool'] + [ '-ex', 'run', '--args'] @@ -1187,6 +1204,7 @@ if __name__ == "__main__": argparse._ArgumentGroup._handle_conflict_ignore = lambda *_: None parser = argparse.ArgumentParser( description="Build and run benches.", + allow_abbrev=False, conflict_handler='ignore') parser.add_argument( '-v', '--verbose', @@ -1315,6 +1333,11 @@ if __name__ == "__main__": action='store_true', help="Drop into gdb on bench failure but stop at the beginning " "of main.") + bench_parser.add_argument( + '--gdb-tool', + type=lambda x: x.split(), + default=GDB_TOOL, + help="Path to gdb tool to use. Defaults to %r." % GDB_TOOL) bench_parser.add_argument( '--exec', type=lambda e: e.split(), @@ -1324,6 +1347,37 @@ if __name__ == "__main__": action='store_true', help="Run under Valgrind to find memory errors. Implicitly sets " "--isolate.") + bench_parser.add_argument( + '--valgrind-tool', + type=lambda x: x.split(), + default=VALGRIND_TOOL, + help="Path to Valgrind tool to use. Defaults to %r." % VALGRIND_TOOL) + bench_parser.add_argument( + '--perf', + help="Run under Linux's perf to sample performance counters, writing " + "samples to this file.") + bench_parser.add_argument( + '--perf-freq', + help="perf sampling frequency. This is passed directly to the perf " + "script.") + bench_parser.add_argument( + '--perf-period', + help="perf sampling period. This is passed directly to the perf " + "script.") + bench_parser.add_argument( + '--perf-events', + help="perf events to record. This is passed directly to the perf " + "script.") + bench_parser.add_argument( + '--perf-script', + type=lambda x: x.split(), + default=PERF_SCRIPT, + help="Path to the perf script to use. Defaults to %r." % PERF_SCRIPT) + bench_parser.add_argument( + '--perf-tool', + type=lambda x: x.split(), + help="Path to the perf tool to use. This is passed directly to the " + "perf script") # compilation flags comp_parser = parser.add_argument_group('compilation options') @@ -1348,7 +1402,7 @@ if __name__ == "__main__": '-o', '--output', help="Output file.") - # runner + bench_ids overlaps bench_paths, so we need to do some munging here + # runner/bench_paths overlap, so need to do some munging here args = parser.parse_intermixed_args() args.bench_paths = [' '.join(args.runner or [])] + args.bench_ids args.runner = args.runner or [RUNNER_PATH] diff --git a/scripts/code.py b/scripts/code.py index 6b373fcf..7e7e960c 100755 --- a/scripts/code.py +++ b/scripts/code.py @@ -5,7 +5,7 @@ # by Linux's Bloat-O-Meter. # # Example: -# ./scripts/code.py lfs.o lfs_util.o -S +# ./scripts/code.py lfs.o lfs_util.o -Ssize # # Copyright (c) 2022, The littlefs authors. # Copyright (c) 2020, Arm Limited. All rights reserved. @@ -14,6 +14,7 @@ import collections as co import csv +import difflib import glob import itertools as it import math as m @@ -25,7 +26,8 @@ import subprocess as sp OBJ_PATHS = ['*.o'] NM_TOOL = ['nm'] -TYPE = 'tTrRdD' +NM_TYPES = 'tTrRdD' +OBJDUMP_TOOL = ['objdump'] # integer fields @@ -135,21 +137,32 @@ def openio(path, mode='r'): def collect(paths, *, nm_tool=NM_TOOL, - type=TYPE, - build_dir=None, + nm_types=NM_TYPES, + objdump_tool=OBJDUMP_TOOL, + sources=None, everything=False, **args): - results = [] - pattern = re.compile( + size_pattern = re.compile( '^(?P[0-9a-fA-F]+)' + - ' (?P[%s])' % re.escape(type) + + ' (?P[%s])' % re.escape(nm_types) + ' (?P.+?)$') + line_pattern = re.compile( + '^\s+(?P[0-9]+)\s+' + '(?:(?P[0-9]+)\s+)?' + '.*\s+' + '(?P[^\s]+)$') + info_pattern = re.compile( + '^(?:.*(?PDW_TAG_[a-z_]+).*' + '|^.*DW_AT_name.*:\s*(?P[^:\s]+)\s*' + '|^.*DW_AT_decl_file.*:\s*(?P[0-9]+)\s*)$') + + results = [] for path in paths: - # map to source file - src_path = re.sub('\.o$', '.c', path) - if build_dir: - src_path = re.sub('%s/*' % re.escape(build_dir), '', - src_path) + # guess the source, if we have debug-info we'll replace this later + file = re.sub('(\.o)?$', '.c', path, 1) + + # find symbol sizes + results_ = [] # note nm-tool may contain extra args cmd = nm_tool + ['--size-sort', path] if args.get('verbose'): @@ -158,21 +171,18 @@ def collect(paths, *, stdout=sp.PIPE, stderr=sp.PIPE if not args.get('verbose') else None, universal_newlines=True, - errors='replace') + errors='replace', + close_fds=False) for line in proc.stdout: - m = pattern.match(line) + m = size_pattern.match(line) if m: func = m.group('func') # discard internal functions if not everything and func.startswith('__'): continue - # discard .8449 suffixes created by optimizer - func = re.sub('\.[0-9]+', '', func) - - results.append(CodeResult( - src_path, func, + results_.append(CodeResult( + file, func, int(m.group('size'), 16))) - proc.wait() if proc.returncode != 0: if not args.get('verbose'): @@ -180,6 +190,121 @@ def collect(paths, *, sys.stdout.write(line) sys.exit(-1) + + # try to figure out the source file if we have debug-info + dirs = {} + files = {} + # note objdump-tool may contain extra args + cmd = objdump_tool + ['--dwarf=rawline', path] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + for line in proc.stdout: + # note that files contain references to dirs, which we + # dereference as soon as we see them as each file table follows a + # dir table + m = line_pattern.match(line) + if m: + if not m.group('dir'): + # found a directory entry + dirs[int(m.group('no'))] = m.group('path') + else: + # found a file entry + dir = int(m.group('dir')) + if dir in dirs: + files[int(m.group('no'))] = os.path.join( + dirs[dir], + m.group('path')) + else: + files[int(m.group('no'))] = m.group('path') + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + # do nothing on error, we don't need objdump to work, source files + # may just be inaccurate + pass + + defs = {} + is_func = False + f_name = None + f_file = None + # note objdump-tool may contain extra args + cmd = objdump_tool + ['--dwarf=info', path] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + for line in proc.stdout: + # state machine here to find definitions + m = info_pattern.match(line) + if m: + if m.group('tag'): + if is_func: + defs[f_name] = files.get(f_file, '?') + is_func = (m.group('tag') == 'DW_TAG_subprogram') + elif m.group('name'): + f_name = m.group('name') + elif m.group('file'): + f_file = int(m.group('file')) + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + # do nothing on error, we don't need objdump to work, source files + # may just be inaccurate + pass + + for r in results_: + # find best matching debug symbol, this may be slightly different + # due to optimizations + if defs: + # exact match? avoid difflib if we can for speed + if r.function in defs: + file = defs[r.function] + else: + _, file = max( + defs.items(), + key=lambda d: difflib.SequenceMatcher(None, + d[0], + r.function, False).ratio()) + else: + file = r.file + + # ignore filtered sources + if sources is not None: + if not any( + os.path.abspath(file) == os.path.abspath(s) + for s in sources): + continue + else: + # default to only cwd + if not everything and not os.path.commonpath([ + os.getcwd(), + os.path.abspath(file)]) == os.getcwd(): + continue + + # simplify path + if os.path.commonpath([ + os.getcwd(), + os.path.abspath(file)]) == os.getcwd(): + file = os.path.relpath(file) + else: + file = os.path.abspath(file) + + results.append(CodeResult(file, r.function, r.size)) + return results @@ -437,7 +562,7 @@ def main(obj_paths, *, paths.append(path) if not paths: - print("error: no .obj files found in %r?" % obj_paths) + print("error: no .o files found in %r?" % obj_paths) sys.exit(-1) results = collect(paths, **args) @@ -469,13 +594,16 @@ def main(obj_paths, *, # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, CodeResult._by + writer = csv.DictWriter(f, + (by if by is not None else CodeResult._by) + ['code_'+k for k in CodeResult._fields]) writer.writeheader() for r in results: writer.writerow( - {k: getattr(r, k) for k in CodeResult._by} - | {'code_'+k: getattr(r, k) for k in CodeResult._fields}) + {k: getattr(r, k) + for k in (by if by is not None else CodeResult._by)} + | {'code_'+k: getattr(r, k) + for k in CodeResult._fields}) # find previous results? if args.get('diff'): @@ -512,7 +640,8 @@ if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Find code size at the function level.") + description="Find code size at the function level.", + allow_abbrev=False) parser.add_argument( 'obj_paths', nargs='*', @@ -579,23 +708,30 @@ if __name__ == "__main__": action='store_true', help="Only show the total.") parser.add_argument( - '-A', '--everything', + '-F', '--source', + dest='sources', + action='append', + help="Only consider definitions in this file. Defaults to anything " + "in the current directory.") + parser.add_argument( + '--everything', action='store_true', help="Include builtin and libc specific symbols.") parser.add_argument( - '--type', - default=TYPE, + '--nm-types', + default=NM_TYPES, help="Type of symbols to report, this uses the same single-character " - "type-names emitted by nm. Defaults to %r." % TYPE) + "type-names emitted by nm. Defaults to %r." % NM_TYPES) parser.add_argument( '--nm-tool', type=lambda x: x.split(), default=NM_TOOL, help="Path to the nm tool to use. Defaults to %r." % NM_TOOL) parser.add_argument( - '--build-dir', - help="Specify the relative build directory. Used to map object files " - "to the correct source files.") + '--objdump-tool', + type=lambda x: x.split(), + default=OBJDUMP_TOOL, + help="Path to the objdump tool to use. Defaults to %r." % OBJDUMP_TOOL) sys.exit(main(**{k: v for k, v in vars(parser.parse_intermixed_args()).items() if v is not None})) diff --git a/scripts/coverage.py b/scripts/coverage.py index 7d36a47e..f74dda82 100755 --- a/scripts/coverage.py +++ b/scripts/coverage.py @@ -3,7 +3,9 @@ # Script to find coverage info after running tests. # # Example: -# ./scripts/coverage.py lfs.t.a.gcda lfs_util.t.a.gcda -s +# ./scripts/coverage.py \ +# lfs.t.a.gcda lfs_util.t.a.gcda \ +# -Flfs.c -Flfs_util.c -slines # # Copyright (c) 2022, The littlefs authors. # Copyright (c) 2020, Arm Limited. All rights reserved. @@ -209,19 +211,13 @@ def openio(path, mode='r'): else: return open(path, mode) -def collect(paths, *, +def collect(gcda_paths, *, gcov_tool=GCOV_TOOL, - build_dir=None, + sources=None, everything=False, **args): results = [] - for path in paths: - # map to source file - src_path = re.sub('\.t\.a\.gcda$', '.c', path) - if build_dir: - src_path = re.sub('%s/*' % re.escape(build_dir), '', - src_path) - + for path in gcda_paths: # get coverage info through gcov's json output # note, gcov-tool may contain extra args cmd = GCOV_TOOL + ['-b', '-t', '--json-format', path] @@ -231,7 +227,8 @@ def collect(paths, *, stdout=sp.PIPE, stderr=sp.PIPE if not args.get('verbose') else None, universal_newlines=True, - errors='replace') + errors='replace', + close_fds=False) data = json.load(proc.stdout) proc.wait() if proc.returncode != 0: @@ -242,12 +239,30 @@ def collect(paths, *, # collect line/branch coverage for file in data['files']: - if file['file'] != src_path: - continue + # ignore filtered sources + if sources is not None: + if not any( + os.path.abspath(file['file']) == os.path.abspath(s) + for s in sources): + continue + else: + # default to only cwd + if not everything and not os.path.commonpath([ + os.getcwd(), + os.path.abspath(file['file'])]) == os.getcwd(): + continue + + # simplify path + if os.path.commonpath([ + os.getcwd(), + os.path.abspath(file['file'])]) == os.getcwd(): + file_name = os.path.relpath(file['file']) + else: + file_name = os.path.abspath(file['file']) for func in file['functions']: func_name = func.get('name', '(inlined)') - # discard internal function (this includes injected test cases) + # discard internal functions (this includes injected test cases) if not everything: if func_name.startswith('__'): continue @@ -255,7 +270,7 @@ def collect(paths, *, # go ahead and add functions, later folding will merge this if # there are other hits on this line results.append(CoverageResult( - src_path, func_name, func['start_line'], + file_name, func_name, func['start_line'], func['execution_count'], 0, Frac(1 if func['execution_count'] > 0 else 0, 1), 0, @@ -271,7 +286,7 @@ def collect(paths, *, # go ahead and add lines, later folding will merge this if # there are other hits on this line results.append(CoverageResult( - src_path, func_name, line['line_number'], + file_name, func_name, line['line_number'], 0, line['count'], 0, Frac(1 if line['count'] > 0 else 0, 1), @@ -519,31 +534,25 @@ def table(Result, results, diff_results=None, *, line[-1])) -def annotate(Result, results, paths, *, +def annotate(Result, results, *, annotate=False, lines=False, branches=False, - build_dir=None, **args): # if neither branches/lines specified, color both if annotate and not lines and not branches: lines, branches = True, True - for path in paths: - # map to source file - src_path = re.sub('\.t\.a\.gcda$', '.c', path) - if build_dir: - src_path = re.sub('%s/*' % re.escape(build_dir), '', - src_path) - + for path in co.OrderedDict.fromkeys(r.file for r in results).keys(): # flatten to line info results = fold(Result, results, by=['file', 'line']) - table = {r.line: r for r in results if r.file == src_path} + table = {r.line: r for r in results if r.file == path} # calculate spans to show if not annotate: spans = [] last = None + func = None for line, r in sorted(table.items()): if ((lines and int(r.hits) == 0) or (branches and r.branches.a < r.branches.b)): @@ -553,27 +562,29 @@ def annotate(Result, results, paths, *, line+1+args['context']) else: if last is not None: - spans.append(last) + spans.append((last, func)) last = range( line-args['context'], line+1+args['context']) + func = r.function if last is not None: - spans.append(last) + spans.append((last, func)) - with open(src_path) as f: + with open(path) as f: skipped = False for i, line in enumerate(f): # skip lines not in spans? - if not annotate and not any(i+1 in s for s in spans): + if not annotate and not any(i+1 in s for s, _ in spans): skipped = True continue if skipped: skipped = False - print('%s@@ %s:%d @@%s' % ( + print('%s@@ %s:%d: %s @@%s' % ( '\x1b[36m' if args['color'] else '', - src_path, + path, i+1, + next(iter(f for _, f in spans)), '\x1b[m' if args['color'] else '')) # build line @@ -659,12 +670,14 @@ def main(gcda_paths, *, # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, CoverageResult._by + writer = csv.DictWriter(f, + (by if by is not None else CoverageResult._by) + ['coverage_'+k for k in CoverageResult._fields]) writer.writeheader() for r in results: writer.writerow( - {k: getattr(r, k) for k in CoverageResult._by} + {k: getattr(r, k) + for k in (by if by is not None else CoverageResult._by)} | {'coverage_'+k: getattr(r, k) for k in CoverageResult._fields}) @@ -698,8 +711,7 @@ def main(gcda_paths, *, or args.get('lines') or args.get('branches')): # annotate sources - annotate(CoverageResult, results, paths, - **args) + annotate(CoverageResult, results, **args) else: # print table table(CoverageResult, results, @@ -724,7 +736,8 @@ if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Find coverage info after running tests.") + description="Find coverage info after running tests.", + allow_abbrev=False) parser.add_argument( 'gcda_paths', nargs='*', @@ -791,15 +804,21 @@ if __name__ == "__main__": action='store_true', help="Only show the total.") parser.add_argument( - '-A', '--everything', + '-F', '--source', + dest='sources', + action='append', + help="Only consider definitions in this file. Defaults to anything " + "in the current directory.") + parser.add_argument( + '--everything', action='store_true', help="Include builtin and libc specific symbols.") parser.add_argument( - '-H', '--hits', + '--hits', action='store_true', help="Show total hits instead of coverage.") parser.add_argument( - '-l', '--annotate', + '-A', '--annotate', action='store_true', help="Show source files annotated with coverage info.") parser.add_argument( @@ -814,7 +833,7 @@ if __name__ == "__main__": '-c', '--context', type=lambda x: int(x, 0), default=3, - help="Show a additional lines of context. Defaults to 3.") + help="Show n additional lines of context. Defaults to 3.") parser.add_argument( '-W', '--width', type=lambda x: int(x, 0), @@ -838,10 +857,6 @@ if __name__ == "__main__": default=GCOV_TOOL, type=lambda x: x.split(), help="Path to the gcov tool to use. Defaults to %r." % GCOV_TOOL) - parser.add_argument( - '--build-dir', - help="Specify the relative build directory. Used to map object files " - "to the correct source files.") sys.exit(main(**{k: v for k, v in vars(parser.parse_intermixed_args()).items() if v is not None})) diff --git a/scripts/data.py b/scripts/data.py index 05ef8681..f95540f3 100755 --- a/scripts/data.py +++ b/scripts/data.py @@ -5,7 +5,7 @@ # by Linux's Bloat-O-Meter. # # Example: -# ./scripts/data.py lfs.o lfs_util.o -S +# ./scripts/data.py lfs.o lfs_util.o -Ssize # # Copyright (c) 2022, The littlefs authors. # Copyright (c) 2020, Arm Limited. All rights reserved. @@ -14,6 +14,7 @@ import collections as co import csv +import difflib import glob import itertools as it import math as m @@ -25,7 +26,8 @@ import subprocess as sp OBJ_PATHS = ['*.o'] NM_TOOL = ['nm'] -TYPE = 'dDbB' +NM_TYPES = 'dDbB' +OBJDUMP_TOOL = ['objdump'] # integer fields @@ -135,21 +137,32 @@ def openio(path, mode='r'): def collect(paths, *, nm_tool=NM_TOOL, - type=TYPE, - build_dir=None, + nm_types=NM_TYPES, + objdump_tool=OBJDUMP_TOOL, + sources=None, everything=False, **args): - results = [] - pattern = re.compile( + size_pattern = re.compile( '^(?P[0-9a-fA-F]+)' + - ' (?P[%s])' % re.escape(type) + + ' (?P[%s])' % re.escape(nm_types) + ' (?P.+?)$') + line_pattern = re.compile( + '^\s+(?P[0-9]+)\s+' + '(?:(?P[0-9]+)\s+)?' + '.*\s+' + '(?P[^\s]+)$') + info_pattern = re.compile( + '^(?:.*(?PDW_TAG_[a-z_]+).*' + '|^.*DW_AT_name.*:\s*(?P[^:\s]+)\s*' + '|^.*DW_AT_decl_file.*:\s*(?P[0-9]+)\s*)$') + + results = [] for path in paths: - # map to source file - src_path = re.sub('\.o$', '.c', path) - if build_dir: - src_path = re.sub('%s/*' % re.escape(build_dir), '', - src_path) + # guess the source, if we have debug-info we'll replace this later + file = re.sub('(\.o)?$', '.c', path, 1) + + # find symbol sizes + results_ = [] # note nm-tool may contain extra args cmd = nm_tool + ['--size-sort', path] if args.get('verbose'): @@ -158,21 +171,18 @@ def collect(paths, *, stdout=sp.PIPE, stderr=sp.PIPE if not args.get('verbose') else None, universal_newlines=True, - errors='replace') + errors='replace', + close_fds=False) for line in proc.stdout: - m = pattern.match(line) + m = size_pattern.match(line) if m: func = m.group('func') # discard internal functions if not everything and func.startswith('__'): continue - # discard .8449 suffixes created by optimizer - func = re.sub('\.[0-9]+', '', func) - - results.append(DataResult( - src_path, func, + results_.append(DataResult( + file, func, int(m.group('size'), 16))) - proc.wait() if proc.returncode != 0: if not args.get('verbose'): @@ -180,6 +190,121 @@ def collect(paths, *, sys.stdout.write(line) sys.exit(-1) + + # try to figure out the source file if we have debug-info + dirs = {} + files = {} + # note objdump-tool may contain extra args + cmd = objdump_tool + ['--dwarf=rawline', path] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + for line in proc.stdout: + # note that files contain references to dirs, which we + # dereference as soon as we see them as each file table follows a + # dir table + m = line_pattern.match(line) + if m: + if not m.group('dir'): + # found a directory entry + dirs[int(m.group('no'))] = m.group('path') + else: + # found a file entry + dir = int(m.group('dir')) + if dir in dirs: + files[int(m.group('no'))] = os.path.join( + dirs[dir], + m.group('path')) + else: + files[int(m.group('no'))] = m.group('path') + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + # do nothing on error, we don't need objdump to work, source files + # may just be inaccurate + pass + + defs = {} + is_func = False + f_name = None + f_file = None + # note objdump-tool may contain extra args + cmd = objdump_tool + ['--dwarf=info', path] + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in cmd)) + proc = sp.Popen(cmd, + stdout=sp.PIPE, + stderr=sp.PIPE if not args.get('verbose') else None, + universal_newlines=True, + errors='replace', + close_fds=False) + for line in proc.stdout: + # state machine here to find definitions + m = info_pattern.match(line) + if m: + if m.group('tag'): + if is_func: + defs[f_name] = files.get(f_file, '?') + is_func = (m.group('tag') == 'DW_TAG_subprogram') + elif m.group('name'): + f_name = m.group('name') + elif m.group('file'): + f_file = int(m.group('file')) + proc.wait() + if proc.returncode != 0: + if not args.get('verbose'): + for line in proc.stderr: + sys.stdout.write(line) + # do nothing on error, we don't need objdump to work, source files + # may just be inaccurate + pass + + for r in results_: + # find best matching debug symbol, this may be slightly different + # due to optimizations + if defs: + # exact match? avoid difflib if we can for speed + if r.function in defs: + file = defs[r.function] + else: + _, file = max( + defs.items(), + key=lambda d: difflib.SequenceMatcher(None, + d[0], + r.function, False).ratio()) + else: + file = r.file + + # ignore filtered sources + if sources is not None: + if not any( + os.path.abspath(file) == os.path.abspath(s) + for s in sources): + continue + else: + # default to only cwd + if not everything and not os.path.commonpath([ + os.getcwd(), + os.path.abspath(file)]) == os.getcwd(): + continue + + # simplify path + if os.path.commonpath([ + os.getcwd(), + os.path.abspath(file)]) == os.getcwd(): + file = os.path.relpath(file) + else: + file = os.path.abspath(file) + + results.append(DataResult(file, r.function, r.size)) + return results @@ -437,7 +562,7 @@ def main(obj_paths, *, paths.append(path) if not paths: - print("error: no .obj files found in %r?" % obj_paths) + print("error: no .o files found in %r?" % obj_paths) sys.exit(-1) results = collect(paths, **args) @@ -469,13 +594,16 @@ def main(obj_paths, *, # write results to CSV if args.get('output'): with openio(args['output'], 'w') as f: - writer = csv.DictWriter(f, DataResult._by + writer = csv.DictWriter(f, + (by if by is not None else DataResult._by) + ['data_'+k for k in DataResult._fields]) writer.writeheader() for r in results: writer.writerow( - {k: getattr(r, k) for k in DataResult._by} - | {'data_'+k: getattr(r, k) for k in DataResult._fields}) + {k: getattr(r, k) + for k in (by if by is not None else DataResult._by)} + | {'data_'+k: getattr(r, k) + for k in DataResult._fields}) # find previous results? if args.get('diff'): @@ -512,7 +640,8 @@ if __name__ == "__main__": import argparse import sys parser = argparse.ArgumentParser( - description="Find data size at the function level.") + description="Find data size at the function level.", + allow_abbrev=False) parser.add_argument( 'obj_paths', nargs='*', @@ -579,23 +708,30 @@ if __name__ == "__main__": action='store_true', help="Only show the total.") parser.add_argument( - '-A', '--everything', + '-F', '--source', + dest='sources', + action='append', + help="Only consider definitions in this file. Defaults to anything " + "in the current directory.") + parser.add_argument( + '--everything', action='store_true', help="Include builtin and libc specific symbols.") parser.add_argument( - '--type', - default=TYPE, + '--nm-types', + default=NM_TYPES, help="Type of symbols to report, this uses the same single-character " - "type-names emitted by nm. Defaults to %r." % TYPE) + "type-names emitted by nm. Defaults to %r." % NM_TYPES) parser.add_argument( '--nm-tool', type=lambda x: x.split(), default=NM_TOOL, help="Path to the nm tool to use. Defaults to %r." % NM_TOOL) parser.add_argument( - '--build-dir', - help="Specify the relative build directory. Used to map object files " - "to the correct source files.") + '--objdump-tool', + type=lambda x: x.split(), + default=OBJDUMP_TOOL, + help="Path to the objdump tool to use. Defaults to %r." % OBJDUMP_TOOL) sys.exit(main(**{k: v for k, v in vars(parser.parse_intermixed_args()).items() if v is not None})) diff --git a/scripts/perf.py b/scripts/perf.py new file mode 100755 index 00000000..3eb3dbc2 --- /dev/null +++ b/scripts/perf.py @@ -0,0 +1,1263 @@ +#!/usr/bin/env python3 +# +# Script to aggregate and report Linux perf results. +# +# Example: +# ./scripts/perf.py -R -obench.perf ./runners/bench_runner +# ./scripts/perf.py bench.perf -Flfs.c -Flfs_util.c -Scycles +# +# Copyright (c) 2022, The littlefs authors. +# SPDX-License-Identifier: BSD-3-Clause +# + +import bisect +import collections as co +import csv +import errno +import fcntl +import functools as ft +import glob +import itertools as it +import math as m +import multiprocessing as mp +import os +import re +import shlex +import shutil +import subprocess as sp +import tempfile +import zipfile + + +PERF_PATHS = ['*.perf'] +PERF_TOOL = ['perf'] +PERF_EVENTS = 'cycles,branch-misses,branches,cache-misses,cache-references' +PERF_FREQ = 100 +OBJDUMP_TOOL = ['objdump'] +THRESHOLD = (0.5, 0.85) + + +# integer fields +class Int(co.namedtuple('Int', 'x')): + __slots__ = () + def __new__(cls, x=0): + if isinstance(x, Int): + return x + if isinstance(x, str): + try: + x = int(x, 0) + except ValueError: + # also accept +-∞ and +-inf + if re.match('^\s*\+?\s*(?:∞|inf)\s*$', x): + x = m.inf + elif re.match('^\s*-\s*(?:∞|inf)\s*$', x): + x = -m.inf + else: + raise + assert isinstance(x, int) or m.isinf(x), x + return super().__new__(cls, x) + + def __str__(self): + if self.x == m.inf: + return '∞' + elif self.x == -m.inf: + return '-∞' + else: + return str(self.x) + + def __int__(self): + assert not m.isinf(self.x) + return self.x + + def __float__(self): + return float(self.x) + + none = '%7s' % '-' + def table(self): + return '%7s' % (self,) + + diff_none = '%7s' % '-' + diff_table = table + + def diff_diff(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + diff = new - old + if diff == +m.inf: + return '%7s' % '+∞' + elif diff == -m.inf: + return '%7s' % '-∞' + else: + return '%+7d' % diff + + def ratio(self, other): + new = self.x if self else 0 + old = other.x if other else 0 + if m.isinf(new) and m.isinf(old): + return 0.0 + elif m.isinf(new): + return +m.inf + elif m.isinf(old): + return -m.inf + elif not old and not new: + return 0.0 + elif not old: + return 1.0 + else: + return (new-old) / old + + def __add__(self, other): + return self.__class__(self.x + other.x) + + def __sub__(self, other): + return self.__class__(self.x - other.x) + + def __mul__(self, other): + return self.__class__(self.x * other.x) + +# perf results +class PerfResult(co.namedtuple('PerfResult', [ + 'file', 'function', 'line', + 'self_cycles', + 'self_bmisses', 'self_branches', + 'self_cmisses', 'self_caches', + 'cycles', + 'bmisses', 'branches', + 'cmisses', 'caches', + 'children', 'parents'])): + _by = ['file', 'function', 'line'] + _fields = [ + 'self_cycles', + 'self_bmisses', 'self_branches', + 'self_cmisses', 'self_caches', + 'cycles', + 'bmisses', 'branches', + 'cmisses', 'caches'] + _types = { + 'self_cycles': Int, + 'self_bmisses': Int, 'self_branches': Int, + 'self_cmisses': Int, 'self_caches': Int, + 'cycles': Int, + 'bmisses': Int, 'branches': Int, + 'cmisses': Int, 'caches': Int} + + __slots__ = () + def __new__(cls, file='', function='', line=0, + self_cycles=0, + self_bmisses=0, self_branches=0, + self_cmisses=0, self_caches=0, + cycles=0, + bmisses=0, branches=0, + cmisses=0, caches=0, + children=set(), parents=set()): + return super().__new__(cls, file, function, int(Int(line)), + Int(self_cycles), + Int(self_bmisses), Int(self_branches), + Int(self_cmisses), Int(self_caches), + Int(cycles), + Int(bmisses), Int(branches), + Int(cmisses), Int(caches), + children, parents) + + def __add__(self, other): + return PerfResult(self.file, self.function, self.line, + self.self_cycles + other.self_cycles, + self.self_bmisses + other.self_bmisses, + self.self_branches + other.self_branches, + self.self_cmisses + other.self_cmisses, + self.self_caches + other.self_caches, + self.cycles + other.cycles, + self.bmisses + other.bmisses, + self.branches + other.branches, + self.cmisses + other.cmisses, + self.caches + other.caches, + self.children | other.children, + self.parents | other.parents) + + +def openio(path, mode='r'): + if path == '-': + if mode == 'r': + return os.fdopen(os.dup(sys.stdin.fileno()), 'r') + else: + return os.fdopen(os.dup(sys.stdout.fileno()), 'w') + else: + return open(path, mode) + +# run perf as a subprocess, storing measurements into a zip file +def record(command, *, + output=None, + perf_freq=PERF_FREQ, + perf_period=None, + perf_events=PERF_EVENTS, + perf_tool=PERF_TOOL, + **args): + if not command: + print('error: no command specified?') + sys.exit(-1) + + if not output: + print('error: no output file specified?') + sys.exit(-1) + + # create a temporary file for perf to write to, as far as I can tell + # this is strictly needed because perf's pipe-mode only works with stdout + with tempfile.NamedTemporaryFile('rb') as f: + # figure out our perf invocation + perf = perf_tool + list(filter(None, [ + 'record', + '-F%s' % perf_freq + if perf_freq is not None + and perf_period is None else None, + '-c%s' % perf_period + if perf_period is not None else None, + '-B', + '-g', + '--all-user', + '-e%s' % perf_events, + '-o%s' % f.name])) + + # run our command + try: + if args.get('verbose'): + print(' '.join(shlex.quote(c) for c in perf + command)) + err = sp.call(perf + command, close_fds=False) + + except KeyboardInterrupt: + err = errno.EOWNERDEAD + + # synchronize access + z = os.open(output, os.O_RDWR | os.O_CREAT) + fcntl.flock(z, fcntl.LOCK_EX) + + # copy measurements into our zip file + with os.fdopen(z, 'r+b') as z: + with zipfile.ZipFile(z, 'a', + compression=zipfile.ZIP_DEFLATED, + compresslevel=1) as z: + with z.open('perf.%d' % os.getpid(), 'w') as g: + shutil.copyfileobj(f, g) + + # forward the return code + return err + + +def collect_decompressed(path, *, + perf_tool=PERF_TOOL, + everything=False, + depth=0, + **args): + sample_pattern = re.compile( + '(?P\w+)' + '\s+(?P\w+)' + '\s+(?P