bench: Tweaked bench.py to include cumulative measurements

This was the one piece needed to be able to replace amor.py with csv.py.
The missing feature in csv.py is the ability to keep track of a
running-sum, but this is a bit of a hack in amor.py considering we
otherwise view csv entries as unordered.

We could add a running-sum to csv.py, or instead, just include a running
sum as a part of our bench output. We have all the information there
anyways, and if it simplifies the mess that is our csv scripts, that's a
win.

---

This also replaces the bench "meas", "iter", and "size" fields with the
slightly simpler "m" (measurement? metric?) and "n" fields. It's up to
the specific benchmark exactly how to interpret "n", but one field is
sufficient for existing scripts.

runners/bench_runner.c

@@ -605,9 +605,8 @@ void bench_permutation(size_t i, uint32_t *buffer, size_t size) {
 
 // bench recording state
 typedef struct bench_record {
-    const char *meas;
+    const char *m;
-    uintmax_t iter;
+    uintmax_t n;
-    uintmax_t size;
     lfs_emubd_io_t last_readed;
     lfs_emubd_io_t last_proged;
     lfs_emubd_io_t last_erased;
@@ -623,7 +622,7 @@ void bench_reset(struct lfs_config *cfg) {
     bench_record_count = 0;
 }
 
-void bench_start(const char *meas, uintmax_t iter, uintmax_t size) {
+void bench_start(const char *m, uintmax_t n) {
     // measure current read/prog/erase
     assert(bench_cfg);
     lfs_emubd_sio_t readed = lfs_emubd_readed(bench_cfg);
@@ -639,15 +638,14 @@ void bench_start(const char *meas, uintmax_t iter, uintmax_t size) {
             sizeof(bench_record_t),
             &bench_record_count,
             &bench_record_capacity);
-    record->meas = meas;
+    record->m = m;
-    record->iter = iter;
+    record->n = n;
-    record->size = size;
     record->last_readed = readed;
     record->last_proged = proged;
     record->last_erased = erased;
 }
 
-void bench_stop(const char *meas) {
+void bench_stop(const char *m) {
     // measure current read/prog/erase
     assert(bench_cfg);
     lfs_emubd_sio_t readed = lfs_emubd_readed(bench_cfg);
@@ -659,12 +657,11 @@ void bench_stop(const char *meas) {
 
     // find our record
     for (size_t i = 0; i < bench_record_count; i++) {
-        if (strcmp(bench_records[i].meas, meas) == 0) {
+        if (strcmp(bench_records[i].m, m) == 0) {
             // print results
-            printf("benched %s %zd %zd %"PRIu64" %"PRIu64" %"PRIu64"\n",
+            printf("benched %s %zd %"PRIu64" %"PRIu64" %"PRIu64"\n",
-                    bench_records[i].meas,
+                    bench_records[i].m,
-                    bench_records[i].iter,
+                    bench_records[i].n,
-                    bench_records[i].size,
                     readed - bench_records[i].last_readed,
                     proged - bench_records[i].last_proged,
                     erased - bench_records[i].last_erased);
@@ -680,28 +677,24 @@ void bench_stop(const char *meas) {
 
     // not found?
     fprintf(stderr, "error: bench stopped before it was started (%s)\n",
-            meas);
+            m);
     assert(false);
     exit(-1);
 }
 
-void bench_result(const char *meas, uintmax_t iter, uintmax_t size,
+void bench_result(const char *m, uintmax_t n, uintmax_t result) {
-        uintmax_t result) {
     // we just print these directly
-    printf("benched %s %zd %zd %"PRIu64"\n",
+    printf("benched %s %zd %"PRIu64"\n",
-            meas,
+            m,
-            iter,
+            n,
-            size,
             result);
 }
 
-void bench_fresult(const char *meas, uintmax_t iter, uintmax_t size,
+void bench_fresult(const char *m, uintmax_t n, double result) {
-        double result) {
     // we just print these directly
-    printf("benched %s %zd %zd %.6f\n",
+    printf("benched %s %zd %.6f\n",
-            meas,
+            m,
-            iter,
+            n,
-            size,
             result);
 }
 

runners/bench_runner.h

@@ -21,24 +21,18 @@ void bench_trace(const char *fmt, ...);
 
 // BENCH_START/BENCH_STOP macros measure readed/proged/erased bytes
 // through emubd
-void bench_start(const char *meas, uintmax_t iter, uintmax_t size);
+void bench_start(const char *m, uintmax_t n);
-void bench_stop(const char *meas);
+void bench_stop(const char *m);
 
-#define BENCH_START(meas, iter, size) \
+#define BENCH_START(m, n) bench_start(m, n)
-    bench_start(meas, iter, size)
+#define BENCH_STOP(m) bench_stop(m)
-#define BENCH_STOP(meas) \
-    bench_stop(meas)
 
 // BENCH_RESULT/BENCH_FRESULT allow for explicit non-io measurements
-void bench_result(const char *meas, uintmax_t iter, uintmax_t size,
+void bench_result(const char *m, uintmax_t n, uintmax_t result);
-        uintmax_t result);
+void bench_fresult(const char *m, uintmax_t n, double result);
-void bench_fresult(const char *meas, uintmax_t iter, uintmax_t size,
-        double result);
 
-#define BENCH_RESULT(meas, iter, size, result) \
+#define BENCH_RESULT(m, n, result) bench_result(m, n, result)
-    bench_result(meas, iter, size, result)
+#define BENCH_FRESULT(m, n, result) bench_fresult(m, n, result)
-#define BENCH_FRESULT(meas, iter, size, result) \
-    bench_fresult(meas, iter, size, result)
 
 
 // note these are indirectly included in any generated files

scripts/bench.py

@@ -1002,9 +1002,8 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
                 '|' '(?P<path>[^:]+):(?P<lineno>\d+):(?P<op_>assert):'
                     ' *(?P<message>.*)'
                 '|' '(?P<op__>benched)'
-                    ' (?P<meas>[^\s]+)'
+                    ' (?P<m>[^\s]+)'
-                    ' (?P<iter>\d+)'
+                    ' (?P<n>\d+)'
-                    ' (?P<size>\d+)'
                     '(?: (?P<readed>[\d\.]+))?'
                     '(?: (?P<proged>[\d\.]+))?'
                     '(?: (?P<erased>[\d\.]+))?'
@@ -1038,6 +1037,9 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
         last_defines = None # fetched on demand
         last_stdout = co.deque(maxlen=args.get('context', 5) + 1)
         last_assert = None
+        readed_ = None
+        proged_ = None
+        erased_ = None
         try:
             while True:
                 # parse a line for state changes
@@ -1068,6 +1070,9 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
                         last_defines = None
                         last_stdout.clear()
                         last_assert = None
+                        readed_ = 0
+                        proged_ = 0
+                        erased_ = 0
                     elif op == 'finished':
                         # force a failure
                         if args.get('fail'):
@@ -1090,9 +1095,8 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
                         if args.get('keep_going'):
                             proc.kill()
                     elif op == 'benched':
-                        meas = m.group('meas')
+                        m_ = m.group('m')
-                        iter = int(m.group('iter'))
+                        n_ = int(m.group('n'))
-                        size = int(m.group('size'))
                         # parse measurements
                         def dat(v):
                             if v is None:
@@ -1101,9 +1105,13 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
                                 return float(v)
                             else:
                                 return int(v)
-                        readed_ = dat(m.group('readed'))
+                        readed__ = dat(m.group('readed'))
-                        proged_ = dat(m.group('proged'))
+                        proged__ = dat(m.group('proged'))
-                        erased_ = dat(m.group('erased'))
+                        erased__ = dat(m.group('erased'))
+                        # keep track of cumulative measurements
+                        readed_ += readed__
+                        proged_ += proged__
+                        erased_ += erased__
                         if output_:
                             # fetch defines if needed, only do this at most
                             # once per perm
@@ -1116,16 +1124,18 @@ def run_stage(name, runner, bench_ids, stdout_, trace_, output_, **args):
                                     'suite': last_suite,
                                     'case': last_case,
                                     **last_defines,
-                                    'meas': meas,
+                                    'm': m_,
-                                    'iter': iter,
+                                    'n': n_,
-                                    'size': size,
+                                    'readed': readed__,
-                                    'readed': readed_,
+                                    'proged': proged__,
-                                    'proged': proged_,
+                                    'erased': erased__,
-                                    'erased': erased_})
+                                    'creaded': readed_,
+                                    'cproged': proged_,
+                                    'cerased': erased_})
                         # keep track of total for summary
-                        readed += readed_
+                        readed += readed__
-                        proged += proged_
+                        proged += proged__
-                        erased += erased_
+                        erased += erased__
         except KeyboardInterrupt:
             proc.kill()
             raise BenchFailure(last_id, 0, list(last_stdout))
@@ -1296,7 +1306,7 @@ def run(runner, bench_ids=[], **args):
     if args.get('output'):
         output = BenchOutput(args['output'],
                 ['suite', 'case'],
-                ['meas', 'iter', 'size', 'readed', 'proged', 'erased'])
+                ['m', 'n', 'readed', 'proged', 'erased'])
 
     # measure runtime
     start = time.time()