1#!/usr/bin/env python3
2#
3# Process img-bench test templates
4#
5# Copyright (c) 2021 Virtuozzo International GmbH.
6#
7# This program is free software; you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation; either version 2 of the License, or
10# (at your option) any later version.
11#
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU General Public License
18# along with this program.  If not, see <http://www.gnu.org/licenses/>.
19#
20
21
22import sys
23import subprocess
24import re
25import json
26
27import simplebench
28from results_to_text import results_to_text
29from table_templater import Templater
30
31
32def bench_func(env, case):
33    test = templater.gen(env['data'], case['data'])
34
35    p = subprocess.run(test, shell=True, stdout=subprocess.PIPE,
36                       stderr=subprocess.STDOUT, universal_newlines=True)
37
38    if p.returncode == 0:
39        try:
40            m = re.search(r'Run completed in (\d+.\d+) seconds.', p.stdout)
41            return {'seconds': float(m.group(1))}
42        except Exception:
43            return {'error': f'failed to parse qemu-img output: {p.stdout}'}
44    else:
45        return {'error': f'qemu-img failed: {p.returncode}: {p.stdout}'}
46
47
48if __name__ == '__main__':
49    if len(sys.argv) > 1:
50        print("""
51Usage: img_bench_templater.py < path/to/test-template.sh
52
53This script generates performance tests from a test template (example below),
54runs them, and displays the results in a table. The template is read from
55stdin.  It must be written in bash and end with a `qemu-img bench` invocation
56(whose result is parsed to get the test instance’s result).
57
58Use the following syntax in the template to create the various different test
59instances:
60
61  column templating: {var1|var2|...} - test will use different values in
62  different columns. You may use several {} constructions in the test, in this
63  case product of all choice-sets will be used.
64
65  row templating: [var1|var2|...] - similar thing to define rows (test-cases)
66
67Test template example:
68
69Assume you want to compare two qemu-img binaries, called qemu-img-old and
70qemu-img-new in your build directory in two test-cases with 4K writes and 64K
71writes. The template may look like this:
72
73qemu_img=/path/to/qemu/build/qemu-img-{old|new}
74$qemu_img create -f qcow2 /ssd/x.qcow2 1G
75$qemu_img bench -c 100 -d 8 [-s 4K|-s 64K] -w -t none -n /ssd/x.qcow2
76
77When passing this to stdin of img_bench_templater.py, the resulting comparison
78table will contain two columns (for two binaries) and two rows (for two
79test-cases).
80
81In addition to displaying the results, script also stores results in JSON
82format into results.json file in current directory.
83""")
84        sys.exit()
85
86    templater = Templater(sys.stdin.read())
87
88    envs = [{'id': ' / '.join(x), 'data': x} for x in templater.columns]
89    cases = [{'id': ' / '.join(x), 'data': x} for x in templater.rows]
90
91    result = simplebench.bench(bench_func, envs, cases, count=5,
92                               initial_run=False)
93    print(results_to_text(result))
94    with open('results.json', 'w') as f:
95        json.dump(result, f, indent=4)
96