xref: /openbmc/qemu/scripts/simplebench/img_bench_templater.py (revision 57b6f58c1d0df757c9311496c32d502925056894)
1 #!/usr/bin/env python3
2 #
3 # Process img-bench test templates
4 #
5 # Copyright (c) 2021 Virtuozzo International GmbH.
6 #
7 # This program is free software; you can redistribute it and/or modify
8 # it under the terms of the GNU General Public License as published by
9 # the Free Software Foundation; either version 2 of the License, or
10 # (at your option) any later version.
11 #
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 # GNU General Public License for more details.
16 #
17 # You should have received a copy of the GNU General Public License
18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
19 #
20 
21 
22 import sys
23 import subprocess
24 import re
25 import json
26 
27 import simplebench
28 from results_to_text import results_to_text
29 from table_templater import Templater
30 
31 
32 def bench_func(env, case):
33     test = templater.gen(env['data'], case['data'])
34 
35     p = subprocess.run(test, shell=True, stdout=subprocess.PIPE,
36                        stderr=subprocess.STDOUT, universal_newlines=True)
37 
38     if p.returncode == 0:
39         try:
40             m = re.search(r'Run completed in (\d+.\d+) seconds.', p.stdout)
41             return {'seconds': float(m.group(1))}
42         except Exception:
43             return {'error': f'failed to parse qemu-img output: {p.stdout}'}
44     else:
45         return {'error': f'qemu-img failed: {p.returncode}: {p.stdout}'}
46 
47 
48 if __name__ == '__main__':
49     if len(sys.argv) > 1:
50         print("""
51 Usage: img_bench_templater.py < path/to/test-template.sh
52 
53 This script generates performance tests from a test template (example below),
54 runs them, and displays the results in a table. The template is read from
55 stdin.  It must be written in bash and end with a `qemu-img bench` invocation
56 (whose result is parsed to get the test instance’s result).
57 
58 Use the following syntax in the template to create the various different test
59 instances:
60 
61   column templating: {var1|var2|...} - test will use different values in
62   different columns. You may use several {} constructions in the test, in this
63   case product of all choice-sets will be used.
64 
65   row templating: [var1|var2|...] - similar thing to define rows (test-cases)
66 
67 Test template example:
68 
69 Assume you want to compare two qemu-img binaries, called qemu-img-old and
70 qemu-img-new in your build directory in two test-cases with 4K writes and 64K
71 writes. The template may look like this:
72 
73 qemu_img=/path/to/qemu/build/qemu-img-{old|new}
74 $qemu_img create -f qcow2 /ssd/x.qcow2 1G
75 $qemu_img bench -c 100 -d 8 [-s 4K|-s 64K] -w -t none -n /ssd/x.qcow2
76 
77 When passing this to stdin of img_bench_templater.py, the resulting comparison
78 table will contain two columns (for two binaries) and two rows (for two
79 test-cases).
80 
81 In addition to displaying the results, script also stores results in JSON
82 format into results.json file in current directory.
83 """)
84         sys.exit()
85 
86     templater = Templater(sys.stdin.read())
87 
88     envs = [{'id': ' / '.join(x), 'data': x} for x in templater.columns]
89     cases = [{'id': ' / '.join(x), 'data': x} for x in templater.rows]
90 
91     result = simplebench.bench(bench_func, envs, cases, count=5,
92                                initial_run=False)
93     print(results_to_text(result))
94     with open('results.json', 'w') as f:
95         json.dump(result, f, indent=4)
96