1 #!/usr/bin/env python3 2 # 3 # Process img-bench test templates 4 # 5 # Copyright (c) 2021 Virtuozzo International GmbH. 6 # 7 # This program is free software; you can redistribute it and/or modify 8 # it under the terms of the GNU General Public License as published by 9 # the Free Software Foundation; either version 2 of the License, or 10 # (at your option) any later version. 11 # 12 # This program is distributed in the hope that it will be useful, 13 # but WITHOUT ANY WARRANTY; without even the implied warranty of 14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 # GNU General Public License for more details. 16 # 17 # You should have received a copy of the GNU General Public License 18 # along with this program. If not, see <http://www.gnu.org/licenses/>. 19 # 20 21 22 import sys 23 import subprocess 24 import re 25 import json 26 27 import simplebench 28 from results_to_text import results_to_text 29 from table_templater import Templater 30 31 32 def bench_func(env, case): 33 test = templater.gen(env['data'], case['data']) 34 35 p = subprocess.run(test, shell=True, stdout=subprocess.PIPE, 36 stderr=subprocess.STDOUT, universal_newlines=True) 37 38 if p.returncode == 0: 39 try: 40 m = re.search(r'Run completed in (\d+.\d+) seconds.', p.stdout) 41 return {'seconds': float(m.group(1))} 42 except Exception: 43 return {'error': f'failed to parse qemu-img output: {p.stdout}'} 44 else: 45 return {'error': f'qemu-img failed: {p.returncode}: {p.stdout}'} 46 47 48 if __name__ == '__main__': 49 if len(sys.argv) > 1: 50 print(""" 51 Usage: img_bench_templater.py < path/to/test-template.sh 52 53 This script generates performance tests from a test template (example below), 54 runs them, and displays the results in a table. The template is read from 55 stdin. It must be written in bash and end with a `qemu-img bench` invocation 56 (whose result is parsed to get the test instance’s result). 57 58 Use the following syntax in the template to create the various different test 59 instances: 60 61 column templating: {var1|var2|...} - test will use different values in 62 different columns. You may use several {} constructions in the test, in this 63 case product of all choice-sets will be used. 64 65 row templating: [var1|var2|...] - similar thing to define rows (test-cases) 66 67 Test template example: 68 69 Assume you want to compare two qemu-img binaries, called qemu-img-old and 70 qemu-img-new in your build directory in two test-cases with 4K writes and 64K 71 writes. The template may look like this: 72 73 qemu_img=/path/to/qemu/build/qemu-img-{old|new} 74 $qemu_img create -f qcow2 /ssd/x.qcow2 1G 75 $qemu_img bench -c 100 -d 8 [-s 4K|-s 64K] -w -t none -n /ssd/x.qcow2 76 77 When passing this to stdin of img_bench_templater.py, the resulting comparison 78 table will contain two columns (for two binaries) and two rows (for two 79 test-cases). 80 81 In addition to displaying the results, script also stores results in JSON 82 format into results.json file in current directory. 83 """) 84 sys.exit() 85 86 templater = Templater(sys.stdin.read()) 87 88 envs = [{'id': ' / '.join(x), 'data': x} for x in templater.columns] 89 cases = [{'id': ' / '.join(x), 'data': x} for x in templater.rows] 90 91 result = simplebench.bench(bench_func, envs, cases, count=5, 92 initial_run=False) 93 print(results_to_text(result)) 94 with open('results.json', 'w') as f: 95 json.dump(result, f, indent=4) 96