1#!/usr/bin/env python
2#
3# Simple benchmarking framework
4#
5# Copyright (c) 2019 Virtuozzo International GmbH.
6#
7# This program is free software; you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation; either version 2 of the License, or
10# (at your option) any later version.
11#
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU General Public License
18# along with this program.  If not, see <http://www.gnu.org/licenses/>.
19#
20
21
22def bench_one(test_func, test_env, test_case, count=5, initial_run=True):
23    """Benchmark one test-case
24
25    test_func   -- benchmarking function with prototype
26                   test_func(env, case), which takes test_env and test_case
27                   arguments and returns {'seconds': int} (which is benchmark
28                   result) on success and {'error': str} on error. Returned
29                   dict may contain any other additional fields.
30    test_env    -- test environment - opaque first argument for test_func
31    test_case   -- test case - opaque second argument for test_func
32    count       -- how many times to call test_func, to calculate average
33    initial_run -- do initial run of test_func, which don't get into result
34
35    Returns dict with the following fields:
36        'runs':     list of test_func results
37        'average':  average seconds per run (exists only if at least one run
38                    succeeded)
39        'delta':    maximum delta between test_func result and the average
40                    (exists only if at least one run succeeded)
41        'n-failed': number of failed runs (exists only if at least one run
42                    failed)
43    """
44    if initial_run:
45        print('  #initial run:')
46        print('   ', test_func(test_env, test_case))
47
48    runs = []
49    for i in range(count):
50        print('  #run {}'.format(i+1))
51        res = test_func(test_env, test_case)
52        print('   ', res)
53        runs.append(res)
54
55    result = {'runs': runs}
56
57    successed = [r for r in runs if ('seconds' in r)]
58    if successed:
59        avg = sum(r['seconds'] for r in successed) / len(successed)
60        result['average'] = avg
61        result['delta'] = max(abs(r['seconds'] - avg) for r in successed)
62
63    if len(successed) < count:
64        result['n-failed'] = count - len(successed)
65
66    return result
67
68
69def ascii_one(result):
70    """Return ASCII representation of bench_one() returned dict."""
71    if 'average' in result:
72        s = '{:.2f} +- {:.2f}'.format(result['average'], result['delta'])
73        if 'n-failed' in result:
74            s += '\n({} failed)'.format(result['n-failed'])
75        return s
76    else:
77        return 'FAILED'
78
79
80def bench(test_func, test_envs, test_cases, *args, **vargs):
81    """Fill benchmark table
82
83    test_func -- benchmarking function, see bench_one for description
84    test_envs -- list of test environments, see bench_one
85    test_cases -- list of test cases, see bench_one
86    args, vargs -- additional arguments for bench_one
87
88    Returns dict with the following fields:
89        'envs':  test_envs
90        'cases': test_cases
91        'tab':   filled 2D array, where cell [i][j] is bench_one result for
92                 test_cases[i] for test_envs[j] (i.e., rows are test cases and
93                 columns are test environments)
94    """
95    tab = {}
96    results = {
97        'envs': test_envs,
98        'cases': test_cases,
99        'tab': tab
100    }
101    n = 1
102    n_tests = len(test_envs) * len(test_cases)
103    for env in test_envs:
104        for case in test_cases:
105            print('Testing {}/{}: {} :: {}'.format(n, n_tests,
106                                                   env['id'], case['id']))
107            if case['id'] not in tab:
108                tab[case['id']] = {}
109            tab[case['id']][env['id']] = bench_one(test_func, env, case,
110                                                   *args, **vargs)
111            n += 1
112
113    print('Done')
114    return results
115
116
117def ascii(results):
118    """Return ASCII representation of bench() returned dict."""
119    from tabulate import tabulate
120
121    tab = [[""] + [c['id'] for c in results['envs']]]
122    for case in results['cases']:
123        row = [case['id']]
124        for env in results['envs']:
125            row.append(ascii_one(results['tab'][case['id']][env['id']]))
126        tab.append(row)
127
128    return tabulate(tab)
129