1#!/usr/bin/env python
2#
3# Simple benchmarking framework
4#
5# Copyright (c) 2019 Virtuozzo International GmbH.
6#
7# This program is free software; you can redistribute it and/or modify
8# it under the terms of the GNU General Public License as published by
9# the Free Software Foundation; either version 2 of the License, or
10# (at your option) any later version.
11#
12# This program is distributed in the hope that it will be useful,
13# but WITHOUT ANY WARRANTY; without even the implied warranty of
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15# GNU General Public License for more details.
16#
17# You should have received a copy of the GNU General Public License
18# along with this program.  If not, see <http://www.gnu.org/licenses/>.
19#
20
21import statistics
22import subprocess
23import time
24
25
26def do_drop_caches():
27    subprocess.run('sync; echo 3 > /proc/sys/vm/drop_caches', shell=True,
28                   check=True)
29
30
31def bench_one(test_func, test_env, test_case, count=5, initial_run=True,
32              slow_limit=100, drop_caches=False):
33    """Benchmark one test-case
34
35    test_func   -- benchmarking function with prototype
36                   test_func(env, case), which takes test_env and test_case
37                   arguments and on success returns dict with 'seconds' or
38                   'iops' (or both) fields, specifying the benchmark result.
39                   If both 'iops' and 'seconds' provided, the 'iops' is
40                   considered the main, and 'seconds' is just an additional
41                   info. On failure test_func should return {'error': str}.
42                   Returned dict may contain any other additional fields.
43    test_env    -- test environment - opaque first argument for test_func
44    test_case   -- test case - opaque second argument for test_func
45    count       -- how many times to call test_func, to calculate average
46    initial_run -- do initial run of test_func, which don't get into result
47    slow_limit  -- stop at slow run (that exceedes the slow_limit by seconds).
48                   (initial run is not measured)
49    drop_caches -- drop caches before each run
50
51    Returns dict with the following fields:
52        'runs':     list of test_func results
53        'dimension': dimension of results, may be 'seconds' or 'iops'
54        'average':  average value (iops or seconds) per run (exists only if at
55                    least one run succeeded)
56        'stdev':    standard deviation of results
57                    (exists only if at least one run succeeded)
58        'n-failed': number of failed runs (exists only if at least one run
59                    failed)
60    """
61    if initial_run:
62        print('  #initial run:')
63        do_drop_caches()
64        print('   ', test_func(test_env, test_case))
65
66    runs = []
67    for i in range(count):
68        t = time.time()
69
70        print('  #run {}'.format(i+1))
71        do_drop_caches()
72        res = test_func(test_env, test_case)
73        print('   ', res)
74        runs.append(res)
75
76        if time.time() - t > slow_limit:
77            print('    - run is too slow, stop here')
78            break
79
80    count = len(runs)
81
82    result = {'runs': runs}
83
84    succeeded = [r for r in runs if ('seconds' in r or 'iops' in r)]
85    if succeeded:
86        if 'iops' in succeeded[0]:
87            assert all('iops' in r for r in succeeded)
88            dim = 'iops'
89        else:
90            assert all('seconds' in r for r in succeeded)
91            assert all('iops' not in r for r in succeeded)
92            dim = 'seconds'
93        result['dimension'] = dim
94        result['average'] = statistics.mean(r[dim] for r in succeeded)
95        if len(succeeded) == 1:
96            result['stdev'] = 0
97        else:
98            result['stdev'] = statistics.stdev(r[dim] for r in succeeded)
99
100    if len(succeeded) < count:
101        result['n-failed'] = count - len(succeeded)
102
103    return result
104
105
106def bench(test_func, test_envs, test_cases, *args, **vargs):
107    """Fill benchmark table
108
109    test_func -- benchmarking function, see bench_one for description
110    test_envs -- list of test environments, see bench_one
111    test_cases -- list of test cases, see bench_one
112    args, vargs -- additional arguments for bench_one
113
114    Returns dict with the following fields:
115        'envs':  test_envs
116        'cases': test_cases
117        'tab':   filled 2D array, where cell [i][j] is bench_one result for
118                 test_cases[i] for test_envs[j] (i.e., rows are test cases and
119                 columns are test environments)
120    """
121    tab = {}
122    results = {
123        'envs': test_envs,
124        'cases': test_cases,
125        'tab': tab
126    }
127    n = 1
128    n_tests = len(test_envs) * len(test_cases)
129    for env in test_envs:
130        for case in test_cases:
131            print('Testing {}/{}: {} :: {}'.format(n, n_tests,
132                                                   env['id'], case['id']))
133            if case['id'] not in tab:
134                tab[case['id']] = {}
135            tab[case['id']][env['id']] = bench_one(test_func, env, case,
136                                                   *args, **vargs)
137            n += 1
138
139    print('Done')
140    return results
141