xref: /openbmc/linux/tools/perf/tests/attr.py (revision c2dea0bc) 
1# SPDX-License-Identifier: GPL-2.0
2
3from __future__ import print_function
4
5import os
6import sys
7import glob
8import optparse
9import platform
10import tempfile
11import logging
12import re
13import shutil
14import subprocess
15
16try:
17    import configparser
18except ImportError:
19    import ConfigParser as configparser
20
21def data_equal(a, b):
22    # Allow multiple values in assignment separated by '|'
23    a_list = a.split('|')
24    b_list = b.split('|')
25
26    for a_item in a_list:
27        for b_item in b_list:
28            if (a_item == b_item):
29                return True
30            elif (a_item == '*') or (b_item == '*'):
31                return True
32
33    return False
34
35class Fail(Exception):
36    def __init__(self, test, msg):
37        self.msg = msg
38        self.test = test
39    def getMsg(self):
40        return '\'%s\' - %s' % (self.test.path, self.msg)
41
42class Notest(Exception):
43    def __init__(self, test, arch):
44        self.arch = arch
45        self.test = test
46    def getMsg(self):
47        return '[%s] \'%s\'' % (self.arch, self.test.path)
48
49class Unsup(Exception):
50    def __init__(self, test):
51        self.test = test
52    def getMsg(self):
53        return '\'%s\'' % self.test.path
54
55class Event(dict):
56    terms = [
57        'cpu',
58        'flags',
59        'type',
60        'size',
61        'config',
62        'sample_period',
63        'sample_type',
64        'read_format',
65        'disabled',
66        'inherit',
67        'pinned',
68        'exclusive',
69        'exclude_user',
70        'exclude_kernel',
71        'exclude_hv',
72        'exclude_idle',
73        'mmap',
74        'comm',
75        'freq',
76        'inherit_stat',
77        'enable_on_exec',
78        'task',
79        'watermark',
80        'precise_ip',
81        'mmap_data',
82        'sample_id_all',
83        'exclude_host',
84        'exclude_guest',
85        'exclude_callchain_kernel',
86        'exclude_callchain_user',
87        'wakeup_events',
88        'bp_type',
89        'config1',
90        'config2',
91        'branch_sample_type',
92        'sample_regs_user',
93        'sample_stack_user',
94    ]
95
96    def add(self, data):
97        for key, val in data:
98            log.debug("      %s = %s" % (key, val))
99            self[key] = val
100
101    def __init__(self, name, data, base):
102        log.debug("    Event %s" % name);
103        self.name  = name;
104        self.group = ''
105        self.add(base)
106        self.add(data)
107
108    def equal(self, other):
109        for t in Event.terms:
110            log.debug("      [%s] %s %s" % (t, self[t], other[t]));
111            if t not in self or t not in other:
112                return False
113            if not data_equal(self[t], other[t]):
114                return False
115        return True
116
117    def optional(self):
118        if 'optional' in self and self['optional'] == '1':
119            return True
120        return False
121
122    def diff(self, other):
123        for t in Event.terms:
124            if t not in self or t not in other:
125                continue
126            if not data_equal(self[t], other[t]):
127                log.warning("expected %s=%s, got %s" % (t, self[t], other[t]))
128
129def parse_version(version):
130    if not version:
131        return None
132    return [int(v) for v in version.split(".")[0:2]]
133
134# Test file description needs to have following sections:
135# [config]
136#   - just single instance in file
137#   - needs to specify:
138#     'command' - perf command name
139#     'args'    - special command arguments
140#     'ret'     - Skip test if Perf doesn't exit with this value (0 by default)
141#     'test_ret'- If set to 'true', fail test instead of skipping for 'ret' argument
142#     'arch'    - architecture specific test (optional)
143#                 comma separated list, ! at the beginning
144#                 negates it.
145#     'auxv'    - Truthy statement that is evaled in the scope of the auxv map. When false,
146#                 the test is skipped. For example 'auxv["AT_HWCAP"] == 10'. (optional)
147#     'kernel_since' - Inclusive kernel version from which the test will start running. Only the
148#                      first two values are supported, for example "6.1" (optional)
149#     'kernel_until' - Exclusive kernel version from which the test will stop running. (optional)
150# [eventX:base]
151#   - one or multiple instances in file
152#   - expected values assignments
153class Test(object):
154    def __init__(self, path, options):
155        parser = configparser.SafeConfigParser()
156        parser.read(path)
157
158        log.warning("running '%s'" % path)
159
160        self.path     = path
161        self.test_dir = options.test_dir
162        self.perf     = options.perf
163        self.command  = parser.get('config', 'command')
164        self.args     = parser.get('config', 'args')
165
166        try:
167            self.ret  = parser.get('config', 'ret')
168        except:
169            self.ret  = 0
170
171        self.test_ret = parser.getboolean('config', 'test_ret', fallback=False)
172
173        try:
174            self.arch  = parser.get('config', 'arch')
175            log.warning("test limitation '%s'" % self.arch)
176        except:
177            self.arch  = ''
178
179        self.auxv = parser.get('config', 'auxv', fallback=None)
180        self.kernel_since = parse_version(parser.get('config', 'kernel_since', fallback=None))
181        self.kernel_until = parse_version(parser.get('config', 'kernel_until', fallback=None))
182        self.expect   = {}
183        self.result   = {}
184        log.debug("  loading expected events");
185        self.load_events(path, self.expect)
186
187    def is_event(self, name):
188        if name.find("event") == -1:
189            return False
190        else:
191            return True
192
193    def skip_test_kernel_since(self):
194        if not self.kernel_since:
195            return False
196        return not self.kernel_since <= parse_version(platform.release())
197
198    def skip_test_kernel_until(self):
199        if not self.kernel_until:
200            return False
201        return not parse_version(platform.release()) < self.kernel_until
202
203    def skip_test_auxv(self):
204        def new_auxv(a, pattern):
205            items = list(filter(None, pattern.split(a)))
206            # AT_HWCAP is hex but doesn't have a prefix, so special case it
207            if items[0] == "AT_HWCAP":
208                value = int(items[-1], 16)
209            else:
210                try:
211                    value = int(items[-1], 0)
212                except:
213                    value = items[-1]
214            return (items[0], value)
215
216        if not self.auxv:
217            return False
218        auxv = subprocess.check_output("LD_SHOW_AUXV=1 sleep 0", shell=True) \
219               .decode(sys.stdout.encoding)
220        pattern = re.compile(r"[: ]+")
221        auxv = dict([new_auxv(a, pattern) for a in auxv.splitlines()])
222        return not eval(self.auxv)
223
224    def skip_test_arch(self, myarch):
225        # If architecture not set always run test
226        if self.arch == '':
227            # log.warning("test for arch %s is ok" % myarch)
228            return False
229
230        # Allow multiple values in assignment separated by ','
231        arch_list = self.arch.split(',')
232
233        # Handle negated list such as !s390x,ppc
234        if arch_list[0][0] == '!':
235            arch_list[0] = arch_list[0][1:]
236            log.warning("excluded architecture list %s" % arch_list)
237            for arch_item in arch_list:
238                # log.warning("test for %s arch is %s" % (arch_item, myarch))
239                if arch_item == myarch:
240                    return True
241            return False
242
243        for arch_item in arch_list:
244            # log.warning("test for architecture '%s' current '%s'" % (arch_item, myarch))
245            if arch_item == myarch:
246                return False
247        return True
248
249    def load_events(self, path, events):
250        parser_event = configparser.SafeConfigParser()
251        parser_event.read(path)
252
253        # The event record section header contains 'event' word,
254        # optionaly followed by ':' allowing to load 'parent
255        # event' first as a base
256        for section in filter(self.is_event, parser_event.sections()):
257
258            parser_items = parser_event.items(section);
259            base_items   = {}
260
261            # Read parent event if there's any
262            if (':' in section):
263                base = section[section.index(':') + 1:]
264                parser_base = configparser.SafeConfigParser()
265                parser_base.read(self.test_dir + '/' + base)
266                base_items = parser_base.items('event')
267
268            e = Event(section, parser_items, base_items)
269            events[section] = e
270
271    def run_cmd(self, tempdir):
272        junk1, junk2, junk3, junk4, myarch = (os.uname())
273
274        if self.skip_test_arch(myarch):
275            raise Notest(self, myarch)
276
277        if self.skip_test_auxv():
278            raise Notest(self, "auxv skip")
279
280        if self.skip_test_kernel_since():
281            raise Notest(self, "old kernel skip")
282
283        if self.skip_test_kernel_until():
284            raise Notest(self, "new kernel skip")
285
286        cmd = "PERF_TEST_ATTR=%s %s %s -o %s/perf.data %s" % (tempdir,
287              self.perf, self.command, tempdir, self.args)
288        ret = os.WEXITSTATUS(os.system(cmd))
289
290        log.info("  '%s' ret '%s', expected '%s'" % (cmd, str(ret), str(self.ret)))
291
292        if not data_equal(str(ret), str(self.ret)):
293            if self.test_ret:
294                raise Fail(self, "Perf exit code failure")
295            else:
296                raise Unsup(self)
297
298    def compare(self, expect, result):
299        match = {}
300
301        log.debug("  compare");
302
303        # For each expected event find all matching
304        # events in result. Fail if there's not any.
305        for exp_name, exp_event in expect.items():
306            exp_list = []
307            res_event = {}
308            log.debug("    matching [%s]" % exp_name)
309            for res_name, res_event in result.items():
310                log.debug("      to [%s]" % res_name)
311                if (exp_event.equal(res_event)):
312                    exp_list.append(res_name)
313                    log.debug("    ->OK")
314                else:
315                    log.debug("    ->FAIL");
316
317            log.debug("    match: [%s] matches %s" % (exp_name, str(exp_list)))
318
319            # we did not any matching event - fail
320            if not exp_list:
321                if exp_event.optional():
322                    log.debug("    %s does not match, but is optional" % exp_name)
323                else:
324                    if not res_event:
325                        log.debug("    res_event is empty");
326                    else:
327                        exp_event.diff(res_event)
328                    raise Fail(self, 'match failure');
329
330            match[exp_name] = exp_list
331
332        # For each defined group in the expected events
333        # check we match the same group in the result.
334        for exp_name, exp_event in expect.items():
335            group = exp_event.group
336
337            if (group == ''):
338                continue
339
340            for res_name in match[exp_name]:
341                res_group = result[res_name].group
342                if res_group not in match[group]:
343                    raise Fail(self, 'group failure')
344
345                log.debug("    group: [%s] matches group leader %s" %
346                         (exp_name, str(match[group])))
347
348        log.debug("  matched")
349
350    def resolve_groups(self, events):
351        for name, event in events.items():
352            group_fd = event['group_fd'];
353            if group_fd == '-1':
354                continue;
355
356            for iname, ievent in events.items():
357                if (ievent['fd'] == group_fd):
358                    event.group = iname
359                    log.debug('[%s] has group leader [%s]' % (name, iname))
360                    break;
361
362    def run(self):
363        tempdir = tempfile.mkdtemp();
364
365        try:
366            # run the test script
367            self.run_cmd(tempdir);
368
369            # load events expectation for the test
370            log.debug("  loading result events");
371            for f in glob.glob(tempdir + '/event*'):
372                self.load_events(f, self.result);
373
374            # resolve group_fd to event names
375            self.resolve_groups(self.expect);
376            self.resolve_groups(self.result);
377
378            # do the expectation - results matching - both ways
379            self.compare(self.expect, self.result)
380            self.compare(self.result, self.expect)
381
382        finally:
383            # cleanup
384            shutil.rmtree(tempdir)
385
386
387def run_tests(options):
388    for f in glob.glob(options.test_dir + '/' + options.test):
389        try:
390            Test(f, options).run()
391        except Unsup as obj:
392            log.warning("unsupp  %s" % obj.getMsg())
393        except Notest as obj:
394            log.warning("skipped %s" % obj.getMsg())
395
396def setup_log(verbose):
397    global log
398    level = logging.CRITICAL
399
400    if verbose == 1:
401        level = logging.WARNING
402    if verbose == 2:
403        level = logging.INFO
404    if verbose >= 3:
405        level = logging.DEBUG
406
407    log = logging.getLogger('test')
408    log.setLevel(level)
409    ch  = logging.StreamHandler()
410    ch.setLevel(level)
411    formatter = logging.Formatter('%(message)s')
412    ch.setFormatter(formatter)
413    log.addHandler(ch)
414
415USAGE = '''%s [OPTIONS]
416  -d dir  # tests dir
417  -p path # perf binary
418  -t test # single test
419  -v      # verbose level
420''' % sys.argv[0]
421
422def main():
423    parser = optparse.OptionParser(usage=USAGE)
424
425    parser.add_option("-t", "--test",
426                      action="store", type="string", dest="test")
427    parser.add_option("-d", "--test-dir",
428                      action="store", type="string", dest="test_dir")
429    parser.add_option("-p", "--perf",
430                      action="store", type="string", dest="perf")
431    parser.add_option("-v", "--verbose",
432                      default=0, action="count", dest="verbose")
433
434    options, args = parser.parse_args()
435    if args:
436        parser.error('FAILED wrong arguments %s' %  ' '.join(args))
437        return -1
438
439    setup_log(options.verbose)
440
441    if not options.test_dir:
442        print('FAILED no -d option specified')
443        sys.exit(-1)
444
445    if not options.test:
446        options.test = 'test*'
447
448    try:
449        run_tests(options)
450
451    except Fail as obj:
452        print("FAILED %s" % obj.getMsg())
453        sys.exit(-1)
454
455    sys.exit(0)
456
457if __name__ == '__main__':
458    main()
459