xref: /openbmc/linux/tools/perf/tests/attr.py (revision c8dbaa22) 
1#! /usr/bin/python
2
3import os
4import sys
5import glob
6import optparse
7import tempfile
8import logging
9import shutil
10import ConfigParser
11
12class Fail(Exception):
13    def __init__(self, test, msg):
14        self.msg = msg
15        self.test = test
16    def getMsg(self):
17        return '\'%s\' - %s' % (self.test.path, self.msg)
18
19class Notest(Exception):
20    def __init__(self, test, arch):
21        self.arch = arch
22        self.test = test
23    def getMsg(self):
24        return '[%s] \'%s\'' % (self.arch, self.test.path)
25
26class Unsup(Exception):
27    def __init__(self, test):
28        self.test = test
29    def getMsg(self):
30        return '\'%s\'' % self.test.path
31
32class Event(dict):
33    terms = [
34        'cpu',
35        'flags',
36        'type',
37        'size',
38        'config',
39        'sample_period',
40        'sample_type',
41        'read_format',
42        'disabled',
43        'inherit',
44        'pinned',
45        'exclusive',
46        'exclude_user',
47        'exclude_kernel',
48        'exclude_hv',
49        'exclude_idle',
50        'mmap',
51        'comm',
52        'freq',
53        'inherit_stat',
54        'enable_on_exec',
55        'task',
56        'watermark',
57        'precise_ip',
58        'mmap_data',
59        'sample_id_all',
60        'exclude_host',
61        'exclude_guest',
62        'exclude_callchain_kernel',
63        'exclude_callchain_user',
64        'wakeup_events',
65        'bp_type',
66        'config1',
67        'config2',
68        'branch_sample_type',
69        'sample_regs_user',
70        'sample_stack_user',
71    ]
72
73    def add(self, data):
74        for key, val in data:
75            log.debug("      %s = %s" % (key, val))
76            self[key] = val
77
78    def __init__(self, name, data, base):
79        log.debug("    Event %s" % name);
80        self.name  = name;
81        self.group = ''
82        self.add(base)
83        self.add(data)
84
85    def compare_data(self, a, b):
86        # Allow multiple values in assignment separated by '|'
87        a_list = a.split('|')
88        b_list = b.split('|')
89
90        for a_item in a_list:
91            for b_item in b_list:
92                if (a_item == b_item):
93                    return True
94                elif (a_item == '*') or (b_item == '*'):
95                    return True
96
97        return False
98
99    def equal(self, other):
100        for t in Event.terms:
101            log.debug("      [%s] %s %s" % (t, self[t], other[t]));
102            if not self.has_key(t) or not other.has_key(t):
103                return False
104            if not self.compare_data(self[t], other[t]):
105                return False
106        return True
107
108    def diff(self, other):
109        for t in Event.terms:
110            if not self.has_key(t) or not other.has_key(t):
111                continue
112            if not self.compare_data(self[t], other[t]):
113		log.warning("expected %s=%s, got %s" % (t, self[t], other[t]))
114
115# Test file description needs to have following sections:
116# [config]
117#   - just single instance in file
118#   - needs to specify:
119#     'command' - perf command name
120#     'args'    - special command arguments
121#     'ret'     - expected command return value (0 by default)
122#     'arch'    - architecture specific test (optional)
123#                 comma separated list, ! at the beginning
124#                 negates it.
125#
126# [eventX:base]
127#   - one or multiple instances in file
128#   - expected values assignments
129class Test(object):
130    def __init__(self, path, options):
131        parser = ConfigParser.SafeConfigParser()
132        parser.read(path)
133
134        log.warning("running '%s'" % path)
135
136        self.path     = path
137        self.test_dir = options.test_dir
138        self.perf     = options.perf
139        self.command  = parser.get('config', 'command')
140        self.args     = parser.get('config', 'args')
141
142        try:
143            self.ret  = parser.get('config', 'ret')
144        except:
145            self.ret  = 0
146
147        try:
148            self.arch  = parser.get('config', 'arch')
149            log.warning("test limitation '%s'" % self.arch)
150        except:
151            self.arch  = ''
152
153        self.expect   = {}
154        self.result   = {}
155        log.debug("  loading expected events");
156        self.load_events(path, self.expect)
157
158    def is_event(self, name):
159        if name.find("event") == -1:
160            return False
161        else:
162            return True
163
164    def skip_test(self, myarch):
165        # If architecture not set always run test
166        if self.arch == '':
167            # log.warning("test for arch %s is ok" % myarch)
168            return False
169
170        # Allow multiple values in assignment separated by ','
171        arch_list = self.arch.split(',')
172
173        # Handle negated list such as !s390x,ppc
174        if arch_list[0][0] == '!':
175            arch_list[0] = arch_list[0][1:]
176            log.warning("excluded architecture list %s" % arch_list)
177            for arch_item in arch_list:
178                # log.warning("test for %s arch is %s" % (arch_item, myarch))
179                if arch_item == myarch:
180                    return True
181            return False
182
183        for arch_item in arch_list:
184            # log.warning("test for architecture '%s' current '%s'" % (arch_item, myarch))
185            if arch_item == myarch:
186                return False
187        return True
188
189    def load_events(self, path, events):
190        parser_event = ConfigParser.SafeConfigParser()
191        parser_event.read(path)
192
193        # The event record section header contains 'event' word,
194        # optionaly followed by ':' allowing to load 'parent
195        # event' first as a base
196        for section in filter(self.is_event, parser_event.sections()):
197
198            parser_items = parser_event.items(section);
199            base_items   = {}
200
201            # Read parent event if there's any
202            if (':' in section):
203                base = section[section.index(':') + 1:]
204                parser_base = ConfigParser.SafeConfigParser()
205                parser_base.read(self.test_dir + '/' + base)
206                base_items = parser_base.items('event')
207
208            e = Event(section, parser_items, base_items)
209            events[section] = e
210
211    def run_cmd(self, tempdir):
212        junk1, junk2, junk3, junk4, myarch = (os.uname())
213
214        if self.skip_test(myarch):
215            raise Notest(self, myarch)
216
217        cmd = "PERF_TEST_ATTR=%s %s %s -o %s/perf.data %s" % (tempdir,
218              self.perf, self.command, tempdir, self.args)
219        ret = os.WEXITSTATUS(os.system(cmd))
220
221        log.info("  '%s' ret %d " % (cmd, ret))
222
223        if ret != int(self.ret):
224            raise Unsup(self)
225
226    def compare(self, expect, result):
227        match = {}
228
229        log.debug("  compare");
230
231        # For each expected event find all matching
232        # events in result. Fail if there's not any.
233        for exp_name, exp_event in expect.items():
234            exp_list = []
235            log.debug("    matching [%s]" % exp_name)
236            for res_name, res_event in result.items():
237                log.debug("      to [%s]" % res_name)
238                if (exp_event.equal(res_event)):
239                    exp_list.append(res_name)
240                    log.debug("    ->OK")
241                else:
242                    log.debug("    ->FAIL");
243
244            log.debug("    match: [%s] matches %s" % (exp_name, str(exp_list)))
245
246            # we did not any matching event - fail
247            if (not exp_list):
248		exp_event.diff(res_event)
249                raise Fail(self, 'match failure');
250
251            match[exp_name] = exp_list
252
253        # For each defined group in the expected events
254        # check we match the same group in the result.
255        for exp_name, exp_event in expect.items():
256            group = exp_event.group
257
258            if (group == ''):
259                continue
260
261            for res_name in match[exp_name]:
262                res_group = result[res_name].group
263                if res_group not in match[group]:
264                    raise Fail(self, 'group failure')
265
266                log.debug("    group: [%s] matches group leader %s" %
267                         (exp_name, str(match[group])))
268
269        log.debug("  matched")
270
271    def resolve_groups(self, events):
272        for name, event in events.items():
273            group_fd = event['group_fd'];
274            if group_fd == '-1':
275                continue;
276
277            for iname, ievent in events.items():
278                if (ievent['fd'] == group_fd):
279                    event.group = iname
280                    log.debug('[%s] has group leader [%s]' % (name, iname))
281                    break;
282
283    def run(self):
284        tempdir = tempfile.mkdtemp();
285
286        try:
287            # run the test script
288            self.run_cmd(tempdir);
289
290            # load events expectation for the test
291            log.debug("  loading result events");
292            for f in glob.glob(tempdir + '/event*'):
293                self.load_events(f, self.result);
294
295            # resolve group_fd to event names
296            self.resolve_groups(self.expect);
297            self.resolve_groups(self.result);
298
299            # do the expectation - results matching - both ways
300            self.compare(self.expect, self.result)
301            self.compare(self.result, self.expect)
302
303        finally:
304            # cleanup
305            shutil.rmtree(tempdir)
306
307
308def run_tests(options):
309    for f in glob.glob(options.test_dir + '/' + options.test):
310        try:
311            Test(f, options).run()
312        except Unsup, obj:
313            log.warning("unsupp  %s" % obj.getMsg())
314        except Notest, obj:
315            log.warning("skipped %s" % obj.getMsg())
316
317def setup_log(verbose):
318    global log
319    level = logging.CRITICAL
320
321    if verbose == 1:
322        level = logging.WARNING
323    if verbose == 2:
324        level = logging.INFO
325    if verbose >= 3:
326        level = logging.DEBUG
327
328    log = logging.getLogger('test')
329    log.setLevel(level)
330    ch  = logging.StreamHandler()
331    ch.setLevel(level)
332    formatter = logging.Formatter('%(message)s')
333    ch.setFormatter(formatter)
334    log.addHandler(ch)
335
336USAGE = '''%s [OPTIONS]
337  -d dir  # tests dir
338  -p path # perf binary
339  -t test # single test
340  -v      # verbose level
341''' % sys.argv[0]
342
343def main():
344    parser = optparse.OptionParser(usage=USAGE)
345
346    parser.add_option("-t", "--test",
347                      action="store", type="string", dest="test")
348    parser.add_option("-d", "--test-dir",
349                      action="store", type="string", dest="test_dir")
350    parser.add_option("-p", "--perf",
351                      action="store", type="string", dest="perf")
352    parser.add_option("-v", "--verbose",
353                      action="count", dest="verbose")
354
355    options, args = parser.parse_args()
356    if args:
357        parser.error('FAILED wrong arguments %s' %  ' '.join(args))
358        return -1
359
360    setup_log(options.verbose)
361
362    if not options.test_dir:
363        print 'FAILED no -d option specified'
364        sys.exit(-1)
365
366    if not options.test:
367        options.test = 'test*'
368
369    try:
370        run_tests(options)
371
372    except Fail, obj:
373        print "FAILED %s" % obj.getMsg();
374        sys.exit(-1)
375
376    sys.exit(0)
377
378if __name__ == '__main__':
379    main()
380