1#! /usr/bin/python 2# SPDX-License-Identifier: GPL-2.0 3 4from __future__ import print_function 5 6import os 7import sys 8import glob 9import optparse 10import tempfile 11import logging 12import shutil 13 14try: 15 import configparser 16except ImportError: 17 import ConfigParser as configparser 18 19def data_equal(a, b): 20 # Allow multiple values in assignment separated by '|' 21 a_list = a.split('|') 22 b_list = b.split('|') 23 24 for a_item in a_list: 25 for b_item in b_list: 26 if (a_item == b_item): 27 return True 28 elif (a_item == '*') or (b_item == '*'): 29 return True 30 31 return False 32 33class Fail(Exception): 34 def __init__(self, test, msg): 35 self.msg = msg 36 self.test = test 37 def getMsg(self): 38 return '\'%s\' - %s' % (self.test.path, self.msg) 39 40class Notest(Exception): 41 def __init__(self, test, arch): 42 self.arch = arch 43 self.test = test 44 def getMsg(self): 45 return '[%s] \'%s\'' % (self.arch, self.test.path) 46 47class Unsup(Exception): 48 def __init__(self, test): 49 self.test = test 50 def getMsg(self): 51 return '\'%s\'' % self.test.path 52 53class Event(dict): 54 terms = [ 55 'cpu', 56 'flags', 57 'type', 58 'size', 59 'config', 60 'sample_period', 61 'sample_type', 62 'read_format', 63 'disabled', 64 'inherit', 65 'pinned', 66 'exclusive', 67 'exclude_user', 68 'exclude_kernel', 69 'exclude_hv', 70 'exclude_idle', 71 'mmap', 72 'comm', 73 'freq', 74 'inherit_stat', 75 'enable_on_exec', 76 'task', 77 'watermark', 78 'precise_ip', 79 'mmap_data', 80 'sample_id_all', 81 'exclude_host', 82 'exclude_guest', 83 'exclude_callchain_kernel', 84 'exclude_callchain_user', 85 'wakeup_events', 86 'bp_type', 87 'config1', 88 'config2', 89 'branch_sample_type', 90 'sample_regs_user', 91 'sample_stack_user', 92 ] 93 94 def add(self, data): 95 for key, val in data: 96 log.debug(" %s = %s" % (key, val)) 97 self[key] = val 98 99 def __init__(self, name, data, base): 100 log.debug(" Event %s" % name); 101 self.name = name; 102 self.group = '' 103 self.add(base) 104 self.add(data) 105 106 def equal(self, other): 107 for t in Event.terms: 108 log.debug(" [%s] %s %s" % (t, self[t], other[t])); 109 if t not in self or t not in other: 110 return False 111 if not data_equal(self[t], other[t]): 112 return False 113 return True 114 115 def optional(self): 116 if 'optional' in self and self['optional'] == '1': 117 return True 118 return False 119 120 def diff(self, other): 121 for t in Event.terms: 122 if t not in self or t not in other: 123 continue 124 if not data_equal(self[t], other[t]): 125 log.warning("expected %s=%s, got %s" % (t, self[t], other[t])) 126 127# Test file description needs to have following sections: 128# [config] 129# - just single instance in file 130# - needs to specify: 131# 'command' - perf command name 132# 'args' - special command arguments 133# 'ret' - expected command return value (0 by default) 134# 'arch' - architecture specific test (optional) 135# comma separated list, ! at the beginning 136# negates it. 137# 138# [eventX:base] 139# - one or multiple instances in file 140# - expected values assignments 141class Test(object): 142 def __init__(self, path, options): 143 parser = configparser.SafeConfigParser() 144 parser.read(path) 145 146 log.warning("running '%s'" % path) 147 148 self.path = path 149 self.test_dir = options.test_dir 150 self.perf = options.perf 151 self.command = parser.get('config', 'command') 152 self.args = parser.get('config', 'args') 153 154 try: 155 self.ret = parser.get('config', 'ret') 156 except: 157 self.ret = 0 158 159 try: 160 self.arch = parser.get('config', 'arch') 161 log.warning("test limitation '%s'" % self.arch) 162 except: 163 self.arch = '' 164 165 self.expect = {} 166 self.result = {} 167 log.debug(" loading expected events"); 168 self.load_events(path, self.expect) 169 170 def is_event(self, name): 171 if name.find("event") == -1: 172 return False 173 else: 174 return True 175 176 def skip_test(self, myarch): 177 # If architecture not set always run test 178 if self.arch == '': 179 # log.warning("test for arch %s is ok" % myarch) 180 return False 181 182 # Allow multiple values in assignment separated by ',' 183 arch_list = self.arch.split(',') 184 185 # Handle negated list such as !s390x,ppc 186 if arch_list[0][0] == '!': 187 arch_list[0] = arch_list[0][1:] 188 log.warning("excluded architecture list %s" % arch_list) 189 for arch_item in arch_list: 190 # log.warning("test for %s arch is %s" % (arch_item, myarch)) 191 if arch_item == myarch: 192 return True 193 return False 194 195 for arch_item in arch_list: 196 # log.warning("test for architecture '%s' current '%s'" % (arch_item, myarch)) 197 if arch_item == myarch: 198 return False 199 return True 200 201 def load_events(self, path, events): 202 parser_event = configparser.SafeConfigParser() 203 parser_event.read(path) 204 205 # The event record section header contains 'event' word, 206 # optionaly followed by ':' allowing to load 'parent 207 # event' first as a base 208 for section in filter(self.is_event, parser_event.sections()): 209 210 parser_items = parser_event.items(section); 211 base_items = {} 212 213 # Read parent event if there's any 214 if (':' in section): 215 base = section[section.index(':') + 1:] 216 parser_base = configparser.SafeConfigParser() 217 parser_base.read(self.test_dir + '/' + base) 218 base_items = parser_base.items('event') 219 220 e = Event(section, parser_items, base_items) 221 events[section] = e 222 223 def run_cmd(self, tempdir): 224 junk1, junk2, junk3, junk4, myarch = (os.uname()) 225 226 if self.skip_test(myarch): 227 raise Notest(self, myarch) 228 229 cmd = "PERF_TEST_ATTR=%s %s %s -o %s/perf.data %s" % (tempdir, 230 self.perf, self.command, tempdir, self.args) 231 ret = os.WEXITSTATUS(os.system(cmd)) 232 233 log.info(" '%s' ret '%s', expected '%s'" % (cmd, str(ret), str(self.ret))) 234 235 if not data_equal(str(ret), str(self.ret)): 236 raise Unsup(self) 237 238 def compare(self, expect, result): 239 match = {} 240 241 log.debug(" compare"); 242 243 # For each expected event find all matching 244 # events in result. Fail if there's not any. 245 for exp_name, exp_event in expect.items(): 246 exp_list = [] 247 res_event = {} 248 log.debug(" matching [%s]" % exp_name) 249 for res_name, res_event in result.items(): 250 log.debug(" to [%s]" % res_name) 251 if (exp_event.equal(res_event)): 252 exp_list.append(res_name) 253 log.debug(" ->OK") 254 else: 255 log.debug(" ->FAIL"); 256 257 log.debug(" match: [%s] matches %s" % (exp_name, str(exp_list))) 258 259 # we did not any matching event - fail 260 if not exp_list: 261 if exp_event.optional(): 262 log.debug(" %s does not match, but is optional" % exp_name) 263 else: 264 if not res_event: 265 log.debug(" res_event is empty"); 266 else: 267 exp_event.diff(res_event) 268 raise Fail(self, 'match failure'); 269 270 match[exp_name] = exp_list 271 272 # For each defined group in the expected events 273 # check we match the same group in the result. 274 for exp_name, exp_event in expect.items(): 275 group = exp_event.group 276 277 if (group == ''): 278 continue 279 280 for res_name in match[exp_name]: 281 res_group = result[res_name].group 282 if res_group not in match[group]: 283 raise Fail(self, 'group failure') 284 285 log.debug(" group: [%s] matches group leader %s" % 286 (exp_name, str(match[group]))) 287 288 log.debug(" matched") 289 290 def resolve_groups(self, events): 291 for name, event in events.items(): 292 group_fd = event['group_fd']; 293 if group_fd == '-1': 294 continue; 295 296 for iname, ievent in events.items(): 297 if (ievent['fd'] == group_fd): 298 event.group = iname 299 log.debug('[%s] has group leader [%s]' % (name, iname)) 300 break; 301 302 def run(self): 303 tempdir = tempfile.mkdtemp(); 304 305 try: 306 # run the test script 307 self.run_cmd(tempdir); 308 309 # load events expectation for the test 310 log.debug(" loading result events"); 311 for f in glob.glob(tempdir + '/event*'): 312 self.load_events(f, self.result); 313 314 # resolve group_fd to event names 315 self.resolve_groups(self.expect); 316 self.resolve_groups(self.result); 317 318 # do the expectation - results matching - both ways 319 self.compare(self.expect, self.result) 320 self.compare(self.result, self.expect) 321 322 finally: 323 # cleanup 324 shutil.rmtree(tempdir) 325 326 327def run_tests(options): 328 for f in glob.glob(options.test_dir + '/' + options.test): 329 try: 330 Test(f, options).run() 331 except Unsup as obj: 332 log.warning("unsupp %s" % obj.getMsg()) 333 except Notest as obj: 334 log.warning("skipped %s" % obj.getMsg()) 335 336def setup_log(verbose): 337 global log 338 level = logging.CRITICAL 339 340 if verbose == 1: 341 level = logging.WARNING 342 if verbose == 2: 343 level = logging.INFO 344 if verbose >= 3: 345 level = logging.DEBUG 346 347 log = logging.getLogger('test') 348 log.setLevel(level) 349 ch = logging.StreamHandler() 350 ch.setLevel(level) 351 formatter = logging.Formatter('%(message)s') 352 ch.setFormatter(formatter) 353 log.addHandler(ch) 354 355USAGE = '''%s [OPTIONS] 356 -d dir # tests dir 357 -p path # perf binary 358 -t test # single test 359 -v # verbose level 360''' % sys.argv[0] 361 362def main(): 363 parser = optparse.OptionParser(usage=USAGE) 364 365 parser.add_option("-t", "--test", 366 action="store", type="string", dest="test") 367 parser.add_option("-d", "--test-dir", 368 action="store", type="string", dest="test_dir") 369 parser.add_option("-p", "--perf", 370 action="store", type="string", dest="perf") 371 parser.add_option("-v", "--verbose", 372 default=0, action="count", dest="verbose") 373 374 options, args = parser.parse_args() 375 if args: 376 parser.error('FAILED wrong arguments %s' % ' '.join(args)) 377 return -1 378 379 setup_log(options.verbose) 380 381 if not options.test_dir: 382 print('FAILED no -d option specified') 383 sys.exit(-1) 384 385 if not options.test: 386 options.test = 'test*' 387 388 try: 389 run_tests(options) 390 391 except Fail as obj: 392 print("FAILED %s" % obj.getMsg()) 393 sys.exit(-1) 394 395 sys.exit(0) 396 397if __name__ == '__main__': 398 main() 399