1*3ad7092fSWeilin Wang#SPDX-License-Identifier: GPL-2.0 2*3ad7092fSWeilin Wangimport re 3*3ad7092fSWeilin Wangimport csv 4*3ad7092fSWeilin Wangimport json 5*3ad7092fSWeilin Wangimport argparse 6*3ad7092fSWeilin Wangfrom pathlib import Path 7*3ad7092fSWeilin Wangimport subprocess 8*3ad7092fSWeilin Wang 9*3ad7092fSWeilin Wangclass Validator: 10*3ad7092fSWeilin Wang def __init__(self, rulefname, reportfname='', t=5, debug=False, datafname='', fullrulefname='', workload='true', metrics=''): 11*3ad7092fSWeilin Wang self.rulefname = rulefname 12*3ad7092fSWeilin Wang self.reportfname = reportfname 13*3ad7092fSWeilin Wang self.rules = None 14*3ad7092fSWeilin Wang self.collectlist=metrics 15*3ad7092fSWeilin Wang self.metrics = set() 16*3ad7092fSWeilin Wang self.tolerance = t 17*3ad7092fSWeilin Wang 18*3ad7092fSWeilin Wang self.workloads = [x for x in workload.split(",") if x] 19*3ad7092fSWeilin Wang self.wlidx = 0 # idx of current workloads 20*3ad7092fSWeilin Wang self.allresults = dict() # metric results of all workload 21*3ad7092fSWeilin Wang self.allignoremetrics = dict() # metrics with no results or negative results 22*3ad7092fSWeilin Wang self.allfailtests = dict() 23*3ad7092fSWeilin Wang self.alltotalcnt = dict() 24*3ad7092fSWeilin Wang self.allpassedcnt = dict() 25*3ad7092fSWeilin Wang self.allerrlist = dict() 26*3ad7092fSWeilin Wang 27*3ad7092fSWeilin Wang self.results = dict() # metric results of current workload 28*3ad7092fSWeilin Wang # vars for test pass/failure statistics 29*3ad7092fSWeilin Wang self.ignoremetrics= set() # metrics with no results or negative results, neg result counts as a failed test 30*3ad7092fSWeilin Wang self.failtests = dict() 31*3ad7092fSWeilin Wang self.totalcnt = 0 32*3ad7092fSWeilin Wang self.passedcnt = 0 33*3ad7092fSWeilin Wang # vars for errors 34*3ad7092fSWeilin Wang self.errlist = list() 35*3ad7092fSWeilin Wang 36*3ad7092fSWeilin Wang # vars for Rule Generator 37*3ad7092fSWeilin Wang self.pctgmetrics = set() # Percentage rule 38*3ad7092fSWeilin Wang 39*3ad7092fSWeilin Wang # vars for debug 40*3ad7092fSWeilin Wang self.datafname = datafname 41*3ad7092fSWeilin Wang self.debug = debug 42*3ad7092fSWeilin Wang self.fullrulefname = fullrulefname 43*3ad7092fSWeilin Wang 44*3ad7092fSWeilin Wang def read_json(self, filename: str) -> dict: 45*3ad7092fSWeilin Wang try: 46*3ad7092fSWeilin Wang with open(Path(filename).resolve(), "r") as f: 47*3ad7092fSWeilin Wang data = json.loads(f.read()) 48*3ad7092fSWeilin Wang except OSError as e: 49*3ad7092fSWeilin Wang print(f"Error when reading file {e}") 50*3ad7092fSWeilin Wang sys.exit() 51*3ad7092fSWeilin Wang 52*3ad7092fSWeilin Wang return data 53*3ad7092fSWeilin Wang 54*3ad7092fSWeilin Wang def json_dump(self, data, output_file): 55*3ad7092fSWeilin Wang parent = Path(output_file).parent 56*3ad7092fSWeilin Wang if not parent.exists(): 57*3ad7092fSWeilin Wang parent.mkdir(parents=True) 58*3ad7092fSWeilin Wang 59*3ad7092fSWeilin Wang with open(output_file, "w+") as output_file: 60*3ad7092fSWeilin Wang json.dump(data, 61*3ad7092fSWeilin Wang output_file, 62*3ad7092fSWeilin Wang ensure_ascii=True, 63*3ad7092fSWeilin Wang indent=4) 64*3ad7092fSWeilin Wang 65*3ad7092fSWeilin Wang def get_results(self, idx:int = 0): 66*3ad7092fSWeilin Wang return self.results[idx] 67*3ad7092fSWeilin Wang 68*3ad7092fSWeilin Wang def get_bounds(self, lb, ub, error, alias={}, ridx:int = 0) -> list: 69*3ad7092fSWeilin Wang """ 70*3ad7092fSWeilin Wang Get bounds and tolerance from lb, ub, and error. 71*3ad7092fSWeilin Wang If missing lb, use 0.0; missing ub, use float('inf); missing error, use self.tolerance. 72*3ad7092fSWeilin Wang 73*3ad7092fSWeilin Wang @param lb: str/float, lower bound 74*3ad7092fSWeilin Wang @param ub: str/float, upper bound 75*3ad7092fSWeilin Wang @param error: float/str, error tolerance 76*3ad7092fSWeilin Wang @returns: lower bound, return inf if the lower bound is a metric value and is not collected 77*3ad7092fSWeilin Wang upper bound, return -1 if the upper bound is a metric value and is not collected 78*3ad7092fSWeilin Wang tolerance, denormalized base on upper bound value 79*3ad7092fSWeilin Wang """ 80*3ad7092fSWeilin Wang # init ubv and lbv to invalid values 81*3ad7092fSWeilin Wang def get_bound_value (bound, initval, ridx): 82*3ad7092fSWeilin Wang val = initval 83*3ad7092fSWeilin Wang if isinstance(bound, int) or isinstance(bound, float): 84*3ad7092fSWeilin Wang val = bound 85*3ad7092fSWeilin Wang elif isinstance(bound, str): 86*3ad7092fSWeilin Wang if bound == '': 87*3ad7092fSWeilin Wang val = float("inf") 88*3ad7092fSWeilin Wang elif bound in alias: 89*3ad7092fSWeilin Wang vall = self.get_value(alias[ub], ridx) 90*3ad7092fSWeilin Wang if vall: 91*3ad7092fSWeilin Wang val = vall[0] 92*3ad7092fSWeilin Wang elif bound.replace('.', '1').isdigit(): 93*3ad7092fSWeilin Wang val = float(bound) 94*3ad7092fSWeilin Wang else: 95*3ad7092fSWeilin Wang print("Wrong bound: {0}".format(bound)) 96*3ad7092fSWeilin Wang else: 97*3ad7092fSWeilin Wang print("Wrong bound: {0}".format(bound)) 98*3ad7092fSWeilin Wang return val 99*3ad7092fSWeilin Wang 100*3ad7092fSWeilin Wang ubv = get_bound_value(ub, -1, ridx) 101*3ad7092fSWeilin Wang lbv = get_bound_value(lb, float('inf'), ridx) 102*3ad7092fSWeilin Wang t = get_bound_value(error, self.tolerance, ridx) 103*3ad7092fSWeilin Wang 104*3ad7092fSWeilin Wang # denormalize error threshold 105*3ad7092fSWeilin Wang denormerr = t * ubv / 100 if ubv != 100 and ubv > 0 else t 106*3ad7092fSWeilin Wang 107*3ad7092fSWeilin Wang return lbv, ubv, denormerr 108*3ad7092fSWeilin Wang 109*3ad7092fSWeilin Wang def get_value(self, name:str, ridx:int = 0) -> list: 110*3ad7092fSWeilin Wang """ 111*3ad7092fSWeilin Wang Get value of the metric from self.results. 112*3ad7092fSWeilin Wang If result of this metric is not provided, the metric name will be added into self.ignoremetics and self.errlist. 113*3ad7092fSWeilin Wang All future test(s) on this metric will fail. 114*3ad7092fSWeilin Wang 115*3ad7092fSWeilin Wang @param name: name of the metric 116*3ad7092fSWeilin Wang @returns: list with value found in self.results; list is empty when not value found. 117*3ad7092fSWeilin Wang """ 118*3ad7092fSWeilin Wang results = [] 119*3ad7092fSWeilin Wang data = self.results[ridx] if ridx in self.results else self.results[0] 120*3ad7092fSWeilin Wang if name not in self.ignoremetrics: 121*3ad7092fSWeilin Wang if name in data: 122*3ad7092fSWeilin Wang results.append(data[name]) 123*3ad7092fSWeilin Wang elif name.replace('.', '1').isdigit(): 124*3ad7092fSWeilin Wang results.append(float(name)) 125*3ad7092fSWeilin Wang else: 126*3ad7092fSWeilin Wang self.errlist.append("Metric '%s' is not collected or the value format is incorrect"%(name)) 127*3ad7092fSWeilin Wang self.ignoremetrics.add(name) 128*3ad7092fSWeilin Wang return results 129*3ad7092fSWeilin Wang 130*3ad7092fSWeilin Wang def check_bound(self, val, lb, ub, err): 131*3ad7092fSWeilin Wang return True if val <= ub + err and val >= lb - err else False 132*3ad7092fSWeilin Wang 133*3ad7092fSWeilin Wang # Positive Value Sanity check 134*3ad7092fSWeilin Wang def pos_val_test(self): 135*3ad7092fSWeilin Wang """ 136*3ad7092fSWeilin Wang Check if metrics value are non-negative. 137*3ad7092fSWeilin Wang One metric is counted as one test. 138*3ad7092fSWeilin Wang Failure: when metric value is negative or not provided. 139*3ad7092fSWeilin Wang Metrics with negative value will be added into the self.failtests['PositiveValueTest'] and self.ignoremetrics. 140*3ad7092fSWeilin Wang """ 141*3ad7092fSWeilin Wang negmetric = set() 142*3ad7092fSWeilin Wang missmetric = set() 143*3ad7092fSWeilin Wang pcnt = 0 144*3ad7092fSWeilin Wang tcnt = 0 145*3ad7092fSWeilin Wang for name, val in self.get_results().items(): 146*3ad7092fSWeilin Wang if val is None or val == '': 147*3ad7092fSWeilin Wang missmetric.add(name) 148*3ad7092fSWeilin Wang self.errlist.append("Metric '%s' is not collected"%(name)) 149*3ad7092fSWeilin Wang elif val < 0: 150*3ad7092fSWeilin Wang negmetric.add("{0}(={1:.4f})".format(name, val)) 151*3ad7092fSWeilin Wang else: 152*3ad7092fSWeilin Wang pcnt += 1 153*3ad7092fSWeilin Wang tcnt += 1 154*3ad7092fSWeilin Wang 155*3ad7092fSWeilin Wang self.failtests['PositiveValueTest']['Total Tests'] = tcnt 156*3ad7092fSWeilin Wang self.failtests['PositiveValueTest']['Passed Tests'] = pcnt 157*3ad7092fSWeilin Wang if len(negmetric) or len(missmetric)> 0: 158*3ad7092fSWeilin Wang self.ignoremetrics.update(negmetric) 159*3ad7092fSWeilin Wang self.ignoremetrics.update(missmetric) 160*3ad7092fSWeilin Wang self.failtests['PositiveValueTest']['Failed Tests'].append({'NegativeValue':list(negmetric), 'MissingValue':list(missmetric)}) 161*3ad7092fSWeilin Wang 162*3ad7092fSWeilin Wang return 163*3ad7092fSWeilin Wang 164*3ad7092fSWeilin Wang def evaluate_formula(self, formula:str, alias:dict, ridx:int = 0): 165*3ad7092fSWeilin Wang """ 166*3ad7092fSWeilin Wang Evaluate the value of formula. 167*3ad7092fSWeilin Wang 168*3ad7092fSWeilin Wang @param formula: the formula to be evaluated 169*3ad7092fSWeilin Wang @param alias: the dict has alias to metric name mapping 170*3ad7092fSWeilin Wang @returns: value of the formula is success; -1 if the one or more metric value not provided 171*3ad7092fSWeilin Wang """ 172*3ad7092fSWeilin Wang stack = [] 173*3ad7092fSWeilin Wang b = 0 174*3ad7092fSWeilin Wang errs = [] 175*3ad7092fSWeilin Wang sign = "+" 176*3ad7092fSWeilin Wang f = str() 177*3ad7092fSWeilin Wang 178*3ad7092fSWeilin Wang #TODO: support parenthesis? 179*3ad7092fSWeilin Wang for i in range(len(formula)): 180*3ad7092fSWeilin Wang if i+1 == len(formula) or formula[i] in ('+', '-', '*', '/'): 181*3ad7092fSWeilin Wang s = alias[formula[b:i]] if i+1 < len(formula) else alias[formula[b:]] 182*3ad7092fSWeilin Wang v = self.get_value(s, ridx) 183*3ad7092fSWeilin Wang if not v: 184*3ad7092fSWeilin Wang errs.append(s) 185*3ad7092fSWeilin Wang else: 186*3ad7092fSWeilin Wang f = f + "{0}(={1:.4f})".format(s, v[0]) 187*3ad7092fSWeilin Wang if sign == "*": 188*3ad7092fSWeilin Wang stack[-1] = stack[-1] * v 189*3ad7092fSWeilin Wang elif sign == "/": 190*3ad7092fSWeilin Wang stack[-1] = stack[-1] / v 191*3ad7092fSWeilin Wang elif sign == '-': 192*3ad7092fSWeilin Wang stack.append(-v[0]) 193*3ad7092fSWeilin Wang else: 194*3ad7092fSWeilin Wang stack.append(v[0]) 195*3ad7092fSWeilin Wang if i + 1 < len(formula): 196*3ad7092fSWeilin Wang sign = formula[i] 197*3ad7092fSWeilin Wang f += sign 198*3ad7092fSWeilin Wang b = i + 1 199*3ad7092fSWeilin Wang 200*3ad7092fSWeilin Wang if len(errs) > 0: 201*3ad7092fSWeilin Wang return -1, "Metric value missing: "+','.join(errs) 202*3ad7092fSWeilin Wang 203*3ad7092fSWeilin Wang val = sum(stack) 204*3ad7092fSWeilin Wang return val, f 205*3ad7092fSWeilin Wang 206*3ad7092fSWeilin Wang # Relationships Tests 207*3ad7092fSWeilin Wang def relationship_test(self, rule: dict): 208*3ad7092fSWeilin Wang """ 209*3ad7092fSWeilin Wang Validate if the metrics follow the required relationship in the rule. 210*3ad7092fSWeilin Wang eg. lower_bound <= eval(formula)<= upper_bound 211*3ad7092fSWeilin Wang One rule is counted as ont test. 212*3ad7092fSWeilin Wang Failure: when one or more metric result(s) not provided, or when formula evaluated outside of upper/lower bounds. 213*3ad7092fSWeilin Wang 214*3ad7092fSWeilin Wang @param rule: dict with metric name(+alias), formula, and required upper and lower bounds. 215*3ad7092fSWeilin Wang """ 216*3ad7092fSWeilin Wang alias = dict() 217*3ad7092fSWeilin Wang for m in rule['Metrics']: 218*3ad7092fSWeilin Wang alias[m['Alias']] = m['Name'] 219*3ad7092fSWeilin Wang lbv, ubv, t = self.get_bounds(rule['RangeLower'], rule['RangeUpper'], rule['ErrorThreshold'], alias, ridx=rule['RuleIndex']) 220*3ad7092fSWeilin Wang val, f = self.evaluate_formula(rule['Formula'], alias, ridx=rule['RuleIndex']) 221*3ad7092fSWeilin Wang if val == -1: 222*3ad7092fSWeilin Wang self.failtests['RelationshipTest']['Failed Tests'].append({'RuleIndex': rule['RuleIndex'], 'Description':f}) 223*3ad7092fSWeilin Wang elif not self.check_bound(val, lbv, ubv, t): 224*3ad7092fSWeilin Wang lb = rule['RangeLower'] 225*3ad7092fSWeilin Wang ub = rule['RangeUpper'] 226*3ad7092fSWeilin Wang if isinstance(lb, str): 227*3ad7092fSWeilin Wang if lb in alias: 228*3ad7092fSWeilin Wang lb = alias[lb] 229*3ad7092fSWeilin Wang if isinstance(ub, str): 230*3ad7092fSWeilin Wang if ub in alias: 231*3ad7092fSWeilin Wang ub = alias[ub] 232*3ad7092fSWeilin Wang self.failtests['RelationshipTest']['Failed Tests'].append({'RuleIndex': rule['RuleIndex'], 'Formula':f, 233*3ad7092fSWeilin Wang 'RangeLower': lb, 'LowerBoundValue': self.get_value(lb), 234*3ad7092fSWeilin Wang 'RangeUpper': ub, 'UpperBoundValue':self.get_value(ub), 235*3ad7092fSWeilin Wang 'ErrorThreshold': t, 'CollectedValue': val}) 236*3ad7092fSWeilin Wang else: 237*3ad7092fSWeilin Wang self.passedcnt += 1 238*3ad7092fSWeilin Wang self.failtests['RelationshipTest']['Passed Tests'] += 1 239*3ad7092fSWeilin Wang self.totalcnt += 1 240*3ad7092fSWeilin Wang self.failtests['RelationshipTest']['Total Tests'] += 1 241*3ad7092fSWeilin Wang 242*3ad7092fSWeilin Wang return 243*3ad7092fSWeilin Wang 244*3ad7092fSWeilin Wang 245*3ad7092fSWeilin Wang # Single Metric Test 246*3ad7092fSWeilin Wang def single_test(self, rule:dict): 247*3ad7092fSWeilin Wang """ 248*3ad7092fSWeilin Wang Validate if the metrics are in the required value range. 249*3ad7092fSWeilin Wang eg. lower_bound <= metrics_value <= upper_bound 250*3ad7092fSWeilin Wang One metric is counted as one test in this type of test. 251*3ad7092fSWeilin Wang One rule may include one or more metrics. 252*3ad7092fSWeilin Wang Failure: when the metric value not provided or the value is outside the bounds. 253*3ad7092fSWeilin Wang This test updates self.total_cnt and records failed tests in self.failtest['SingleMetricTest']. 254*3ad7092fSWeilin Wang 255*3ad7092fSWeilin Wang @param rule: dict with metrics to validate and the value range requirement 256*3ad7092fSWeilin Wang """ 257*3ad7092fSWeilin Wang lbv, ubv, t = self.get_bounds(rule['RangeLower'], rule['RangeUpper'], rule['ErrorThreshold']) 258*3ad7092fSWeilin Wang metrics = rule['Metrics'] 259*3ad7092fSWeilin Wang passcnt = 0 260*3ad7092fSWeilin Wang totalcnt = 0 261*3ad7092fSWeilin Wang faillist = [] 262*3ad7092fSWeilin Wang for m in metrics: 263*3ad7092fSWeilin Wang totalcnt += 1 264*3ad7092fSWeilin Wang result = self.get_value(m['Name']) 265*3ad7092fSWeilin Wang if len(result) > 0 and self.check_bound(result[0], lbv, ubv, t): 266*3ad7092fSWeilin Wang passcnt += 1 267*3ad7092fSWeilin Wang else: 268*3ad7092fSWeilin Wang faillist.append({'MetricName':m['Name'], 'CollectedValue':result}) 269*3ad7092fSWeilin Wang 270*3ad7092fSWeilin Wang self.totalcnt += totalcnt 271*3ad7092fSWeilin Wang self.passedcnt += passcnt 272*3ad7092fSWeilin Wang self.failtests['SingleMetricTest']['Total Tests'] += totalcnt 273*3ad7092fSWeilin Wang self.failtests['SingleMetricTest']['Passed Tests'] += passcnt 274*3ad7092fSWeilin Wang if len(faillist) != 0: 275*3ad7092fSWeilin Wang self.failtests['SingleMetricTest']['Failed Tests'].append({'RuleIndex':rule['RuleIndex'], 276*3ad7092fSWeilin Wang 'RangeLower': rule['RangeLower'], 277*3ad7092fSWeilin Wang 'RangeUpper': rule['RangeUpper'], 278*3ad7092fSWeilin Wang 'ErrorThreshold':rule['ErrorThreshold'], 279*3ad7092fSWeilin Wang 'Failure':faillist}) 280*3ad7092fSWeilin Wang 281*3ad7092fSWeilin Wang return 282*3ad7092fSWeilin Wang 283*3ad7092fSWeilin Wang def create_report(self): 284*3ad7092fSWeilin Wang """ 285*3ad7092fSWeilin Wang Create final report and write into a JSON file. 286*3ad7092fSWeilin Wang """ 287*3ad7092fSWeilin Wang alldata = list() 288*3ad7092fSWeilin Wang for i in range(0, len(self.workloads)): 289*3ad7092fSWeilin Wang reportstas = {"Total Rule Count": self.alltotalcnt[i], "Passed Rule Count": self.allpassedcnt[i]} 290*3ad7092fSWeilin Wang data = {"Metric Validation Statistics": reportstas, "Tests in Category": self.allfailtests[i], 291*3ad7092fSWeilin Wang "Errors":self.allerrlist[i]} 292*3ad7092fSWeilin Wang alldata.append({"Workload": self.workloads[i], "Report": data}) 293*3ad7092fSWeilin Wang 294*3ad7092fSWeilin Wang json_str = json.dumps(alldata, indent=4) 295*3ad7092fSWeilin Wang print("Test validation finished. Final report: ") 296*3ad7092fSWeilin Wang print(json_str) 297*3ad7092fSWeilin Wang 298*3ad7092fSWeilin Wang if self.debug: 299*3ad7092fSWeilin Wang allres = [{"Workload": self.workloads[i], "Results": self.allresults[i]} for i in range(0, len(self.workloads))] 300*3ad7092fSWeilin Wang self.json_dump(allres, self.datafname) 301*3ad7092fSWeilin Wang 302*3ad7092fSWeilin Wang def check_rule(self, testtype, metric_list): 303*3ad7092fSWeilin Wang """ 304*3ad7092fSWeilin Wang Check if the rule uses metric(s) that not exist in current platform. 305*3ad7092fSWeilin Wang 306*3ad7092fSWeilin Wang @param metric_list: list of metrics from the rule. 307*3ad7092fSWeilin Wang @return: False when find one metric out in Metric file. (This rule should not skipped.) 308*3ad7092fSWeilin Wang True when all metrics used in the rule are found in Metric file. 309*3ad7092fSWeilin Wang """ 310*3ad7092fSWeilin Wang if testtype == "RelationshipTest": 311*3ad7092fSWeilin Wang for m in metric_list: 312*3ad7092fSWeilin Wang if m['Name'] not in self.metrics: 313*3ad7092fSWeilin Wang return False 314*3ad7092fSWeilin Wang return True 315*3ad7092fSWeilin Wang 316*3ad7092fSWeilin Wang # Start of Collector and Converter 317*3ad7092fSWeilin Wang def convert(self, data: list, idx: int): 318*3ad7092fSWeilin Wang """ 319*3ad7092fSWeilin Wang Convert collected metric data from the -j output to dict of {metric_name:value}. 320*3ad7092fSWeilin Wang """ 321*3ad7092fSWeilin Wang for json_string in data: 322*3ad7092fSWeilin Wang try: 323*3ad7092fSWeilin Wang result =json.loads(json_string) 324*3ad7092fSWeilin Wang if "metric-unit" in result and result["metric-unit"] != "(null)" and result["metric-unit"] != "": 325*3ad7092fSWeilin Wang name = result["metric-unit"].split(" ")[1] if len(result["metric-unit"].split(" ")) > 1 \ 326*3ad7092fSWeilin Wang else result["metric-unit"] 327*3ad7092fSWeilin Wang if idx not in self.results: self.results[idx] = dict() 328*3ad7092fSWeilin Wang self.results[idx][name.lower()] = float(result["metric-value"]) 329*3ad7092fSWeilin Wang except ValueError as error: 330*3ad7092fSWeilin Wang continue 331*3ad7092fSWeilin Wang return 332*3ad7092fSWeilin Wang 333*3ad7092fSWeilin Wang def collect_perf(self, data_file: str, workload: str): 334*3ad7092fSWeilin Wang """ 335*3ad7092fSWeilin Wang Collect metric data with "perf stat -M" on given workload with -a and -j. 336*3ad7092fSWeilin Wang """ 337*3ad7092fSWeilin Wang self.results = dict() 338*3ad7092fSWeilin Wang tool = 'perf' 339*3ad7092fSWeilin Wang print(f"Starting perf collection") 340*3ad7092fSWeilin Wang print(f"Workload: {workload}") 341*3ad7092fSWeilin Wang collectlist = dict() 342*3ad7092fSWeilin Wang if self.collectlist != "": 343*3ad7092fSWeilin Wang collectlist[0] = {x for x in self.collectlist.split(",")} 344*3ad7092fSWeilin Wang else: 345*3ad7092fSWeilin Wang collectlist[0] = set(list(self.metrics)) 346*3ad7092fSWeilin Wang # Create metric set for relationship rules 347*3ad7092fSWeilin Wang for rule in self.rules: 348*3ad7092fSWeilin Wang if rule["TestType"] == "RelationshipTest": 349*3ad7092fSWeilin Wang metrics = [m["Name"] for m in rule["Metrics"]] 350*3ad7092fSWeilin Wang if not any(m not in collectlist[0] for m in metrics): 351*3ad7092fSWeilin Wang collectlist[rule["RuleIndex"]] = set(metrics) 352*3ad7092fSWeilin Wang 353*3ad7092fSWeilin Wang for idx, metrics in collectlist.items(): 354*3ad7092fSWeilin Wang if idx == 0: wl = "sleep 0.5".split() 355*3ad7092fSWeilin Wang else: wl = workload.split() 356*3ad7092fSWeilin Wang for metric in metrics: 357*3ad7092fSWeilin Wang command = [tool, 'stat', '-j', '-M', f"{metric}", "-a"] 358*3ad7092fSWeilin Wang command.extend(wl) 359*3ad7092fSWeilin Wang cmd = subprocess.run(command, stderr=subprocess.PIPE, encoding='utf-8') 360*3ad7092fSWeilin Wang data = [x+'}' for x in cmd.stderr.split('}\n') if x] 361*3ad7092fSWeilin Wang self.convert(data, idx) 362*3ad7092fSWeilin Wang # End of Collector and Converter 363*3ad7092fSWeilin Wang 364*3ad7092fSWeilin Wang # Start of Rule Generator 365*3ad7092fSWeilin Wang def parse_perf_metrics(self): 366*3ad7092fSWeilin Wang """ 367*3ad7092fSWeilin Wang Read and parse perf metric file: 368*3ad7092fSWeilin Wang 1) find metrics with '1%' or '100%' as ScaleUnit for Percent check 369*3ad7092fSWeilin Wang 2) create metric name list 370*3ad7092fSWeilin Wang """ 371*3ad7092fSWeilin Wang command = ['perf', 'list', '-j', '--details', 'metrics'] 372*3ad7092fSWeilin Wang cmd = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf-8') 373*3ad7092fSWeilin Wang try: 374*3ad7092fSWeilin Wang data = json.loads(cmd.stdout) 375*3ad7092fSWeilin Wang for m in data: 376*3ad7092fSWeilin Wang if 'MetricName' not in m: 377*3ad7092fSWeilin Wang print("Warning: no metric name") 378*3ad7092fSWeilin Wang continue 379*3ad7092fSWeilin Wang name = m['MetricName'] 380*3ad7092fSWeilin Wang self.metrics.add(name) 381*3ad7092fSWeilin Wang if 'ScaleUnit' in m and (m['ScaleUnit'] == '1%' or m['ScaleUnit'] == '100%'): 382*3ad7092fSWeilin Wang self.pctgmetrics.add(name.lower()) 383*3ad7092fSWeilin Wang except ValueError as error: 384*3ad7092fSWeilin Wang print(f"Error when parsing metric data") 385*3ad7092fSWeilin Wang sys.exit() 386*3ad7092fSWeilin Wang 387*3ad7092fSWeilin Wang return 388*3ad7092fSWeilin Wang 389*3ad7092fSWeilin Wang def create_rules(self): 390*3ad7092fSWeilin Wang """ 391*3ad7092fSWeilin Wang Create full rules which includes: 392*3ad7092fSWeilin Wang 1) All the rules from the "relationshi_rules" file 393*3ad7092fSWeilin Wang 2) SingleMetric rule for all the 'percent' metrics 394*3ad7092fSWeilin Wang 395*3ad7092fSWeilin Wang Reindex all the rules to avoid repeated RuleIndex 396*3ad7092fSWeilin Wang """ 397*3ad7092fSWeilin Wang self.rules = self.read_json(self.rulefname)['RelationshipRules'] 398*3ad7092fSWeilin Wang pctgrule = {'RuleIndex':0, 399*3ad7092fSWeilin Wang 'TestType':'SingleMetricTest', 400*3ad7092fSWeilin Wang 'RangeLower':'0', 401*3ad7092fSWeilin Wang 'RangeUpper': '100', 402*3ad7092fSWeilin Wang 'ErrorThreshold': self.tolerance, 403*3ad7092fSWeilin Wang 'Description':'Metrics in percent unit have value with in [0, 100]', 404*3ad7092fSWeilin Wang 'Metrics': [{'Name': m} for m in self.pctgmetrics]} 405*3ad7092fSWeilin Wang self.rules.append(pctgrule) 406*3ad7092fSWeilin Wang 407*3ad7092fSWeilin Wang # Re-index all rules to avoid repeated RuleIndex 408*3ad7092fSWeilin Wang idx = 1 409*3ad7092fSWeilin Wang for r in self.rules: 410*3ad7092fSWeilin Wang r['RuleIndex'] = idx 411*3ad7092fSWeilin Wang idx += 1 412*3ad7092fSWeilin Wang 413*3ad7092fSWeilin Wang if self.debug: 414*3ad7092fSWeilin Wang #TODO: need to test and generate file name correctly 415*3ad7092fSWeilin Wang data = {'RelationshipRules':self.rules, 'SupportedMetrics': [{"MetricName": name} for name in self.metrics]} 416*3ad7092fSWeilin Wang self.json_dump(data, self.fullrulefname) 417*3ad7092fSWeilin Wang 418*3ad7092fSWeilin Wang return 419*3ad7092fSWeilin Wang # End of Rule Generator 420*3ad7092fSWeilin Wang 421*3ad7092fSWeilin Wang def _storewldata(self, key): 422*3ad7092fSWeilin Wang ''' 423*3ad7092fSWeilin Wang Store all the data of one workload into the corresponding data structure for all workloads. 424*3ad7092fSWeilin Wang @param key: key to the dictionaries (index of self.workloads). 425*3ad7092fSWeilin Wang ''' 426*3ad7092fSWeilin Wang self.allresults[key] = self.results 427*3ad7092fSWeilin Wang self.allignoremetrics[key] = self.ignoremetrics 428*3ad7092fSWeilin Wang self.allfailtests[key] = self.failtests 429*3ad7092fSWeilin Wang self.alltotalcnt[key] = self.totalcnt 430*3ad7092fSWeilin Wang self.allpassedcnt[key] = self.passedcnt 431*3ad7092fSWeilin Wang self.allerrlist[key] = self.errlist 432*3ad7092fSWeilin Wang 433*3ad7092fSWeilin Wang #Initialize data structures before data validation of each workload 434*3ad7092fSWeilin Wang def _init_data(self): 435*3ad7092fSWeilin Wang 436*3ad7092fSWeilin Wang testtypes = ['PositiveValueTest', 'RelationshipTest', 'SingleMetricTest'] 437*3ad7092fSWeilin Wang self.results = dict() 438*3ad7092fSWeilin Wang self.ignoremetrics= set() 439*3ad7092fSWeilin Wang self.errlist = list() 440*3ad7092fSWeilin Wang self.failtests = {k:{'Total Tests':0, 'Passed Tests':0, 'Failed Tests':[]} for k in testtypes} 441*3ad7092fSWeilin Wang self.totalcnt = 0 442*3ad7092fSWeilin Wang self.passedcnt = 0 443*3ad7092fSWeilin Wang 444*3ad7092fSWeilin Wang def test(self): 445*3ad7092fSWeilin Wang ''' 446*3ad7092fSWeilin Wang The real entry point of the test framework. 447*3ad7092fSWeilin Wang This function loads the validation rule JSON file and Standard Metric file to create rules for 448*3ad7092fSWeilin Wang testing and namemap dictionaries. 449*3ad7092fSWeilin Wang It also reads in result JSON file for testing. 450*3ad7092fSWeilin Wang 451*3ad7092fSWeilin Wang In the test process, it passes through each rule and launch correct test function bases on the 452*3ad7092fSWeilin Wang 'TestType' field of the rule. 453*3ad7092fSWeilin Wang 454*3ad7092fSWeilin Wang The final report is written into a JSON file. 455*3ad7092fSWeilin Wang ''' 456*3ad7092fSWeilin Wang self.parse_perf_metrics() 457*3ad7092fSWeilin Wang self.create_rules() 458*3ad7092fSWeilin Wang for i in range(0, len(self.workloads)): 459*3ad7092fSWeilin Wang self._init_data() 460*3ad7092fSWeilin Wang self.collect_perf(self.datafname, self.workloads[i]) 461*3ad7092fSWeilin Wang # Run positive value test 462*3ad7092fSWeilin Wang self.pos_val_test() 463*3ad7092fSWeilin Wang for r in self.rules: 464*3ad7092fSWeilin Wang # skip rules that uses metrics not exist in this platform 465*3ad7092fSWeilin Wang testtype = r['TestType'] 466*3ad7092fSWeilin Wang if not self.check_rule(testtype, r['Metrics']): 467*3ad7092fSWeilin Wang continue 468*3ad7092fSWeilin Wang if testtype == 'RelationshipTest': 469*3ad7092fSWeilin Wang self.relationship_test(r) 470*3ad7092fSWeilin Wang elif testtype == 'SingleMetricTest': 471*3ad7092fSWeilin Wang self.single_test(r) 472*3ad7092fSWeilin Wang else: 473*3ad7092fSWeilin Wang print("Unsupported Test Type: ", testtype) 474*3ad7092fSWeilin Wang self.errlist.append("Unsupported Test Type from rule: " + r['RuleIndex']) 475*3ad7092fSWeilin Wang self._storewldata(i) 476*3ad7092fSWeilin Wang print("Workload: ", self.workloads[i]) 477*3ad7092fSWeilin Wang print("Total metrics collected: ", self.failtests['PositiveValueTest']['Total Tests']) 478*3ad7092fSWeilin Wang print("Non-negative metric count: ", self.failtests['PositiveValueTest']['Passed Tests']) 479*3ad7092fSWeilin Wang print("Total Test Count: ", self.totalcnt) 480*3ad7092fSWeilin Wang print("Passed Test Count: ", self.passedcnt) 481*3ad7092fSWeilin Wang 482*3ad7092fSWeilin Wang self.create_report() 483*3ad7092fSWeilin Wang return sum(self.alltotalcnt.values()) != sum(self.allpassedcnt.values()) 484*3ad7092fSWeilin Wang# End of Class Validator 485*3ad7092fSWeilin Wang 486*3ad7092fSWeilin Wang 487*3ad7092fSWeilin Wangdef main() -> None: 488*3ad7092fSWeilin Wang parser = argparse.ArgumentParser(description="Launch metric value validation") 489*3ad7092fSWeilin Wang 490*3ad7092fSWeilin Wang parser.add_argument("-rule", help="Base validation rule file", required=True) 491*3ad7092fSWeilin Wang parser.add_argument("-output_dir", help="Path for validator output file, report file", required=True) 492*3ad7092fSWeilin Wang parser.add_argument("-debug", help="Debug run, save intermediate data to files", action="store_true", default=False) 493*3ad7092fSWeilin Wang parser.add_argument("-wl", help="Workload to run while data collection", default="true") 494*3ad7092fSWeilin Wang parser.add_argument("-m", help="Metric list to validate", default="") 495*3ad7092fSWeilin Wang args = parser.parse_args() 496*3ad7092fSWeilin Wang outpath = Path(args.output_dir) 497*3ad7092fSWeilin Wang reportf = Path.joinpath(outpath, 'perf_report.json') 498*3ad7092fSWeilin Wang fullrule = Path.joinpath(outpath, 'full_rule.json') 499*3ad7092fSWeilin Wang datafile = Path.joinpath(outpath, 'perf_data.json') 500*3ad7092fSWeilin Wang 501*3ad7092fSWeilin Wang validator = Validator(args.rule, reportf, debug=args.debug, 502*3ad7092fSWeilin Wang datafname=datafile, fullrulefname=fullrule, workload=args.wl, 503*3ad7092fSWeilin Wang metrics=args.m) 504*3ad7092fSWeilin Wang ret = validator.test() 505*3ad7092fSWeilin Wang 506*3ad7092fSWeilin Wang return ret 507*3ad7092fSWeilin Wang 508*3ad7092fSWeilin Wang 509*3ad7092fSWeilin Wangif __name__ == "__main__": 510*3ad7092fSWeilin Wang import sys 511*3ad7092fSWeilin Wang sys.exit(main()) 512*3ad7092fSWeilin Wang 513*3ad7092fSWeilin Wang 514*3ad7092fSWeilin Wang 515