xref: /openbmc/linux/tools/perf/tests/shell/lib/perf_metric_validation.py (revision 1203a63d)

#SPDX-License-Identifier: GPL-2.0
import re
import csv
import json
import argparse
from pathlib import Path
import subprocess

class Validator:
    def __init__(self, rulefname, reportfname='', t=5, debug=False, datafname='', fullrulefname='', workload='true', metrics=''):
        self.rulefname = rulefname
        self.reportfname = reportfname
        self.rules = None
        self.collectlist:str = metrics
        self.metrics = self.__set_metrics(metrics)
        self.skiplist = set()
        self.tolerance = t

        self.workloads = [x for x in workload.split(",") if x]
        self.wlidx = 0 # idx of current workloads
        self.allresults = dict() # metric results of all workload
        self.allignoremetrics = dict() # metrics with no results or negative results
        self.allfailtests = dict()
        self.alltotalcnt = dict()
        self.allpassedcnt = dict()
        self.allerrlist = dict()

        self.results = dict() # metric results of current workload
        # vars for test pass/failure statistics
        self.ignoremetrics= set() # metrics with no results or negative results, neg result counts as a failed test
        self.failtests = dict()
        self.totalcnt = 0
        self.passedcnt = 0
        # vars for errors
        self.errlist = list()

        # vars for Rule Generator
        self.pctgmetrics = set() # Percentage rule

        # vars for debug
        self.datafname = datafname
        self.debug = debug
        self.fullrulefname = fullrulefname

    def __set_metrics(self, metrics=''):
        if metrics != '':
            return set(metrics.split(","))
        else:
            return set()

    def read_json(self, filename: str) -> dict:
        try:
            with open(Path(filename).resolve(), "r") as f:
                data = json.loads(f.read())
        except OSError as e:
            print(f"Error when reading file {e}")
            sys.exit()

        return data

    def json_dump(self, data, output_file):
        parent = Path(output_file).parent
        if not parent.exists():
            parent.mkdir(parents=True)

        with open(output_file, "w+") as output_file:
            json.dump(data,
                      output_file,
                      ensure_ascii=True,
                      indent=4)

    def get_results(self, idx:int = 0):
        return self.results[idx]

    def get_bounds(self, lb, ub, error, alias={}, ridx:int = 0) -> list:
        """
        Get bounds and tolerance from lb, ub, and error.
        If missing lb, use 0.0; missing ub, use float('inf); missing error, use self.tolerance.

        @param lb: str/float, lower bound
        @param ub: str/float, upper bound
        @param error: float/str, error tolerance
        @returns: lower bound, return inf if the lower bound is a metric value and is not collected
                  upper bound, return -1 if the upper bound is a metric value and is not collected
                  tolerance, denormalized base on upper bound value
        """
        # init ubv and lbv to invalid values
        def get_bound_value (bound, initval, ridx):
            val = initval
            if isinstance(bound, int) or isinstance(bound, float):
                val = bound
            elif isinstance(bound, str):
                if bound == '':
                    val = float("inf")
                elif bound in alias:
                    vall = self.get_value(alias[ub], ridx)
                    if vall:
                        val = vall[0]
                elif bound.replace('.', '1').isdigit():
                    val = float(bound)
                else:
                    print("Wrong bound: {0}".format(bound))
            else:
                print("Wrong bound: {0}".format(bound))
            return val

        ubv = get_bound_value(ub, -1, ridx)
        lbv = get_bound_value(lb, float('inf'), ridx)
        t = get_bound_value(error, self.tolerance, ridx)

        # denormalize error threshold
        denormerr = t * ubv / 100 if ubv != 100 and ubv > 0 else t

        return lbv, ubv, denormerr

    def get_value(self, name:str, ridx:int = 0) -> list:
        """
        Get value of the metric from self.results.
        If result of this metric is not provided, the metric name will be added into self.ignoremetics and self.errlist.
        All future test(s) on this metric will fail.

        @param name: name of the metric
        @returns: list with value found in self.results; list is empty when value is not found.
        """
        results = []
        data = self.results[ridx] if ridx in self.results else self.results[0]
        if name not in self.ignoremetrics:
            if name in data:
                results.append(data[name])
            elif name.replace('.', '1').isdigit():
                results.append(float(name))
            else:
                self.ignoremetrics.add(name)
        return results

    def check_bound(self, val, lb, ub, err):
        return True if val <= ub + err and val >= lb - err else False

    # Positive Value Sanity check
    def pos_val_test(self):
        """
        Check if metrics value are non-negative.
        One metric is counted as one test.
        Failure: when metric value is negative or not provided.
        Metrics with negative value will be added into the self.failtests['PositiveValueTest'] and self.ignoremetrics.
        """
        negmetric = dict()
        pcnt = 0
        tcnt = 0
        rerun = list()
        for name, val in self.get_results().items():
            if val < 0:
                negmetric[name] = val
                rerun.append(name)
            else:
                pcnt += 1
            tcnt += 1
        if len(rerun) > 0 and len(rerun) < 20:
            second_results = dict()
            self.second_test(rerun, second_results)
            for name, val in second_results.items():
                if name not in negmetric: continue
                if val >= 0:
                    del negmetric[name]
                    pcnt += 1

        self.failtests['PositiveValueTest']['Total Tests'] = tcnt
        self.failtests['PositiveValueTest']['Passed Tests'] = pcnt
        if len(negmetric.keys()):
            self.ignoremetrics.update(negmetric.keys())
            negmessage = ["{0}(={1:.4f})".format(name, val) for name, val in negmetric.items()]
            self.failtests['PositiveValueTest']['Failed Tests'].append({'NegativeValue': negmessage})

        return

    def evaluate_formula(self, formula:str, alias:dict, ridx:int = 0):
        """
        Evaluate the value of formula.

        @param formula: the formula to be evaluated
        @param alias: the dict has alias to metric name mapping
        @returns: value of the formula is success; -1 if the one or more metric value not provided
        """
        stack = []
        b = 0
        errs = []
        sign = "+"
        f = str()

        #TODO: support parenthesis?
        for i in range(len(formula)):
            if i+1 == len(formula) or formula[i] in ('+', '-', '*', '/'):
                s = alias[formula[b:i]] if i+1 < len(formula) else alias[formula[b:]]
                v = self.get_value(s, ridx)
                if not v:
                    errs.append(s)
                else:
                    f = f + "{0}(={1:.4f})".format(s, v[0])
                    if sign == "*":
                        stack[-1] = stack[-1] * v
                    elif sign == "/":
                        stack[-1] = stack[-1] / v
                    elif sign == '-':
                        stack.append(-v[0])
                    else:
                        stack.append(v[0])
                if i + 1 < len(formula):
                    sign = formula[i]
                    f += sign
                    b = i + 1

        if len(errs) > 0:
            return -1, "Metric value missing: "+','.join(errs)

        val = sum(stack)
        return val, f

    # Relationships Tests
    def relationship_test(self, rule: dict):
        """
        Validate if the metrics follow the required relationship in the rule.
        eg. lower_bound <= eval(formula)<= upper_bound
        One rule is counted as ont test.
        Failure: when one or more metric result(s) not provided, or when formula evaluated outside of upper/lower bounds.

        @param rule: dict with metric name(+alias), formula, and required upper and lower bounds.
        """
        alias = dict()
        for m in rule['Metrics']:
            alias[m['Alias']] = m['Name']
        lbv, ubv, t = self.get_bounds(rule['RangeLower'], rule['RangeUpper'], rule['ErrorThreshold'], alias, ridx=rule['RuleIndex'])
        val, f = self.evaluate_formula(rule['Formula'], alias, ridx=rule['RuleIndex'])
        if val == -1:
            self.failtests['RelationshipTest']['Failed Tests'].append({'RuleIndex': rule['RuleIndex'], 'Description':f})
        elif not self.check_bound(val, lbv, ubv, t):
            lb = rule['RangeLower']
            ub = rule['RangeUpper']
            if isinstance(lb, str):
                if lb in alias:
                    lb = alias[lb]
            if isinstance(ub, str):
                if ub in alias:
                    ub = alias[ub]
            self.failtests['RelationshipTest']['Failed Tests'].append({'RuleIndex': rule['RuleIndex'], 'Formula':f,
                                                                       'RangeLower': lb, 'LowerBoundValue': self.get_value(lb),
                                                                       'RangeUpper': ub, 'UpperBoundValue':self.get_value(ub),
                                                                       'ErrorThreshold': t, 'CollectedValue': val})
        else:
            self.passedcnt += 1
            self.failtests['RelationshipTest']['Passed Tests'] += 1
        self.totalcnt += 1
        self.failtests['RelationshipTest']['Total Tests'] += 1

        return


    # Single Metric Test
    def single_test(self, rule:dict):
        """
        Validate if the metrics are in the required value range.
        eg. lower_bound <= metrics_value <= upper_bound
        One metric is counted as one test in this type of test.
        One rule may include one or more metrics.
        Failure: when the metric value not provided or the value is outside the bounds.
        This test updates self.total_cnt and records failed tests in self.failtest['SingleMetricTest'].

        @param rule: dict with metrics to validate and the value range requirement
        """
        lbv, ubv, t = self.get_bounds(rule['RangeLower'], rule['RangeUpper'], rule['ErrorThreshold'])
        metrics = rule['Metrics']
        passcnt = 0
        totalcnt = 0
        faillist = list()
        failures = dict()
        rerun = list()
        for m in metrics:
            totalcnt += 1
            result = self.get_value(m['Name'])
            if len(result) > 0 and self.check_bound(result[0], lbv, ubv, t) or m['Name'] in self.skiplist:
                passcnt += 1
            else:
                failures[m['Name']] = result
                rerun.append(m['Name'])

        if len(rerun) > 0 and len(rerun) < 20:
            second_results = dict()
            self.second_test(rerun, second_results)
            for name, val in second_results.items():
                if name not in failures: continue
                if self.check_bound(val, lbv, ubv, t):
                    passcnt += 1
                    del failures[name]
                else:
                    failures[name] = val
                    self.results[0][name] = val

        self.totalcnt += totalcnt
        self.passedcnt += passcnt
        self.failtests['SingleMetricTest']['Total Tests'] += totalcnt
        self.failtests['SingleMetricTest']['Passed Tests'] += passcnt
        if len(failures.keys()) != 0:
            faillist = [{'MetricName':name, 'CollectedValue':val} for name, val in failures.items()]
            self.failtests['SingleMetricTest']['Failed Tests'].append({'RuleIndex':rule['RuleIndex'],
                                                                       'RangeLower': rule['RangeLower'],
                                                                       'RangeUpper': rule['RangeUpper'],
                                                                       'ErrorThreshold':rule['ErrorThreshold'],
                                                                       'Failure':faillist})

        return

    def create_report(self):
        """
        Create final report and write into a JSON file.
        """
        alldata = list()
        for i in range(0, len(self.workloads)):
            reportstas = {"Total Rule Count": self.alltotalcnt[i], "Passed Rule Count": self.allpassedcnt[i]}
            data = {"Metric Validation Statistics": reportstas, "Tests in Category": self.allfailtests[i],
                    "Errors":self.allerrlist[i]}
            alldata.append({"Workload": self.workloads[i], "Report": data})

        json_str = json.dumps(alldata, indent=4)
        print("Test validation finished. Final report: ")
        print(json_str)

        if self.debug:
            allres = [{"Workload": self.workloads[i], "Results": self.allresults[i]} for i in range(0, len(self.workloads))]
            self.json_dump(allres, self.datafname)

    def check_rule(self, testtype, metric_list):
        """
        Check if the rule uses metric(s) that not exist in current platform.

        @param metric_list: list of metrics from the rule.
        @return: False when find one metric out in Metric file. (This rule should not skipped.)
                 True when all metrics used in the rule are found in Metric file.
        """
        if testtype == "RelationshipTest":
            for m in metric_list:
                if m['Name'] not in self.metrics:
                    return False
        return True

    # Start of Collector and Converter
    def convert(self, data: list, metricvalues:dict):
        """
        Convert collected metric data from the -j output to dict of {metric_name:value}.
        """
        for json_string in data:
            try:
                result =json.loads(json_string)
                if "metric-unit" in result and result["metric-unit"] != "(null)" and result["metric-unit"] != "":
                    name = result["metric-unit"].split("  ")[1] if len(result["metric-unit"].split("  ")) > 1 \
                        else result["metric-unit"]
                    metricvalues[name.lower()] = float(result["metric-value"])
            except ValueError as error:
                continue
        return

    def _run_perf(self, metric, workload: str):
        tool = 'perf'
        command = [tool, 'stat', '-j', '-M', f"{metric}", "-a"]
        wl = workload.split()
        command.extend(wl)
        print(" ".join(command))
        cmd = subprocess.run(command, stderr=subprocess.PIPE, encoding='utf-8')
        data = [x+'}' for x in cmd.stderr.split('}\n') if x]
        return data


    def collect_perf(self, workload: str):
        """
        Collect metric data with "perf stat -M" on given workload with -a and -j.
        """
        self.results = dict()
        print(f"Starting perf collection")
        print(f"Long workload: {workload}")
        collectlist = dict()
        if self.collectlist != "":
            collectlist[0] = {x for x in self.collectlist.split(",")}
        else:
            collectlist[0] = set(list(self.metrics))
        # Create metric set for relationship rules
        for rule in self.rules:
            if rule["TestType"] == "RelationshipTest":
                metrics = [m["Name"] for m in rule["Metrics"]]
                if not any(m not in collectlist[0] for m in metrics):
                    collectlist[rule["RuleIndex"]] = [",".join(list(set(metrics)))]

        for idx, metrics in collectlist.items():
            if idx == 0: wl = "true"
            else: wl = workload
            for metric in metrics:
                data = self._run_perf(metric, wl)
                if idx not in self.results: self.results[idx] = dict()
                self.convert(data, self.results[idx])
        return

    def second_test(self, collectlist, second_results):
        workload = self.workloads[self.wlidx]
        for metric in collectlist:
            data = self._run_perf(metric, workload)
            self.convert(data, second_results)

    # End of Collector and Converter

    # Start of Rule Generator
    def parse_perf_metrics(self):
        """
        Read and parse perf metric file:
        1) find metrics with '1%' or '100%' as ScaleUnit for Percent check
        2) create metric name list
        """
        command = ['perf', 'list', '-j', '--details', 'metrics']
        cmd = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf-8')
        try:
            data = json.loads(cmd.stdout)
            for m in data:
                if 'MetricName' not in m:
                    print("Warning: no metric name")
                    continue
                name = m['MetricName'].lower()
                self.metrics.add(name)
                if 'ScaleUnit' in m and (m['ScaleUnit'] == '1%' or m['ScaleUnit'] == '100%'):
                    self.pctgmetrics.add(name.lower())
        except ValueError as error:
            print(f"Error when parsing metric data")
            sys.exit()

        return

    def remove_unsupported_rules(self, rules):
        new_rules = []
        for rule in rules:
            add_rule = True
            for m in rule["Metrics"]:
                if m["Name"] in self.skiplist or m["Name"] not in self.metrics:
                    add_rule = False
                    break
            if add_rule:
                new_rules.append(rule)
        return new_rules

    def create_rules(self):
        """
        Create full rules which includes:
        1) All the rules from the "relationshi_rules" file
        2) SingleMetric rule for all the 'percent' metrics

        Reindex all the rules to avoid repeated RuleIndex
        """
        data = self.read_json(self.rulefname)
        rules = data['RelationshipRules']
        self.skiplist = set([name.lower() for name in data['SkipList']])
        self.rules = self.remove_unsupported_rules(rules)
        pctgrule = {'RuleIndex':0,
                    'TestType':'SingleMetricTest',
                    'RangeLower':'0',
                    'RangeUpper': '100',
                    'ErrorThreshold': self.tolerance,
                    'Description':'Metrics in percent unit have value with in [0, 100]',
                    'Metrics': [{'Name': m.lower()} for m in self.pctgmetrics]}
        self.rules.append(pctgrule)

        # Re-index all rules to avoid repeated RuleIndex
        idx = 1
        for r in self.rules:
            r['RuleIndex'] = idx
            idx += 1

        if self.debug:
            #TODO: need to test and generate file name correctly
            data = {'RelationshipRules':self.rules, 'SupportedMetrics': [{"MetricName": name} for name in self.metrics]}
            self.json_dump(data, self.fullrulefname)

        return
    # End of Rule Generator

    def _storewldata(self, key):
        '''
        Store all the data of one workload into the corresponding data structure for all workloads.
        @param key: key to the dictionaries (index of self.workloads).
        '''
        self.allresults[key] = self.results
        self.allignoremetrics[key] = self.ignoremetrics
        self.allfailtests[key] = self.failtests
        self.alltotalcnt[key] = self.totalcnt
        self.allpassedcnt[key] = self.passedcnt
        self.allerrlist[key] = self.errlist

    #Initialize data structures before data validation of each workload
    def _init_data(self):

        testtypes = ['PositiveValueTest', 'RelationshipTest', 'SingleMetricTest']
        self.results = dict()
        self.ignoremetrics= set()
        self.errlist = list()
        self.failtests = {k:{'Total Tests':0, 'Passed Tests':0, 'Failed Tests':[]} for k in testtypes}
        self.totalcnt = 0
        self.passedcnt = 0

    def test(self):
        '''
        The real entry point of the test framework.
        This function loads the validation rule JSON file and Standard Metric file to create rules for
        testing and namemap dictionaries.
        It also reads in result JSON file for testing.

        In the test process, it passes through each rule and launch correct test function bases on the
        'TestType' field of the rule.

        The final report is written into a JSON file.
        '''
        if not self.collectlist:
            self.parse_perf_metrics()
        self.create_rules()
        for i in range(0, len(self.workloads)):
            self.wlidx = i
            self._init_data()
            self.collect_perf(self.workloads[i])
            # Run positive value test
            self.pos_val_test()
            for r in self.rules:
                # skip rules that uses metrics not exist in this platform
                testtype = r['TestType']
                if not self.check_rule(testtype, r['Metrics']):
                    continue
                if  testtype == 'RelationshipTest':
                    self.relationship_test(r)
                elif testtype == 'SingleMetricTest':
                    self.single_test(r)
                else:
                    print("Unsupported Test Type: ", testtype)
                    self.errlist.append("Unsupported Test Type from rule: " + r['RuleIndex'])
            self._storewldata(i)
            print("Workload: ", self.workloads[i])
            print("Total metrics collected: ", self.failtests['PositiveValueTest']['Total Tests'])
            print("Non-negative metric count: ", self.failtests['PositiveValueTest']['Passed Tests'])
            print("Total Test Count: ", self.totalcnt)
            print("Passed Test Count: ", self.passedcnt)

        self.create_report()
        return sum(self.alltotalcnt.values()) != sum(self.allpassedcnt.values())
# End of Class Validator


def main() -> None:
    parser = argparse.ArgumentParser(description="Launch metric value validation")

    parser.add_argument("-rule", help="Base validation rule file", required=True)
    parser.add_argument("-output_dir", help="Path for validator output file, report file", required=True)
    parser.add_argument("-debug", help="Debug run, save intermediate data to files", action="store_true", default=False)
    parser.add_argument("-wl", help="Workload to run while data collection", default="true")
    parser.add_argument("-m", help="Metric list to validate", default="")
    args = parser.parse_args()
    outpath = Path(args.output_dir)
    reportf = Path.joinpath(outpath, 'perf_report.json')
    fullrule = Path.joinpath(outpath, 'full_rule.json')
    datafile = Path.joinpath(outpath, 'perf_data.json')

    validator = Validator(args.rule, reportf, debug=args.debug,
                        datafname=datafile, fullrulefname=fullrule, workload=args.wl,
                        metrics=args.m)
    ret = validator.test()

    return ret


if __name__ == "__main__":
    import sys
    sys.exit(main())