1# resulttool - regression analysis 2# 3# Copyright (c) 2019, Intel Corporation. 4# Copyright (c) 2019, Linux Foundation 5# 6# SPDX-License-Identifier: GPL-2.0-only 7# 8 9import resulttool.resultutils as resultutils 10import json 11 12from oeqa.utils.git import GitRepo 13import oeqa.utils.gitarchive as gitarchive 14 15def compare_result(logger, base_name, target_name, base_result, target_result): 16 base_result = base_result.get('result') 17 target_result = target_result.get('result') 18 result = {} 19 if base_result and target_result: 20 for k in base_result: 21 base_testcase = base_result[k] 22 base_status = base_testcase.get('status') 23 if base_status: 24 target_testcase = target_result.get(k, {}) 25 target_status = target_testcase.get('status') 26 if base_status != target_status: 27 result[k] = {'base': base_status, 'target': target_status} 28 else: 29 logger.error('Failed to retrieved base test case status: %s' % k) 30 if result: 31 resultstring = "Regression: %s\n %s\n" % (base_name, target_name) 32 for k in sorted(result): 33 resultstring += ' %s: %s -> %s\n' % (k, result[k]['base'], result[k]['target']) 34 else: 35 resultstring = "Match: %s\n %s" % (base_name, target_name) 36 return result, resultstring 37 38def get_results(logger, source): 39 return resultutils.load_resultsdata(source, configmap=resultutils.regression_map) 40 41def regression(args, logger): 42 base_results = get_results(logger, args.base_result) 43 target_results = get_results(logger, args.target_result) 44 45 regression_common(args, logger, base_results, target_results) 46 47def regression_common(args, logger, base_results, target_results): 48 if args.base_result_id: 49 base_results = resultutils.filter_resultsdata(base_results, args.base_result_id) 50 if args.target_result_id: 51 target_results = resultutils.filter_resultsdata(target_results, args.target_result_id) 52 53 matches = [] 54 regressions = [] 55 notfound = [] 56 57 for a in base_results: 58 if a in target_results: 59 base = list(base_results[a].keys()) 60 target = list(target_results[a].keys()) 61 # We may have multiple base/targets which are for different configurations. Start by 62 # removing any pairs which match 63 for c in base.copy(): 64 for b in target.copy(): 65 res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b]) 66 if not res: 67 matches.append(resstr) 68 base.remove(c) 69 target.remove(b) 70 break 71 # Should only now see regressions, we may not be able to match multiple pairs directly 72 for c in base: 73 for b in target: 74 res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b]) 75 if res: 76 regressions.append(resstr) 77 else: 78 notfound.append("%s not found in target" % a) 79 print("\n".join(sorted(matches))) 80 print("\n".join(sorted(regressions))) 81 print("\n".join(sorted(notfound))) 82 83 return 0 84 85def regression_git(args, logger): 86 base_results = {} 87 target_results = {} 88 89 tag_name = "{branch}/{commit_number}-g{commit}/{tag_number}" 90 repo = GitRepo(args.repo) 91 92 revs = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch) 93 94 if args.branch2: 95 revs2 = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch2) 96 if not len(revs2): 97 logger.error("No revisions found to compare against") 98 return 1 99 if not len(revs): 100 logger.error("No revision to report on found") 101 return 1 102 else: 103 if len(revs) < 2: 104 logger.error("Only %d tester revisions found, unable to generate report" % len(revs)) 105 return 1 106 107 # Pick revisions 108 if args.commit: 109 if args.commit_number: 110 logger.warning("Ignoring --commit-number as --commit was specified") 111 index1 = gitarchive.rev_find(revs, 'commit', args.commit) 112 elif args.commit_number: 113 index1 = gitarchive.rev_find(revs, 'commit_number', args.commit_number) 114 else: 115 index1 = len(revs) - 1 116 117 if args.branch2: 118 revs2.append(revs[index1]) 119 index1 = len(revs2) - 1 120 revs = revs2 121 122 if args.commit2: 123 if args.commit_number2: 124 logger.warning("Ignoring --commit-number2 as --commit2 was specified") 125 index2 = gitarchive.rev_find(revs, 'commit', args.commit2) 126 elif args.commit_number2: 127 index2 = gitarchive.rev_find(revs, 'commit_number', args.commit_number2) 128 else: 129 if index1 > 0: 130 index2 = index1 - 1 131 # Find the closest matching commit number for comparision 132 # In future we could check the commit is a common ancestor and 133 # continue back if not but this good enough for now 134 while index2 > 0 and revs[index2].commit_number > revs[index1].commit_number: 135 index2 = index2 - 1 136 else: 137 logger.error("Unable to determine the other commit, use " 138 "--commit2 or --commit-number2 to specify it") 139 return 1 140 141 logger.info("Comparing:\n%s\nto\n%s\n" % (revs[index1], revs[index2])) 142 143 base_results = resultutils.git_get_result(repo, revs[index1][2]) 144 target_results = resultutils.git_get_result(repo, revs[index2][2]) 145 146 regression_common(args, logger, base_results, target_results) 147 148 return 0 149 150def register_commands(subparsers): 151 """Register subcommands from this plugin""" 152 153 parser_build = subparsers.add_parser('regression', help='regression file/directory analysis', 154 description='regression analysis comparing the base set of results to the target results', 155 group='analysis') 156 parser_build.set_defaults(func=regression) 157 parser_build.add_argument('base_result', 158 help='base result file/directory/URL for the comparison') 159 parser_build.add_argument('target_result', 160 help='target result file/directory/URL to compare with') 161 parser_build.add_argument('-b', '--base-result-id', default='', 162 help='(optional) filter the base results to this result ID') 163 parser_build.add_argument('-t', '--target-result-id', default='', 164 help='(optional) filter the target results to this result ID') 165 166 parser_build = subparsers.add_parser('regression-git', help='regression git analysis', 167 description='regression analysis comparing base result set to target ' 168 'result set', 169 group='analysis') 170 parser_build.set_defaults(func=regression_git) 171 parser_build.add_argument('repo', 172 help='the git repository containing the data') 173 parser_build.add_argument('-b', '--base-result-id', default='', 174 help='(optional) default select regression based on configurations unless base result ' 175 'id was provided') 176 parser_build.add_argument('-t', '--target-result-id', default='', 177 help='(optional) default select regression based on configurations unless target result ' 178 'id was provided') 179 180 parser_build.add_argument('--branch', '-B', default='master', help="Branch to find commit in") 181 parser_build.add_argument('--branch2', help="Branch to find comparision revisions in") 182 parser_build.add_argument('--commit', help="Revision to search for") 183 parser_build.add_argument('--commit-number', help="Revision number to search for, redundant if --commit is specified") 184 parser_build.add_argument('--commit2', help="Revision to compare with") 185 parser_build.add_argument('--commit-number2', help="Revision number to compare with, redundant if --commit2 is specified") 186 187