1# resulttool - regression analysis
2#
3# Copyright (c) 2019, Intel Corporation.
4# Copyright (c) 2019, Linux Foundation
5#
6# SPDX-License-Identifier: GPL-2.0-only
7#
8
9import resulttool.resultutils as resultutils
10import json
11
12from oeqa.utils.git import GitRepo
13import oeqa.utils.gitarchive as gitarchive
14
15def compare_result(logger, base_name, target_name, base_result, target_result):
16    base_result = base_result.get('result')
17    target_result = target_result.get('result')
18    result = {}
19    if base_result and target_result:
20        for k in base_result:
21            base_testcase = base_result[k]
22            base_status = base_testcase.get('status')
23            if base_status:
24                target_testcase = target_result.get(k, {})
25                target_status = target_testcase.get('status')
26                if base_status != target_status:
27                    result[k] = {'base': base_status, 'target': target_status}
28            else:
29                logger.error('Failed to retrieved base test case status: %s' % k)
30    if result:
31        resultstring = "Regression: %s\n            %s\n" % (base_name, target_name)
32        for k in sorted(result):
33            resultstring += '    %s: %s -> %s\n' % (k, result[k]['base'], result[k]['target'])
34    else:
35        resultstring = "Match: %s\n       %s" % (base_name, target_name)
36    return result, resultstring
37
38def get_results(logger, source):
39    return resultutils.load_resultsdata(source, configmap=resultutils.regression_map)
40
41def regression(args, logger):
42    base_results = get_results(logger, args.base_result)
43    target_results = get_results(logger, args.target_result)
44
45    regression_common(args, logger, base_results, target_results)
46
47def regression_common(args, logger, base_results, target_results):
48    if args.base_result_id:
49        base_results = resultutils.filter_resultsdata(base_results, args.base_result_id)
50    if args.target_result_id:
51        target_results = resultutils.filter_resultsdata(target_results, args.target_result_id)
52
53    matches = []
54    regressions = []
55    notfound = []
56
57    for a in base_results:
58        if a in target_results:
59            base = list(base_results[a].keys())
60            target = list(target_results[a].keys())
61            # We may have multiple base/targets which are for different configurations. Start by
62            # removing any pairs which match
63            for c in base.copy():
64                for b in target.copy():
65                    res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b])
66                    if not res:
67                        matches.append(resstr)
68                        base.remove(c)
69                        target.remove(b)
70                        break
71            # Should only now see regressions, we may not be able to match multiple pairs directly
72            for c in base:
73                for b in target:
74                    res, resstr = compare_result(logger, c, b, base_results[a][c], target_results[a][b])
75                    if res:
76                        regressions.append(resstr)
77        else:
78            notfound.append("%s not found in target" % a)
79    print("\n".join(sorted(matches)))
80    print("\n".join(sorted(regressions)))
81    print("\n".join(sorted(notfound)))
82
83    return 0
84
85def regression_git(args, logger):
86    base_results = {}
87    target_results = {}
88
89    tag_name = "{branch}/{commit_number}-g{commit}/{tag_number}"
90    repo = GitRepo(args.repo)
91
92    revs = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch)
93
94    if args.branch2:
95        revs2 = gitarchive.get_test_revs(logger, repo, tag_name, branch=args.branch2)
96        if not len(revs2):
97            logger.error("No revisions found to compare against")
98            return 1
99        if not len(revs):
100            logger.error("No revision to report on found")
101            return 1
102    else:
103        if len(revs) < 2:
104            logger.error("Only %d tester revisions found, unable to generate report" % len(revs))
105            return 1
106
107    # Pick revisions
108    if args.commit:
109        if args.commit_number:
110            logger.warning("Ignoring --commit-number as --commit was specified")
111        index1 = gitarchive.rev_find(revs, 'commit', args.commit)
112    elif args.commit_number:
113        index1 = gitarchive.rev_find(revs, 'commit_number', args.commit_number)
114    else:
115        index1 = len(revs) - 1
116
117    if args.branch2:
118        revs2.append(revs[index1])
119        index1 = len(revs2) - 1
120        revs = revs2
121
122    if args.commit2:
123        if args.commit_number2:
124            logger.warning("Ignoring --commit-number2 as --commit2 was specified")
125        index2 = gitarchive.rev_find(revs, 'commit', args.commit2)
126    elif args.commit_number2:
127        index2 = gitarchive.rev_find(revs, 'commit_number', args.commit_number2)
128    else:
129        if index1 > 0:
130            index2 = index1 - 1
131            # Find the closest matching commit number for comparision
132            # In future we could check the commit is a common ancestor and
133            # continue back if not but this good enough for now
134            while index2 > 0 and revs[index2].commit_number > revs[index1].commit_number:
135                index2 = index2 - 1
136        else:
137            logger.error("Unable to determine the other commit, use "
138                      "--commit2 or --commit-number2 to specify it")
139            return 1
140
141    logger.info("Comparing:\n%s\nto\n%s\n" % (revs[index1], revs[index2]))
142
143    base_results = resultutils.git_get_result(repo, revs[index1][2])
144    target_results = resultutils.git_get_result(repo, revs[index2][2])
145
146    regression_common(args, logger, base_results, target_results)
147
148    return 0
149
150def register_commands(subparsers):
151    """Register subcommands from this plugin"""
152
153    parser_build = subparsers.add_parser('regression', help='regression file/directory analysis',
154                                         description='regression analysis comparing the base set of results to the target results',
155                                         group='analysis')
156    parser_build.set_defaults(func=regression)
157    parser_build.add_argument('base_result',
158                              help='base result file/directory/URL for the comparison')
159    parser_build.add_argument('target_result',
160                              help='target result file/directory/URL to compare with')
161    parser_build.add_argument('-b', '--base-result-id', default='',
162                              help='(optional) filter the base results to this result ID')
163    parser_build.add_argument('-t', '--target-result-id', default='',
164                              help='(optional) filter the target results to this result ID')
165
166    parser_build = subparsers.add_parser('regression-git', help='regression git analysis',
167                                         description='regression analysis comparing base result set to target '
168                                                     'result set',
169                                         group='analysis')
170    parser_build.set_defaults(func=regression_git)
171    parser_build.add_argument('repo',
172                              help='the git repository containing the data')
173    parser_build.add_argument('-b', '--base-result-id', default='',
174                              help='(optional) default select regression based on configurations unless base result '
175                                   'id was provided')
176    parser_build.add_argument('-t', '--target-result-id', default='',
177                              help='(optional) default select regression based on configurations unless target result '
178                                   'id was provided')
179
180    parser_build.add_argument('--branch', '-B', default='master', help="Branch to find commit in")
181    parser_build.add_argument('--branch2', help="Branch to find comparision revisions in")
182    parser_build.add_argument('--commit', help="Revision to search for")
183    parser_build.add_argument('--commit-number', help="Revision number to search for, redundant if --commit is specified")
184    parser_build.add_argument('--commit2', help="Revision to compare with")
185    parser_build.add_argument('--commit-number2', help="Revision number to compare with, redundant if --commit2 is specified")
186
187