1#!/usr/bin/env python3
2
3# Yocto Project test results management tool
4# This script is an thin layer over resulttool to manage tes results and regression reports.
5# Its main feature is to translate tags or branch names to revisions SHA1, and then to run resulttool
6# with those computed revisions
7#
8# Copyright (C) 2023 OpenEmbedded Contributors
9#
10# SPDX-License-Identifier: MIT
11#
12
13import sys
14import os
15import argparse
16import subprocess
17import tempfile
18import lib.scriptutils as scriptutils
19
20script_path = os.path.dirname(os.path.realpath(__file__))
21poky_path = os.path.abspath(os.path.join(script_path, ".."))
22resulttool = os.path.abspath(os.path.join(script_path, "resulttool"))
23logger = scriptutils.logger_create(sys.argv[0])
24testresults_default_url="git://git.yoctoproject.org/yocto-testresults"
25
26def create_workdir():
27    workdir = tempfile.mkdtemp(prefix='yocto-testresults-query.')
28    logger.info(f"Shallow-cloning testresults in {workdir}")
29    subprocess.check_call(["git", "clone", testresults_default_url, workdir, "--depth", "1"])
30    return workdir
31
32def get_sha1(pokydir, revision):
33    try:
34        rev = subprocess.check_output(["git", "rev-list", "-n", "1", revision], cwd=pokydir).decode('utf-8').strip()
35        logger.info(f"SHA-1 revision for {revision} in {pokydir} is {rev}")
36        return rev
37    except subprocess.CalledProcessError:
38        logger.error(f"Can not find SHA-1 for {revision} in {pokydir}")
39        return None
40
41def get_branch(tag):
42    # The tags in test results repository, as returned by git rev-list, have the following form:
43    # refs/tags/<branch>/<count>-g<sha1>/<num>
44    return '/'.join(tag.split("/")[2:-2])
45
46def fetch_testresults(workdir, sha1):
47    logger.info(f"Fetching test results for {sha1} in {workdir}")
48    rawtags = subprocess.check_output(["git", "ls-remote", "--refs", "--tags", "origin", f"*{sha1}*"], cwd=workdir).decode('utf-8').strip()
49    if not rawtags:
50        raise Exception(f"No reference found for commit {sha1} in {workdir}")
51    branch = ""
52    for rev in [rawtag.split()[1] for rawtag in rawtags.splitlines()]:
53        if not branch:
54            branch = get_branch(rev)
55        logger.info(f"Fetching matching revision: {rev}")
56        subprocess.check_call(["git", "fetch", "--depth", "1", "origin", f"{rev}:{rev}"], cwd=workdir)
57    return branch
58
59def compute_regression_report(workdir, basebranch, baserevision, targetbranch, targetrevision, args):
60    logger.info(f"Running resulttool regression between SHA1 {baserevision} and {targetrevision}")
61    command = [resulttool, "regression-git", "--branch", basebranch, "--commit", baserevision, "--branch2", targetbranch, "--commit2", targetrevision, workdir]
62    if args.limit:
63        command.extend(["-l", args.limit])
64    report = subprocess.check_output(command).decode("utf-8")
65    return report
66
67def print_report_with_header(report, baseversion, baserevision, targetversion, targetrevision):
68    print("========================== Regression report ==============================")
69    print(f'{"=> Target:": <16}{targetversion: <16}({targetrevision})')
70    print(f'{"=> Base:": <16}{baseversion: <16}({baserevision})')
71    print("===========================================================================\n")
72    print(report, end='')
73
74def regression(args):
75    logger.info(f"Compute regression report between {args.base} and {args.target}")
76    if args.testresultsdir:
77        workdir = args.testresultsdir
78    else:
79        workdir = create_workdir()
80
81    try:
82        baserevision = get_sha1(poky_path, args.base)
83        targetrevision = get_sha1(poky_path, args.target)
84        if not baserevision or not targetrevision:
85            logger.error("One or more revision(s) missing. You might be targeting nonexistant tags/branches, or are in wrong repository (you must use Poky and not oe-core)")
86            if not args.testresultsdir:
87                subprocess.check_call(["rm", "-rf",  workdir])
88            sys.exit(1)
89        basebranch = fetch_testresults(workdir, baserevision)
90        targetbranch = fetch_testresults(workdir, targetrevision)
91        report = compute_regression_report(workdir, basebranch, baserevision, targetbranch, targetrevision, args)
92        print_report_with_header(report, args.base, baserevision, args.target, targetrevision)
93    finally:
94        if not args.testresultsdir:
95            subprocess.check_call(["rm", "-rf",  workdir])
96
97def main():
98    parser = argparse.ArgumentParser(description="Yocto Project test results helper")
99    subparsers = parser.add_subparsers(
100        help="Supported commands for test results helper",
101        required=True)
102    parser_regression_report = subparsers.add_parser(
103        "regression-report",
104        help="Generate regression report between two fixed revisions. Revisions can be branch name or tag")
105    parser_regression_report.add_argument(
106        'base',
107        help="Revision or tag against which to compare results (i.e: the older)")
108    parser_regression_report.add_argument(
109        'target',
110        help="Revision or tag to compare against the base (i.e: the newer)")
111    parser_regression_report.add_argument(
112        '-t',
113        '--testresultsdir',
114        help=f"An existing test results directory. {sys.argv[0]} will automatically clone it and use default branch if not provided")
115    parser_regression_report.add_argument(
116        '-l',
117        '--limit',
118        help=f"Maximum number of changes to display per test. Can be set to 0 to print all changes")
119    parser_regression_report.set_defaults(func=regression)
120
121    args = parser.parse_args()
122    args.func(args)
123
124if __name__ == '__main__':
125    try:
126        ret =  main()
127    except Exception:
128        ret = 1
129        import traceback
130        traceback.print_exc()
131    sys.exit(ret)
132