1#!/usr/bin/env python3
2
3# Yocto Project test results management tool
4# This script is an thin layer over resulttool to manage tes results and regression reports.
5# Its main feature is to translate tags or branch names to revisions SHA1, and then to run resulttool
6# with those computed revisions
7#
8# Copyright (C) 2023 OpenEmbedded Contributors
9#
10# SPDX-License-Identifier: MIT
11#
12
13import sys
14import os
15import argparse
16import subprocess
17import tempfile
18import lib.scriptutils as scriptutils
19
20script_path = os.path.dirname(os.path.realpath(__file__))
21poky_path = os.path.abspath(os.path.join(script_path, ".."))
22resulttool = os.path.abspath(os.path.join(script_path, "resulttool"))
23logger = scriptutils.logger_create(sys.argv[0])
24testresults_default_url="git://git.yoctoproject.org/yocto-testresults"
25
26def create_workdir():
27    workdir = tempfile.mkdtemp(prefix='yocto-testresults-query.')
28    logger.info(f"Shallow-cloning testresults in {workdir}")
29    subprocess.check_call(["git", "clone", testresults_default_url, workdir, "--depth", "1"])
30    return workdir
31
32def get_sha1(pokydir, revision):
33    try:
34        rev = subprocess.check_output(["git", "rev-list", "-n", "1", revision], cwd=pokydir).decode('utf-8').strip()
35        logger.info(f"SHA-1 revision for {revision} in {pokydir} is {rev}")
36        return rev
37    except subprocess.CalledProcessError:
38        logger.error(f"Can not find SHA-1 for {revision} in {pokydir}")
39        return None
40
41def fetch_testresults(workdir, sha1):
42    logger.info(f"Fetching test results for {sha1} in {workdir}")
43    rawtags = subprocess.check_output(["git", "ls-remote", "--refs", "--tags", "origin", f"*{sha1}*"], cwd=workdir).decode('utf-8').strip()
44    if not rawtags:
45        raise Exception(f"No reference found for commit {sha1} in {workdir}")
46    for rev in [rawtag.split()[1] for rawtag in rawtags.splitlines()]:
47        logger.info(f"Fetching matching revisions: {rev}")
48        subprocess.check_call(["git", "fetch", "--depth", "1", "origin", f"{rev}:{rev}"], cwd=workdir)
49
50def compute_regression_report(workdir, baserevision, targetrevision):
51    logger.info(f"Running resulttool regression between SHA1 {baserevision} and {targetrevision}")
52    report = subprocess.check_output([resulttool, "regression-git", "--commit", baserevision, "--commit2", targetrevision, workdir]).decode("utf-8")
53    return report
54
55def print_report_with_header(report, baseversion, baserevision, targetversion, targetrevision):
56    print("========================== Regression report ==============================")
57    print(f'{"=> Target:": <16}{targetversion: <16}({targetrevision})')
58    print(f'{"=> Base:": <16}{baseversion: <16}({baserevision})')
59    print("===========================================================================\n")
60    print(report, end='')
61
62def regression(args):
63    logger.info(f"Compute regression report between {args.base} and {args.target}")
64    if args.testresultsdir:
65        workdir = args.testresultsdir
66    else:
67        workdir = create_workdir()
68
69    try:
70        baserevision = get_sha1(poky_path, args.base)
71        targetrevision = get_sha1(poky_path, args.target)
72        if not baserevision or not targetrevision:
73            logger.error("One or more revision(s) missing. You might be targeting nonexistant tags/branches, or are in wrong repository (you must use Poky and not oe-core)")
74            if not args.testresultsdir:
75                subprocess.check_call(["rm", "-rf",  workdir])
76            sys.exit(1)
77        fetch_testresults(workdir, baserevision)
78        fetch_testresults(workdir, targetrevision)
79        report = compute_regression_report(workdir, baserevision, targetrevision)
80        print_report_with_header(report, args.base, baserevision, args.target, targetrevision)
81    finally:
82        if not args.testresultsdir:
83            subprocess.check_call(["rm", "-rf",  workdir])
84
85def main():
86    parser = argparse.ArgumentParser(description="Yocto Project test results helper")
87    subparsers = parser.add_subparsers(
88        help="Supported commands for test results helper",
89        required=True)
90    parser_regression_report = subparsers.add_parser(
91        "regression-report",
92        help="Generate regression report between two fixed revisions. Revisions can be branch name or tag")
93    parser_regression_report.add_argument(
94        'base',
95        help="Revision or tag against which to compare results (i.e: the older)")
96    parser_regression_report.add_argument(
97        'target',
98        help="Revision or tag to compare against the base (i.e: the newer)")
99    parser_regression_report.add_argument(
100        '-t',
101        '--testresultsdir',
102        help=f"An existing test results directory. {sys.argv[0]} will automatically clone it and use default branch if not provided")
103    parser_regression_report.set_defaults(func=regression)
104
105    args = parser.parse_args()
106    args.func(args)
107
108if __name__ == '__main__':
109    try:
110        ret =  main()
111    except Exception:
112        ret = 1
113        import traceback
114        traceback.print_exc()
115    sys.exit(ret)
116