1#!/usr/bin/env python3
2
3# Yocto Project test results management tool
4# This script is an thin layer over resulttool to manage tes results and regression reports.
5# Its main feature is to translate tags or branch names to revisions SHA1, and then to run resulttool
6# with those computed revisions
7#
8# Copyright (C) 2023 OpenEmbedded Contributors
9#
10# SPDX-License-Identifier: MIT
11#
12
13import sys
14import os
15import argparse
16import subprocess
17import tempfile
18import lib.scriptutils as scriptutils
19
20script_path = os.path.dirname(os.path.realpath(__file__))
21poky_path = os.path.abspath(os.path.join(script_path, ".."))
22resulttool = os.path.abspath(os.path.join(script_path, "resulttool"))
23logger = scriptutils.logger_create(sys.argv[0])
24testresults_default_url="git://git.yoctoproject.org/yocto-testresults"
25
26def create_workdir():
27    workdir = tempfile.mkdtemp(prefix='yocto-testresults-query.')
28    logger.info(f"Shallow-cloning testresults in {workdir}")
29    subprocess.check_call(["git", "clone", testresults_default_url, workdir, "--depth", "1"])
30    return workdir
31
32def get_sha1(pokydir, revision):
33    try:
34        rev = subprocess.check_output(["git", "rev-list", "-n", "1", revision], cwd=pokydir).decode('utf-8').strip()
35        logger.info(f"SHA-1 revision for {revision} in {pokydir} is {rev}")
36        return rev
37    except subprocess.CalledProcessError:
38        logger.error(f"Can not find SHA-1 for {revision} in {pokydir}")
39        return None
40
41def get_branch(tag):
42    # The tags in test results repository, as returned by git rev-list, have the following form:
43    # refs/tags/<branch>/<count>-g<sha1>/<num>
44    return '/'.join(tag.split("/")[2:-2])
45
46def fetch_testresults(workdir, sha1):
47    logger.info(f"Fetching test results for {sha1} in {workdir}")
48    rawtags = subprocess.check_output(["git", "ls-remote", "--refs", "--tags", "origin", f"*{sha1}*"], cwd=workdir).decode('utf-8').strip()
49    if not rawtags:
50        raise Exception(f"No reference found for commit {sha1} in {workdir}")
51    branch = ""
52    for rev in [rawtag.split()[1] for rawtag in rawtags.splitlines()]:
53        if not branch:
54            branch = get_branch(rev)
55        logger.info(f"Fetching matching revision: {rev}")
56        subprocess.check_call(["git", "fetch", "--depth", "1", "origin", f"{rev}:{rev}"], cwd=workdir)
57    return branch
58
59def compute_regression_report(workdir, basebranch, baserevision, targetbranch, targetrevision):
60    logger.info(f"Running resulttool regression between SHA1 {baserevision} and {targetrevision}")
61    report = subprocess.check_output([resulttool, "regression-git", "--branch", basebranch, "--commit", baserevision, "--branch2", targetbranch, "--commit2", targetrevision, workdir]).decode("utf-8")
62    return report
63
64def print_report_with_header(report, baseversion, baserevision, targetversion, targetrevision):
65    print("========================== Regression report ==============================")
66    print(f'{"=> Target:": <16}{targetversion: <16}({targetrevision})')
67    print(f'{"=> Base:": <16}{baseversion: <16}({baserevision})')
68    print("===========================================================================\n")
69    print(report, end='')
70
71def regression(args):
72    logger.info(f"Compute regression report between {args.base} and {args.target}")
73    if args.testresultsdir:
74        workdir = args.testresultsdir
75    else:
76        workdir = create_workdir()
77
78    try:
79        baserevision = get_sha1(poky_path, args.base)
80        targetrevision = get_sha1(poky_path, args.target)
81        if not baserevision or not targetrevision:
82            logger.error("One or more revision(s) missing. You might be targeting nonexistant tags/branches, or are in wrong repository (you must use Poky and not oe-core)")
83            if not args.testresultsdir:
84                subprocess.check_call(["rm", "-rf",  workdir])
85            sys.exit(1)
86        basebranch = fetch_testresults(workdir, baserevision)
87        targetbranch = fetch_testresults(workdir, targetrevision)
88        report = compute_regression_report(workdir, basebranch, baserevision, targetbranch, targetrevision)
89        print_report_with_header(report, args.base, baserevision, args.target, targetrevision)
90    finally:
91        if not args.testresultsdir:
92            subprocess.check_call(["rm", "-rf",  workdir])
93
94def main():
95    parser = argparse.ArgumentParser(description="Yocto Project test results helper")
96    subparsers = parser.add_subparsers(
97        help="Supported commands for test results helper",
98        required=True)
99    parser_regression_report = subparsers.add_parser(
100        "regression-report",
101        help="Generate regression report between two fixed revisions. Revisions can be branch name or tag")
102    parser_regression_report.add_argument(
103        'base',
104        help="Revision or tag against which to compare results (i.e: the older)")
105    parser_regression_report.add_argument(
106        'target',
107        help="Revision or tag to compare against the base (i.e: the newer)")
108    parser_regression_report.add_argument(
109        '-t',
110        '--testresultsdir',
111        help=f"An existing test results directory. {sys.argv[0]} will automatically clone it and use default branch if not provided")
112    parser_regression_report.set_defaults(func=regression)
113
114    args = parser.parse_args()
115    args.func(args)
116
117if __name__ == '__main__':
118    try:
119        ret =  main()
120    except Exception:
121        ret = 1
122        import traceback
123        traceback.print_exc()
124    sys.exit(ret)
125