1#!/usr/bin/env python3 2 3# Yocto Project test results management tool 4# This script is an thin layer over resulttool to manage tes results and regression reports. 5# Its main feature is to translate tags or branch names to revisions SHA1, and then to run resulttool 6# with those computed revisions 7# 8# Copyright (C) 2023 OpenEmbedded Contributors 9# 10# SPDX-License-Identifier: MIT 11# 12 13import sys 14import os 15import argparse 16import subprocess 17import tempfile 18import lib.scriptutils as scriptutils 19 20script_path = os.path.dirname(os.path.realpath(__file__)) 21poky_path = os.path.abspath(os.path.join(script_path, "..")) 22resulttool = os.path.abspath(os.path.join(script_path, "resulttool")) 23logger = scriptutils.logger_create(sys.argv[0]) 24testresults_default_url="git://git.yoctoproject.org/yocto-testresults" 25 26def create_workdir(): 27 workdir = tempfile.mkdtemp(prefix='yocto-testresults-query.') 28 logger.info(f"Shallow-cloning testresults in {workdir}") 29 subprocess.check_call(["git", "clone", testresults_default_url, workdir, "--depth", "1"]) 30 return workdir 31 32def get_sha1(pokydir, revision): 33 try: 34 rev = subprocess.check_output(["git", "rev-list", "-n", "1", revision], cwd=pokydir).decode('utf-8').strip() 35 logger.info(f"SHA-1 revision for {revision} in {pokydir} is {rev}") 36 return rev 37 except subprocess.CalledProcessError: 38 logger.error(f"Can not find SHA-1 for {revision} in {pokydir}") 39 return None 40 41def fetch_testresults(workdir, sha1): 42 logger.info(f"Fetching test results for {sha1} in {workdir}") 43 rawtags = subprocess.check_output(["git", "ls-remote", "--refs", "--tags", "origin", f"*{sha1}*"], cwd=workdir).decode('utf-8').strip() 44 if not rawtags: 45 raise Exception(f"No reference found for commit {sha1} in {workdir}") 46 for rev in [rawtag.split()[1] for rawtag in rawtags.splitlines()]: 47 logger.info(f"Fetching matching revisions: {rev}") 48 subprocess.check_call(["git", "fetch", "--depth", "1", "origin", f"{rev}:{rev}"], cwd=workdir) 49 50def compute_regression_report(workdir, baserevision, targetrevision): 51 logger.info(f"Running resulttool regression between SHA1 {baserevision} and {targetrevision}") 52 report = subprocess.check_output([resulttool, "regression-git", "--commit", baserevision, "--commit2", targetrevision, workdir]).decode("utf-8") 53 return report 54 55def print_report_with_header(report, baseversion, baserevision, targetversion, targetrevision): 56 print("========================== Regression report ==============================") 57 print(f'{"=> Target:": <16}{targetversion: <16}({targetrevision})') 58 print(f'{"=> Base:": <16}{baseversion: <16}({baserevision})') 59 print("===========================================================================\n") 60 print(report, end='') 61 62def regression(args): 63 logger.info(f"Compute regression report between {args.base} and {args.target}") 64 if args.testresultsdir: 65 workdir = args.testresultsdir 66 else: 67 workdir = create_workdir() 68 69 try: 70 baserevision = get_sha1(poky_path, args.base) 71 targetrevision = get_sha1(poky_path, args.target) 72 if not baserevision or not targetrevision: 73 logger.error("One or more revision(s) missing. You might be targeting nonexistant tags/branches, or are in wrong repository (you must use Poky and not oe-core)") 74 if not args.testresultsdir: 75 subprocess.check_call(["rm", "-rf", workdir]) 76 sys.exit(1) 77 fetch_testresults(workdir, baserevision) 78 fetch_testresults(workdir, targetrevision) 79 report = compute_regression_report(workdir, baserevision, targetrevision) 80 print_report_with_header(report, args.base, baserevision, args.target, targetrevision) 81 finally: 82 if not args.testresultsdir: 83 subprocess.check_call(["rm", "-rf", workdir]) 84 85def main(): 86 parser = argparse.ArgumentParser(description="Yocto Project test results helper") 87 subparsers = parser.add_subparsers( 88 help="Supported commands for test results helper", 89 required=True) 90 parser_regression_report = subparsers.add_parser( 91 "regression-report", 92 help="Generate regression report between two fixed revisions. Revisions can be branch name or tag") 93 parser_regression_report.add_argument( 94 'base', 95 help="Revision or tag against which to compare results (i.e: the older)") 96 parser_regression_report.add_argument( 97 'target', 98 help="Revision or tag to compare against the base (i.e: the newer)") 99 parser_regression_report.add_argument( 100 '-t', 101 '--testresultsdir', 102 help=f"An existing test results directory. {sys.argv[0]} will automatically clone it and use default branch if not provided") 103 parser_regression_report.set_defaults(func=regression) 104 105 args = parser.parse_args() 106 args.func(args) 107 108if __name__ == '__main__': 109 try: 110 ret = main() 111 except Exception: 112 ret = 1 113 import traceback 114 traceback.print_exc() 115 sys.exit(ret) 116