1*6aa7eec5SAndrew Geissler#!/usr/bin/env python3 2*6aa7eec5SAndrew Geissler 3*6aa7eec5SAndrew Geissler# Yocto Project test results management tool 4*6aa7eec5SAndrew Geissler# This script is an thin layer over resulttool to manage tes results and regression reports. 5*6aa7eec5SAndrew Geissler# Its main feature is to translate tags or branch names to revisions SHA1, and then to run resulttool 6*6aa7eec5SAndrew Geissler# with those computed revisions 7*6aa7eec5SAndrew Geissler# 8*6aa7eec5SAndrew Geissler# Copyright (C) 2023 OpenEmbedded Contributors 9*6aa7eec5SAndrew Geissler# 10*6aa7eec5SAndrew Geissler# SPDX-License-Identifier: MIT 11*6aa7eec5SAndrew Geissler# 12*6aa7eec5SAndrew Geissler 13*6aa7eec5SAndrew Geisslerimport sys 14*6aa7eec5SAndrew Geisslerimport os 15*6aa7eec5SAndrew Geisslerimport argparse 16*6aa7eec5SAndrew Geisslerimport subprocess 17*6aa7eec5SAndrew Geisslerimport tempfile 18*6aa7eec5SAndrew Geisslerimport lib.scriptutils as scriptutils 19*6aa7eec5SAndrew Geissler 20*6aa7eec5SAndrew Geisslerscript_path = os.path.dirname(os.path.realpath(__file__)) 21*6aa7eec5SAndrew Geisslerpoky_path = os.path.abspath(os.path.join(script_path, "..")) 22*6aa7eec5SAndrew Geisslerresulttool = os.path.abspath(os.path.join(script_path, "resulttool")) 23*6aa7eec5SAndrew Geisslerlogger = scriptutils.logger_create(sys.argv[0]) 24*6aa7eec5SAndrew Geisslertestresults_default_url="git://git.yoctoproject.org/yocto-testresults" 25*6aa7eec5SAndrew Geissler 26*6aa7eec5SAndrew Geisslerdef create_workdir(): 27*6aa7eec5SAndrew Geissler workdir = tempfile.mkdtemp(prefix='yocto-testresults-query.') 28*6aa7eec5SAndrew Geissler logger.info(f"Shallow-cloning testresults in {workdir}") 29*6aa7eec5SAndrew Geissler subprocess.check_call(["git", "clone", testresults_default_url, workdir, "--depth", "1"]) 30*6aa7eec5SAndrew Geissler return workdir 31*6aa7eec5SAndrew Geissler 32*6aa7eec5SAndrew Geisslerdef get_sha1(pokydir, revision): 33*6aa7eec5SAndrew Geissler try: 34*6aa7eec5SAndrew Geissler rev = subprocess.check_output(["git", "rev-list", "-n", "1", revision], cwd=pokydir).decode('utf-8').strip() 35*6aa7eec5SAndrew Geissler logger.info(f"SHA-1 revision for {revision} in {pokydir} is {rev}") 36*6aa7eec5SAndrew Geissler return rev 37*6aa7eec5SAndrew Geissler except subprocess.CalledProcessError: 38*6aa7eec5SAndrew Geissler logger.error(f"Can not find SHA-1 for {revision} in {pokydir}") 39*6aa7eec5SAndrew Geissler return None 40*6aa7eec5SAndrew Geissler 41*6aa7eec5SAndrew Geisslerdef fetch_testresults(workdir, sha1): 42*6aa7eec5SAndrew Geissler logger.info(f"Fetching test results for {sha1} in {workdir}") 43*6aa7eec5SAndrew Geissler rawtags = subprocess.check_output(["git", "ls-remote", "--refs", "--tags", "origin", f"*{sha1}*"], cwd=workdir).decode('utf-8').strip() 44*6aa7eec5SAndrew Geissler if not rawtags: 45*6aa7eec5SAndrew Geissler raise Exception(f"No reference found for commit {sha1} in {workdir}") 46*6aa7eec5SAndrew Geissler for rev in [rawtag.split()[1] for rawtag in rawtags.splitlines()]: 47*6aa7eec5SAndrew Geissler logger.info(f"Fetching matching revisions: {rev}") 48*6aa7eec5SAndrew Geissler subprocess.check_call(["git", "fetch", "--depth", "1", "origin", f"{rev}:{rev}"], cwd=workdir) 49*6aa7eec5SAndrew Geissler 50*6aa7eec5SAndrew Geisslerdef compute_regression_report(workdir, baserevision, targetrevision): 51*6aa7eec5SAndrew Geissler logger.info(f"Running resulttool regression between SHA1 {baserevision} and {targetrevision}") 52*6aa7eec5SAndrew Geissler report = subprocess.check_output([resulttool, "regression-git", "--commit", baserevision, "--commit2", targetrevision, workdir]).decode("utf-8") 53*6aa7eec5SAndrew Geissler return report 54*6aa7eec5SAndrew Geissler 55*6aa7eec5SAndrew Geisslerdef print_report_with_header(report, baseversion, baserevision, targetversion, targetrevision): 56*6aa7eec5SAndrew Geissler print("========================== Regression report ==============================") 57*6aa7eec5SAndrew Geissler print(f'{"=> Target:": <16}{targetversion: <16}({targetrevision})') 58*6aa7eec5SAndrew Geissler print(f'{"=> Base:": <16}{baseversion: <16}({baserevision})') 59*6aa7eec5SAndrew Geissler print("===========================================================================\n") 60*6aa7eec5SAndrew Geissler print(report, end='') 61*6aa7eec5SAndrew Geissler 62*6aa7eec5SAndrew Geisslerdef regression(args): 63*6aa7eec5SAndrew Geissler logger.info(f"Compute regression report between {args.base} and {args.target}") 64*6aa7eec5SAndrew Geissler if args.testresultsdir: 65*6aa7eec5SAndrew Geissler workdir = args.testresultsdir 66*6aa7eec5SAndrew Geissler else: 67*6aa7eec5SAndrew Geissler workdir = create_workdir() 68*6aa7eec5SAndrew Geissler 69*6aa7eec5SAndrew Geissler try: 70*6aa7eec5SAndrew Geissler baserevision = get_sha1(poky_path, args.base) 71*6aa7eec5SAndrew Geissler targetrevision = get_sha1(poky_path, args.target) 72*6aa7eec5SAndrew Geissler if not baserevision or not targetrevision: 73*6aa7eec5SAndrew Geissler logger.error("One or more revision(s) missing. You might be targeting nonexistant tags/branches, or are in wrong repository (you must use Poky and not oe-core)") 74*6aa7eec5SAndrew Geissler if not args.testresultsdir: 75*6aa7eec5SAndrew Geissler subprocess.check_call(["rm", "-rf", workdir]) 76*6aa7eec5SAndrew Geissler sys.exit(1) 77*6aa7eec5SAndrew Geissler fetch_testresults(workdir, baserevision) 78*6aa7eec5SAndrew Geissler fetch_testresults(workdir, targetrevision) 79*6aa7eec5SAndrew Geissler report = compute_regression_report(workdir, baserevision, targetrevision) 80*6aa7eec5SAndrew Geissler print_report_with_header(report, args.base, baserevision, args.target, targetrevision) 81*6aa7eec5SAndrew Geissler finally: 82*6aa7eec5SAndrew Geissler if not args.testresultsdir: 83*6aa7eec5SAndrew Geissler subprocess.check_call(["rm", "-rf", workdir]) 84*6aa7eec5SAndrew Geissler 85*6aa7eec5SAndrew Geisslerdef main(): 86*6aa7eec5SAndrew Geissler parser = argparse.ArgumentParser(description="Yocto Project test results helper") 87*6aa7eec5SAndrew Geissler subparsers = parser.add_subparsers( 88*6aa7eec5SAndrew Geissler help="Supported commands for test results helper", 89*6aa7eec5SAndrew Geissler required=True) 90*6aa7eec5SAndrew Geissler parser_regression_report = subparsers.add_parser( 91*6aa7eec5SAndrew Geissler "regression-report", 92*6aa7eec5SAndrew Geissler help="Generate regression report between two fixed revisions. Revisions can be branch name or tag") 93*6aa7eec5SAndrew Geissler parser_regression_report.add_argument( 94*6aa7eec5SAndrew Geissler 'base', 95*6aa7eec5SAndrew Geissler help="Revision or tag against which to compare results (i.e: the older)") 96*6aa7eec5SAndrew Geissler parser_regression_report.add_argument( 97*6aa7eec5SAndrew Geissler 'target', 98*6aa7eec5SAndrew Geissler help="Revision or tag to compare against the base (i.e: the newer)") 99*6aa7eec5SAndrew Geissler parser_regression_report.add_argument( 100*6aa7eec5SAndrew Geissler '-t', 101*6aa7eec5SAndrew Geissler '--testresultsdir', 102*6aa7eec5SAndrew Geissler help=f"An existing test results directory. {sys.argv[0]} will automatically clone it and use default branch if not provided") 103*6aa7eec5SAndrew Geissler parser_regression_report.set_defaults(func=regression) 104*6aa7eec5SAndrew Geissler 105*6aa7eec5SAndrew Geissler args = parser.parse_args() 106*6aa7eec5SAndrew Geissler args.func(args) 107*6aa7eec5SAndrew Geissler 108*6aa7eec5SAndrew Geisslerif __name__ == '__main__': 109*6aa7eec5SAndrew Geissler try: 110*6aa7eec5SAndrew Geissler ret = main() 111*6aa7eec5SAndrew Geissler except Exception: 112*6aa7eec5SAndrew Geissler ret = 1 113*6aa7eec5SAndrew Geissler import traceback 114*6aa7eec5SAndrew Geissler traceback.print_exc() 115*6aa7eec5SAndrew Geissler sys.exit(ret) 116