1#!/usr/bin/env python3 2 3# Yocto Project test results management tool 4# This script is an thin layer over resulttool to manage tes results and regression reports. 5# Its main feature is to translate tags or branch names to revisions SHA1, and then to run resulttool 6# with those computed revisions 7# 8# Copyright (C) 2023 OpenEmbedded Contributors 9# 10# SPDX-License-Identifier: MIT 11# 12 13import sys 14import os 15import argparse 16import subprocess 17import tempfile 18import lib.scriptutils as scriptutils 19 20script_path = os.path.dirname(os.path.realpath(__file__)) 21poky_path = os.path.abspath(os.path.join(script_path, "..")) 22resulttool = os.path.abspath(os.path.join(script_path, "resulttool")) 23logger = scriptutils.logger_create(sys.argv[0]) 24testresults_default_url="git://git.yoctoproject.org/yocto-testresults" 25 26def create_workdir(): 27 workdir = tempfile.mkdtemp(prefix='yocto-testresults-query.') 28 logger.info(f"Shallow-cloning testresults in {workdir}") 29 subprocess.check_call(["git", "clone", testresults_default_url, workdir, "--depth", "1"]) 30 return workdir 31 32def get_sha1(pokydir, revision): 33 try: 34 rev = subprocess.check_output(["git", "rev-list", "-n", "1", revision], cwd=pokydir).decode('utf-8').strip() 35 logger.info(f"SHA-1 revision for {revision} in {pokydir} is {rev}") 36 return rev 37 except subprocess.CalledProcessError: 38 logger.error(f"Can not find SHA-1 for {revision} in {pokydir}") 39 return None 40 41def get_branch(tag): 42 # The tags in test results repository, as returned by git rev-list, have the following form: 43 # refs/tags/<branch>/<count>-g<sha1>/<num> 44 return '/'.join(tag.split("/")[2:-2]) 45 46def fetch_testresults(workdir, sha1): 47 logger.info(f"Fetching test results for {sha1} in {workdir}") 48 rawtags = subprocess.check_output(["git", "ls-remote", "--refs", "--tags", "origin", f"*{sha1}*"], cwd=workdir).decode('utf-8').strip() 49 if not rawtags: 50 raise Exception(f"No reference found for commit {sha1} in {workdir}") 51 branch = "" 52 for rev in [rawtag.split()[1] for rawtag in rawtags.splitlines()]: 53 if not branch: 54 branch = get_branch(rev) 55 logger.info(f"Fetching matching revision: {rev}") 56 subprocess.check_call(["git", "fetch", "--depth", "1", "origin", f"{rev}:{rev}"], cwd=workdir) 57 return branch 58 59def compute_regression_report(workdir, basebranch, baserevision, targetbranch, targetrevision): 60 logger.info(f"Running resulttool regression between SHA1 {baserevision} and {targetrevision}") 61 report = subprocess.check_output([resulttool, "regression-git", "--branch", basebranch, "--commit", baserevision, "--branch2", targetbranch, "--commit2", targetrevision, workdir]).decode("utf-8") 62 return report 63 64def print_report_with_header(report, baseversion, baserevision, targetversion, targetrevision): 65 print("========================== Regression report ==============================") 66 print(f'{"=> Target:": <16}{targetversion: <16}({targetrevision})') 67 print(f'{"=> Base:": <16}{baseversion: <16}({baserevision})') 68 print("===========================================================================\n") 69 print(report, end='') 70 71def regression(args): 72 logger.info(f"Compute regression report between {args.base} and {args.target}") 73 if args.testresultsdir: 74 workdir = args.testresultsdir 75 else: 76 workdir = create_workdir() 77 78 try: 79 baserevision = get_sha1(poky_path, args.base) 80 targetrevision = get_sha1(poky_path, args.target) 81 if not baserevision or not targetrevision: 82 logger.error("One or more revision(s) missing. You might be targeting nonexistant tags/branches, or are in wrong repository (you must use Poky and not oe-core)") 83 if not args.testresultsdir: 84 subprocess.check_call(["rm", "-rf", workdir]) 85 sys.exit(1) 86 basebranch = fetch_testresults(workdir, baserevision) 87 targetbranch = fetch_testresults(workdir, targetrevision) 88 report = compute_regression_report(workdir, basebranch, baserevision, targetbranch, targetrevision) 89 print_report_with_header(report, args.base, baserevision, args.target, targetrevision) 90 finally: 91 if not args.testresultsdir: 92 subprocess.check_call(["rm", "-rf", workdir]) 93 94def main(): 95 parser = argparse.ArgumentParser(description="Yocto Project test results helper") 96 subparsers = parser.add_subparsers( 97 help="Supported commands for test results helper", 98 required=True) 99 parser_regression_report = subparsers.add_parser( 100 "regression-report", 101 help="Generate regression report between two fixed revisions. Revisions can be branch name or tag") 102 parser_regression_report.add_argument( 103 'base', 104 help="Revision or tag against which to compare results (i.e: the older)") 105 parser_regression_report.add_argument( 106 'target', 107 help="Revision or tag to compare against the base (i.e: the newer)") 108 parser_regression_report.add_argument( 109 '-t', 110 '--testresultsdir', 111 help=f"An existing test results directory. {sys.argv[0]} will automatically clone it and use default branch if not provided") 112 parser_regression_report.set_defaults(func=regression) 113 114 args = parser.parse_args() 115 args.func(args) 116 117if __name__ == '__main__': 118 try: 119 ret = main() 120 except Exception: 121 ret = 1 122 import traceback 123 traceback.print_exc() 124 sys.exit(ret) 125