1#!/usr/bin/python 2 3# This script generates the unit test coverage report for openbmc project. 4# 5# Usage: 6# get_unit_test_report.py target_dir [url_file] 7# 8# Positional arguments: 9# target_dir Target directory in pwd to place all cloned repos and logs. 10# url_file Text file containing url of repositories. Optional. 11# By using this argument, the user can get a report only for 12# specific repositories given in the file. 13# Refer ./scripts/repositories.txt 14# 15# Examples: 16# get_unit_test_report.py target_dir 17# get_unit_test_report.py target_dir repositories.txt 18# 19# Output format: 20# 21# ***********************************OUTPUT*********************************** 22# https://github.com/openbmc/phosphor-dbus-monitor.git NO 23# https://github.com/openbmc/phosphor-sel-logger.git;protocol=git NO 24# ***********************************OUTPUT*********************************** 25# 26# Other outputs and errors are redirected to output.log and debug.log in 27# target_dir. 28 29import argparse 30import logging 31import os 32import re 33import shutil 34import subprocess 35 36import requests 37 38# Repo list not expected to contain UT. Will be moved to a file in future. 39skip_list = [ 40 "openbmc-tools", 41 "inarp", 42 "openbmc", 43 "openbmc.github.io", 44 "phosphor-ecc", 45 "phosphor-pcie-presence", 46 "phosphor-u-boot-env-mgr", 47 "rrd-ipmi-blob", 48 "librrdplus", 49 "openpower-inventory-upload", 50 "openpower-logging", 51 "openpower-power-control", 52 "docs", 53 "openbmc-test-automation", 54 "openbmc-build-scripts", 55 "skeleton", 56 "linux", 57 # Not active, expected to be archived soon. 58 "ibm-pldm-oem", 59] 60 61 62# Create parser. 63text = """%(prog)s target_dir [url_file] 64 65Example usages: 66get_unit_test_report.py target_dir 67get_unit_test_report.py target_dir repositories.txt""" 68 69parser = argparse.ArgumentParser( 70 usage=text, description="Script generates the unit test coverage report" 71) 72parser.add_argument( 73 "target_dir", 74 type=str, 75 help="""Name of a non-existing directory in pwd to store all 76 cloned repos, logs and UT reports""", 77) 78parser.add_argument( 79 "url_file", 80 type=str, 81 nargs="?", 82 help="""Text file containing url of repositories. 83 By using this argument, the user can get a report only for 84 specific repositories given in the file. 85 Refer ./scripts/repositories.txt""", 86) 87args = parser.parse_args() 88 89input_urls = [] 90if args.url_file: 91 try: 92 # Get URLs from the file. 93 with open(args.url_file) as reader: 94 file_content = reader.read().splitlines() 95 input_urls = list(filter(lambda x: x, file_content)) 96 if not (input_urls): 97 print("Input file {} is empty. Quitting...".format(args.url_file)) 98 quit() 99 except IOError as e: 100 print( 101 "Issue in reading file '{}'. Reason: {}".format( 102 args.url_file, str(e) 103 ) 104 ) 105 quit() 106 107 108# Create target working directory. 109pwd = os.getcwd() 110working_dir = os.path.join(pwd, args.target_dir) 111try: 112 os.mkdir(working_dir) 113except OSError: 114 answer = input( 115 "Target directory " 116 + working_dir 117 + " already exists. " 118 + "Do you want to delete [Y/N]: " 119 ) 120 if answer == "Y": 121 try: 122 shutil.rmtree(working_dir) 123 os.mkdir(working_dir) 124 except OSError as e: 125 print(str(e)) 126 quit() 127 else: 128 print("Exiting....") 129 quit() 130 131# Create log directory. 132log_dir = os.path.join(working_dir, "logs") 133try: 134 os.mkdir(log_dir) 135except OSError as e: 136 print("Unable to create log directory: " + log_dir) 137 print(str(e)) 138 quit() 139 140 141# Log files 142debug_file = os.path.join(log_dir, "debug.log") 143output_file = os.path.join(log_dir, "output.log") 144logging.basicConfig( 145 format="%(levelname)s - %(message)s", 146 level=logging.DEBUG, 147 filename=debug_file, 148) 149logger = logging.getLogger(__name__) 150 151# Create handlers 152console_handler = logging.StreamHandler() 153file_handler = logging.FileHandler(output_file) 154console_handler.setLevel(logging.INFO) 155file_handler.setLevel(logging.INFO) 156 157# Create formatters and add it to handlers 158log_format = logging.Formatter("%(message)s") 159console_handler.setFormatter(log_format) 160file_handler.setFormatter(log_format) 161 162# Add handlers to the logger 163logger.addHandler(console_handler) 164logger.addHandler(file_handler) 165 166 167# Create report directory. 168report_dir = os.path.join(working_dir, "reports") 169try: 170 os.mkdir(report_dir) 171except OSError as e: 172 logger.error("Unable to create report directory: " + report_dir) 173 logger.error(str(e)) 174 quit() 175 176# Clone OpenBmc build scripts. 177try: 178 output = subprocess.check_output( 179 "git clone https://github.com/openbmc/openbmc-build-scripts.git", 180 shell=True, 181 cwd=working_dir, 182 stderr=subprocess.STDOUT, 183 ) 184 logger.debug(output) 185except subprocess.CalledProcessError as e: 186 logger.error(e.output) 187 logger.error(e.cmd) 188 logger.error("Unable to clone openbmc-build-scripts") 189 quit() 190 191repo_data = [] 192if input_urls: 193 api_url = "https://api.github.com/repos/openbmc/" 194 for url in input_urls: 195 try: 196 repo_name = url.strip().split("/")[-1].split(";")[0].split(".")[0] 197 except IndexError as e: 198 logger.error("ERROR: Unable to get sandbox name for url " + url) 199 logger.error("Reason: " + str(e)) 200 continue 201 202 try: 203 resp = requests.get(api_url + repo_name) 204 if resp.status_code != 200: 205 logger.info(api_url + repo_name + " ==> " + resp.reason) 206 continue 207 repo_data.extend([resp.json()]) 208 except ValueError: 209 logger.error("ERROR: Failed to get response for " + repo_name) 210 logger.error(resp) 211 continue 212 213else: 214 # Get number of pages. 215 resp = requests.head("https://api.github.com/users/openbmc/repos") 216 if resp.status_code != 200: 217 logger.error("Error! Unable to get repositories") 218 logger.error(resp.status_code) 219 logger.error(resp.reason) 220 quit() 221 num_of_pages = int(resp.links["last"]["url"].split("page=")[-1]) 222 logger.debug("No. of pages: " + str(num_of_pages)) 223 224 # Fetch data from all pages. 225 for page in range(1, num_of_pages + 1): 226 resp = requests.get( 227 "https://api.github.com/users/openbmc/repos?page=" + str(page) 228 ) 229 data = resp.json() 230 repo_data.extend(data) 231 232 233# Get URLs and their archive status from response. 234url_info = {} 235for repo in repo_data: 236 try: 237 url_info[repo["clone_url"]] = repo["archived"] 238 except KeyError: 239 logger.error("Failed to get archived status of {}".format(repo)) 240 url_info[repo["clone_url"]] = False 241 continue 242logger.debug(url_info) 243repo_count = len(url_info) 244logger.info("Number of repositories (Including archived): " + str(repo_count)) 245 246# Clone repository and run unit test. 247coverage_report = [] 248counter = 0 249tested_report_count = 0 250coverage_count = 0 251unit_test_count = 0 252no_report_count = 0 253error_count = 0 254skip_count = 0 255archive_count = 0 256url_list = sorted(url_info) 257for url in url_list: 258 ut_status = "NO" 259 skip = False 260 if url_info[url]: 261 ut_status = "ARCHIVED" 262 skip = True 263 else: 264 try: 265 # Eg: url = "https://github.com/openbmc/u-boot.git" 266 # sandbox_name = "u-boot" 267 sandbox_name = ( 268 url.strip().split("/")[-1].split(";")[0].split(".")[0] 269 ) 270 except IndexError as e: 271 logger.error("ERROR: Unable to get sandbox name for url " + url) 272 logger.error("Reason: " + str(e)) 273 continue 274 275 if sandbox_name in skip_list or re.match(r"meta-", sandbox_name): 276 logger.debug("SKIPPING: " + sandbox_name) 277 skip = True 278 ut_status = "SKIPPED" 279 else: 280 checkout_cmd = "rm -rf " + sandbox_name + ";git clone " + url 281 try: 282 subprocess.check_output( 283 checkout_cmd, 284 shell=True, 285 cwd=working_dir, 286 stderr=subprocess.STDOUT, 287 ) 288 except subprocess.CalledProcessError as e: 289 logger.debug(e.output) 290 logger.debug(e.cmd) 291 logger.debug("Failed to clone " + sandbox_name) 292 ut_status = "ERROR" 293 skip = True 294 if not (skip): 295 docker_cmd = ( 296 "WORKSPACE=$(pwd) UNIT_TEST_PKG=" 297 + sandbox_name 298 + " " 299 + "./openbmc-build-scripts/run-unit-test-docker.sh" 300 ) 301 try: 302 result = subprocess.check_output( 303 docker_cmd, 304 cwd=working_dir, 305 shell=True, 306 stderr=subprocess.STDOUT, 307 ) 308 logger.debug(result) 309 logger.debug("UT BUILD COMPLETED FOR: " + sandbox_name) 310 311 except subprocess.CalledProcessError as e: 312 logger.debug(e.output) 313 logger.debug(e.cmd) 314 logger.debug("UT BUILD EXITED FOR: " + sandbox_name) 315 ut_status = "ERROR" 316 317 folder_name = os.path.join(working_dir, sandbox_name) 318 repo_report_dir = os.path.join(report_dir, sandbox_name) 319 320 report_names = ("coveragereport", "test-suite.log", "LastTest.log") 321 find_cmd = "".join( 322 "find " + folder_name + " -name " + report + ";" 323 for report in report_names 324 ) 325 try: 326 result = subprocess.check_output(find_cmd, shell=True) 327 result = result.decode("utf-8") 328 except subprocess.CalledProcessError as e: 329 logger.debug(e.output) 330 logger.debug(e.cmd) 331 logger.debug("CMD TO FIND REPORT FAILED FOR: " + sandbox_name) 332 ut_status = "ERROR" 333 334 if ut_status != "ERROR": 335 if result: 336 if result.__contains__("coveragereport"): 337 ut_status = "YES, COVERAGE" 338 coverage_count += 1 339 elif "test-suite.log" in result: 340 ut_status = "YES, UNIT TEST" 341 unit_test_count += 1 342 elif "LastTest.log" in result: 343 file_names = result.splitlines() 344 for file in file_names: 345 pattern_count_cmd = ( 346 "sed -n '/Start testing/,/End testing/p;' " 347 + file 348 + "|wc -l" 349 ) 350 try: 351 num_of_lines = subprocess.check_output( 352 pattern_count_cmd, shell=True 353 ) 354 except subprocess.CalledProcessError as e: 355 logger.debug(e.output) 356 logger.debug(e.cmd) 357 logger.debug( 358 "CONTENT CHECK FAILED FOR: " + sandbox_name 359 ) 360 ut_status = "ERROR" 361 362 if int(num_of_lines.strip()) > 5: 363 ut_status = "YES, UNIT TEST" 364 unit_test_count += 1 365 366 if "YES" in ut_status: 367 tested_report_count += 1 368 result = result.splitlines() 369 for file_path in result: 370 destination = os.path.dirname( 371 os.path.join( 372 report_dir, os.path.relpath(file_path, working_dir) 373 ) 374 ) 375 copy_cmd = ( 376 "mkdir -p " 377 + destination 378 + ";cp -rf " 379 + file_path.strip() 380 + " " 381 + destination 382 ) 383 try: 384 subprocess.check_output(copy_cmd, shell=True) 385 except subprocess.CalledProcessError as e: 386 logger.debug(e.output) 387 logger.debug(e.cmd) 388 logger.info("FAILED TO COPY REPORTS FOR: " + sandbox_name) 389 390 if ut_status == "ERROR": 391 error_count += 1 392 elif ut_status == "NO": 393 no_report_count += 1 394 elif ut_status == "SKIPPED": 395 skip_count += 1 396 elif ut_status == "ARCHIVED": 397 archive_count += 1 398 399 coverage_report.append("{:<65}{:<10}".format(url.strip(), ut_status)) 400 counter += 1 401 logger.info(str(counter) + " in " + str(repo_count) + " completed") 402 403logger.info("*" * 30 + "UNIT TEST COVERAGE REPORT" + "*" * 30) 404for res in coverage_report: 405 logger.info(res) 406logger.info("*" * 30 + "UNIT TEST COVERAGE REPORT" + "*" * 30) 407 408logger.info("REPORTS: " + report_dir) 409logger.info("LOGS: " + log_dir) 410logger.info("*" * 85) 411logger.info("SUMMARY: ") 412logger.info("TOTAL REPOSITORIES : " + str(repo_count)) 413logger.info("TESTED REPOSITORIES : " + str(tested_report_count)) 414logger.info("ERROR : " + str(error_count)) 415logger.info("COVERAGE REPORT : " + str(coverage_count)) 416logger.info("UNIT TEST REPORT : " + str(unit_test_count)) 417logger.info("NO REPORT : " + str(no_report_count)) 418logger.info("SKIPPED : " + str(skip_count)) 419logger.info("ARCHIVED : " + str(archive_count)) 420logger.info("*" * 85) 421