xref: /openbmc/openbmc-build-scripts/scripts/get_unit_test_report.py (revision 185d5b59480b7fa4b94bf110337a1c0c61e1cdab) 
1#!/usr/bin/python
2
3# This script generates the unit test coverage report for openbmc project.
4#
5# Usage:
6# get_unit_test_report.py target_dir
7#
8# Description of arguments:
9# target_dir  Target directory in pwd to place all cloned repos and logs.
10#
11# Eg: get_unit_test_report.py target_dir
12#
13# Output format:
14#
15# ***********************************OUTPUT***********************************
16# https://github.com/openbmc/phosphor-dbus-monitor.git               NO
17# https://github.com/openbmc/phosphor-sel-logger.git;protocol=git    NO
18# ***********************************OUTPUT***********************************
19#
20# Other outputs and errors are redirected to output.log and debug.log in target_dir.
21
22import argparse
23import logging
24import os
25import re
26import requests
27import shutil
28import sys
29import subprocess
30
31
32# Create parser.
33parser = argparse.ArgumentParser(usage='%(prog)s file target_dir',
34                                 description="Script generates the unit test coverage report")
35parser.add_argument("target_dir", type=str,
36                    help='''Name of a non-existing directory in pwd to store all
37                            cloned repos, logs and UT reports''')
38args = parser.parse_args()
39
40
41# Create target working directory.
42pwd = os.getcwd()
43working_dir = os.path.join(pwd, args.target_dir)
44try:
45    os.mkdir(working_dir)
46except OSError as e:
47    answer = raw_input("Target directory " + working_dir + " already exists. "
48                       + "Do you want to delete [Y/N]: ")
49    if answer == "Y":
50        try:
51            shutil.rmtree(working_dir)
52            os.mkdir(working_dir)
53        except OSError as e:
54            print(str(e))
55            quit()
56    else:
57        print("Exiting....")
58        quit()
59
60# Create log directory.
61log_dir = os.path.join(working_dir, "logs")
62try:
63    os.mkdir(log_dir)
64except OSError as e:
65    print("Unable to create log directory: " + log_dir)
66    print(str(e))
67    quit()
68
69
70# Repo list not expected to contain UT. Will be moved to a file in future.
71skip_list = ["openbmc-tools", "inarp", "openbmc", "openbmc.github.io",
72             "phosphor-ecc", "phosphor-pcie-presence", "phosphor-u-boot-env-mgr",
73             "rrd-ipmi-blob", "librrdplus", "openpower-inventory-upload",
74             "openpower-logging", "openpower-power-control", "docs",
75             "openbmc-test-automation", "openbmc-build-scripts", "skeleton",
76             "linux",
77             # Not active, expected to be archived soon.
78             "ibm-pldm-oem"]
79
80# Log files
81debug_file = os.path.join(log_dir, "debug.log")
82output_file = os.path.join(log_dir, "output.log")
83logging.basicConfig(format='%(levelname)s - %(message)s', level=logging.DEBUG,
84                    filename=debug_file)
85logger = logging.getLogger(__name__)
86
87# Create handlers
88console_handler = logging.StreamHandler()
89file_handler = logging.FileHandler(output_file)
90console_handler.setLevel(logging.INFO)
91file_handler.setLevel(logging.INFO)
92
93# Create formatters and add it to handlers
94log_format = logging.Formatter('%(message)s')
95console_handler.setFormatter(log_format)
96file_handler.setFormatter(log_format)
97
98# Add handlers to the logger
99logger.addHandler(console_handler)
100logger.addHandler(file_handler)
101
102
103# Create report directory.
104report_dir = os.path.join(working_dir, "reports")
105try:
106    os.mkdir(report_dir)
107except OSError as e:
108    logger.error("Unable to create report directory: " + report_dir)
109    logger.error(str(e))
110    quit()
111
112# Clone OpenBmc build scripts.
113try:
114    output = subprocess.check_output("git clone https://github.com/openbmc/openbmc-build-scripts.git",
115                                     shell=True, cwd=working_dir, stderr=subprocess.STDOUT)
116    logger.debug(output)
117except subprocess.CalledProcessError as e:
118    logger.debug(e.output)
119    logger.debug(e.cmd)
120    logger.debug("Unable to clone openbmc-build-scripts")
121    quit()
122
123# Get number of pages.
124resp = requests.head('https://api.github.com/users/openbmc/repos')
125if resp.status_code != 200:
126    logger.error("Error! Unable to get repositories")
127    logger.debug(resp.status_code)
128    logger.debug(resp.text)
129    quit()
130num_of_pages = int(resp.links['last']['url'].split('page=')[-1])
131logger.debug("No. of pages: " + str(num_of_pages))
132
133# Fetch data from all pages.
134repo_data = []
135for page in range(1, num_of_pages+1):
136    resp = requests.get('https://api.github.com/users/openbmc/repos?page=' + str(page))
137    data = resp.json()
138    repo_data.extend(data)
139
140# Get URLs and their archive status from response.
141url_info = {}
142for repo in repo_data:
143    url_info[repo["clone_url"]] = repo["archived"]
144logger.debug(url_info)
145repo_count = len(url_info)
146logger.info("Number of repositories (Including archived): " + str(repo_count))
147
148# Clone repository and run unit test.
149coverage_report = []
150counter = 0
151tested_report_count = 0
152coverage_count = 0
153unit_test_count = 0
154no_report_count = 0
155error_count = 0
156skip_count = 0
157archive_count = 0
158url_list = sorted(url_info)
159for url in url_list:
160    ut_status = "NO"
161    skip = False
162    if url_info[url]:
163        ut_status = "ARCHIVED"
164        skip = True
165    else:
166        try:
167            # Eg: url = "https://github.com/openbmc/u-boot.git"
168            #     sandbox_name = "u-boot"
169            sandbox_name = url.strip().split('/')[-1].split(";")[0].split(".")[0]
170        except IndexError as e:
171            logger.debug("ERROR: Unable to get sandbox name for url " + url)
172            logger.debug("Reason: " + str(e))
173
174        if (sandbox_name in skip_list or
175            re.match(r'meta-', sandbox_name)):
176            logger.debug("SKIPPING: " + sandbox_name)
177            skip = True
178            ut_status = "SKIPPED"
179        else:
180            checkout_cmd = "rm -rf " + sandbox_name + ";git clone " + url
181            try:
182                subprocess.check_output(checkout_cmd, shell=True, cwd=working_dir,
183                                                 stderr=subprocess.STDOUT)
184            except subprocess.CalledProcessError as e:
185                logger.debug(e.output)
186                logger.debug(e.cmd)
187                logger.debug("Failed to clone " + sandbox_name)
188                ut_status = "ERROR"
189                skip = True
190    if not(skip):
191        docker_cmd = "WORKSPACE=$(pwd) UNIT_TEST_PKG=" + sandbox_name + " " + \
192                     "./openbmc-build-scripts/run-unit-test-docker.sh"
193        try:
194            result = subprocess.check_output(docker_cmd, cwd=working_dir, shell=True,
195                                             stderr=subprocess.STDOUT)
196            logger.debug(result)
197            logger.debug("UT BUILD COMPLETED FOR: " + sandbox_name)
198
199        except subprocess.CalledProcessError as e:
200            logger.debug(e.output)
201            logger.debug(e.cmd)
202            logger.debug("UT BUILD EXITED FOR: " + sandbox_name)
203            ut_status = "ERROR"
204
205        folder_name = os.path.join(working_dir, sandbox_name)
206        repo_report_dir = os.path.join(report_dir, sandbox_name)
207
208        report_names = ("coveragereport", "test-suite.log", "LastTest.log")
209        find_cmd = "".join("find " + folder_name + " -name " + report + ";"
210                           for report in report_names)
211        try:
212            result = subprocess.check_output(find_cmd, shell=True)
213        except subprocess.CalledProcessError as e:
214            logger.debug(e.output)
215            logger.debug(e.cmd)
216            logger.debug("CMD TO FIND REPORT FAILED FOR: " + sandbox_name)
217            ut_status = "ERROR"
218
219        if ut_status != "ERROR":
220            if result:
221                if result.__contains__("coveragereport"):
222                    ut_status = "YES, COVERAGE"
223                    coverage_count += 1
224                elif "test-suite.log" in result:
225                    ut_status = "YES, UNIT TEST"
226                    unit_test_count += 1
227                elif "LastTest.log" in result:
228                    file_names = result.splitlines()
229                    for file in file_names:
230                        pattern_count_cmd = "sed -n '/Start testing/,/End testing/p;' " + \
231                              file + "|wc -l"
232                        try:
233                            num_of_lines = subprocess.check_output(pattern_count_cmd,
234                                                                   shell=True)
235                        except subprocess.CalledProcessError as e:
236                            logger.debug(e.output)
237                            logger.debug(e.cmd)
238                            logger.debug("CONTENT CHECK FAILED FOR: " + sandbox_name)
239                            ut_status = "ERROR"
240
241                        if int(num_of_lines.strip()) > 5:
242                            ut_status = "YES, UNIT TEST"
243                            unit_test_count += 1
244
245        if "YES" in ut_status:
246            tested_report_count += 1
247            result = result.splitlines()
248            for file_path in result:
249                destination = os.path.dirname(os.path.join(report_dir,
250                                                           os.path.relpath(file_path,
251                                                                           working_dir)))
252                copy_cmd = "mkdir -p " + destination + ";cp -rf " + \
253                           file_path.strip() + " " + destination
254                try:
255                    subprocess.check_output(copy_cmd, shell=True)
256                except subprocess.CalledProcessError as e:
257                    logger.debug(e.output)
258                    logger.debug(e.cmd)
259                    logger.info("FAILED TO COPY REPORTS FOR: " + sandbox_name)
260
261    if ut_status == "ERROR":
262        error_count += 1
263    elif ut_status == "NO":
264        no_report_count += 1
265    elif ut_status == "SKIPPED":
266        skip_count += 1
267    elif ut_status == "ARCHIVED":
268        archive_count += 1
269
270    coverage_report.append("{:<65}{:<10}".format(url.strip(), ut_status))
271    counter += 1
272    logger.info(str(counter) + " in " + str(repo_count) + " completed")
273
274logger.info("*" * 30 + "UNIT TEST COVERAGE REPORT" + "*" * 30)
275for res in coverage_report:
276    logger.info(res)
277logger.info("*" * 30 + "UNIT TEST COVERAGE REPORT" + "*" * 30)
278
279logger.info("REPORTS: " + report_dir)
280logger.info("LOGS: " + log_dir)
281logger.info("*" * 85)
282logger.info("SUMMARY: ")
283logger.info("TOTAL REPOSITORIES     : " + str(repo_count))
284logger.info("TESTED REPOSITORIES    : " + str(tested_report_count))
285logger.info("ERROR                  : " + str(error_count))
286logger.info("COVERAGE REPORT        : " + str(coverage_count))
287logger.info("UNIT TEST REPORT       : " + str(unit_test_count))
288logger.info("NO REPORT              : " + str(no_report_count))
289logger.info("SKIPPED                : " + str(skip_count))
290logger.info("ARCHIVED               : " + str(archive_count))
291logger.info("*" * 85)
292