xref: /openbmc/openbmc-build-scripts/scripts/get_unit_test_report.py (revision 229b76a95f87af60c976a0c0dfe84716c9ce5318) 
1#!/usr/bin/python
2
3# This script generates the unit test coverage report.
4#
5# Usage:
6# get_unit_test_report.py file target_dir
7#
8# Description of arguments:
9# file        File with repository URLs, one on each line without quotes.
10#             Eg: git://github.com/openbmc/ibm-dbus-interfaces
11# target_dir  Target directory in pwd to place all cloned repos and logs.
12#
13# Eg: get_unit_test_report.py repo_names target_dir
14#
15# Output format:
16#
17# ***********************************OUTPUT***********************************
18# https://github.com/openbmc/phosphor-dbus-monitor.git               NO
19# https://github.com/openbmc/phosphor-sel-logger.git;protocol=git    NO
20# ***********************************OUTPUT***********************************
21#
22# Other outputs and errors are redirected to output.log and debug.log in target_dir.
23
24import argparse
25import logging
26import os
27import shutil
28import sys
29import subprocess
30
31
32# Create parser.
33parser = argparse.ArgumentParser(usage='%(prog)s file target_dir',
34                                 description="Script generates the unit test coverage report")
35
36parser.add_argument("file", type=str,
37                    help='''Text file containing repository links separated by
38                            new line
39                            Eg: git://github.com/openbmc/ibm-dbus-interfaces''')
40
41parser.add_argument("target_dir", type=str,
42                    help='''Name of a non-existing directory in pwd to store all
43                            cloned repos, logs and UT reports''')
44args = parser.parse_args()
45
46
47# Create target working directory.
48pwd = os.getcwd()
49working_dir = os.path.join(pwd, args.target_dir)
50try:
51    os.mkdir(working_dir)
52except OSError as e:
53    answer = raw_input("Target directory " + working_dir + " already exists. "
54                       + "Do you want to delete [Y/N]: ")
55    if answer == "Y":
56        try:
57            shutil.rmtree(working_dir)
58            os.mkdir(working_dir)
59        except OSError as e:
60            print(str(e))
61            quit()
62    else:
63        print("Exiting....")
64        quit()
65
66# Create log directory.
67log_dir = os.path.join(working_dir, "logs")
68try:
69    os.mkdir(log_dir)
70except OSError as e:
71    print("Unable to create log directory: " + log_dir)
72    print(str(e))
73    quit()
74
75
76# Log files
77debug_file = os.path.join(log_dir, "debug.log")
78output_file = os.path.join(log_dir, "output.log")
79logging.basicConfig(format='%(levelname)s - %(message)s', level=logging.DEBUG,
80                    filename=debug_file)
81logger = logging.getLogger(__name__)
82
83# Create handlers
84console_handler = logging.StreamHandler()
85file_handler = logging.FileHandler(output_file)
86console_handler.setLevel(logging.INFO)
87file_handler.setLevel(logging.INFO)
88
89# Create formatters and add it to handlers
90log_format = logging.Formatter('%(message)s')
91console_handler.setFormatter(log_format)
92file_handler.setFormatter(log_format)
93
94# Add handlers to the logger
95logger.addHandler(console_handler)
96logger.addHandler(file_handler)
97
98
99# Create report directory.
100report_dir = os.path.join(working_dir, "reports")
101try:
102    os.mkdir(report_dir)
103except OSError as e:
104    logger.error("Unable to create report directory: " + report_dir)
105    logger.error(str(e))
106    quit()
107
108# Clone OpenBmc build scripts.
109try:
110    output = subprocess.check_output("git clone https://github.com/openbmc/openbmc-build-scripts.git",
111                                     shell=True, cwd=working_dir, stderr=subprocess.STDOUT)
112    logger.debug(output)
113except subprocess.CalledProcessError as e:
114    logger.debug(e.output)
115    logger.debug(e.cmd)
116    logger.debug("Unable to clone openbmc-build-scripts")
117    quit()
118
119# Read URLs from input file.
120handle = open(args.file)
121url_list = handle.readlines()
122repo_count = len(url_list)
123logger.info("Number of repositories: " + str(repo_count))
124
125# Clone repository and run unit test.
126coverage_report = []
127counter = 0
128total_report_count = 0
129coverage_count = 0
130unit_test_count = 0
131no_report_count = 0
132error_count = 0
133for url in url_list:
134    ci_exists = "NO"
135    skip = False
136    sandbox_name = url.strip().split('/')[-1].split(";")[0].split(".")[0]
137    checkout_cmd = "git clone " + url
138
139    try:
140        result = subprocess.check_output(checkout_cmd, shell=True, cwd=working_dir,
141                                         stderr=subprocess.STDOUT)
142    except subprocess.CalledProcessError as e:
143        logger.debug(e.output)
144        logger.debug(e.cmd)
145        logger.debug("Failed to clone " + sandbox_name)
146        ci_exists = "ERROR"
147        skip = True
148
149    if not(skip):
150        docker_cmd = "WORKSPACE=$(pwd) UNIT_TEST_PKG=" + sandbox_name + " " + \
151                     "./openbmc-build-scripts/run-unit-test-docker.sh"
152        try:
153            result = subprocess.check_output(docker_cmd, cwd=working_dir, shell=True,
154                                             stderr=subprocess.STDOUT)
155            logger.debug(result)
156            logger.debug("UT BUILD COMPLETED FOR: " + sandbox_name)
157
158        except subprocess.CalledProcessError as e:
159            logger.debug(e.output)
160            logger.debug(e.cmd)
161            logger.debug("UT BUILD EXITED FOR: " + sandbox_name)
162            ci_exists = "ERROR"
163
164        folder_name = os.path.join(working_dir, sandbox_name)
165        repo_report_dir = os.path.join(report_dir, sandbox_name)
166
167        report_names = ("coveragereport", "test-suite.log")
168        find_cmd = "".join("find " + folder_name + " -name " + report + ";"
169                           for report in report_names)
170        result = subprocess.check_output(find_cmd, shell=True)
171        if result:
172            total_report_count += 1
173            ci_exists = "YES"
174            if result.__contains__("coveragereport"):
175                ci_exists += ", COVERAGE"
176                coverage_count += 1
177            if result.__contains__("test-suite.log"):
178                ci_exists += ", UNIT TEST"
179                unit_test_count += 1
180
181            result = result.splitlines()
182            for file_path in result:
183                destination = os.path.dirname(os.path.join(report_dir,
184                                                           os.path.relpath(file_path,
185                                                                           working_dir)))
186                copy_cmd = "mkdir -p " + destination + ";cp -rf " + \
187                           file_path.strip() + " " + destination
188                subprocess.check_output(copy_cmd, shell=True)
189    if ci_exists == "ERROR":
190        error_count += 1
191    elif ci_exists == "NO":
192        no_report_count += 1
193
194    coverage_report.append("{:<65}{:<10}".format(url.strip(), ci_exists))
195    counter += 1
196    logger.info(str(counter) + " in " + str(repo_count) + " completed")
197
198logger.info("*" * 30 + "UNIT TEST COVERAGE REPORT" + "*" * 30)
199for res in coverage_report:
200    logger.info(res)
201logger.info("*" * 30 + "UNIT TEST COVERAGE REPORT" + "*" * 30)
202
203logger.info("REPORTS: " + report_dir)
204logger.info("LOGS: " + log_dir)
205logger.info("*" * 85)
206logger.info("SUMMARY: ")
207logger.info("TOTAL REPOSITORIES     : " + str(repo_count))
208logger.info("TESTED REPOSITORIES    : " + str(total_report_count))
209logger.info("ERROR                  : " + str(error_count))
210logger.info("COVERAGE REPORT        : " + str(coverage_count))
211logger.info("UNIT TEST REPORT       : " + str(unit_test_count))
212logger.info("NO REPORT              : " + str(no_report_count))
213logger.info("*" * 85)
214