1#!/usr/bin/python
2
3# This script generates the unit test coverage report for openbmc project.
4#
5# Usage:
6# get_unit_test_report.py target_dir [url_file]
7#
8# Positional arguments:
9# target_dir  Target directory in pwd to place all cloned repos and logs.
10# url_file    Text file containing url of repositories. Optional.
11#             By using this argument, the user can get a report only for
12#             specific repositories given in the file.
13#             Refer ./scripts/repositories.txt
14#
15# Examples:
16#     get_unit_test_report.py target_dir
17#     get_unit_test_report.py target_dir repositories.txt
18#
19# Output format:
20#
21# ***********************************OUTPUT***********************************
22# https://github.com/openbmc/phosphor-dbus-monitor.git               NO
23# https://github.com/openbmc/phosphor-sel-logger.git;protocol=git    NO
24# ***********************************OUTPUT***********************************
25#
26# Other outputs and errors are redirected to output.log and debug.log in
27# target_dir.
28
29import argparse
30import logging
31import os
32import re
33import shutil
34import subprocess
35
36import requests
37
38# Repo list not expected to contain UT. Will be moved to a file in future.
39skip_list = [
40    "openbmc-tools",
41    "inarp",
42    "openbmc",
43    "openbmc.github.io",
44    "phosphor-ecc",
45    "phosphor-pcie-presence",
46    "phosphor-u-boot-env-mgr",
47    "rrd-ipmi-blob",
48    "librrdplus",
49    "openpower-inventory-upload",
50    "openpower-logging",
51    "openpower-power-control",
52    "docs",
53    "openbmc-test-automation",
54    "openbmc-build-scripts",
55    "skeleton",
56    "linux",
57    # Not active, expected to be archived soon.
58    "ibm-pldm-oem",
59]
60
61
62# Create parser.
63text = """%(prog)s target_dir [url_file]
64
65Example usages:
66get_unit_test_report.py target_dir
67get_unit_test_report.py target_dir repositories.txt"""
68
69parser = argparse.ArgumentParser(
70    usage=text, description="Script generates the unit test coverage report"
71)
72parser.add_argument(
73    "target_dir",
74    type=str,
75    help="""Name of a non-existing directory in pwd to store all
76                cloned repos, logs and UT reports""",
77)
78parser.add_argument(
79    "url_file",
80    type=str,
81    nargs="?",
82    help="""Text file containing url of repositories.
83                By using this argument, the user can get a report only for
84                specific repositories given in the file.
85                Refer ./scripts/repositories.txt""",
86)
87args = parser.parse_args()
88
89input_urls = []
90if args.url_file:
91    try:
92        # Get URLs from the file.
93        with open(args.url_file) as reader:
94            file_content = reader.read().splitlines()
95            input_urls = list(filter(lambda x: x, file_content))
96        if not (input_urls):
97            print("Input file {} is empty. Quitting...".format(args.url_file))
98            quit()
99    except IOError as e:
100        print(
101            "Issue in reading file '{}'. Reason: {}".format(
102                args.url_file, str(e)
103            )
104        )
105        quit()
106
107
108# Create target working directory.
109pwd = os.getcwd()
110working_dir = os.path.join(pwd, args.target_dir)
111try:
112    os.mkdir(working_dir)
113except OSError:
114    answer = input(
115        "Target directory "
116        + working_dir
117        + " already exists. "
118        + "Do you want to delete [Y/N]: "
119    )
120    if answer == "Y":
121        try:
122            shutil.rmtree(working_dir)
123            os.mkdir(working_dir)
124        except OSError as e:
125            print(str(e))
126            quit()
127    else:
128        print("Exiting....")
129        quit()
130
131# Create log directory.
132log_dir = os.path.join(working_dir, "logs")
133try:
134    os.mkdir(log_dir)
135except OSError as e:
136    print("Unable to create log directory: " + log_dir)
137    print(str(e))
138    quit()
139
140
141# Log files
142debug_file = os.path.join(log_dir, "debug.log")
143output_file = os.path.join(log_dir, "output.log")
144logging.basicConfig(
145    format="%(levelname)s - %(message)s",
146    level=logging.DEBUG,
147    filename=debug_file,
148)
149logger = logging.getLogger(__name__)
150
151# Create handlers
152console_handler = logging.StreamHandler()
153file_handler = logging.FileHandler(output_file)
154console_handler.setLevel(logging.INFO)
155file_handler.setLevel(logging.INFO)
156
157# Create formatters and add it to handlers
158log_format = logging.Formatter("%(message)s")
159console_handler.setFormatter(log_format)
160file_handler.setFormatter(log_format)
161
162# Add handlers to the logger
163logger.addHandler(console_handler)
164logger.addHandler(file_handler)
165
166
167# Create report directory.
168report_dir = os.path.join(working_dir, "reports")
169try:
170    os.mkdir(report_dir)
171except OSError as e:
172    logger.error("Unable to create report directory: " + report_dir)
173    logger.error(str(e))
174    quit()
175
176# Clone OpenBmc build scripts.
177try:
178    output = subprocess.check_output(
179        "git clone https://github.com/openbmc/openbmc-build-scripts.git",
180        shell=True,
181        cwd=working_dir,
182        stderr=subprocess.STDOUT,
183    )
184    logger.debug(output)
185except subprocess.CalledProcessError as e:
186    logger.error(e.output)
187    logger.error(e.cmd)
188    logger.error("Unable to clone openbmc-build-scripts")
189    quit()
190
191repo_data = []
192if input_urls:
193    api_url = "https://api.github.com/repos/openbmc/"
194    for url in input_urls:
195        try:
196            repo_name = url.strip().split("/")[-1].split(";")[0].split(".")[0]
197        except IndexError as e:
198            logger.error("ERROR: Unable to get sandbox name for url " + url)
199            logger.error("Reason: " + str(e))
200            continue
201
202        try:
203            resp = requests.get(api_url + repo_name)
204            if resp.status_code != 200:
205                logger.info(api_url + repo_name + " ==> " + resp.reason)
206                continue
207            repo_data.extend([resp.json()])
208        except ValueError:
209            logger.error("ERROR: Failed to get response for " + repo_name)
210            logger.error(resp)
211            continue
212
213else:
214    # Get number of pages.
215    resp = requests.head("https://api.github.com/users/openbmc/repos")
216    if resp.status_code != 200:
217        logger.error("Error! Unable to get repositories")
218        logger.error(resp.status_code)
219        logger.error(resp.reason)
220        quit()
221    num_of_pages = int(resp.links["last"]["url"].split("page=")[-1])
222    logger.debug("No. of pages: " + str(num_of_pages))
223
224    # Fetch data from all pages.
225    for page in range(1, num_of_pages + 1):
226        resp = requests.get(
227            "https://api.github.com/users/openbmc/repos?page=" + str(page)
228        )
229        data = resp.json()
230        repo_data.extend(data)
231
232
233# Get URLs and their archive status from response.
234url_info = {}
235for repo in repo_data:
236    try:
237        url_info[repo["clone_url"]] = repo["archived"]
238    except KeyError:
239        logger.error("Failed to get archived status of {}".format(repo))
240        url_info[repo["clone_url"]] = False
241        continue
242logger.debug(url_info)
243repo_count = len(url_info)
244logger.info("Number of repositories (Including archived): " + str(repo_count))
245
246# Clone repository and run unit test.
247coverage_report = []
248counter = 0
249tested_report_count = 0
250coverage_count = 0
251unit_test_count = 0
252no_report_count = 0
253error_count = 0
254skip_count = 0
255archive_count = 0
256url_list = sorted(url_info)
257for url in url_list:
258    ut_status = "NO"
259    skip = False
260    if url_info[url]:
261        ut_status = "ARCHIVED"
262        skip = True
263    else:
264        try:
265            # Eg: url = "https://github.com/openbmc/u-boot.git"
266            #     sandbox_name = "u-boot"
267            sandbox_name = (
268                url.strip().split("/")[-1].split(";")[0].split(".")[0]
269            )
270        except IndexError as e:
271            logger.error("ERROR: Unable to get sandbox name for url " + url)
272            logger.error("Reason: " + str(e))
273            continue
274
275        if sandbox_name in skip_list or re.match(r"meta-", sandbox_name):
276            logger.debug("SKIPPING: " + sandbox_name)
277            skip = True
278            ut_status = "SKIPPED"
279        else:
280            checkout_cmd = "rm -rf " + sandbox_name + ";git clone " + url
281            try:
282                subprocess.check_output(
283                    checkout_cmd,
284                    shell=True,
285                    cwd=working_dir,
286                    stderr=subprocess.STDOUT,
287                )
288            except subprocess.CalledProcessError as e:
289                logger.debug(e.output)
290                logger.debug(e.cmd)
291                logger.debug("Failed to clone " + sandbox_name)
292                ut_status = "ERROR"
293                skip = True
294    if not (skip):
295        docker_cmd = (
296            "WORKSPACE=$(pwd) UNIT_TEST_PKG="
297            + sandbox_name
298            + " "
299            + "./openbmc-build-scripts/run-unit-test-docker.sh"
300        )
301        try:
302            result = subprocess.check_output(
303                docker_cmd,
304                cwd=working_dir,
305                shell=True,
306                stderr=subprocess.STDOUT,
307            )
308            logger.debug(result)
309            logger.debug("UT BUILD COMPLETED FOR: " + sandbox_name)
310
311        except subprocess.CalledProcessError as e:
312            logger.debug(e.output)
313            logger.debug(e.cmd)
314            logger.debug("UT BUILD EXITED FOR: " + sandbox_name)
315            ut_status = "ERROR"
316
317        folder_name = os.path.join(working_dir, sandbox_name)
318        repo_report_dir = os.path.join(report_dir, sandbox_name)
319
320        report_names = ("coveragereport", "test-suite.log", "LastTest.log")
321        find_cmd = "".join(
322            "find " + folder_name + " -name " + report + ";"
323            for report in report_names
324        )
325        try:
326            result = subprocess.check_output(find_cmd, shell=True)
327            result = result.decode("utf-8")
328        except subprocess.CalledProcessError as e:
329            logger.debug(e.output)
330            logger.debug(e.cmd)
331            logger.debug("CMD TO FIND REPORT FAILED FOR: " + sandbox_name)
332            ut_status = "ERROR"
333
334        if ut_status != "ERROR":
335            if result:
336                if result.__contains__("coveragereport"):
337                    ut_status = "YES, COVERAGE"
338                    coverage_count += 1
339                elif "test-suite.log" in result:
340                    ut_status = "YES, UNIT TEST"
341                    unit_test_count += 1
342                elif "LastTest.log" in result:
343                    file_names = result.splitlines()
344                    for file in file_names:
345                        pattern_count_cmd = (
346                            "sed -n '/Start testing/,/End testing/p;' "
347                            + file
348                            + "|wc -l"
349                        )
350                        try:
351                            num_of_lines = subprocess.check_output(
352                                pattern_count_cmd, shell=True
353                            )
354                        except subprocess.CalledProcessError as e:
355                            logger.debug(e.output)
356                            logger.debug(e.cmd)
357                            logger.debug(
358                                "CONTENT CHECK FAILED FOR: " + sandbox_name
359                            )
360                            ut_status = "ERROR"
361
362                        if int(num_of_lines.strip()) > 5:
363                            ut_status = "YES, UNIT TEST"
364                            unit_test_count += 1
365
366        if "YES" in ut_status:
367            tested_report_count += 1
368            result = result.splitlines()
369            for file_path in result:
370                destination = os.path.dirname(
371                    os.path.join(
372                        report_dir, os.path.relpath(file_path, working_dir)
373                    )
374                )
375                copy_cmd = (
376                    "mkdir -p "
377                    + destination
378                    + ";cp -rf "
379                    + file_path.strip()
380                    + " "
381                    + destination
382                )
383                try:
384                    subprocess.check_output(copy_cmd, shell=True)
385                except subprocess.CalledProcessError as e:
386                    logger.debug(e.output)
387                    logger.debug(e.cmd)
388                    logger.info("FAILED TO COPY REPORTS FOR: " + sandbox_name)
389
390    if ut_status == "ERROR":
391        error_count += 1
392    elif ut_status == "NO":
393        no_report_count += 1
394    elif ut_status == "SKIPPED":
395        skip_count += 1
396    elif ut_status == "ARCHIVED":
397        archive_count += 1
398
399    coverage_report.append("{:<65}{:<10}".format(url.strip(), ut_status))
400    counter += 1
401    logger.info(str(counter) + " in " + str(repo_count) + " completed")
402
403logger.info("*" * 30 + "UNIT TEST COVERAGE REPORT" + "*" * 30)
404for res in coverage_report:
405    logger.info(res)
406logger.info("*" * 30 + "UNIT TEST COVERAGE REPORT" + "*" * 30)
407
408logger.info("REPORTS: " + report_dir)
409logger.info("LOGS: " + log_dir)
410logger.info("*" * 85)
411logger.info("SUMMARY: ")
412logger.info("TOTAL REPOSITORIES     : " + str(repo_count))
413logger.info("TESTED REPOSITORIES    : " + str(tested_report_count))
414logger.info("ERROR                  : " + str(error_count))
415logger.info("COVERAGE REPORT        : " + str(coverage_count))
416logger.info("UNIT TEST REPORT       : " + str(unit_test_count))
417logger.info("NO REPORT              : " + str(no_report_count))
418logger.info("SKIPPED                : " + str(skip_count))
419logger.info("ARCHIVED               : " + str(archive_count))
420logger.info("*" * 85)
421