1#!/usr/bin/env python3
2
3r"""
4Use robot framework API to extract test result data from output.xml generated
5by robot tests. For more information on the Robot Framework API, see
6http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html
7"""
8
9import csv
10import datetime
11import getopt
12import os
13import re
14import stat
15import sys
16from xml.etree import ElementTree
17
18import robot.errors
19from gen_arg import *
20from gen_print import *
21from gen_valid import *
22from robot.api import ExecutionResult
23from robot.result.visitor import ResultVisitor
24
25# Remove the python library path to restore with local project path later.
26save_path_0 = sys.path[0]
27del sys.path[0]
28sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib"))
29
30# Restore sys.path[0].
31sys.path.insert(0, save_path_0)
32
33
34this_program = sys.argv[0]
35info = " For more information:  " + this_program + "  -h"
36if len(sys.argv) == 1:
37    print(info)
38    sys.exit(1)
39
40
41parser = argparse.ArgumentParser(
42    usage=info,
43    description=(
44        "%(prog)s uses a robot framework API to extract test result    data"
45        " from output.xml generated by robot tests. For more information on"
46        " the    Robot Framework API, see   "
47        " http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html"
48    ),
49    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
50    prefix_chars="-+",
51)
52
53parser.add_argument(
54    "--source",
55    "-s",
56    help=(
57        "The output.xml robot test result file path.  This parameter is       "
58        "    required."
59    ),
60)
61
62parser.add_argument(
63    "--dest",
64    "-d",
65    help=(
66        "The directory path where the generated .csv files will go.  This     "
67        "      parameter is required."
68    ),
69)
70
71parser.add_argument(
72    "--version_id",
73    help=(
74        "Driver version of openbmc firmware which was used during test,       "
75        '   e.g. "v2.1-215-g6e7eacb".  This parameter is required.'
76    ),
77)
78
79parser.add_argument(
80    "--platform",
81    help=(
82        "OpenBMC platform which was used during test,          e.g."
83        ' "Witherspoon".  This parameter is required.'
84    ),
85)
86
87parser.add_argument(
88    "--level",
89    help=(
90        "OpenBMC release level which was used during test,          e.g."
91        ' "Master", "OBMC920".  This parameter is required.'
92    ),
93)
94
95parser.add_argument(
96    "--test_phase",
97    help=(
98        'Name of testing phase, e.g. "DVT", "SVT", etc.          This'
99        " parameter is optional."
100    ),
101    default="FVT",
102)
103
104parser.add_argument(
105    "--subsystem",
106    help=(
107        'Name of the subsystem, e.g. "OPENBMC" etc.          This parameter is'
108        " optional."
109    ),
110    default="OPENBMC",
111)
112
113parser.add_argument(
114    "--processor",
115    help='Name of processor, e.g. "P9". This parameter is optional.',
116    default="OPENPOWER",
117)
118
119
120# Populate stock_list with options we want.
121stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)]
122
123
124def exit_function(signal_number=0, frame=None):
125    r"""
126    Execute whenever the program ends normally or with the signals that we
127    catch (i.e. TERM, INT).
128    """
129
130    dprint_executing()
131
132    dprint_var(signal_number)
133
134    qprint_pgm_footer()
135
136
137def signal_handler(signal_number, frame):
138    r"""
139    Handle signals.  Without a function to catch a SIGTERM or SIGINT, the
140    program would terminate immediately with return code 143 and without
141    calling the exit_function.
142    """
143
144    # Our convention is to set up exit_function with atexit.register() so
145    # there is no need to explicitly call exit_function from here.
146
147    dprint_executing()
148
149    # Calling exit prevents us from returning to the code that was running
150    # when the signal was received.
151    exit(0)
152
153
154def validate_parms():
155    r"""
156    Validate program parameters, etc.  Return True or False (i.e. pass/fail)
157    accordingly.
158    """
159
160    if not valid_file_path(source):
161        return False
162
163    if not valid_dir_path(dest):
164        return False
165
166    gen_post_validation(exit_function, signal_handler)
167
168    return True
169
170
171def parse_output_xml(
172    xml_file_path,
173    csv_dir_path,
174    version_id,
175    platform,
176    level,
177    test_phase,
178    processor,
179):
180    r"""
181    Parse the robot-generated output.xml file and extract various test
182    output data. Put the extracted information into a csv file in the "dest"
183    folder.
184
185    Description of argument(s):
186    xml_file_path                   The path to a Robot-generated output.xml
187                                    file.
188    csv_dir_path                    The path to the directory that is to
189                                    contain the .csv files generated by
190                                    this function.
191    version_id                      Version of the openbmc firmware
192                                    (e.g. "v2.1-215-g6e7eacb").
193    platform                        Platform of the openbmc system.
194    level                           Release level of the OpenBMC system
195                                    (e.g. "Master").
196    """
197
198    # Initialize tallies
199    total_critical_tc = 0
200    total_critical_passed = 0
201    total_critical_failed = 0
202    total_non_critical_tc = 0
203    total_non_critical_passed = 0
204    total_non_critical_failed = 0
205
206    result = ExecutionResult(xml_file_path)
207    result.configure(
208        stat_config={
209            "suite_stat_level": 2,
210            "tag_stat_combine": "tagANDanother",
211        }
212    )
213
214    stats = result.statistics
215    print("--------------------------------------")
216    try:
217        total_critical_tc = (
218            stats.total.critical.passed + stats.total.critical.failed
219        )
220        total_critical_passed = stats.total.critical.passed
221        total_critical_failed = stats.total.critical.failed
222    except AttributeError:
223        pass
224
225    try:
226        total_non_critical_tc = stats.total.passed + stats.total.failed
227        total_non_critical_passed = stats.total.passed
228        total_non_critical_failed = stats.total.failed
229    except AttributeError:
230        pass
231
232    print(
233        "Total Test Count:\t %d" % (total_non_critical_tc + total_critical_tc)
234    )
235
236    print("Total Critical Test Failed:\t %d" % total_critical_failed)
237    print("Total Critical Test Passed:\t %d" % total_critical_passed)
238    print("Total Non-Critical Test Failed:\t %d" % total_non_critical_failed)
239    print("Total Non-Critical Test Passed:\t %d" % total_non_critical_passed)
240    print("Test Start Time:\t %s" % result.suite.starttime)
241    print("Test End Time:\t\t %s" % result.suite.endtime)
242    print("--------------------------------------")
243
244    # Use ResultVisitor object and save off the test data info
245    class TestResult(ResultVisitor):
246        def __init__(self):
247            self.testData = []
248
249        def visit_test(self, test):
250            self.testData += [test]
251
252    collectDataObj = TestResult()
253    result.visit(collectDataObj)
254
255    # Write the result statistics attributes to CSV file
256    l_csvlist = []
257
258    # Default Test data
259    l_test_type = test_phase
260
261    l_pse_rel = "Master"
262    if level:
263        l_pse_rel = level
264
265    l_env = "HW"
266    l_proc = processor
267    l_platform_type = ""
268    l_func_area = ""
269
270    # System data from XML meta data
271    # l_system_info = get_system_details(xml_file_path)
272
273    # First let us try to collect information from keyboard input
274    # If keyboard input cannot give both information, then find from xml file.
275    if version_id and platform:
276        l_driver = version_id
277        l_platform_type = platform
278        print("BMC Version_id:%s" % version_id)
279        print("BMC Platform:%s" % platform)
280    else:
281        # System data from XML meta data
282        l_system_info = get_system_details(xml_file_path)
283        l_driver = l_system_info[0]
284        l_platform_type = l_system_info[1]
285
286    # Driver version id and platform are mandatorily required for CSV file
287    # generation. If any one is not avaulable, exit CSV file generation
288    # process.
289    if l_driver and l_platform_type:
290        print("Driver and system info set.")
291    else:
292        print(
293            "Both driver and system info need to be set.                CSV"
294            " file is not generated."
295        )
296        sys.exit()
297
298    # Default header
299    l_header = [
300        "test_start",
301        "test_end",
302        "subsys",
303        "test_type",
304        "test_result",
305        "test_name",
306        "pse_rel",
307        "driver",
308        "env",
309        "proc",
310        "platform_type",
311        "test_func_area",
312    ]
313
314    l_csvlist.append(l_header)
315
316    # Generate CSV file onto the path with current time stamp
317    l_base_dir = csv_dir_path
318    l_timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S")
319    # Example: 2017-02-20-08-47-22_Witherspoon.csv
320    l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv"
321
322    print("Writing data into csv file:%s" % l_csvfile)
323
324    for testcase in collectDataObj.testData:
325        # Functional Area: Suite Name
326        # Test Name: Test Case Name
327        l_func_area = str(testcase.parent).split(" ", 1)[1]
328        l_test_name = str(testcase)
329
330        # Test Result pass=0 fail=1
331        if testcase.status == "PASS":
332            l_test_result = 0
333        else:
334            l_test_result = 1
335
336        # Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S"
337        l_stime = xml_to_csv_time(testcase.starttime)
338        l_etime = xml_to_csv_time(testcase.endtime)
339        # Data Sequence: test_start,test_end,subsys,test_type,
340        #                test_result,test_name,pse_rel,driver,
341        #                env,proc,platform_type,test_func_area,
342        l_data = [
343            l_stime,
344            l_etime,
345            subsystem,
346            l_test_type,
347            l_test_result,
348            l_test_name,
349            l_pse_rel,
350            l_driver,
351            l_env,
352            l_proc,
353            l_platform_type,
354            l_func_area,
355        ]
356        l_csvlist.append(l_data)
357
358    # Open the file and write to the CSV file
359    l_file = open(l_csvfile, "w")
360    l_writer = csv.writer(l_file, lineterminator="\n")
361    l_writer.writerows(l_csvlist)
362    l_file.close()
363    # Set file permissions 666.
364    perm = (
365        stat.S_IRUSR
366        + stat.S_IWUSR
367        + stat.S_IRGRP
368        + stat.S_IWGRP
369        + stat.S_IROTH
370        + stat.S_IWOTH
371    )
372    os.chmod(l_csvfile, perm)
373
374
375def xml_to_csv_time(xml_datetime):
376    r"""
377    Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format
378    and return it.
379
380    Description of argument(s):
381    datetime                        The date in the following format: %Y%m%d
382                                    %H:%M:%S.%f (This is the format
383                                    typically found in an XML file.)
384
385    The date returned will be in the following format: %Y-%m-%d-%H-%M-%S
386    """
387
388    # 20170206 05:05:19.342
389    l_str = datetime.datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f")
390    # 2017-02-06-05-05-19
391    l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S")
392    return str(l_str)
393
394
395def get_system_details(xml_file_path):
396    r"""
397    Get the system data from output.xml generated by robot and return it.
398    The list returned will be in the following order: [driver,platform]
399
400    Description of argument(s):
401    xml_file_path                   The relative or absolute path to the
402                                    output.xml file.
403    """
404
405    bmc_version_id = ""
406    bmc_platform = ""
407    with open(xml_file_path, "rt") as output:
408        tree = ElementTree.parse(output)
409
410    for node in tree.iter("msg"):
411        # /etc/os-release output is logged in the XML as msg
412        # Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79"
413        if "${output} = VERSION_ID=" in node.text:
414            # Get BMC version (e.g. v1.99.1-96-g2a46570)
415            bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1]
416
417        # Platform is logged in the XML as msg.
418        # Example: ${bmc_model} = Witherspoon BMC
419        if "${bmc_model} = " in node.text:
420            bmc_platform = node.text.split(" = ")[1]
421
422    print_vars(bmc_version_id, bmc_platform)
423    return [str(bmc_version_id), str(bmc_platform)]
424
425
426def main():
427    if not gen_get_options(parser, stock_list):
428        return False
429
430    if not validate_parms():
431        return False
432
433    qprint_pgm_header()
434
435    parse_output_xml(
436        source, dest, version_id, platform, level, test_phase, processor
437    )
438
439    return True
440
441
442# Main
443
444if not main():
445    exit(1)
446