1#!/usr/bin/env python3
2
3r"""
4Use robot framework API to extract test result data from output.xml generated
5by robot tests. For more information on the Robot Framework API, see
6http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html
7"""
8
9import csv
10import datetime
11import os
12import stat
13import sys
14from xml.etree import ElementTree
15
16from robot.api import ExecutionResult
17from robot.result.visitor import ResultVisitor
18
19# Remove the python library path to restore with local project path later.
20SAVE_PATH_0 = sys.path[0]
21del sys.path[0]
22sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib"))
23
24from gen_arg import *  # NOQA
25from gen_print import *  # NOQA
26from gen_valid import *  # NOQA
27
28# Restore sys.path[0].
29sys.path.insert(0, SAVE_PATH_0)
30
31
32this_program = sys.argv[0]
33info = " For more information:  " + this_program + "  -h"
34if len(sys.argv) == 1:
35    print(info)
36    sys.exit(1)
37
38
39parser = argparse.ArgumentParser(
40    usage=info,
41    description=(
42        "%(prog)s uses a robot framework API to extract test result    data"
43        " from output.xml generated by robot tests. For more information on"
44        " the    Robot Framework API, see   "
45        " http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html"
46    ),
47    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
48    prefix_chars="-+",
49)
50
51parser.add_argument(
52    "--source",
53    "-s",
54    help=(
55        "The output.xml robot test result file path.  This parameter is       "
56        "    required."
57    ),
58)
59
60parser.add_argument(
61    "--dest",
62    "-d",
63    help=(
64        "The directory path where the generated .csv files will go.  This     "
65        "      parameter is required."
66    ),
67)
68
69parser.add_argument(
70    "--version_id",
71    help=(
72        "Driver version of openbmc firmware which was used during test,       "
73        '   e.g. "v2.1-215-g6e7eacb".  This parameter is required.'
74    ),
75)
76
77parser.add_argument(
78    "--platform",
79    help=(
80        "OpenBMC platform which was used during test,          e.g."
81        ' "Witherspoon".  This parameter is required.'
82    ),
83)
84
85parser.add_argument(
86    "--level",
87    help=(
88        "OpenBMC release level which was used during test,          e.g."
89        ' "Master", "OBMC920".  This parameter is required.'
90    ),
91)
92
93parser.add_argument(
94    "--test_phase",
95    help=(
96        'Name of testing phase, e.g. "DVT", "SVT", etc.          This'
97        " parameter is optional."
98    ),
99    default="FVT",
100)
101
102parser.add_argument(
103    "--subsystem",
104    help=(
105        'Name of the subsystem, e.g. "OPENBMC" etc.          This parameter is'
106        " optional."
107    ),
108    default="OPENBMC",
109)
110
111parser.add_argument(
112    "--processor",
113    help='Name of processor, e.g. "P9". This parameter is optional.',
114    default="OPENPOWER",
115)
116
117
118# Populate stock_list with options we want.
119stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)]
120
121
122def exit_function(signal_number=0, frame=None):
123    r"""
124    Execute whenever the program ends normally or with the signals that we
125    catch (i.e. TERM, INT).
126    """
127
128    dprint_executing()
129
130    dprint_var(signal_number)
131
132    qprint_pgm_footer()
133
134
135def signal_handler(signal_number, frame):
136    r"""
137    Handle signals.  Without a function to catch a SIGTERM or SIGINT, the
138    program would terminate immediately with return code 143 and without
139    calling the exit_function.
140    """
141
142    # Our convention is to set up exit_function with atexit.register() so
143    # there is no need to explicitly call exit_function from here.
144
145    dprint_executing()
146
147    # Calling exit prevents us from returning to the code that was running
148    # when the signal was received.
149    sys.exit(0)
150
151
152def validate_parms():
153    r"""
154    Validate program parameters, etc.  Return True or False (i.e. pass/fail)
155    accordingly.
156    """
157
158    if not valid_file_path(source):
159        return False
160
161    if not valid_dir_path(dest):
162        return False
163
164    gen_post_validation(exit_function, signal_handler)
165
166    return True
167
168
169def parse_output_xml(
170    xml_file_path,
171    csv_dir_path,
172    version_id,
173    platform,
174    level,
175    test_phase,
176    processor,
177):
178    r"""
179    Parse the robot-generated output.xml file and extract various test
180    output data. Put the extracted information into a csv file in the "dest"
181    folder.
182
183    Description of argument(s):
184    xml_file_path                   The path to a Robot-generated output.xml
185                                    file.
186    csv_dir_path                    The path to the directory that is to
187                                    contain the .csv files generated by
188                                    this function.
189    version_id                      Version of the openbmc firmware
190                                    (e.g. "v2.1-215-g6e7eacb").
191    platform                        Platform of the openbmc system.
192    level                           Release level of the OpenBMC system
193                                    (e.g. "Master").
194    """
195
196    # Initialize tallies
197    total_critical_tc = 0
198    total_critical_passed = 0
199    total_critical_failed = 0
200    total_non_critical_tc = 0
201    total_non_critical_passed = 0
202    total_non_critical_failed = 0
203
204    result = ExecutionResult(xml_file_path)
205    result.configure(
206        stat_config={
207            "suite_stat_level": 2,
208            "tag_stat_combine": "tagANDanother",
209        }
210    )
211
212    stats = result.statistics
213    print("--------------------------------------")
214    try:
215        total_critical_tc = (
216            stats.total.critical.passed + stats.total.critical.failed
217        )
218        total_critical_passed = stats.total.critical.passed
219        total_critical_failed = stats.total.critical.failed
220    except AttributeError:
221        pass
222
223    try:
224        total_non_critical_tc = stats.total.passed + stats.total.failed
225        total_non_critical_passed = stats.total.passed
226        total_non_critical_failed = stats.total.failed
227    except AttributeError:
228        pass
229
230    print(
231        "Total Test Count:\t %d" % (total_non_critical_tc + total_critical_tc)
232    )
233
234    print("Total Critical Test Failed:\t %d" % total_critical_failed)
235    print("Total Critical Test Passed:\t %d" % total_critical_passed)
236    print("Total Non-Critical Test Failed:\t %d" % total_non_critical_failed)
237    print("Total Non-Critical Test Passed:\t %d" % total_non_critical_passed)
238    print("Test Start Time:\t %s" % result.suite.starttime)
239    print("Test End Time:\t\t %s" % result.suite.endtime)
240    print("--------------------------------------")
241
242    # Use ResultVisitor object and save off the test data info.
243    class TestResult(ResultVisitor):
244        r"""
245        Class methods to save off the test data information.
246        """
247
248        def __init__(self):
249            self.test_data = []
250
251        def visit_test(self, test):
252            self.test_data += [test]
253
254    collect_data_obj = TestResult()
255    result.visit(collect_data_obj)
256
257    # Write the result statistics attributes to CSV file
258    l_csvlist = []
259
260    # Default Test data
261    l_test_type = test_phase
262
263    l_pse_rel = "Master"
264    if level:
265        l_pse_rel = level
266
267    l_env = "HW"
268    l_proc = processor
269    l_platform_type = ""
270    l_func_area = ""
271
272    # System data from XML meta data
273    # l_system_info = get_system_details(xml_file_path)
274
275    # First let us try to collect information from keyboard input
276    # If keyboard input cannot give both information, then find from xml file.
277    if version_id and platform:
278        l_driver = version_id
279        l_platform_type = platform
280        print("BMC Version_id:%s" % version_id)
281        print("BMC Platform:%s" % platform)
282    else:
283        # System data from XML meta data
284        l_system_info = get_system_details(xml_file_path)
285        l_driver = l_system_info[0]
286        l_platform_type = l_system_info[1]
287
288    # Driver version id and platform are mandatorily required for CSV file
289    # generation. If any one is not avaulable, exit CSV file generation
290    # process.
291    if l_driver and l_platform_type:
292        print("Driver and system info set.")
293    else:
294        print(
295            "Both driver and system info need to be set.                CSV"
296            " file is not generated."
297        )
298        sys.exit()
299
300    # Default header
301    l_header = [
302        "test_start",
303        "test_end",
304        "subsys",
305        "test_type",
306        "test_result",
307        "test_name",
308        "pse_rel",
309        "driver",
310        "env",
311        "proc",
312        "platform_type",
313        "test_func_area",
314    ]
315
316    l_csvlist.append(l_header)
317
318    # Generate CSV file onto the path with current time stamp
319    l_base_dir = csv_dir_path
320    l_timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S")
321    # Example: 2017-02-20-08-47-22_Witherspoon.csv
322    l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv"
323
324    print("Writing data into csv file:%s" % l_csvfile)
325
326    for testcase in collect_data_obj.test_data:
327        # Functional Area: Suite Name
328        # Test Name: Test Case Name
329        l_func_area = str(testcase.parent).split(" ", 1)[1]
330        l_test_name = str(testcase)
331
332        # Test Result pass=0 fail=1
333        if testcase.status == "PASS":
334            l_test_result = 0
335        elif testcase.status == "SKIP":
336            # Skipped test result should not be mark pass or fail.
337            continue
338        else:
339            l_test_result = 1
340
341        # Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S"
342        l_stime = xml_to_csv_time(testcase.starttime)
343        l_etime = xml_to_csv_time(testcase.endtime)
344        # Data Sequence: test_start,test_end,subsys,test_type,
345        #                test_result,test_name,pse_rel,driver,
346        #                env,proc,platform_type,test_func_area,
347        l_data = [
348            l_stime,
349            l_etime,
350            subsystem,
351            l_test_type,
352            l_test_result,
353            l_test_name,
354            l_pse_rel,
355            l_driver,
356            l_env,
357            l_proc,
358            l_platform_type,
359            l_func_area,
360        ]
361        l_csvlist.append(l_data)
362
363    # Open the file and write to the CSV file
364    with open(l_csvfile, "w", encoding="utf8") as l_file:
365        l_writer = csv.writer(l_file, lineterminator="\n")
366        l_writer.writerows(l_csvlist)
367        l_file.close()
368
369    # Set file permissions 666.
370    perm = (
371        stat.S_IRUSR
372        + stat.S_IWUSR
373        + stat.S_IRGRP
374        + stat.S_IWGRP
375        + stat.S_IROTH
376        + stat.S_IWOTH
377    )
378    os.chmod(l_csvfile, perm)
379
380
381def xml_to_csv_time(xml_datetime):
382    r"""
383    Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format
384    and return it.
385
386    Description of argument(s):
387    datetime                        The date in the following format: %Y%m%d
388                                    %H:%M:%S.%f (This is the format
389                                    typically found in an XML file.)
390
391    The date returned will be in the following format: %Y-%m-%d-%H-%M-%S
392    """
393
394    # 20170206 05:05:19.342
395    l_str = datetime.datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f")
396    # 2017-02-06-05-05-19
397    l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S")
398    return str(l_str)
399
400
401def get_system_details(xml_file_path):
402    r"""
403    Get the system data from output.xml generated by robot and return it.
404    The list returned will be in the following order: [driver,platform]
405
406    Description of argument(s):
407    xml_file_path                   The relative or absolute path to the
408                                    output.xml file.
409    """
410
411    bmc_version_id = ""
412    bmc_platform = ""
413    with open(xml_file_path, "rt", encoding="utf-8") as output:
414        tree = ElementTree.parse(output)
415
416    for node in tree.iter("msg"):
417        # /etc/os-release output is logged in the XML as msg
418        # Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79"
419        if "${output} = VERSION_ID=" in node.text:
420            # Get BMC version (e.g. v1.99.1-96-g2a46570)
421            bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1]
422
423        # Platform is logged in the XML as msg.
424        # Example: ${bmc_model} = Witherspoon BMC
425        if "${bmc_model} = " in node.text:
426            bmc_platform = node.text.split(" = ")[1]
427
428    print_vars(bmc_version_id, bmc_platform)
429    return [str(bmc_version_id), str(bmc_platform)]
430
431
432def main():
433    r"""
434    Main caller.
435    """
436
437    if not gen_get_options(parser, stock_list):
438        return False
439
440    if not validate_parms():
441        return False
442
443    qprint_pgm_header()
444
445    parse_output_xml(
446        source, dest, version_id, platform, level, test_phase, processor
447    )
448
449    return True
450
451
452# Main
453
454if not main():
455    sys.exit(1)
456