1#!/usr/bin/env python3
2
3r"""
4Use robot framework API to extract test result data from output.xml generated
5by robot tests. For more information on the Robot Framework API, see
6http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html
7"""
8
9import csv
10import datetime
11import getopt
12import os
13import re
14import stat
15import sys
16from xml.etree import ElementTree
17
18import robot.errors
19from robot.api import ExecutionResult
20from robot.result.visitor import ResultVisitor
21
22# Remove the python library path to restore with local project path later.
23save_path_0 = sys.path[0]
24del sys.path[0]
25sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib"))
26
27from gen_arg import *  # NOQA
28from gen_print import *  # NOQA
29from gen_valid import *  # NOQA
30
31# Restore sys.path[0].
32sys.path.insert(0, save_path_0)
33
34
35this_program = sys.argv[0]
36info = " For more information:  " + this_program + "  -h"
37if len(sys.argv) == 1:
38    print(info)
39    sys.exit(1)
40
41
42parser = argparse.ArgumentParser(
43    usage=info,
44    description=(
45        "%(prog)s uses a robot framework API to extract test result    data"
46        " from output.xml generated by robot tests. For more information on"
47        " the    Robot Framework API, see   "
48        " http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html"
49    ),
50    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
51    prefix_chars="-+",
52)
53
54parser.add_argument(
55    "--source",
56    "-s",
57    help=(
58        "The output.xml robot test result file path.  This parameter is       "
59        "    required."
60    ),
61)
62
63parser.add_argument(
64    "--dest",
65    "-d",
66    help=(
67        "The directory path where the generated .csv files will go.  This     "
68        "      parameter is required."
69    ),
70)
71
72parser.add_argument(
73    "--version_id",
74    help=(
75        "Driver version of openbmc firmware which was used during test,       "
76        '   e.g. "v2.1-215-g6e7eacb".  This parameter is required.'
77    ),
78)
79
80parser.add_argument(
81    "--platform",
82    help=(
83        "OpenBMC platform which was used during test,          e.g."
84        ' "Witherspoon".  This parameter is required.'
85    ),
86)
87
88parser.add_argument(
89    "--level",
90    help=(
91        "OpenBMC release level which was used during test,          e.g."
92        ' "Master", "OBMC920".  This parameter is required.'
93    ),
94)
95
96parser.add_argument(
97    "--test_phase",
98    help=(
99        'Name of testing phase, e.g. "DVT", "SVT", etc.          This'
100        " parameter is optional."
101    ),
102    default="FVT",
103)
104
105parser.add_argument(
106    "--subsystem",
107    help=(
108        'Name of the subsystem, e.g. "OPENBMC" etc.          This parameter is'
109        " optional."
110    ),
111    default="OPENBMC",
112)
113
114parser.add_argument(
115    "--processor",
116    help='Name of processor, e.g. "P9". This parameter is optional.',
117    default="OPENPOWER",
118)
119
120
121# Populate stock_list with options we want.
122stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)]
123
124
125def exit_function(signal_number=0, frame=None):
126    r"""
127    Execute whenever the program ends normally or with the signals that we
128    catch (i.e. TERM, INT).
129    """
130
131    dprint_executing()
132
133    dprint_var(signal_number)
134
135    qprint_pgm_footer()
136
137
138def signal_handler(signal_number, frame):
139    r"""
140    Handle signals.  Without a function to catch a SIGTERM or SIGINT, the
141    program would terminate immediately with return code 143 and without
142    calling the exit_function.
143    """
144
145    # Our convention is to set up exit_function with atexit.register() so
146    # there is no need to explicitly call exit_function from here.
147
148    dprint_executing()
149
150    # Calling exit prevents us from returning to the code that was running
151    # when the signal was received.
152    exit(0)
153
154
155def validate_parms():
156    r"""
157    Validate program parameters, etc.  Return True or False (i.e. pass/fail)
158    accordingly.
159    """
160
161    if not valid_file_path(source):
162        return False
163
164    if not valid_dir_path(dest):
165        return False
166
167    gen_post_validation(exit_function, signal_handler)
168
169    return True
170
171
172def parse_output_xml(
173    xml_file_path,
174    csv_dir_path,
175    version_id,
176    platform,
177    level,
178    test_phase,
179    processor,
180):
181    r"""
182    Parse the robot-generated output.xml file and extract various test
183    output data. Put the extracted information into a csv file in the "dest"
184    folder.
185
186    Description of argument(s):
187    xml_file_path                   The path to a Robot-generated output.xml
188                                    file.
189    csv_dir_path                    The path to the directory that is to
190                                    contain the .csv files generated by
191                                    this function.
192    version_id                      Version of the openbmc firmware
193                                    (e.g. "v2.1-215-g6e7eacb").
194    platform                        Platform of the openbmc system.
195    level                           Release level of the OpenBMC system
196                                    (e.g. "Master").
197    """
198
199    # Initialize tallies
200    total_critical_tc = 0
201    total_critical_passed = 0
202    total_critical_failed = 0
203    total_non_critical_tc = 0
204    total_non_critical_passed = 0
205    total_non_critical_failed = 0
206
207    result = ExecutionResult(xml_file_path)
208    result.configure(
209        stat_config={
210            "suite_stat_level": 2,
211            "tag_stat_combine": "tagANDanother",
212        }
213    )
214
215    stats = result.statistics
216    print("--------------------------------------")
217    try:
218        total_critical_tc = (
219            stats.total.critical.passed + stats.total.critical.failed
220        )
221        total_critical_passed = stats.total.critical.passed
222        total_critical_failed = stats.total.critical.failed
223    except AttributeError:
224        pass
225
226    try:
227        total_non_critical_tc = stats.total.passed + stats.total.failed
228        total_non_critical_passed = stats.total.passed
229        total_non_critical_failed = stats.total.failed
230    except AttributeError:
231        pass
232
233    print(
234        "Total Test Count:\t %d" % (total_non_critical_tc + total_critical_tc)
235    )
236
237    print("Total Critical Test Failed:\t %d" % total_critical_failed)
238    print("Total Critical Test Passed:\t %d" % total_critical_passed)
239    print("Total Non-Critical Test Failed:\t %d" % total_non_critical_failed)
240    print("Total Non-Critical Test Passed:\t %d" % total_non_critical_passed)
241    print("Test Start Time:\t %s" % result.suite.starttime)
242    print("Test End Time:\t\t %s" % result.suite.endtime)
243    print("--------------------------------------")
244
245    # Use ResultVisitor object and save off the test data info
246    class TestResult(ResultVisitor):
247        def __init__(self):
248            self.testData = []
249
250        def visit_test(self, test):
251            self.testData += [test]
252
253    collectDataObj = TestResult()
254    result.visit(collectDataObj)
255
256    # Write the result statistics attributes to CSV file
257    l_csvlist = []
258
259    # Default Test data
260    l_test_type = test_phase
261
262    l_pse_rel = "Master"
263    if level:
264        l_pse_rel = level
265
266    l_env = "HW"
267    l_proc = processor
268    l_platform_type = ""
269    l_func_area = ""
270
271    # System data from XML meta data
272    # l_system_info = get_system_details(xml_file_path)
273
274    # First let us try to collect information from keyboard input
275    # If keyboard input cannot give both information, then find from xml file.
276    if version_id and platform:
277        l_driver = version_id
278        l_platform_type = platform
279        print("BMC Version_id:%s" % version_id)
280        print("BMC Platform:%s" % platform)
281    else:
282        # System data from XML meta data
283        l_system_info = get_system_details(xml_file_path)
284        l_driver = l_system_info[0]
285        l_platform_type = l_system_info[1]
286
287    # Driver version id and platform are mandatorily required for CSV file
288    # generation. If any one is not avaulable, exit CSV file generation
289    # process.
290    if l_driver and l_platform_type:
291        print("Driver and system info set.")
292    else:
293        print(
294            "Both driver and system info need to be set.                CSV"
295            " file is not generated."
296        )
297        sys.exit()
298
299    # Default header
300    l_header = [
301        "test_start",
302        "test_end",
303        "subsys",
304        "test_type",
305        "test_result",
306        "test_name",
307        "pse_rel",
308        "driver",
309        "env",
310        "proc",
311        "platform_type",
312        "test_func_area",
313    ]
314
315    l_csvlist.append(l_header)
316
317    # Generate CSV file onto the path with current time stamp
318    l_base_dir = csv_dir_path
319    l_timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S")
320    # Example: 2017-02-20-08-47-22_Witherspoon.csv
321    l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv"
322
323    print("Writing data into csv file:%s" % l_csvfile)
324
325    for testcase in collectDataObj.testData:
326        # Functional Area: Suite Name
327        # Test Name: Test Case Name
328        l_func_area = str(testcase.parent).split(" ", 1)[1]
329        l_test_name = str(testcase)
330
331        # Test Result pass=0 fail=1
332        if testcase.status == "PASS":
333            l_test_result = 0
334        elif testcase.status == "SKIP":
335            # Skipped test result should not be mark pass or fail.
336            continue
337        else:
338            l_test_result = 1
339
340        # Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S"
341        l_stime = xml_to_csv_time(testcase.starttime)
342        l_etime = xml_to_csv_time(testcase.endtime)
343        # Data Sequence: test_start,test_end,subsys,test_type,
344        #                test_result,test_name,pse_rel,driver,
345        #                env,proc,platform_type,test_func_area,
346        l_data = [
347            l_stime,
348            l_etime,
349            subsystem,
350            l_test_type,
351            l_test_result,
352            l_test_name,
353            l_pse_rel,
354            l_driver,
355            l_env,
356            l_proc,
357            l_platform_type,
358            l_func_area,
359        ]
360        l_csvlist.append(l_data)
361
362    # Open the file and write to the CSV file
363    l_file = open(l_csvfile, "w")
364    l_writer = csv.writer(l_file, lineterminator="\n")
365    l_writer.writerows(l_csvlist)
366    l_file.close()
367    # Set file permissions 666.
368    perm = (
369        stat.S_IRUSR
370        + stat.S_IWUSR
371        + stat.S_IRGRP
372        + stat.S_IWGRP
373        + stat.S_IROTH
374        + stat.S_IWOTH
375    )
376    os.chmod(l_csvfile, perm)
377
378
379def xml_to_csv_time(xml_datetime):
380    r"""
381    Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format
382    and return it.
383
384    Description of argument(s):
385    datetime                        The date in the following format: %Y%m%d
386                                    %H:%M:%S.%f (This is the format
387                                    typically found in an XML file.)
388
389    The date returned will be in the following format: %Y-%m-%d-%H-%M-%S
390    """
391
392    # 20170206 05:05:19.342
393    l_str = datetime.datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f")
394    # 2017-02-06-05-05-19
395    l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S")
396    return str(l_str)
397
398
399def get_system_details(xml_file_path):
400    r"""
401    Get the system data from output.xml generated by robot and return it.
402    The list returned will be in the following order: [driver,platform]
403
404    Description of argument(s):
405    xml_file_path                   The relative or absolute path to the
406                                    output.xml file.
407    """
408
409    bmc_version_id = ""
410    bmc_platform = ""
411    with open(xml_file_path, "rt") as output:
412        tree = ElementTree.parse(output)
413
414    for node in tree.iter("msg"):
415        # /etc/os-release output is logged in the XML as msg
416        # Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79"
417        if "${output} = VERSION_ID=" in node.text:
418            # Get BMC version (e.g. v1.99.1-96-g2a46570)
419            bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1]
420
421        # Platform is logged in the XML as msg.
422        # Example: ${bmc_model} = Witherspoon BMC
423        if "${bmc_model} = " in node.text:
424            bmc_platform = node.text.split(" = ")[1]
425
426    print_vars(bmc_version_id, bmc_platform)
427    return [str(bmc_version_id), str(bmc_platform)]
428
429
430def main():
431    if not gen_get_options(parser, stock_list):
432        return False
433
434    if not validate_parms():
435        return False
436
437    qprint_pgm_header()
438
439    parse_output_xml(
440        source, dest, version_id, platform, level, test_phase, processor
441    )
442
443    return True
444
445
446# Main
447
448if not main():
449    exit(1)
450