1#!/usr/bin/env python3
2
3r"""
4Use robot framework API to extract test result data from output.xml generated
5by robot tests. For more information on the Robot Framework API, see
6http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html
7"""
8
9import csv
10import datetime
11import getopt
12import os
13import re
14import stat
15import sys
16from xml.etree import ElementTree
17
18import robot.errors
19from robot.api import ExecutionResult
20from robot.result.visitor import ResultVisitor
21
22# Remove the python library path to restore with local project path later.
23save_path_0 = sys.path[0]
24del sys.path[0]
25sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib"))
26
27from gen_arg import *  # NOQA
28from gen_print import *  # NOQA
29from gen_valid import *  # NOQA
30
31# Restore sys.path[0].
32sys.path.insert(0, save_path_0)
33
34
35this_program = sys.argv[0]
36info = " For more information:  " + this_program + "  -h"
37if len(sys.argv) == 1:
38    print(info)
39    sys.exit(1)
40
41
42parser = argparse.ArgumentParser(
43    usage=info,
44    description=(
45        "%(prog)s uses a robot framework API to extract test result    data"
46        " from output.xml generated by robot tests. For more information on"
47        " the    Robot Framework API, see   "
48        " http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html"
49    ),
50    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
51    prefix_chars="-+",
52)
53
54parser.add_argument(
55    "--source",
56    "-s",
57    help=(
58        "The output.xml robot test result file path.  This parameter is       "
59        "    required."
60    ),
61)
62
63parser.add_argument(
64    "--dest",
65    "-d",
66    help=(
67        "The directory path where the generated .csv files will go.  This     "
68        "      parameter is required."
69    ),
70)
71
72parser.add_argument(
73    "--version_id",
74    help=(
75        "Driver version of openbmc firmware which was used during test,       "
76        '   e.g. "v2.1-215-g6e7eacb".  This parameter is required.'
77    ),
78)
79
80parser.add_argument(
81    "--platform",
82    help=(
83        "OpenBMC platform which was used during test,          e.g."
84        ' "Witherspoon".  This parameter is required.'
85    ),
86)
87
88parser.add_argument(
89    "--level",
90    help=(
91        "OpenBMC release level which was used during test,          e.g."
92        ' "Master", "OBMC920".  This parameter is required.'
93    ),
94)
95
96parser.add_argument(
97    "--test_phase",
98    help=(
99        'Name of testing phase, e.g. "DVT", "SVT", etc.          This'
100        " parameter is optional."
101    ),
102    default="FVT",
103)
104
105parser.add_argument(
106    "--subsystem",
107    help=(
108        'Name of the subsystem, e.g. "OPENBMC" etc.          This parameter is'
109        " optional."
110    ),
111    default="OPENBMC",
112)
113
114parser.add_argument(
115    "--processor",
116    help='Name of processor, e.g. "P9". This parameter is optional.',
117    default="OPENPOWER",
118)
119
120
121# Populate stock_list with options we want.
122stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)]
123
124
125def exit_function(signal_number=0, frame=None):
126    r"""
127    Execute whenever the program ends normally or with the signals that we
128    catch (i.e. TERM, INT).
129    """
130
131    dprint_executing()
132
133    dprint_var(signal_number)
134
135    qprint_pgm_footer()
136
137
138def signal_handler(signal_number, frame):
139    r"""
140    Handle signals.  Without a function to catch a SIGTERM or SIGINT, the
141    program would terminate immediately with return code 143 and without
142    calling the exit_function.
143    """
144
145    # Our convention is to set up exit_function with atexit.register() so
146    # there is no need to explicitly call exit_function from here.
147
148    dprint_executing()
149
150    # Calling exit prevents us from returning to the code that was running
151    # when the signal was received.
152    exit(0)
153
154
155def validate_parms():
156    r"""
157    Validate program parameters, etc.  Return True or False (i.e. pass/fail)
158    accordingly.
159    """
160
161    if not valid_file_path(source):
162        return False
163
164    if not valid_dir_path(dest):
165        return False
166
167    gen_post_validation(exit_function, signal_handler)
168
169    return True
170
171
172def parse_output_xml(
173    xml_file_path,
174    csv_dir_path,
175    version_id,
176    platform,
177    level,
178    test_phase,
179    processor,
180):
181    r"""
182    Parse the robot-generated output.xml file and extract various test
183    output data. Put the extracted information into a csv file in the "dest"
184    folder.
185
186    Description of argument(s):
187    xml_file_path                   The path to a Robot-generated output.xml
188                                    file.
189    csv_dir_path                    The path to the directory that is to
190                                    contain the .csv files generated by
191                                    this function.
192    version_id                      Version of the openbmc firmware
193                                    (e.g. "v2.1-215-g6e7eacb").
194    platform                        Platform of the openbmc system.
195    level                           Release level of the OpenBMC system
196                                    (e.g. "Master").
197    """
198
199    # Initialize tallies
200    total_critical_tc = 0
201    total_critical_passed = 0
202    total_critical_failed = 0
203    total_non_critical_tc = 0
204    total_non_critical_passed = 0
205    total_non_critical_failed = 0
206
207    result = ExecutionResult(xml_file_path)
208    result.configure(
209        stat_config={
210            "suite_stat_level": 2,
211            "tag_stat_combine": "tagANDanother",
212        }
213    )
214
215    stats = result.statistics
216    print("--------------------------------------")
217    try:
218        total_critical_tc = (
219            stats.total.critical.passed + stats.total.critical.failed
220        )
221        total_critical_passed = stats.total.critical.passed
222        total_critical_failed = stats.total.critical.failed
223    except AttributeError:
224        pass
225
226    try:
227        total_non_critical_tc = stats.total.passed + stats.total.failed
228        total_non_critical_passed = stats.total.passed
229        total_non_critical_failed = stats.total.failed
230    except AttributeError:
231        pass
232
233    print(
234        "Total Test Count:\t %d" % (total_non_critical_tc + total_critical_tc)
235    )
236
237    print("Total Critical Test Failed:\t %d" % total_critical_failed)
238    print("Total Critical Test Passed:\t %d" % total_critical_passed)
239    print("Total Non-Critical Test Failed:\t %d" % total_non_critical_failed)
240    print("Total Non-Critical Test Passed:\t %d" % total_non_critical_passed)
241    print("Test Start Time:\t %s" % result.suite.starttime)
242    print("Test End Time:\t\t %s" % result.suite.endtime)
243    print("--------------------------------------")
244
245    # Use ResultVisitor object and save off the test data info
246    class TestResult(ResultVisitor):
247        def __init__(self):
248            self.testData = []
249
250        def visit_test(self, test):
251            self.testData += [test]
252
253    collectDataObj = TestResult()
254    result.visit(collectDataObj)
255
256    # Write the result statistics attributes to CSV file
257    l_csvlist = []
258
259    # Default Test data
260    l_test_type = test_phase
261
262    l_pse_rel = "Master"
263    if level:
264        l_pse_rel = level
265
266    l_env = "HW"
267    l_proc = processor
268    l_platform_type = ""
269    l_func_area = ""
270
271    # System data from XML meta data
272    # l_system_info = get_system_details(xml_file_path)
273
274    # First let us try to collect information from keyboard input
275    # If keyboard input cannot give both information, then find from xml file.
276    if version_id and platform:
277        l_driver = version_id
278        l_platform_type = platform
279        print("BMC Version_id:%s" % version_id)
280        print("BMC Platform:%s" % platform)
281    else:
282        # System data from XML meta data
283        l_system_info = get_system_details(xml_file_path)
284        l_driver = l_system_info[0]
285        l_platform_type = l_system_info[1]
286
287    # Driver version id and platform are mandatorily required for CSV file
288    # generation. If any one is not avaulable, exit CSV file generation
289    # process.
290    if l_driver and l_platform_type:
291        print("Driver and system info set.")
292    else:
293        print(
294            "Both driver and system info need to be set.                CSV"
295            " file is not generated."
296        )
297        sys.exit()
298
299    # Default header
300    l_header = [
301        "test_start",
302        "test_end",
303        "subsys",
304        "test_type",
305        "test_result",
306        "test_name",
307        "pse_rel",
308        "driver",
309        "env",
310        "proc",
311        "platform_type",
312        "test_func_area",
313    ]
314
315    l_csvlist.append(l_header)
316
317    # Generate CSV file onto the path with current time stamp
318    l_base_dir = csv_dir_path
319    l_timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S")
320    # Example: 2017-02-20-08-47-22_Witherspoon.csv
321    l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv"
322
323    print("Writing data into csv file:%s" % l_csvfile)
324
325    for testcase in collectDataObj.testData:
326        # Functional Area: Suite Name
327        # Test Name: Test Case Name
328        l_func_area = str(testcase.parent).split(" ", 1)[1]
329        l_test_name = str(testcase)
330
331        # Test Result pass=0 fail=1
332        if testcase.status == "PASS":
333            l_test_result = 0
334        else:
335            l_test_result = 1
336
337        # Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S"
338        l_stime = xml_to_csv_time(testcase.starttime)
339        l_etime = xml_to_csv_time(testcase.endtime)
340        # Data Sequence: test_start,test_end,subsys,test_type,
341        #                test_result,test_name,pse_rel,driver,
342        #                env,proc,platform_type,test_func_area,
343        l_data = [
344            l_stime,
345            l_etime,
346            subsystem,
347            l_test_type,
348            l_test_result,
349            l_test_name,
350            l_pse_rel,
351            l_driver,
352            l_env,
353            l_proc,
354            l_platform_type,
355            l_func_area,
356        ]
357        l_csvlist.append(l_data)
358
359    # Open the file and write to the CSV file
360    l_file = open(l_csvfile, "w")
361    l_writer = csv.writer(l_file, lineterminator="\n")
362    l_writer.writerows(l_csvlist)
363    l_file.close()
364    # Set file permissions 666.
365    perm = (
366        stat.S_IRUSR
367        + stat.S_IWUSR
368        + stat.S_IRGRP
369        + stat.S_IWGRP
370        + stat.S_IROTH
371        + stat.S_IWOTH
372    )
373    os.chmod(l_csvfile, perm)
374
375
376def xml_to_csv_time(xml_datetime):
377    r"""
378    Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format
379    and return it.
380
381    Description of argument(s):
382    datetime                        The date in the following format: %Y%m%d
383                                    %H:%M:%S.%f (This is the format
384                                    typically found in an XML file.)
385
386    The date returned will be in the following format: %Y-%m-%d-%H-%M-%S
387    """
388
389    # 20170206 05:05:19.342
390    l_str = datetime.datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f")
391    # 2017-02-06-05-05-19
392    l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S")
393    return str(l_str)
394
395
396def get_system_details(xml_file_path):
397    r"""
398    Get the system data from output.xml generated by robot and return it.
399    The list returned will be in the following order: [driver,platform]
400
401    Description of argument(s):
402    xml_file_path                   The relative or absolute path to the
403                                    output.xml file.
404    """
405
406    bmc_version_id = ""
407    bmc_platform = ""
408    with open(xml_file_path, "rt") as output:
409        tree = ElementTree.parse(output)
410
411    for node in tree.iter("msg"):
412        # /etc/os-release output is logged in the XML as msg
413        # Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79"
414        if "${output} = VERSION_ID=" in node.text:
415            # Get BMC version (e.g. v1.99.1-96-g2a46570)
416            bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1]
417
418        # Platform is logged in the XML as msg.
419        # Example: ${bmc_model} = Witherspoon BMC
420        if "${bmc_model} = " in node.text:
421            bmc_platform = node.text.split(" = ")[1]
422
423    print_vars(bmc_version_id, bmc_platform)
424    return [str(bmc_version_id), str(bmc_platform)]
425
426
427def main():
428    if not gen_get_options(parser, stock_list):
429        return False
430
431    if not validate_parms():
432        return False
433
434    qprint_pgm_header()
435
436    parse_output_xml(
437        source, dest, version_id, platform, level, test_phase, processor
438    )
439
440    return True
441
442
443# Main
444
445if not main():
446    exit(1)
447