1#!/usr/bin/env python
2
3r"""
4Use robot framework API to extract test result data from output.xml generated
5by robot tests. For more information on the Robot Framework API, see
6http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html
7"""
8
9import sys
10import os
11import getopt
12import csv
13import robot.errors
14import re
15import stat
16from datetime import datetime
17from robot.api import ExecutionResult
18from robot.result.visitor import ResultVisitor
19from xml.etree import ElementTree
20
21# Remove the python library path to restore with local project path later.
22save_path_0 = sys.path[0]
23del sys.path[0]
24sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib"))
25
26from gen_arg import *
27from gen_print import *
28from gen_valid import *
29
30# Restore sys.path[0].
31sys.path.insert(0, save_path_0)
32
33
34this_program = sys.argv[0]
35info = " For more information:  " + this_program + '  -h'
36if len(sys.argv) == 1:
37    print (info)
38    sys.exit(1)
39
40
41parser = argparse.ArgumentParser(
42    usage=info,
43    description="%(prog)s uses a robot framework API to extract test result\
44    data from output.xml generated by robot tests. For more information on the\
45    Robot Framework API, see\
46    http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html",
47    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
48    prefix_chars='-+')
49
50parser.add_argument(
51    '--source',
52    '-s',
53    help='The output.xml robot test result file path.  This parameter is \
54          required.')
55
56parser.add_argument(
57    '--dest',
58    '-d',
59    help='The directory path where the generated .csv files will go.  This \
60          parameter is required.')
61
62parser.add_argument(
63    '--version_id',
64    help='Driver version of openbmc firmware which was used during test,\
65          e.g. "v2.1-215-g6e7eacb".  This parameter is required.')
66
67parser.add_argument(
68    '--platform',
69    help='OpenBMC platform which was used during test,\
70          e.g. "Witherspoon".  This parameter is required.')
71
72parser.add_argument(
73    '--level',
74    help='OpenBMC release level which was used during test,\
75          e.g. "Master", "OBMC920".  This parameter is required.')
76
77parser.add_argument(
78    '--test_phase',
79    help='Name of testing phase, e.g. "DVT", "SVT", etc.\
80          This parameter is optional.',
81    default="FVT")
82
83parser.add_argument(
84    '--processor',
85    help='Name of processor, e.g. "P9". This parameter is optional.',
86    default="OPENPOWER")
87
88
89# Populate stock_list with options we want.
90stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)]
91
92
93def exit_function(signal_number=0,
94                  frame=None):
95    r"""
96    Execute whenever the program ends normally or with the signals that we
97    catch (i.e. TERM, INT).
98    """
99
100    dprint_executing()
101
102    dprint_var(signal_number)
103
104    qprint_pgm_footer()
105
106
107def signal_handler(signal_number,
108                   frame):
109    r"""
110    Handle signals.  Without a function to catch a SIGTERM or SIGINT, the
111    program would terminate immediately with return code 143 and without
112    calling the exit_function.
113    """
114
115    # Our convention is to set up exit_function with atexit.register() so
116    # there is no need to explicitly call exit_function from here.
117
118    dprint_executing()
119
120    # Calling exit prevents us from returning to the code that was running
121    # when the signal was received.
122    exit(0)
123
124
125def validate_parms():
126    r"""
127    Validate program parameters, etc.  Return True or False (i.e. pass/fail)
128    accordingly.
129    """
130
131    if not valid_file_path(source):
132        return False
133
134    if not valid_dir_path(dest):
135        return False
136
137    gen_post_validation(exit_function, signal_handler)
138
139    return True
140
141
142def parse_output_xml(xml_file_path, csv_dir_path, version_id, platform, level,
143                     test_phase, processor):
144    r"""
145    Parse the robot-generated output.xml file and extract various test
146    output data. Put the extracted information into a csv file in the "dest"
147    folder.
148
149    Description of argument(s):
150    xml_file_path                   The path to a Robot-generated output.xml
151                                    file.
152    csv_dir_path                    The path to the directory that is to
153                                    contain the .csv files generated by
154                                    this function.
155    version_id                      Version of the openbmc firmware
156                                    (e.g. "v2.1-215-g6e7eacb").
157    platform                        Platform of the openbmc system.
158    level                           Release level of the OpenBMC system
159                                    (e.g. "Master").
160    """
161
162    result = ExecutionResult(xml_file_path)
163    result.configure(stat_config={'suite_stat_level': 2,
164                                  'tag_stat_combine': 'tagANDanother'})
165
166    stats = result.statistics
167    print("--------------------------------------")
168    total_tc = stats.total.critical.passed + stats.total.critical.failed
169    print("Total Test Count:\t %d" % total_tc)
170    print("Total Test Failed:\t %d" % stats.total.critical.failed)
171    print("Total Test Passed:\t %d" % stats.total.critical.passed)
172    print("Test Start Time:\t %s" % result.suite.starttime)
173    print("Test End Time:\t\t %s" % result.suite.endtime)
174    print("--------------------------------------")
175
176    # Use ResultVisitor object and save off the test data info
177    class TestResult(ResultVisitor):
178        def __init__(self):
179            self.testData = []
180
181        def visit_test(self, test):
182            self.testData += [test]
183
184    collectDataObj = TestResult()
185    result.visit(collectDataObj)
186
187    # Write the result statistics attributes to CSV file
188    l_csvlist = []
189
190    # Default Test data
191    l_subsys = 'OPENBMC'
192    l_test_type = test_phase
193
194    l_pse_rel = 'Master'
195    if level:
196        l_pse_rel = level
197
198    l_env = 'HW'
199    l_proc = processor
200    l_platform_type = ""
201    l_func_area = ""
202
203    # System data from XML meta data
204    # l_system_info = get_system_details(xml_file_path)
205
206    # First let us try to collect information from keyboard input
207    # If keyboard input cannot give both information, then find from xml file.
208    if version_id and platform:
209        l_driver = version_id
210        l_platform_type = platform
211        print("BMC Version_id:%s" % version_id)
212        print("BMC Platform:%s" % platform)
213    else:
214        # System data from XML meta data
215        l_system_info = get_system_details(xml_file_path)
216        l_driver = l_system_info[0]
217        l_platform_type = l_system_info[1]
218
219    # Driver version id and platform are mandatorily required for CSV file
220    # generation. If any one is not avaulable, exit CSV file generation
221    # process.
222    if l_driver and l_platform_type:
223        print("Driver and system info set.")
224    else:
225        print("Both driver and system info need to be set.\
226                CSV file is not generated.")
227        sys.exit()
228
229    # Default header
230    l_header = ['test_start', 'test_end', 'subsys', 'test_type',
231                'test_result', 'test_name', 'pse_rel', 'driver',
232                'env', 'proc', 'platform_type', 'test_func_area']
233
234    l_csvlist.append(l_header)
235
236    # Generate CSV file onto the path with current time stamp
237    l_base_dir = csv_dir_path
238    l_timestamp = datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S")
239    # Example: 2017-02-20-08-47-22_Witherspoon.csv
240    l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv"
241
242    print("Writing data into csv file:%s" % l_csvfile)
243
244    for testcase in collectDataObj.testData:
245        # Functional Area: Suite Name
246        # Test Name: Test Case Name
247        l_func_area = str(testcase.parent).split(' ', 1)[1]
248        l_test_name = str(testcase)
249
250        # Test Result pass=0 fail=1
251        if testcase.status == 'PASS':
252            l_test_result = 0
253        else:
254            l_test_result = 1
255
256        # Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S"
257        l_stime = xml_to_csv_time(testcase.starttime)
258        l_etime = xml_to_csv_time(testcase.endtime)
259        # Data Sequence: test_start,test_end,subsys,test_type,
260        #                test_result,test_name,pse_rel,driver,
261        #                env,proc,platform_type,test_func_area,
262        l_data = [l_stime, l_etime, l_subsys, l_test_type, l_test_result,
263                  l_test_name, l_pse_rel, l_driver, l_env, l_proc,
264                  l_platform_type, l_func_area]
265        l_csvlist.append(l_data)
266
267    # Open the file and write to the CSV file
268    l_file = open(l_csvfile, "w")
269    l_writer = csv.writer(l_file, lineterminator='\n')
270    l_writer.writerows(l_csvlist)
271    l_file.close()
272    # Set file permissions 666.
273    perm = stat.S_IRUSR + stat.S_IWUSR + stat.S_IRGRP + stat.S_IWGRP + stat.S_IROTH + stat.S_IWOTH
274    os.chmod(l_csvfile, perm)
275
276
277def xml_to_csv_time(xml_datetime):
278    r"""
279    Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format
280    and return it.
281
282    Description of argument(s):
283    datetime                        The date in the following format: %Y%m%d
284                                    %H:%M:%S.%f (This is the format
285                                    typically found in an XML file.)
286
287    The date returned will be in the following format: %Y-%m-%d-%H-%M-%S
288    """
289
290    # 20170206 05:05:19.342
291    l_str = datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f")
292    # 2017-02-06-05-05-19
293    l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S")
294    return str(l_str)
295
296
297def get_system_details(xml_file_path):
298    r"""
299    Get the system data from output.xml generated by robot and return it.
300    The list returned will be in the following order: [driver,platform]
301
302    Description of argument(s):
303    xml_file_path                   The relative or absolute path to the
304                                    output.xml file.
305    """
306
307    bmc_version_id = ""
308    bmc_platform = ""
309    with open(xml_file_path, 'rt') as output:
310        tree = ElementTree.parse(output)
311
312    for node in tree.iter('msg'):
313        # /etc/os-release output is logged in the XML as msg
314        # Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79"
315        if '${output} = VERSION_ID=' in node.text:
316            # Get BMC version (e.g. v1.99.1-96-g2a46570)
317            bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1]
318
319        # Platform is logged in the XML as msg.
320        # Example: ${bmc_model} = Witherspoon BMC
321        if '${bmc_model} = ' in node.text:
322            bmc_platform = node.text.split(" = ")[1]
323
324    print_vars(bmc_version_id, bmc_platform)
325    return [str(bmc_version_id), str(bmc_platform)]
326
327
328def main():
329
330    if not gen_get_options(parser, stock_list):
331        return False
332
333    if not validate_parms():
334        return False
335
336    qprint_pgm_header()
337
338    parse_output_xml(source, dest, version_id, platform, level,
339                     test_phase, processor)
340
341    return True
342
343
344# Main
345
346if not main():
347    exit(1)
348