1#!/usr/bin/env python
2
3r"""
4Use robot framework API to extract test result data from output.xml generated
5by robot tests. For more information on the Robot Framework API, see
6http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html
7"""
8
9import sys
10import os
11import getopt
12import csv
13import robot.errors
14import re
15import stat
16import datetime
17from robot.api import ExecutionResult
18from robot.result.visitor import ResultVisitor
19from xml.etree import ElementTree
20
21# Remove the python library path to restore with local project path later.
22save_path_0 = sys.path[0]
23del sys.path[0]
24sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib"))
25
26from gen_arg import *
27from gen_print import *
28from gen_valid import *
29
30# Restore sys.path[0].
31sys.path.insert(0, save_path_0)
32
33
34this_program = sys.argv[0]
35info = " For more information:  " + this_program + '  -h'
36if len(sys.argv) == 1:
37    print(info)
38    sys.exit(1)
39
40
41parser = argparse.ArgumentParser(
42    usage=info,
43    description="%(prog)s uses a robot framework API to extract test result\
44    data from output.xml generated by robot tests. For more information on the\
45    Robot Framework API, see\
46    http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html",
47    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
48    prefix_chars='-+')
49
50parser.add_argument(
51    '--source',
52    '-s',
53    help='The output.xml robot test result file path.  This parameter is \
54          required.')
55
56parser.add_argument(
57    '--dest',
58    '-d',
59    help='The directory path where the generated .csv files will go.  This \
60          parameter is required.')
61
62parser.add_argument(
63    '--version_id',
64    help='Driver version of openbmc firmware which was used during test,\
65          e.g. "v2.1-215-g6e7eacb".  This parameter is required.')
66
67parser.add_argument(
68    '--platform',
69    help='OpenBMC platform which was used during test,\
70          e.g. "Witherspoon".  This parameter is required.')
71
72parser.add_argument(
73    '--level',
74    help='OpenBMC release level which was used during test,\
75          e.g. "Master", "OBMC920".  This parameter is required.')
76
77parser.add_argument(
78    '--test_phase',
79    help='Name of testing phase, e.g. "DVT", "SVT", etc.\
80          This parameter is optional.',
81    default="FVT")
82
83parser.add_argument(
84    '--subsystem',
85    help='Name of the subsystem, e.g. "OPENBMC" etc.\
86          This parameter is optional.',
87    default="OPENBMC")
88
89parser.add_argument(
90    '--processor',
91    help='Name of processor, e.g. "P9". This parameter is optional.',
92    default="OPENPOWER")
93
94
95# Populate stock_list with options we want.
96stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)]
97
98
99def exit_function(signal_number=0,
100                  frame=None):
101    r"""
102    Execute whenever the program ends normally or with the signals that we
103    catch (i.e. TERM, INT).
104    """
105
106    dprint_executing()
107
108    dprint_var(signal_number)
109
110    qprint_pgm_footer()
111
112
113def signal_handler(signal_number,
114                   frame):
115    r"""
116    Handle signals.  Without a function to catch a SIGTERM or SIGINT, the
117    program would terminate immediately with return code 143 and without
118    calling the exit_function.
119    """
120
121    # Our convention is to set up exit_function with atexit.register() so
122    # there is no need to explicitly call exit_function from here.
123
124    dprint_executing()
125
126    # Calling exit prevents us from returning to the code that was running
127    # when the signal was received.
128    exit(0)
129
130
131def validate_parms():
132    r"""
133    Validate program parameters, etc.  Return True or False (i.e. pass/fail)
134    accordingly.
135    """
136
137    if not valid_file_path(source):
138        return False
139
140    if not valid_dir_path(dest):
141        return False
142
143    gen_post_validation(exit_function, signal_handler)
144
145    return True
146
147
148def parse_output_xml(xml_file_path, csv_dir_path, version_id, platform, level,
149                     test_phase, processor):
150    r"""
151    Parse the robot-generated output.xml file and extract various test
152    output data. Put the extracted information into a csv file in the "dest"
153    folder.
154
155    Description of argument(s):
156    xml_file_path                   The path to a Robot-generated output.xml
157                                    file.
158    csv_dir_path                    The path to the directory that is to
159                                    contain the .csv files generated by
160                                    this function.
161    version_id                      Version of the openbmc firmware
162                                    (e.g. "v2.1-215-g6e7eacb").
163    platform                        Platform of the openbmc system.
164    level                           Release level of the OpenBMC system
165                                    (e.g. "Master").
166    """
167
168    result = ExecutionResult(xml_file_path)
169    result.configure(stat_config={'suite_stat_level': 2,
170                                  'tag_stat_combine': 'tagANDanother'})
171
172    stats = result.statistics
173    print("--------------------------------------")
174    total_tc = stats.total.critical.passed + stats.total.critical.failed
175    print("Total Test Count:\t %d" % total_tc)
176    print("Total Test Failed:\t %d" % stats.total.critical.failed)
177    print("Total Test Passed:\t %d" % stats.total.critical.passed)
178    print("Test Start Time:\t %s" % result.suite.starttime)
179    print("Test End Time:\t\t %s" % result.suite.endtime)
180    print("--------------------------------------")
181
182    # Use ResultVisitor object and save off the test data info
183    class TestResult(ResultVisitor):
184        def __init__(self):
185            self.testData = []
186
187        def visit_test(self, test):
188            self.testData += [test]
189
190    collectDataObj = TestResult()
191    result.visit(collectDataObj)
192
193    # Write the result statistics attributes to CSV file
194    l_csvlist = []
195
196    # Default Test data
197    l_test_type = test_phase
198
199    l_pse_rel = 'Master'
200    if level:
201        l_pse_rel = level
202
203    l_env = 'HW'
204    l_proc = processor
205    l_platform_type = ""
206    l_func_area = ""
207
208    # System data from XML meta data
209    # l_system_info = get_system_details(xml_file_path)
210
211    # First let us try to collect information from keyboard input
212    # If keyboard input cannot give both information, then find from xml file.
213    if version_id and platform:
214        l_driver = version_id
215        l_platform_type = platform
216        print("BMC Version_id:%s" % version_id)
217        print("BMC Platform:%s" % platform)
218    else:
219        # System data from XML meta data
220        l_system_info = get_system_details(xml_file_path)
221        l_driver = l_system_info[0]
222        l_platform_type = l_system_info[1]
223
224    # Driver version id and platform are mandatorily required for CSV file
225    # generation. If any one is not avaulable, exit CSV file generation
226    # process.
227    if l_driver and l_platform_type:
228        print("Driver and system info set.")
229    else:
230        print("Both driver and system info need to be set.\
231                CSV file is not generated.")
232        sys.exit()
233
234    # Default header
235    l_header = ['test_start', 'test_end', 'subsys', 'test_type',
236                'test_result', 'test_name', 'pse_rel', 'driver',
237                'env', 'proc', 'platform_type', 'test_func_area']
238
239    l_csvlist.append(l_header)
240
241    # Generate CSV file onto the path with current time stamp
242    l_base_dir = csv_dir_path
243    l_timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S")
244    # Example: 2017-02-20-08-47-22_Witherspoon.csv
245    l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv"
246
247    print("Writing data into csv file:%s" % l_csvfile)
248
249    for testcase in collectDataObj.testData:
250        # Functional Area: Suite Name
251        # Test Name: Test Case Name
252        l_func_area = str(testcase.parent).split(' ', 1)[1]
253        l_test_name = str(testcase)
254
255        # Test Result pass=0 fail=1
256        if testcase.status == 'PASS':
257            l_test_result = 0
258        else:
259            l_test_result = 1
260
261        # Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S"
262        l_stime = xml_to_csv_time(testcase.starttime)
263        l_etime = xml_to_csv_time(testcase.endtime)
264        # Data Sequence: test_start,test_end,subsys,test_type,
265        #                test_result,test_name,pse_rel,driver,
266        #                env,proc,platform_type,test_func_area,
267        l_data = [l_stime, l_etime, subsystem, l_test_type, l_test_result,
268                  l_test_name, l_pse_rel, l_driver, l_env, l_proc,
269                  l_platform_type, l_func_area]
270        l_csvlist.append(l_data)
271
272    # Open the file and write to the CSV file
273    l_file = open(l_csvfile, "w")
274    l_writer = csv.writer(l_file, lineterminator='\n')
275    l_writer.writerows(l_csvlist)
276    l_file.close()
277    # Set file permissions 666.
278    perm = stat.S_IRUSR + stat.S_IWUSR + stat.S_IRGRP + stat.S_IWGRP + stat.S_IROTH + stat.S_IWOTH
279    os.chmod(l_csvfile, perm)
280
281
282def xml_to_csv_time(xml_datetime):
283    r"""
284    Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format
285    and return it.
286
287    Description of argument(s):
288    datetime                        The date in the following format: %Y%m%d
289                                    %H:%M:%S.%f (This is the format
290                                    typically found in an XML file.)
291
292    The date returned will be in the following format: %Y-%m-%d-%H-%M-%S
293    """
294
295    # 20170206 05:05:19.342
296    l_str = datetime.datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f")
297    # 2017-02-06-05-05-19
298    l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S")
299    return str(l_str)
300
301
302def get_system_details(xml_file_path):
303    r"""
304    Get the system data from output.xml generated by robot and return it.
305    The list returned will be in the following order: [driver,platform]
306
307    Description of argument(s):
308    xml_file_path                   The relative or absolute path to the
309                                    output.xml file.
310    """
311
312    bmc_version_id = ""
313    bmc_platform = ""
314    with open(xml_file_path, 'rt') as output:
315        tree = ElementTree.parse(output)
316
317    for node in tree.iter('msg'):
318        # /etc/os-release output is logged in the XML as msg
319        # Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79"
320        if '${output} = VERSION_ID=' in node.text:
321            # Get BMC version (e.g. v1.99.1-96-g2a46570)
322            bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1]
323
324        # Platform is logged in the XML as msg.
325        # Example: ${bmc_model} = Witherspoon BMC
326        if '${bmc_model} = ' in node.text:
327            bmc_platform = node.text.split(" = ")[1]
328
329    print_vars(bmc_version_id, bmc_platform)
330    return [str(bmc_version_id), str(bmc_platform)]
331
332
333def main():
334
335    if not gen_get_options(parser, stock_list):
336        return False
337
338    if not validate_parms():
339        return False
340
341    qprint_pgm_header()
342
343    parse_output_xml(source, dest, version_id, platform, level,
344                     test_phase, processor)
345
346    return True
347
348
349# Main
350
351if not main():
352    exit(1)
353