1#!/usr/bin/env python
2
3r"""
4Use robot framework API to extract test result data from output.xml generated
5by robot tests. For more information on the Robot Framework API, see
6http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html
7"""
8
9import sys
10import os
11import getopt
12import csv
13import robot.errors
14import re
15from datetime import datetime
16from robot.api import ExecutionResult
17from robot.result.visitor import ResultVisitor
18from xml.etree import ElementTree
19
20# Remove the python library path to restore with local project path later.
21save_path_0 = sys.path[0]
22del sys.path[0]
23sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib"))
24
25from gen_arg import *
26from gen_print import *
27from gen_valid import *
28
29# Restore sys.path[0].
30sys.path.insert(0, save_path_0)
31
32
33this_program = sys.argv[0]
34info = " For more information:  " + this_program + '  -h'
35if len(sys.argv) == 1:
36    print (info)
37    sys.exit(1)
38
39
40parser = argparse.ArgumentParser(
41    usage=info,
42    description="%(prog)s uses a robot framework API to extract test result\
43    data from output.xml generated by robot tests. For more information on the\
44    Robot Framework API, see\
45    http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html",
46    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
47    prefix_chars='-+')
48
49parser.add_argument(
50    '--source',
51    '-s',
52    help='The output.xml robot test result file path.  This parameter is \
53          required.')
54
55parser.add_argument(
56    '--dest',
57    '-d',
58    help='The directory path where the generated .csv files will go.  This \
59          parameter is required.')
60
61parser.add_argument(
62    '--version_id',
63    help='Driver version of openbmc firmware which was used during test,\
64          e.g. "v2.1-215-g6e7eacb".  This parameter is required.')
65
66parser.add_argument(
67    '--platform',
68    help='OpenBMC platform which was used during test,\
69          e.g. "Witherspoon".  This parameter is required.')
70
71parser.add_argument(
72    '--level',
73    help='OpenBMC release level which was used during test,\
74          e.g. "Master", "OBMC920".  This parameter is required.')
75
76parser.add_argument(
77    '--test_phase',
78    help='Name of testing phase, e.g. "DVT", "SVT", etc.\
79          This parameter is optional.',
80    default="FVT")
81
82parser.add_argument(
83    '--processor',
84    help='Name of processor, e.g. "P9". This parameter is optional.',
85    default="OPENPOWER")
86
87
88# Populate stock_list with options we want.
89stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)]
90
91
92def exit_function(signal_number=0,
93                  frame=None):
94    r"""
95    Execute whenever the program ends normally or with the signals that we
96    catch (i.e. TERM, INT).
97    """
98
99    dprint_executing()
100
101    dprint_var(signal_number)
102
103    qprint_pgm_footer()
104
105
106def signal_handler(signal_number,
107                   frame):
108    r"""
109    Handle signals.  Without a function to catch a SIGTERM or SIGINT, the
110    program would terminate immediately with return code 143 and without
111    calling the exit_function.
112    """
113
114    # Our convention is to set up exit_function with atexit.register() so
115    # there is no need to explicitly call exit_function from here.
116
117    dprint_executing()
118
119    # Calling exit prevents us from returning to the code that was running
120    # when the signal was received.
121    exit(0)
122
123
124def validate_parms():
125    r"""
126    Validate program parameters, etc.  Return True or False (i.e. pass/fail)
127    accordingly.
128    """
129
130    if not valid_file_path(source):
131        return False
132
133    if not valid_dir_path(dest):
134        return False
135
136    gen_post_validation(exit_function, signal_handler)
137
138    return True
139
140
141def parse_output_xml(xml_file_path, csv_dir_path, version_id, platform, level,
142                     test_phase, processor):
143    r"""
144    Parse the robot-generated output.xml file and extract various test
145    output data. Put the extracted information into a csv file in the "dest"
146    folder.
147
148    Description of argument(s):
149    xml_file_path                   The path to a Robot-generated output.xml
150                                    file.
151    csv_dir_path                    The path to the directory that is to
152                                    contain the .csv files generated by
153                                    this function.
154    version_id                      Version of the openbmc firmware
155                                    (e.g. "v2.1-215-g6e7eacb").
156    platform                        Platform of the openbmc system.
157    level                           Release level of the OpenBMC system
158                                    (e.g. "Master").
159    """
160
161    result = ExecutionResult(xml_file_path)
162    result.configure(stat_config={'suite_stat_level': 2,
163                                  'tag_stat_combine': 'tagANDanother'})
164
165    stats = result.statistics
166    print("--------------------------------------")
167    total_tc = stats.total.critical.passed + stats.total.critical.failed
168    print("Total Test Count:\t %d" % total_tc)
169    print("Total Test Failed:\t %d" % stats.total.critical.failed)
170    print("Total Test Passed:\t %d" % stats.total.critical.passed)
171    print("Test Start Time:\t %s" % result.suite.starttime)
172    print("Test End Time:\t\t %s" % result.suite.endtime)
173    print("--------------------------------------")
174
175    # Use ResultVisitor object and save off the test data info
176    class TestResult(ResultVisitor):
177        def __init__(self):
178            self.testData = []
179
180        def visit_test(self, test):
181            self.testData += [test]
182
183    collectDataObj = TestResult()
184    result.visit(collectDataObj)
185
186    # Write the result statistics attributes to CSV file
187    l_csvlist = []
188
189    # Default Test data
190    l_subsys = 'OPENBMC'
191    l_test_type = test_phase
192
193    l_pse_rel = 'Master'
194    if level:
195        l_pse_rel = level
196
197    l_env = 'HW'
198    l_proc = processor
199    l_platform_type = ""
200    l_func_area = ""
201
202    # System data from XML meta data
203    # l_system_info = get_system_details(xml_file_path)
204
205    # First let us try to collect information from keyboard input
206    # If keyboard input cannot give both information, then find from xml file.
207    if version_id and platform:
208        l_driver = version_id
209        l_platform_type = platform
210        print("BMC Version_id:%s" % version_id)
211        print("BMC Platform:%s" % platform)
212    else:
213        # System data from XML meta data
214        l_system_info = get_system_details(xml_file_path)
215        l_driver = l_system_info[0]
216        l_platform_type = l_system_info[1]
217
218    # Driver version id and platform are mandatorily required for CSV file
219    # generation. If any one is not avaulable, exit CSV file generation
220    # process.
221    if l_driver and l_platform_type:
222        print("Driver and system info set.")
223    else:
224        print("Both driver and system info need to be set.\
225                CSV file is not generated.")
226        sys.exit()
227
228    # Default header
229    l_header = ['test_start', 'test_end', 'subsys', 'test_type',
230                'test_result', 'test_name', 'pse_rel', 'driver',
231                'env', 'proc', 'platform_type', 'test_func_area']
232
233    l_csvlist.append(l_header)
234
235    # Generate CSV file onto the path with current time stamp
236    l_base_dir = csv_dir_path
237    l_timestamp = datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S")
238    # Example: 2017-02-20-08-47-22_Witherspoon.csv
239    l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv"
240
241    print("Writing data into csv file:%s" % l_csvfile)
242
243    for testcase in collectDataObj.testData:
244        # Functional Area: Suite Name
245        # Test Name: Test Case Name
246        l_func_area = str(testcase.parent).split(' ', 1)[1]
247        l_test_name = str(testcase)
248
249        # Test Result pass=0 fail=1
250        if testcase.status == 'PASS':
251            l_test_result = 0
252        else:
253            l_test_result = 1
254
255        # Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S"
256        l_stime = xml_to_csv_time(testcase.starttime)
257        l_etime = xml_to_csv_time(testcase.endtime)
258        # Data Sequence: test_start,test_end,subsys,test_type,
259        #                test_result,test_name,pse_rel,driver,
260        #                env,proc,platform_type,test_func_area,
261        l_data = [l_stime, l_etime, l_subsys, l_test_type, l_test_result,
262                  l_test_name, l_pse_rel, l_driver, l_env, l_proc,
263                  l_platform_type, l_func_area]
264        l_csvlist.append(l_data)
265
266    # Open the file and write to the CSV file
267    l_file = open(l_csvfile, "w")
268    l_writer = csv.writer(l_file, lineterminator='\n')
269    l_writer.writerows(l_csvlist)
270    l_file.close()
271
272
273def xml_to_csv_time(xml_datetime):
274    r"""
275    Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format
276    and return it.
277
278    Description of argument(s):
279    datetime                        The date in the following format: %Y%m%d
280                                    %H:%M:%S.%f (This is the format
281                                    typically found in an XML file.)
282
283    The date returned will be in the following format: %Y-%m-%d-%H-%M-%S
284    """
285
286    # 20170206 05:05:19.342
287    l_str = datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f")
288    # 2017-02-06-05-05-19
289    l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S")
290    return str(l_str)
291
292
293def get_system_details(xml_file_path):
294    r"""
295    Get the system data from output.xml generated by robot and return it.
296    The list returned will be in the following order: [driver,platform]
297
298    Description of argument(s):
299    xml_file_path                   The relative or absolute path to the
300                                    output.xml file.
301    """
302
303    bmc_version_id = ""
304    bmc_platform = ""
305    with open(xml_file_path, 'rt') as output:
306        tree = ElementTree.parse(output)
307
308    for node in tree.iter('msg'):
309        # /etc/os-release output is logged in the XML as msg
310        # Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79"
311        if '${output} = VERSION_ID=' in node.text:
312            # Get BMC version (e.g. v1.99.1-96-g2a46570)
313            bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1]
314
315        # Platform is logged in the XML as msg.
316        # Example: ${bmc_model} = Witherspoon BMC
317        if '${bmc_model} = ' in node.text:
318            bmc_platform = node.text.split(" = ")[1]
319
320    print_vars(bmc_version_id, bmc_platform)
321    return [str(bmc_version_id), str(bmc_platform)]
322
323
324def main():
325
326    if not gen_get_options(parser, stock_list):
327        return False
328
329    if not validate_parms():
330        return False
331
332    qprint_pgm_header()
333
334    parse_output_xml(source, dest, version_id, platform, level,
335                     test_phase, processor)
336
337    return True
338
339
340# Main
341
342if not main():
343    exit(1)
344