1#!/usr/bin/env python3
2
3r"""
4Use robot framework API to extract test result data from output.xml generated
5by robot tests. For more information on the Robot Framework API, see
6http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html
7"""
8
9from robot.api import ExecutionResult
10from robot.result.visitor import ResultVisitor
11from xml.etree import ElementTree
12
13import sys
14import os
15import getopt
16import csv
17import robot.errors
18import re
19import stat
20import datetime
21
22# Remove the python library path to restore with local project path later.
23save_path_0 = sys.path[0]
24del sys.path[0]
25sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib"))
26
27from gen_arg import *    # NOQA
28from gen_print import *  # NOQA
29from gen_valid import *  # NOQA
30
31# Restore sys.path[0].
32sys.path.insert(0, save_path_0)
33
34
35this_program = sys.argv[0]
36info = " For more information:  " + this_program + '  -h'
37if len(sys.argv) == 1:
38    print(info)
39    sys.exit(1)
40
41
42parser = argparse.ArgumentParser(
43    usage=info,
44    description="%(prog)s uses a robot framework API to extract test result\
45    data from output.xml generated by robot tests. For more information on the\
46    Robot Framework API, see\
47    http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html",
48    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
49    prefix_chars='-+')
50
51parser.add_argument(
52    '--source',
53    '-s',
54    help='The output.xml robot test result file path.  This parameter is \
55          required.')
56
57parser.add_argument(
58    '--dest',
59    '-d',
60    help='The directory path where the generated .csv files will go.  This \
61          parameter is required.')
62
63parser.add_argument(
64    '--version_id',
65    help='Driver version of openbmc firmware which was used during test,\
66          e.g. "v2.1-215-g6e7eacb".  This parameter is required.')
67
68parser.add_argument(
69    '--platform',
70    help='OpenBMC platform which was used during test,\
71          e.g. "Witherspoon".  This parameter is required.')
72
73parser.add_argument(
74    '--level',
75    help='OpenBMC release level which was used during test,\
76          e.g. "Master", "OBMC920".  This parameter is required.')
77
78parser.add_argument(
79    '--test_phase',
80    help='Name of testing phase, e.g. "DVT", "SVT", etc.\
81          This parameter is optional.',
82    default="FVT")
83
84parser.add_argument(
85    '--subsystem',
86    help='Name of the subsystem, e.g. "OPENBMC" etc.\
87          This parameter is optional.',
88    default="OPENBMC")
89
90parser.add_argument(
91    '--processor',
92    help='Name of processor, e.g. "P9". This parameter is optional.',
93    default="OPENPOWER")
94
95
96# Populate stock_list with options we want.
97stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)]
98
99
100def exit_function(signal_number=0,
101                  frame=None):
102    r"""
103    Execute whenever the program ends normally or with the signals that we
104    catch (i.e. TERM, INT).
105    """
106
107    dprint_executing()
108
109    dprint_var(signal_number)
110
111    qprint_pgm_footer()
112
113
114def signal_handler(signal_number,
115                   frame):
116    r"""
117    Handle signals.  Without a function to catch a SIGTERM or SIGINT, the
118    program would terminate immediately with return code 143 and without
119    calling the exit_function.
120    """
121
122    # Our convention is to set up exit_function with atexit.register() so
123    # there is no need to explicitly call exit_function from here.
124
125    dprint_executing()
126
127    # Calling exit prevents us from returning to the code that was running
128    # when the signal was received.
129    exit(0)
130
131
132def validate_parms():
133    r"""
134    Validate program parameters, etc.  Return True or False (i.e. pass/fail)
135    accordingly.
136    """
137
138    if not valid_file_path(source):
139        return False
140
141    if not valid_dir_path(dest):
142        return False
143
144    gen_post_validation(exit_function, signal_handler)
145
146    return True
147
148
149def parse_output_xml(xml_file_path, csv_dir_path, version_id, platform, level,
150                     test_phase, processor):
151    r"""
152    Parse the robot-generated output.xml file and extract various test
153    output data. Put the extracted information into a csv file in the "dest"
154    folder.
155
156    Description of argument(s):
157    xml_file_path                   The path to a Robot-generated output.xml
158                                    file.
159    csv_dir_path                    The path to the directory that is to
160                                    contain the .csv files generated by
161                                    this function.
162    version_id                      Version of the openbmc firmware
163                                    (e.g. "v2.1-215-g6e7eacb").
164    platform                        Platform of the openbmc system.
165    level                           Release level of the OpenBMC system
166                                    (e.g. "Master").
167    """
168
169    # Initialize tallies
170    total_critical_tc = 0
171    total_critical_passed = 0
172    total_critical_failed = 0
173    total_non_critical_tc = 0
174    total_non_critical_passed = 0
175    total_non_critical_failed = 0
176
177    result = ExecutionResult(xml_file_path)
178    result.configure(stat_config={'suite_stat_level': 2,
179                                  'tag_stat_combine': 'tagANDanother'})
180
181    stats = result.statistics
182    print("--------------------------------------")
183    try:
184        total_critical_tc = stats.total.critical.passed + stats.total.critical.failed
185        total_critical_passed = stats.total.critical.passed
186        total_critical_failed = stats.total.critical.failed
187    except AttributeError:
188        pass
189
190    try:
191        total_non_critical_tc = stats.total.passed + stats.total.failed
192        total_non_critical_passed = stats.total.passed
193        total_non_critical_failed = stats.total.failed
194    except AttributeError:
195        pass
196
197    print("Total Test Count:\t %d" % (total_non_critical_tc + total_critical_tc))
198
199    print("Total Critical Test Failed:\t %d" % total_critical_failed)
200    print("Total Critical Test Passed:\t %d" % total_critical_passed)
201    print("Total Non-Critical Test Failed:\t %d" % total_non_critical_failed)
202    print("Total Non-Critical Test Passed:\t %d" % total_non_critical_passed)
203    print("Test Start Time:\t %s" % result.suite.starttime)
204    print("Test End Time:\t\t %s" % result.suite.endtime)
205    print("--------------------------------------")
206
207    # Use ResultVisitor object and save off the test data info
208    class TestResult(ResultVisitor):
209        def __init__(self):
210            self.testData = []
211
212        def visit_test(self, test):
213            self.testData += [test]
214
215    collectDataObj = TestResult()
216    result.visit(collectDataObj)
217
218    # Write the result statistics attributes to CSV file
219    l_csvlist = []
220
221    # Default Test data
222    l_test_type = test_phase
223
224    l_pse_rel = 'Master'
225    if level:
226        l_pse_rel = level
227
228    l_env = 'HW'
229    l_proc = processor
230    l_platform_type = ""
231    l_func_area = ""
232
233    # System data from XML meta data
234    # l_system_info = get_system_details(xml_file_path)
235
236    # First let us try to collect information from keyboard input
237    # If keyboard input cannot give both information, then find from xml file.
238    if version_id and platform:
239        l_driver = version_id
240        l_platform_type = platform
241        print("BMC Version_id:%s" % version_id)
242        print("BMC Platform:%s" % platform)
243    else:
244        # System data from XML meta data
245        l_system_info = get_system_details(xml_file_path)
246        l_driver = l_system_info[0]
247        l_platform_type = l_system_info[1]
248
249    # Driver version id and platform are mandatorily required for CSV file
250    # generation. If any one is not avaulable, exit CSV file generation
251    # process.
252    if l_driver and l_platform_type:
253        print("Driver and system info set.")
254    else:
255        print("Both driver and system info need to be set.\
256                CSV file is not generated.")
257        sys.exit()
258
259    # Default header
260    l_header = ['test_start', 'test_end', 'subsys', 'test_type',
261                'test_result', 'test_name', 'pse_rel', 'driver',
262                'env', 'proc', 'platform_type', 'test_func_area']
263
264    l_csvlist.append(l_header)
265
266    # Generate CSV file onto the path with current time stamp
267    l_base_dir = csv_dir_path
268    l_timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S")
269    # Example: 2017-02-20-08-47-22_Witherspoon.csv
270    l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv"
271
272    print("Writing data into csv file:%s" % l_csvfile)
273
274    for testcase in collectDataObj.testData:
275        # Functional Area: Suite Name
276        # Test Name: Test Case Name
277        l_func_area = str(testcase.parent).split(' ', 1)[1]
278        l_test_name = str(testcase)
279
280        # Test Result pass=0 fail=1
281        if testcase.status == 'PASS':
282            l_test_result = 0
283        else:
284            l_test_result = 1
285
286        # Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S"
287        l_stime = xml_to_csv_time(testcase.starttime)
288        l_etime = xml_to_csv_time(testcase.endtime)
289        # Data Sequence: test_start,test_end,subsys,test_type,
290        #                test_result,test_name,pse_rel,driver,
291        #                env,proc,platform_type,test_func_area,
292        l_data = [l_stime, l_etime, subsystem, l_test_type, l_test_result,
293                  l_test_name, l_pse_rel, l_driver, l_env, l_proc,
294                  l_platform_type, l_func_area]
295        l_csvlist.append(l_data)
296
297    # Open the file and write to the CSV file
298    l_file = open(l_csvfile, "w")
299    l_writer = csv.writer(l_file, lineterminator='\n')
300    l_writer.writerows(l_csvlist)
301    l_file.close()
302    # Set file permissions 666.
303    perm = stat.S_IRUSR + stat.S_IWUSR + stat.S_IRGRP + stat.S_IWGRP + stat.S_IROTH + stat.S_IWOTH
304    os.chmod(l_csvfile, perm)
305
306
307def xml_to_csv_time(xml_datetime):
308    r"""
309    Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format
310    and return it.
311
312    Description of argument(s):
313    datetime                        The date in the following format: %Y%m%d
314                                    %H:%M:%S.%f (This is the format
315                                    typically found in an XML file.)
316
317    The date returned will be in the following format: %Y-%m-%d-%H-%M-%S
318    """
319
320    # 20170206 05:05:19.342
321    l_str = datetime.datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f")
322    # 2017-02-06-05-05-19
323    l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S")
324    return str(l_str)
325
326
327def get_system_details(xml_file_path):
328    r"""
329    Get the system data from output.xml generated by robot and return it.
330    The list returned will be in the following order: [driver,platform]
331
332    Description of argument(s):
333    xml_file_path                   The relative or absolute path to the
334                                    output.xml file.
335    """
336
337    bmc_version_id = ""
338    bmc_platform = ""
339    with open(xml_file_path, 'rt') as output:
340        tree = ElementTree.parse(output)
341
342    for node in tree.iter('msg'):
343        # /etc/os-release output is logged in the XML as msg
344        # Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79"
345        if '${output} = VERSION_ID=' in node.text:
346            # Get BMC version (e.g. v1.99.1-96-g2a46570)
347            bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1]
348
349        # Platform is logged in the XML as msg.
350        # Example: ${bmc_model} = Witherspoon BMC
351        if '${bmc_model} = ' in node.text:
352            bmc_platform = node.text.split(" = ")[1]
353
354    print_vars(bmc_version_id, bmc_platform)
355    return [str(bmc_version_id), str(bmc_platform)]
356
357
358def main():
359
360    if not gen_get_options(parser, stock_list):
361        return False
362
363    if not validate_parms():
364        return False
365
366    qprint_pgm_header()
367
368    parse_output_xml(source, dest, version_id, platform, level,
369                     test_phase, processor)
370
371    return True
372
373
374# Main
375
376if not main():
377    exit(1)
378