1#!/usr/bin/env python3 2 3r""" 4Use robot framework API to extract test result data from output.xml generated 5by robot tests. For more information on the Robot Framework API, see 6http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html 7""" 8 9import sys 10import os 11import getopt 12import csv 13import robot.errors 14import re 15import stat 16import datetime 17from robot.api import ExecutionResult 18from robot.result.visitor import ResultVisitor 19from xml.etree import ElementTree 20 21# Remove the python library path to restore with local project path later. 22save_path_0 = sys.path[0] 23del sys.path[0] 24sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib")) 25 26from gen_arg import * 27from gen_print import * 28from gen_valid import * 29 30# Restore sys.path[0]. 31sys.path.insert(0, save_path_0) 32 33 34this_program = sys.argv[0] 35info = " For more information: " + this_program + ' -h' 36if len(sys.argv) == 1: 37 print(info) 38 sys.exit(1) 39 40 41parser = argparse.ArgumentParser( 42 usage=info, 43 description="%(prog)s uses a robot framework API to extract test result\ 44 data from output.xml generated by robot tests. For more information on the\ 45 Robot Framework API, see\ 46 http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html", 47 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 48 prefix_chars='-+') 49 50parser.add_argument( 51 '--source', 52 '-s', 53 help='The output.xml robot test result file path. This parameter is \ 54 required.') 55 56parser.add_argument( 57 '--dest', 58 '-d', 59 help='The directory path where the generated .csv files will go. This \ 60 parameter is required.') 61 62parser.add_argument( 63 '--version_id', 64 help='Driver version of openbmc firmware which was used during test,\ 65 e.g. "v2.1-215-g6e7eacb". This parameter is required.') 66 67parser.add_argument( 68 '--platform', 69 help='OpenBMC platform which was used during test,\ 70 e.g. "Witherspoon". This parameter is required.') 71 72parser.add_argument( 73 '--level', 74 help='OpenBMC release level which was used during test,\ 75 e.g. "Master", "OBMC920". This parameter is required.') 76 77parser.add_argument( 78 '--test_phase', 79 help='Name of testing phase, e.g. "DVT", "SVT", etc.\ 80 This parameter is optional.', 81 default="FVT") 82 83parser.add_argument( 84 '--subsystem', 85 help='Name of the subsystem, e.g. "OPENBMC" etc.\ 86 This parameter is optional.', 87 default="OPENBMC") 88 89parser.add_argument( 90 '--processor', 91 help='Name of processor, e.g. "P9". This parameter is optional.', 92 default="OPENPOWER") 93 94 95# Populate stock_list with options we want. 96stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)] 97 98 99def exit_function(signal_number=0, 100 frame=None): 101 r""" 102 Execute whenever the program ends normally or with the signals that we 103 catch (i.e. TERM, INT). 104 """ 105 106 dprint_executing() 107 108 dprint_var(signal_number) 109 110 qprint_pgm_footer() 111 112 113def signal_handler(signal_number, 114 frame): 115 r""" 116 Handle signals. Without a function to catch a SIGTERM or SIGINT, the 117 program would terminate immediately with return code 143 and without 118 calling the exit_function. 119 """ 120 121 # Our convention is to set up exit_function with atexit.register() so 122 # there is no need to explicitly call exit_function from here. 123 124 dprint_executing() 125 126 # Calling exit prevents us from returning to the code that was running 127 # when the signal was received. 128 exit(0) 129 130 131def validate_parms(): 132 r""" 133 Validate program parameters, etc. Return True or False (i.e. pass/fail) 134 accordingly. 135 """ 136 137 if not valid_file_path(source): 138 return False 139 140 if not valid_dir_path(dest): 141 return False 142 143 gen_post_validation(exit_function, signal_handler) 144 145 return True 146 147 148def parse_output_xml(xml_file_path, csv_dir_path, version_id, platform, level, 149 test_phase, processor): 150 r""" 151 Parse the robot-generated output.xml file and extract various test 152 output data. Put the extracted information into a csv file in the "dest" 153 folder. 154 155 Description of argument(s): 156 xml_file_path The path to a Robot-generated output.xml 157 file. 158 csv_dir_path The path to the directory that is to 159 contain the .csv files generated by 160 this function. 161 version_id Version of the openbmc firmware 162 (e.g. "v2.1-215-g6e7eacb"). 163 platform Platform of the openbmc system. 164 level Release level of the OpenBMC system 165 (e.g. "Master"). 166 """ 167 168 # Initialize tallies 169 total_critical_tc = 0 170 total_critical_passed = 0 171 total_critical_failed = 0 172 total_non_critical_tc = 0 173 total_non_critical_passed = 0 174 total_non_critical_failed = 0 175 176 result = ExecutionResult(xml_file_path) 177 result.configure(stat_config={'suite_stat_level': 2, 178 'tag_stat_combine': 'tagANDanother'}) 179 180 stats = result.statistics 181 print("--------------------------------------") 182 try: 183 total_critical_tc = stats.total.critical.passed + stats.total.critical.failed 184 total_critical_passed = stats.total.critical.passed 185 total_critical_failed = stats.total.critical.failed 186 except AttributeError: 187 pass 188 189 try: 190 total_non_critical_tc = stats.total.passed + stats.total.failed 191 total_non_critical_passed = stats.total.passed 192 total_non_critical_failed = stats.total.failed 193 except AttributeError: 194 pass 195 196 print("Total Test Count:\t %d" % (total_non_critical_tc + total_critical_tc)) 197 198 print("Total Critical Test Failed:\t %d" % total_critical_failed) 199 print("Total Critical Test Passed:\t %d" % total_critical_passed) 200 print("Total Non-Critical Test Failed:\t %d" % total_non_critical_failed) 201 print("Total Non-Critical Test Passed:\t %d" % total_non_critical_passed) 202 print("Test Start Time:\t %s" % result.suite.starttime) 203 print("Test End Time:\t\t %s" % result.suite.endtime) 204 print("--------------------------------------") 205 206 # Use ResultVisitor object and save off the test data info 207 class TestResult(ResultVisitor): 208 def __init__(self): 209 self.testData = [] 210 211 def visit_test(self, test): 212 self.testData += [test] 213 214 collectDataObj = TestResult() 215 result.visit(collectDataObj) 216 217 # Write the result statistics attributes to CSV file 218 l_csvlist = [] 219 220 # Default Test data 221 l_test_type = test_phase 222 223 l_pse_rel = 'Master' 224 if level: 225 l_pse_rel = level 226 227 l_env = 'HW' 228 l_proc = processor 229 l_platform_type = "" 230 l_func_area = "" 231 232 # System data from XML meta data 233 # l_system_info = get_system_details(xml_file_path) 234 235 # First let us try to collect information from keyboard input 236 # If keyboard input cannot give both information, then find from xml file. 237 if version_id and platform: 238 l_driver = version_id 239 l_platform_type = platform 240 print("BMC Version_id:%s" % version_id) 241 print("BMC Platform:%s" % platform) 242 else: 243 # System data from XML meta data 244 l_system_info = get_system_details(xml_file_path) 245 l_driver = l_system_info[0] 246 l_platform_type = l_system_info[1] 247 248 # Driver version id and platform are mandatorily required for CSV file 249 # generation. If any one is not avaulable, exit CSV file generation 250 # process. 251 if l_driver and l_platform_type: 252 print("Driver and system info set.") 253 else: 254 print("Both driver and system info need to be set.\ 255 CSV file is not generated.") 256 sys.exit() 257 258 # Default header 259 l_header = ['test_start', 'test_end', 'subsys', 'test_type', 260 'test_result', 'test_name', 'pse_rel', 'driver', 261 'env', 'proc', 'platform_type', 'test_func_area'] 262 263 l_csvlist.append(l_header) 264 265 # Generate CSV file onto the path with current time stamp 266 l_base_dir = csv_dir_path 267 l_timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S") 268 # Example: 2017-02-20-08-47-22_Witherspoon.csv 269 l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv" 270 271 print("Writing data into csv file:%s" % l_csvfile) 272 273 for testcase in collectDataObj.testData: 274 # Functional Area: Suite Name 275 # Test Name: Test Case Name 276 l_func_area = str(testcase.parent).split(' ', 1)[1] 277 l_test_name = str(testcase) 278 279 # Test Result pass=0 fail=1 280 if testcase.status == 'PASS': 281 l_test_result = 0 282 else: 283 l_test_result = 1 284 285 # Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S" 286 l_stime = xml_to_csv_time(testcase.starttime) 287 l_etime = xml_to_csv_time(testcase.endtime) 288 # Data Sequence: test_start,test_end,subsys,test_type, 289 # test_result,test_name,pse_rel,driver, 290 # env,proc,platform_type,test_func_area, 291 l_data = [l_stime, l_etime, subsystem, l_test_type, l_test_result, 292 l_test_name, l_pse_rel, l_driver, l_env, l_proc, 293 l_platform_type, l_func_area] 294 l_csvlist.append(l_data) 295 296 # Open the file and write to the CSV file 297 l_file = open(l_csvfile, "w") 298 l_writer = csv.writer(l_file, lineterminator='\n') 299 l_writer.writerows(l_csvlist) 300 l_file.close() 301 # Set file permissions 666. 302 perm = stat.S_IRUSR + stat.S_IWUSR + stat.S_IRGRP + stat.S_IWGRP + stat.S_IROTH + stat.S_IWOTH 303 os.chmod(l_csvfile, perm) 304 305 306def xml_to_csv_time(xml_datetime): 307 r""" 308 Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format 309 and return it. 310 311 Description of argument(s): 312 datetime The date in the following format: %Y%m%d 313 %H:%M:%S.%f (This is the format 314 typically found in an XML file.) 315 316 The date returned will be in the following format: %Y-%m-%d-%H-%M-%S 317 """ 318 319 # 20170206 05:05:19.342 320 l_str = datetime.datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f") 321 # 2017-02-06-05-05-19 322 l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S") 323 return str(l_str) 324 325 326def get_system_details(xml_file_path): 327 r""" 328 Get the system data from output.xml generated by robot and return it. 329 The list returned will be in the following order: [driver,platform] 330 331 Description of argument(s): 332 xml_file_path The relative or absolute path to the 333 output.xml file. 334 """ 335 336 bmc_version_id = "" 337 bmc_platform = "" 338 with open(xml_file_path, 'rt') as output: 339 tree = ElementTree.parse(output) 340 341 for node in tree.iter('msg'): 342 # /etc/os-release output is logged in the XML as msg 343 # Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79" 344 if '${output} = VERSION_ID=' in node.text: 345 # Get BMC version (e.g. v1.99.1-96-g2a46570) 346 bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1] 347 348 # Platform is logged in the XML as msg. 349 # Example: ${bmc_model} = Witherspoon BMC 350 if '${bmc_model} = ' in node.text: 351 bmc_platform = node.text.split(" = ")[1] 352 353 print_vars(bmc_version_id, bmc_platform) 354 return [str(bmc_version_id), str(bmc_platform)] 355 356 357def main(): 358 359 if not gen_get_options(parser, stock_list): 360 return False 361 362 if not validate_parms(): 363 return False 364 365 qprint_pgm_header() 366 367 parse_output_xml(source, dest, version_id, platform, level, 368 test_phase, processor) 369 370 return True 371 372 373# Main 374 375if not main(): 376 exit(1) 377