1#!/usr/bin/env python3 2 3r""" 4Use robot framework API to extract test result data from output.xml generated 5by robot tests. For more information on the Robot Framework API, see 6https://robot-framework.readthedocs.io/en/stable/autodoc/robot.result.html 7""" 8 9import csv 10import datetime 11import os 12import stat 13import sys 14from xml.etree import ElementTree 15 16from robot.api import ExecutionResult 17from robot.result.visitor import ResultVisitor 18 19# Remove the python library path to restore with local project path later. 20SAVE_PATH_0 = sys.path[0] 21del sys.path[0] 22sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib")) 23 24from gen_arg import * # NOQA 25from gen_print import * # NOQA 26from gen_valid import * # NOQA 27 28# Restore sys.path[0]. 29sys.path.insert(0, SAVE_PATH_0) 30 31 32this_program = sys.argv[0] 33info = " For more information: " + this_program + " -h" 34if len(sys.argv) == 1: 35 print(info) 36 sys.exit(1) 37 38 39parser = argparse.ArgumentParser( 40 usage=info, 41 description=( 42 "%(prog)s uses a robot framework API to extract test result data" 43 " from output.xml generated by robot tests. For more information on" 44 " the Robot Framework API, see " 45 " https://robot-framework.readthedocs.io/en/stable/autodoc/robot.result.html" 46 ), 47 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 48 prefix_chars="-+", 49) 50 51parser.add_argument( 52 "--source", 53 "-s", 54 help=( 55 "The output.xml robot test result file path. This parameter is " 56 " required." 57 ), 58) 59 60parser.add_argument( 61 "--dest", 62 "-d", 63 help=( 64 "The directory path where the generated .csv files will go. This" 65 " parameter is required." 66 ), 67) 68 69parser.add_argument( 70 "--version_id", 71 help=( 72 "Driver version of openbmc firmware which was used during test," 73 ' e.g. "v2.1-215-g6e7eacb". This parameter is required.' 74 ), 75) 76 77parser.add_argument( 78 "--platform", 79 help=( 80 "OpenBMC platform which was used during test, e.g." 81 ' "Witherspoon". This parameter is required.' 82 ), 83) 84 85parser.add_argument( 86 "--level", 87 help=( 88 "OpenBMC release level which was used during test, e.g." 89 ' "Master", "OBMC920". This parameter is required.' 90 ), 91) 92 93parser.add_argument( 94 "--test_phase", 95 help=( 96 'Name of testing phase, e.g. "CI", "Regression", etc. This' 97 " parameter is optional." 98 ), 99 default="FVT", 100) 101 102parser.add_argument( 103 "--subsystem", 104 help=( 105 'Name of the subsystem, e.g. "OPENBMC" etc. This parameter is' 106 " optional." 107 ), 108 default="OPENBMC", 109) 110 111parser.add_argument( 112 "--processor", 113 help='Name of processor, e.g. "XY". This parameter is optional.', 114 default="OPENPOWER", 115) 116 117 118# Populate stock_list with options we want. 119stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)] 120 121 122def exit_function(signal_number=0): 123 r""" 124 Execute whenever the program ends normally or with the signals that we 125 catch (i.e. TERM, INT). 126 """ 127 128 dprint_executing() 129 130 dprint_var(signal_number) 131 132 qprint_pgm_footer() 133 134 135def signal_handler(): 136 r""" 137 Handle signals. Without a function to catch a SIGTERM or SIGINT, the 138 program would terminate immediately with return code 143 and without 139 calling the exit_function. 140 """ 141 142 # Our convention is to set up exit_function with atexit.register() so 143 # there is no need to explicitly call exit_function from here. 144 145 dprint_executing() 146 147 # Calling exit prevents us from returning to the code that was running 148 # when the signal was received. 149 sys.exit(0) 150 151 152def validate_parms(): 153 r""" 154 Validate program parameters, etc. Return True or False (i.e. pass/fail) 155 accordingly. 156 """ 157 158 if not valid_file_path(source): 159 return False 160 161 if not valid_dir_path(dest): 162 return False 163 164 gen_post_validation(exit_function, signal_handler) 165 166 return True 167 168 169def parse_output_xml( 170 xml_file_path, 171 csv_dir_path, 172 version_id, 173 platform, 174 level, 175 test_phase, 176 processor, 177): 178 r""" 179 Parse the robot-generated output.xml file and extract various test 180 output data. Put the extracted information into a csv file in the "dest" 181 folder. 182 183 Description of argument(s): 184 xml_file_path The path to a Robot-generated output.xml 185 file. 186 csv_dir_path The path to the directory that is to 187 contain the .csv files generated by 188 this function. 189 version_id Version of the openbmc firmware 190 (e.g. "v2.1-215-g6e7eacb"). 191 platform Platform of the openbmc system. 192 level Release level of the OpenBMC system 193 (e.g. "Master"). 194 test_phase Name of the test phase 195 e.g. "CI", "Regression", etc. 196 processor Name of processor, e.g. "XY". 197 """ 198 199 # Initialize tallies 200 total_critical_tc = 0 201 total_critical_passed = 0 202 total_critical_failed = 0 203 total_non_critical_tc = 0 204 total_non_critical_passed = 0 205 total_non_critical_failed = 0 206 207 result = ExecutionResult(xml_file_path) 208 result.configure( 209 stat_config={ 210 "suite_stat_level": 2, 211 "tag_stat_combine": "tagANDanother", 212 } 213 ) 214 215 stats = result.statistics 216 print("--------------------------------------") 217 try: 218 total_critical_tc = ( 219 stats.total.critical.passed + stats.total.critical.failed 220 ) 221 total_critical_passed = stats.total.critical.passed 222 total_critical_failed = stats.total.critical.failed 223 except AttributeError: 224 pass 225 226 try: 227 total_non_critical_tc = stats.total.passed + stats.total.failed 228 total_non_critical_passed = stats.total.passed 229 total_non_critical_failed = stats.total.failed 230 except AttributeError: 231 pass 232 233 print( 234 "Total Test Count:\t %d" % (total_non_critical_tc + total_critical_tc) 235 ) 236 237 print("Total Critical Test Failed:\t %d" % total_critical_failed) 238 print("Total Critical Test Passed:\t %d" % total_critical_passed) 239 print("Total Non-Critical Test Failed:\t %d" % total_non_critical_failed) 240 print("Total Non-Critical Test Passed:\t %d" % total_non_critical_passed) 241 print("Test Start Time:\t %s" % result.suite.starttime) 242 print("Test End Time:\t\t %s" % result.suite.endtime) 243 print("--------------------------------------") 244 245 # Use ResultVisitor object and save off the test data info. 246 class TestResult(ResultVisitor): 247 r""" 248 Class methods to save off the test data information. 249 """ 250 251 def __init__(self): 252 self.test_data = [] 253 254 def visit_test(self, test): 255 self.test_data += [test] 256 257 collect_data_obj = TestResult() 258 result.visit(collect_data_obj) 259 260 # Write the result statistics attributes to CSV file 261 l_csvlist = [] 262 263 # Default Test data 264 l_test_type = test_phase 265 266 l_pse_rel = "Master" 267 if level: 268 l_pse_rel = level 269 270 l_env = "HW" 271 l_proc = processor 272 l_platform_type = "" 273 l_func_area = "" 274 275 # System data from XML meta data 276 # l_system_info = get_system_details(xml_file_path) 277 278 # First let us try to collect information from keyboard input 279 # If keyboard input cannot give both information, then find from xml file. 280 if version_id and platform: 281 l_driver = version_id 282 l_platform_type = platform 283 print("BMC Version_id:%s" % version_id) 284 print("BMC Platform:%s" % platform) 285 else: 286 # System data from XML meta data 287 l_system_info = get_system_details(xml_file_path) 288 l_driver = l_system_info[0] 289 l_platform_type = l_system_info[1] 290 291 # Driver version id and platform are mandatorily required for CSV file 292 # generation. If any one is not avaulable, exit CSV file generation 293 # process. 294 if l_driver and l_platform_type: 295 print("Driver and system info set.") 296 else: 297 print( 298 "Both driver and system info need to be set. CSV" 299 " file is not generated." 300 ) 301 sys.exit() 302 303 # Default header 304 l_header = [ 305 "test_start", 306 "test_end", 307 "subsys", 308 "test_type", 309 "test_result", 310 "test_name", 311 "pse_rel", 312 "driver", 313 "env", 314 "proc", 315 "platform_type", 316 "test_func_area", 317 ] 318 319 l_csvlist.append(l_header) 320 321 # Generate CSV file onto the path with current time stamp 322 l_base_dir = csv_dir_path 323 l_timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S") 324 # Example: 2017-02-20-08-47-22_Witherspoon.csv 325 l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv" 326 327 print("Writing data into csv file:%s" % l_csvfile) 328 329 for testcase in collect_data_obj.test_data: 330 # Functional Area: Suite Name 331 # Test Name: Test Case Name 332 l_func_area = str(testcase.parent).split(" ", 1)[1] 333 l_test_name = str(testcase) 334 335 # Test Result pass=0 fail=1 336 if testcase.status == "PASS": 337 l_test_result = 0 338 elif testcase.status == "SKIP": 339 # Skipped test result should not be mark pass or fail. 340 continue 341 else: 342 l_test_result = 1 343 344 # Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S" 345 l_stime = xml_to_csv_time(testcase.starttime) 346 l_etime = xml_to_csv_time(testcase.endtime) 347 # Data Sequence: test_start,test_end,subsys,test_type, 348 # test_result,test_name,pse_rel,driver, 349 # env,proc,platform_type,test_func_area, 350 l_data = [ 351 l_stime, 352 l_etime, 353 subsystem, 354 l_test_type, 355 l_test_result, 356 l_test_name, 357 l_pse_rel, 358 l_driver, 359 l_env, 360 l_proc, 361 l_platform_type, 362 l_func_area, 363 ] 364 l_csvlist.append(l_data) 365 366 # Open the file and write to the CSV file 367 with open(l_csvfile, "w", encoding="utf8") as l_file: 368 l_writer = csv.writer(l_file, lineterminator="\n") 369 l_writer.writerows(l_csvlist) 370 l_file.close() 371 372 # Set file permissions 666. 373 perm = ( 374 stat.S_IRUSR 375 + stat.S_IWUSR 376 + stat.S_IRGRP 377 + stat.S_IWGRP 378 + stat.S_IROTH 379 + stat.S_IWOTH 380 ) 381 os.chmod(l_csvfile, perm) 382 383 384def xml_to_csv_time(xml_datetime): 385 r""" 386 Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format 387 and return it. 388 389 Description of argument(s): 390 xml_datetime The date in the following format: %Y%m%d 391 %H:%M:%S.%f (This is the format 392 typically found in an XML file.) 393 394 The date returned will be in the following format: %Y-%m-%d-%H-%M-%S 395 """ 396 397 # 20170206 05:05:19.342 398 l_str = datetime.datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f") 399 # 2017-02-06-05-05-19 400 l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S") 401 return str(l_str) 402 403 404def get_system_details(xml_file_path): 405 r""" 406 Get the system data from output.xml generated by robot and return it. 407 The list returned will be in the following order: [driver,platform] 408 409 Description of argument(s): 410 xml_file_path The relative or absolute path to the 411 output.xml file. 412 """ 413 414 bmc_version_id = "" 415 bmc_platform = "" 416 with open(xml_file_path, "rt", encoding="utf-8") as output: 417 tree = ElementTree.parse(output) 418 419 for node in tree.iter("msg"): 420 # /etc/os-release output is logged in the XML as msg 421 # Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79" 422 if "${output} = VERSION_ID=" in node.text: 423 # Get BMC version (e.g. v1.99.1-96-g2a46570) 424 bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1] 425 426 # Platform is logged in the XML as msg. 427 # Example: ${bmc_model} = Witherspoon BMC 428 if "${bmc_model} = " in node.text: 429 bmc_platform = node.text.split(" = ")[1] 430 431 print_vars(bmc_version_id, bmc_platform) 432 return [str(bmc_version_id), str(bmc_platform)] 433 434 435def main(): 436 r""" 437 Main caller. 438 """ 439 440 if not gen_get_options(parser, stock_list): 441 return False 442 443 if not validate_parms(): 444 return False 445 446 qprint_pgm_header() 447 448 parse_output_xml( 449 source, dest, version_id, platform, level, test_phase, processor 450 ) 451 452 return True 453 454 455# Main 456 457if not main(): 458 sys.exit(1) 459