1#!/usr/bin/env python3 2 3r""" 4Use robot framework API to extract test result data from output.xml generated 5by robot tests. For more information on the Robot Framework API, see 6http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html 7""" 8 9import csv 10import datetime 11import getopt 12import os 13import re 14import stat 15import sys 16from xml.etree import ElementTree 17 18import robot.errors 19from gen_arg import * 20from gen_print import * 21from gen_valid import * 22from robot.api import ExecutionResult 23from robot.result.visitor import ResultVisitor 24 25# Remove the python library path to restore with local project path later. 26save_path_0 = sys.path[0] 27del sys.path[0] 28sys.path.append(os.path.join(os.path.dirname(__file__), "../../lib")) 29 30# Restore sys.path[0]. 31sys.path.insert(0, save_path_0) 32 33 34this_program = sys.argv[0] 35info = " For more information: " + this_program + " -h" 36if len(sys.argv) == 1: 37 print(info) 38 sys.exit(1) 39 40 41parser = argparse.ArgumentParser( 42 usage=info, 43 description=( 44 "%(prog)s uses a robot framework API to extract test result data" 45 " from output.xml generated by robot tests. For more information on" 46 " the Robot Framework API, see " 47 " http://robot-framework.readthedocs.io/en/3.0/autodoc/robot.result.html" 48 ), 49 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 50 prefix_chars="-+", 51) 52 53parser.add_argument( 54 "--source", 55 "-s", 56 help=( 57 "The output.xml robot test result file path. This parameter is " 58 " required." 59 ), 60) 61 62parser.add_argument( 63 "--dest", 64 "-d", 65 help=( 66 "The directory path where the generated .csv files will go. This " 67 " parameter is required." 68 ), 69) 70 71parser.add_argument( 72 "--version_id", 73 help=( 74 "Driver version of openbmc firmware which was used during test, " 75 ' e.g. "v2.1-215-g6e7eacb". This parameter is required.' 76 ), 77) 78 79parser.add_argument( 80 "--platform", 81 help=( 82 "OpenBMC platform which was used during test, e.g." 83 ' "Witherspoon". This parameter is required.' 84 ), 85) 86 87parser.add_argument( 88 "--level", 89 help=( 90 "OpenBMC release level which was used during test, e.g." 91 ' "Master", "OBMC920". This parameter is required.' 92 ), 93) 94 95parser.add_argument( 96 "--test_phase", 97 help=( 98 'Name of testing phase, e.g. "DVT", "SVT", etc. This' 99 " parameter is optional." 100 ), 101 default="FVT", 102) 103 104parser.add_argument( 105 "--subsystem", 106 help=( 107 'Name of the subsystem, e.g. "OPENBMC" etc. This parameter is' 108 " optional." 109 ), 110 default="OPENBMC", 111) 112 113parser.add_argument( 114 "--processor", 115 help='Name of processor, e.g. "P9". This parameter is optional.', 116 default="OPENPOWER", 117) 118 119 120# Populate stock_list with options we want. 121stock_list = [("test_mode", 0), ("quiet", 0), ("debug", 0)] 122 123 124def exit_function(signal_number=0, frame=None): 125 r""" 126 Execute whenever the program ends normally or with the signals that we 127 catch (i.e. TERM, INT). 128 """ 129 130 dprint_executing() 131 132 dprint_var(signal_number) 133 134 qprint_pgm_footer() 135 136 137def signal_handler(signal_number, frame): 138 r""" 139 Handle signals. Without a function to catch a SIGTERM or SIGINT, the 140 program would terminate immediately with return code 143 and without 141 calling the exit_function. 142 """ 143 144 # Our convention is to set up exit_function with atexit.register() so 145 # there is no need to explicitly call exit_function from here. 146 147 dprint_executing() 148 149 # Calling exit prevents us from returning to the code that was running 150 # when the signal was received. 151 exit(0) 152 153 154def validate_parms(): 155 r""" 156 Validate program parameters, etc. Return True or False (i.e. pass/fail) 157 accordingly. 158 """ 159 160 if not valid_file_path(source): 161 return False 162 163 if not valid_dir_path(dest): 164 return False 165 166 gen_post_validation(exit_function, signal_handler) 167 168 return True 169 170 171def parse_output_xml( 172 xml_file_path, 173 csv_dir_path, 174 version_id, 175 platform, 176 level, 177 test_phase, 178 processor, 179): 180 r""" 181 Parse the robot-generated output.xml file and extract various test 182 output data. Put the extracted information into a csv file in the "dest" 183 folder. 184 185 Description of argument(s): 186 xml_file_path The path to a Robot-generated output.xml 187 file. 188 csv_dir_path The path to the directory that is to 189 contain the .csv files generated by 190 this function. 191 version_id Version of the openbmc firmware 192 (e.g. "v2.1-215-g6e7eacb"). 193 platform Platform of the openbmc system. 194 level Release level of the OpenBMC system 195 (e.g. "Master"). 196 """ 197 198 # Initialize tallies 199 total_critical_tc = 0 200 total_critical_passed = 0 201 total_critical_failed = 0 202 total_non_critical_tc = 0 203 total_non_critical_passed = 0 204 total_non_critical_failed = 0 205 206 result = ExecutionResult(xml_file_path) 207 result.configure( 208 stat_config={ 209 "suite_stat_level": 2, 210 "tag_stat_combine": "tagANDanother", 211 } 212 ) 213 214 stats = result.statistics 215 print("--------------------------------------") 216 try: 217 total_critical_tc = ( 218 stats.total.critical.passed + stats.total.critical.failed 219 ) 220 total_critical_passed = stats.total.critical.passed 221 total_critical_failed = stats.total.critical.failed 222 except AttributeError: 223 pass 224 225 try: 226 total_non_critical_tc = stats.total.passed + stats.total.failed 227 total_non_critical_passed = stats.total.passed 228 total_non_critical_failed = stats.total.failed 229 except AttributeError: 230 pass 231 232 print( 233 "Total Test Count:\t %d" % (total_non_critical_tc + total_critical_tc) 234 ) 235 236 print("Total Critical Test Failed:\t %d" % total_critical_failed) 237 print("Total Critical Test Passed:\t %d" % total_critical_passed) 238 print("Total Non-Critical Test Failed:\t %d" % total_non_critical_failed) 239 print("Total Non-Critical Test Passed:\t %d" % total_non_critical_passed) 240 print("Test Start Time:\t %s" % result.suite.starttime) 241 print("Test End Time:\t\t %s" % result.suite.endtime) 242 print("--------------------------------------") 243 244 # Use ResultVisitor object and save off the test data info 245 class TestResult(ResultVisitor): 246 def __init__(self): 247 self.testData = [] 248 249 def visit_test(self, test): 250 self.testData += [test] 251 252 collectDataObj = TestResult() 253 result.visit(collectDataObj) 254 255 # Write the result statistics attributes to CSV file 256 l_csvlist = [] 257 258 # Default Test data 259 l_test_type = test_phase 260 261 l_pse_rel = "Master" 262 if level: 263 l_pse_rel = level 264 265 l_env = "HW" 266 l_proc = processor 267 l_platform_type = "" 268 l_func_area = "" 269 270 # System data from XML meta data 271 # l_system_info = get_system_details(xml_file_path) 272 273 # First let us try to collect information from keyboard input 274 # If keyboard input cannot give both information, then find from xml file. 275 if version_id and platform: 276 l_driver = version_id 277 l_platform_type = platform 278 print("BMC Version_id:%s" % version_id) 279 print("BMC Platform:%s" % platform) 280 else: 281 # System data from XML meta data 282 l_system_info = get_system_details(xml_file_path) 283 l_driver = l_system_info[0] 284 l_platform_type = l_system_info[1] 285 286 # Driver version id and platform are mandatorily required for CSV file 287 # generation. If any one is not avaulable, exit CSV file generation 288 # process. 289 if l_driver and l_platform_type: 290 print("Driver and system info set.") 291 else: 292 print( 293 "Both driver and system info need to be set. CSV" 294 " file is not generated." 295 ) 296 sys.exit() 297 298 # Default header 299 l_header = [ 300 "test_start", 301 "test_end", 302 "subsys", 303 "test_type", 304 "test_result", 305 "test_name", 306 "pse_rel", 307 "driver", 308 "env", 309 "proc", 310 "platform_type", 311 "test_func_area", 312 ] 313 314 l_csvlist.append(l_header) 315 316 # Generate CSV file onto the path with current time stamp 317 l_base_dir = csv_dir_path 318 l_timestamp = datetime.datetime.utcnow().strftime("%Y-%m-%d-%H-%M-%S") 319 # Example: 2017-02-20-08-47-22_Witherspoon.csv 320 l_csvfile = l_base_dir + l_timestamp + "_" + l_platform_type + ".csv" 321 322 print("Writing data into csv file:%s" % l_csvfile) 323 324 for testcase in collectDataObj.testData: 325 # Functional Area: Suite Name 326 # Test Name: Test Case Name 327 l_func_area = str(testcase.parent).split(" ", 1)[1] 328 l_test_name = str(testcase) 329 330 # Test Result pass=0 fail=1 331 if testcase.status == "PASS": 332 l_test_result = 0 333 else: 334 l_test_result = 1 335 336 # Format datetime from robot output.xml to "%Y-%m-%d-%H-%M-%S" 337 l_stime = xml_to_csv_time(testcase.starttime) 338 l_etime = xml_to_csv_time(testcase.endtime) 339 # Data Sequence: test_start,test_end,subsys,test_type, 340 # test_result,test_name,pse_rel,driver, 341 # env,proc,platform_type,test_func_area, 342 l_data = [ 343 l_stime, 344 l_etime, 345 subsystem, 346 l_test_type, 347 l_test_result, 348 l_test_name, 349 l_pse_rel, 350 l_driver, 351 l_env, 352 l_proc, 353 l_platform_type, 354 l_func_area, 355 ] 356 l_csvlist.append(l_data) 357 358 # Open the file and write to the CSV file 359 l_file = open(l_csvfile, "w") 360 l_writer = csv.writer(l_file, lineterminator="\n") 361 l_writer.writerows(l_csvlist) 362 l_file.close() 363 # Set file permissions 666. 364 perm = ( 365 stat.S_IRUSR 366 + stat.S_IWUSR 367 + stat.S_IRGRP 368 + stat.S_IWGRP 369 + stat.S_IROTH 370 + stat.S_IWOTH 371 ) 372 os.chmod(l_csvfile, perm) 373 374 375def xml_to_csv_time(xml_datetime): 376 r""" 377 Convert the time from %Y%m%d %H:%M:%S.%f format to %Y-%m-%d-%H-%M-%S format 378 and return it. 379 380 Description of argument(s): 381 datetime The date in the following format: %Y%m%d 382 %H:%M:%S.%f (This is the format 383 typically found in an XML file.) 384 385 The date returned will be in the following format: %Y-%m-%d-%H-%M-%S 386 """ 387 388 # 20170206 05:05:19.342 389 l_str = datetime.datetime.strptime(xml_datetime, "%Y%m%d %H:%M:%S.%f") 390 # 2017-02-06-05-05-19 391 l_str = l_str.strftime("%Y-%m-%d-%H-%M-%S") 392 return str(l_str) 393 394 395def get_system_details(xml_file_path): 396 r""" 397 Get the system data from output.xml generated by robot and return it. 398 The list returned will be in the following order: [driver,platform] 399 400 Description of argument(s): 401 xml_file_path The relative or absolute path to the 402 output.xml file. 403 """ 404 405 bmc_version_id = "" 406 bmc_platform = "" 407 with open(xml_file_path, "rt") as output: 408 tree = ElementTree.parse(output) 409 410 for node in tree.iter("msg"): 411 # /etc/os-release output is logged in the XML as msg 412 # Example: ${output} = VERSION_ID="v1.99.2-71-gbc49f79" 413 if "${output} = VERSION_ID=" in node.text: 414 # Get BMC version (e.g. v1.99.1-96-g2a46570) 415 bmc_version_id = str(node.text.split("VERSION_ID=")[1])[1:-1] 416 417 # Platform is logged in the XML as msg. 418 # Example: ${bmc_model} = Witherspoon BMC 419 if "${bmc_model} = " in node.text: 420 bmc_platform = node.text.split(" = ")[1] 421 422 print_vars(bmc_version_id, bmc_platform) 423 return [str(bmc_version_id), str(bmc_platform)] 424 425 426def main(): 427 if not gen_get_options(parser, stock_list): 428 return False 429 430 if not validate_parms(): 431 return False 432 433 qprint_pgm_header() 434 435 parse_output_xml( 436 source, dest, version_id, platform, level, test_phase, processor 437 ) 438 439 return True 440 441 442# Main 443 444if not main(): 445 exit(1) 446