1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import cPickle as pickle 14import socket 15 16from robot.utils import DotDict 17from robot.libraries.BuiltIn import BuiltIn 18 19from boot_data import * 20import gen_print as gp 21import gen_robot_print as grp 22import gen_robot_plug_in as grpi 23import gen_robot_valid as grv 24import gen_misc as gm 25import gen_cmd as gc 26import gen_robot_keyword as grk 27import state as st 28import var_stack as vs 29 30base_path = os.path.dirname(os.path.dirname( 31 imp.find_module("gen_robot_print")[1])) +\ 32 os.sep 33sys.path.append(base_path + "extended/") 34import run_keyword as rk 35 36# Setting master_pid correctly influences the behavior of plug-ins like 37# DB_Logging 38program_pid = os.getpid() 39master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 40pgm_name = re.sub('\.py$', '', os.path.basename(__file__)) 41 42# Set up boot data structures. 43boot_table = create_boot_table() 44valid_boot_types = create_valid_boot_list(boot_table) 45 46boot_lists = read_boot_lists() 47last_ten = [] 48 49state = st.return_state_constant('default_state') 50cp_setup_called = 0 51next_boot = "" 52base_tool_dir_path = os.path.normpath(os.environ.get( 53 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 54 55ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 56boot_success = 0 57status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 58if status_dir_path != "": 59 status_dir_path = os.path.normpath(status_dir_path) + os.sep 60default_power_on = "REST Power On" 61default_power_off = "REST Power Off" 62boot_count = 0 63 64LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 65ffdc_prefix = "" 66boot_start_time = "" 67boot_end_time = "" 68save_stack = vs.var_stack('save_stack') 69main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 70 71 72def process_host(host, 73 host_var_name=""): 74 r""" 75 Process a host by getting the associated host name and IP address and 76 setting them in global variables. 77 78 If the caller does not pass the host_var_name, this function will try to 79 figure out the name of the variable used by the caller for the host parm. 80 Callers are advised to explicitly specify the host_var_name when calling 81 with an exec command. In such cases, the get_arg_name cannot figure out 82 the host variable name. 83 84 This function will then create similar global variable names by 85 removing "_host" and appending "_host_name" or "_ip" to the host variable 86 name. 87 88 Example: 89 90 If a call is made like this: 91 process_host(openbmc_host) 92 93 Global variables openbmc_host_name and openbmc_ip will be set. 94 95 Description of argument(s): 96 host A host name or IP. The name of the variable used should 97 have a suffix of "_host". 98 host_var_name The name of the variable being used as the host parm. 99 """ 100 101 if host_var_name == "": 102 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 103 104 host_name_var_name = re.sub("host", "host_name", host_var_name) 105 ip_var_name = re.sub("host", "ip", host_var_name) 106 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 107 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 108 host + "')" 109 exec(cmd_buf) 110 111 112def process_pgm_parms(): 113 r""" 114 Process the program parameters by assigning them all to corresponding 115 globals. Also, set some global values that depend on program parameters. 116 """ 117 118 # Program parameter processing. 119 # Assign all program parms to python variables which are global to this 120 # module. 121 122 global parm_list 123 parm_list = BuiltIn().get_variable_value("${parm_list}") 124 # The following subset of parms should be processed as integers. 125 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 126 'boot_fail_threshold', 'delete_errlogs', 'quiet', 'test_mode', 127 'debug'] 128 for parm in parm_list: 129 if parm in int_list: 130 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 131 "}\", \"0\"))" 132 else: 133 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 134 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 135 gp.dpissuing(cmd_buf) 136 exec(cmd_buf) 137 if re.match(r".*_host$", parm): 138 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 139 exec(cmd_buf) 140 if re.match(r".*_password$", parm): 141 # Register the value of any parm whose name ends in _password. 142 # This will cause the print functions to replace passwords with 143 # asterisks in the output. 144 cmd_buf = "gp.register_passwords(" + parm + ")" 145 exec(cmd_buf) 146 147 global ffdc_dir_path_style 148 global boot_list 149 global boot_stack 150 global boot_results_file_path 151 global boot_results 152 global ffdc_list_file_path 153 global ffdc_report_list_path 154 global ffdc_summary_list_path 155 156 if ffdc_dir_path_style == "": 157 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 158 159 # Convert these program parms to lists for easier processing.. 160 boot_list = filter(None, boot_list.split(":")) 161 boot_stack = filter(None, boot_stack.split(":")) 162 163 cleanup_boot_results_file() 164 boot_results_file_path = create_boot_results_file_path(pgm_name, 165 openbmc_nickname, 166 master_pid) 167 168 if os.path.isfile(boot_results_file_path): 169 # We've been called before in this run so we'll load the saved 170 # boot_results object. 171 boot_results = pickle.load(open(boot_results_file_path, 'rb')) 172 else: 173 boot_results = boot_results(boot_table, boot_pass, boot_fail) 174 175 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 176 "/FFDC_FILE_LIST" 177 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 178 "/FFDC_REPORT_FILE_LIST" 179 180 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 181 "/FFDC_SUMMARY_FILE_LIST" 182 183 184def initial_plug_in_setup(): 185 r""" 186 Initialize all plug-in environment variables which do not change for the 187 duration of the program. 188 189 """ 190 191 global LOG_LEVEL 192 BuiltIn().set_log_level("NONE") 193 194 BuiltIn().set_global_variable("${master_pid}", master_pid) 195 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 196 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 197 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 198 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 199 ffdc_list_file_path) 200 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 201 ffdc_report_list_path) 202 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 203 ffdc_summary_list_path) 204 205 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 206 ffdc_dir_path_style) 207 BuiltIn().set_global_variable("${FFDC_CHECK}", 208 ffdc_check) 209 210 # For each program parameter, set the corresponding AUTOBOOT_ environment 211 # variable value. Also, set an AUTOBOOT_ environment variable for every 212 # element in additional_values. 213 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 214 "status_dir_path", "base_tool_dir_path", 215 "ffdc_list_file_path", "ffdc_report_list_path", 216 "ffdc_summary_list_path"] 217 218 plug_in_vars = parm_list + additional_values 219 220 for var_name in plug_in_vars: 221 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 222 var_name = var_name.upper() 223 if var_value is None: 224 var_value = "" 225 os.environ["AUTOBOOT_" + var_name] = str(var_value) 226 227 BuiltIn().set_log_level(LOG_LEVEL) 228 229 # Make sure the ffdc list directory exists. 230 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 231 if not os.path.exists(ffdc_list_dir_path): 232 os.makedirs(ffdc_list_dir_path) 233 234 235def plug_in_setup(): 236 r""" 237 Initialize all changing plug-in environment variables for use by the 238 plug-in programs. 239 """ 240 241 global LOG_LEVEL 242 global test_really_running 243 244 BuiltIn().set_log_level("NONE") 245 246 boot_pass, boot_fail = boot_results.return_total_pass_fail() 247 if boot_pass > 1: 248 test_really_running = 1 249 else: 250 test_really_running = 0 251 252 BuiltIn().set_global_variable("${test_really_running}", 253 test_really_running) 254 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 255 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 256 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 257 BuiltIn().set_global_variable("${boot_success}", boot_success) 258 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 259 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 260 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 261 262 # For each program parameter, set the corresponding AUTOBOOT_ environment 263 # variable value. Also, set an AUTOBOOT_ environment variable for every 264 # element in additional_values. 265 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 266 "boot_fail", "test_really_running", "ffdc_prefix", 267 "boot_start_time", "boot_end_time"] 268 269 plug_in_vars = additional_values 270 271 for var_name in plug_in_vars: 272 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 273 var_name = var_name.upper() 274 if var_value is None: 275 var_value = "" 276 os.environ["AUTOBOOT_" + var_name] = str(var_value) 277 278 if debug: 279 shell_rc, out_buf = \ 280 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 281 282 BuiltIn().set_log_level(LOG_LEVEL) 283 284 285def pre_boot_plug_in_setup(): 286 287 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 288 try: 289 os.remove(ffdc_list_file_path) 290 except OSError: 291 pass 292 293 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 294 try: 295 os.remove(ffdc_report_list_path) 296 except OSError: 297 pass 298 299 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 300 try: 301 os.remove(ffdc_summary_list_path) 302 except OSError: 303 pass 304 305 global ffdc_prefix 306 307 seconds = time.time() 308 loc_time = time.localtime(seconds) 309 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 310 311 ffdc_prefix = openbmc_nickname + "." + time_string 312 313 314def setup(): 315 r""" 316 Do general program setup tasks. 317 """ 318 319 global cp_setup_called 320 global transitional_boot_selected 321 322 gp.qprintn() 323 324 transitional_boot_selected = False 325 326 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 327 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 328 # If we can't find process_plug_in_packages.py, ssh_pw or 329 # validate_plug_ins.py, then we don't have our repo bin in PATH. 330 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" + 331 " ssh_pw validate_plug_ins.py", quiet=1, 332 print_output=0, show_err=0) 333 if shell_rc != 0: 334 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 335 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 336 if robot_pgm_dir_path not in sys.path: 337 sys.path.append(robot_pgm_dir_path) 338 PYTHONPATH = os.environ.get("PYTHONPATH", "") 339 if PYTHONPATH == "": 340 os.environ['PYTHONPATH'] = robot_pgm_dir_path 341 else: 342 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 343 344 validate_parms() 345 346 grp.rqprint_pgm_header() 347 348 grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF") 349 350 initial_plug_in_setup() 351 352 plug_in_setup() 353 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 354 call_point='setup') 355 if rc != 0: 356 error_message = "Plug-in setup failed.\n" 357 grp.rprint_error_report(error_message) 358 BuiltIn().fail(error_message) 359 # Setting cp_setup_called lets our Teardown know that it needs to call 360 # the cleanup plug-in call point. 361 cp_setup_called = 1 362 363 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 364 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 365 # FFDC_LOG_PATH is used by "FFDC" keyword. 366 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 367 368 # Also printed by FFDC. 369 global host_name 370 global host_ip 371 host = socket.gethostname() 372 host_name, host_ip = gm.get_host_name_ip(host) 373 374 gp.dprint_var(boot_table, 1) 375 gp.dprint_var(boot_lists) 376 377 378def validate_parms(): 379 r""" 380 Validate all program parameters. 381 """ 382 383 process_pgm_parms() 384 385 gp.qprintn() 386 387 global openbmc_model 388 grv.rvalid_value("openbmc_host") 389 grv.rvalid_value("openbmc_username") 390 grv.rvalid_value("openbmc_password") 391 if os_host != "": 392 grv.rvalid_value("os_username") 393 grv.rvalid_value("os_password") 394 395 if pdu_host != "": 396 grv.rvalid_value("pdu_username") 397 grv.rvalid_value("pdu_password") 398 grv.rvalid_integer("pdu_slot_no") 399 if openbmc_serial_host != "": 400 grv.rvalid_integer("openbmc_serial_port") 401 if openbmc_model == "": 402 status, ret_values =\ 403 grk.run_key_u("Get BMC System Model") 404 openbmc_model = ret_values 405 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 406 grv.rvalid_value("openbmc_model") 407 grv.rvalid_integer("max_num_tests") 408 grv.rvalid_integer("boot_pass") 409 grv.rvalid_integer("boot_fail") 410 411 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 412 BuiltIn().set_global_variable("${plug_in_packages_list}", 413 plug_in_packages_list) 414 415 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) 416 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 417 error_message = "You must provide either a value for either the" +\ 418 " boot_list or the boot_stack parm.\n" 419 BuiltIn().fail(gp.sprint_error(error_message)) 420 421 valid_boot_list(boot_list, valid_boot_types) 422 valid_boot_list(boot_stack, valid_boot_types) 423 424 selected_PDU_boots = list(set(boot_list + boot_stack) & 425 set(boot_lists['PDU_reboot'])) 426 427 if len(selected_PDU_boots) > 0 and pdu_host == "": 428 error_message = "You have selected the following boots which" +\ 429 " require a PDU host but no value for pdu_host:\n" 430 error_message += gp.sprint_var(selected_PDU_boots) 431 error_message += gp.sprint_var(pdu_host, 2) 432 BuiltIn().fail(gp.sprint_error(error_message)) 433 434 return 435 436 437def my_get_state(): 438 r""" 439 Get the system state plus a little bit of wrapping. 440 """ 441 442 global state 443 444 req_states = ['epoch_seconds'] + st.default_req_states 445 446 gp.qprint_timen("Getting system state.") 447 if test_mode: 448 state['epoch_seconds'] = int(time.time()) 449 else: 450 state = st.get_state(req_states=req_states, quiet=quiet) 451 gp.qprint_var(state) 452 453 454def valid_state(): 455 r""" 456 Verify that our state dictionary contains no blank values. If we don't get 457 valid state data, we cannot continue to work. 458 """ 459 460 if st.compare_states(state, st.invalid_state_match, 'or'): 461 error_message = "The state dictionary contains blank fields which" +\ 462 " is illegal.\n" + gp.sprint_var(state) 463 BuiltIn().fail(gp.sprint_error(error_message)) 464 465 466def select_boot(): 467 r""" 468 Select a boot test to be run based on our current state and return the 469 chosen boot type. 470 471 Description of arguments: 472 state The state of the machine. 473 """ 474 475 global transitional_boot_selected 476 global boot_stack 477 478 gp.qprint_timen("Selecting a boot test.") 479 480 if transitional_boot_selected and not boot_success: 481 prior_boot = next_boot 482 boot_candidate = boot_stack.pop() 483 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" + 484 " transition to a valid state for '" + boot_candidate + 485 "' which was at the top of the boot_stack. Since" + 486 " the '" + next_boot + "' failed, the '" + 487 boot_candidate + "' has been removed from the stack" + 488 " to avoid and endless failure loop.") 489 if len(boot_stack) == 0: 490 return "" 491 492 my_get_state() 493 valid_state() 494 495 transitional_boot_selected = False 496 stack_popped = 0 497 if len(boot_stack) > 0: 498 stack_popped = 1 499 gp.qprint_dashes() 500 gp.qprint_var(boot_stack) 501 gp.qprint_dashes() 502 skip_boot_printed = 0 503 while len(boot_stack) > 0: 504 boot_candidate = boot_stack.pop() 505 if stack_mode == 'normal': 506 break 507 else: 508 if st.compare_states(state, boot_table[boot_candidate]['end']): 509 if not skip_boot_printed: 510 gp.qprint_var(stack_mode) 511 gp.qprintn() 512 gp.qprint_timen("Skipping the following boot tests" + 513 " which are unnecessary since their" + 514 " required end states match the" + 515 " current machine state:") 516 skip_boot_printed = 1 517 gp.qprint_var(boot_candidate) 518 boot_candidate = "" 519 if boot_candidate == "": 520 gp.qprint_dashes() 521 gp.qprint_var(boot_stack) 522 gp.qprint_dashes() 523 return boot_candidate 524 if st.compare_states(state, boot_table[boot_candidate]['start']): 525 gp.qprint_timen("The machine state is valid for a '" + 526 boot_candidate + "' boot test.") 527 gp.qprint_dashes() 528 gp.qprint_var(boot_stack) 529 gp.qprint_dashes() 530 return boot_candidate 531 else: 532 gp.qprint_timen("The machine state does not match the required" + 533 " starting state for a '" + boot_candidate + 534 "' boot test:") 535 gp.qprint_varx("boot_table[" + boot_candidate + "][start]", 536 boot_table[boot_candidate]['start'], 1) 537 boot_stack.append(boot_candidate) 538 transitional_boot_selected = True 539 popped_boot = boot_candidate 540 541 # Loop through your list selecting a boot_candidates 542 boot_candidates = [] 543 for boot_candidate in boot_list: 544 if st.compare_states(state, boot_table[boot_candidate]['start']): 545 if stack_popped: 546 if st.compare_states(boot_table[boot_candidate]['end'], 547 boot_table[popped_boot]['start']): 548 boot_candidates.append(boot_candidate) 549 else: 550 boot_candidates.append(boot_candidate) 551 552 if len(boot_candidates) == 0: 553 gp.qprint_timen("The user's boot list contained no boot tests" + 554 " which are valid for the current machine state.") 555 boot_candidate = default_power_on 556 if not st.compare_states(state, boot_table[default_power_on]['start']): 557 boot_candidate = default_power_off 558 boot_candidates.append(boot_candidate) 559 gp.qprint_timen("Using default '" + boot_candidate + 560 "' boot type to transition to valid state.") 561 562 gp.dprint_var(boot_candidates) 563 564 # Randomly select a boot from the candidate list. 565 boot = random.choice(boot_candidates) 566 567 return boot 568 569 570def print_last_boots(): 571 r""" 572 Print the last ten boots done with their time stamps. 573 """ 574 575 # indent 0, 90 chars wide, linefeed, char is "=" 576 gp.qprint_dashes(0, 90) 577 gp.qprintn("Last 10 boots:\n") 578 579 for boot_entry in last_ten: 580 grp.rqprint(boot_entry) 581 gp.qprint_dashes(0, 90) 582 583 584def print_defect_report(ffdc_file_list): 585 r""" 586 Print a defect report. 587 588 Description of argument(s): 589 ffdc_file_list A list of files which were collected by our ffdc functions. 590 """ 591 592 # Making deliberate choice to NOT run plug_in_setup(). We don't want 593 # ffdc_prefix updated. 594 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 595 call_point='ffdc_report', stop_on_plug_in_failure=0) 596 597 # Get additional header data which may have been created by ffdc plug-ins. 598 # Also, delete the individual header files to cleanup. 599 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 600 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 601 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 602 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 603 show_err=0) 604 605 # Get additional summary data which may have been created by ffdc plug-ins. 606 # Also, delete the individual header files to cleanup. 607 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 608 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 609 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 610 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 611 show_err=0) 612 613 # ffdc_list_file_path contains a list of any ffdc files created by plug- 614 # ins, etc. Read that data into a list. 615 try: 616 plug_in_ffdc_list = \ 617 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 618 plug_in_ffdc_list = filter(None, plug_in_ffdc_list) 619 except IOError: 620 plug_in_ffdc_list = [] 621 622 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 623 # in. Eliminate duplicates and sort the list. 624 ffdc_file_list = list(set(ffdc_file_list + plug_in_ffdc_list)) 625 ffdc_file_list.sort() 626 627 if status_file_path != "": 628 ffdc_file_list.insert(0, status_file_path) 629 630 # Convert the list to a printable list. 631 printable_ffdc_file_list = "\n".join(ffdc_file_list) 632 633 # Open ffdc_file_list for writing. We will write a complete list of 634 # FFDC files to it for possible use by plug-ins like cp_stop_check. 635 ffdc_list_file = open(ffdc_list_file_path, 'w') 636 ffdc_list_file.write(printable_ffdc_file_list + "\n") 637 ffdc_list_file.close() 638 639 indent = 0 640 width = 90 641 linefeed = 1 642 char = "=" 643 644 gp.qprintn() 645 gp.qprint_dashes(indent, width, linefeed, char) 646 gp.qprintn("Copy this data to the defect:\n") 647 648 if len(more_header_info) > 0: 649 gp.qprintn(more_header_info) 650 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 651 openbmc_host_name, openbmc_ip, openbmc_username, 652 openbmc_password, os_host, os_host_name, os_ip, os_username, 653 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 654 pdu_password, pdu_slot_no, openbmc_serial_host, 655 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 656 657 gp.qprintn() 658 print_last_boots() 659 gp.qprintn() 660 gp.qprint_var(state) 661 gp.qprintn() 662 gp.qprintn("FFDC data files:") 663 gp.qprintn(printable_ffdc_file_list) 664 gp.qprintn() 665 666 if len(ffdc_summary_info) > 0: 667 gp.qprintn(ffdc_summary_info) 668 669 gp.qprint_dashes(indent, width, linefeed, char) 670 671 672def my_ffdc(): 673 r""" 674 Collect FFDC data. 675 """ 676 677 global state 678 679 plug_in_setup() 680 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 681 call_point='ffdc', stop_on_plug_in_failure=0) 682 683 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 684 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" + 685 AUTOBOOT_FFDC_PREFIX + 686 " ffdc_function_list=" + 687 ffdc_function_list, ignore=1) 688 if status != 'PASS': 689 gp.qprint_error("Call to ffdc failed.\n") 690 691 my_get_state() 692 693 print_defect_report(ffdc_file_list) 694 695 696def print_test_start_message(boot_keyword): 697 r""" 698 Print a message indicating what boot test is about to run. 699 700 Description of arguments: 701 boot_keyword The name of the boot which is to be run 702 (e.g. "BMC Power On"). 703 """ 704 705 global last_ten 706 global boot_start_time 707 708 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 709 710 # Set boot_start_time for use by plug-ins. 711 boot_start_time = doing_msg[1:33] 712 gp.qprint_var(boot_start_time) 713 714 gp.qprint(doing_msg) 715 716 last_ten.append(doing_msg) 717 718 if len(last_ten) > 10: 719 del last_ten[0] 720 721 722def run_boot(boot): 723 r""" 724 Run the specified boot. 725 726 Description of arguments: 727 boot The name of the boot test to be performed. 728 """ 729 730 global state 731 732 print_test_start_message(boot) 733 734 plug_in_setup() 735 rc, shell_rc, failed_plug_in_name = \ 736 grpi.rprocess_plug_in_packages(call_point="pre_boot") 737 if rc != 0: 738 error_message = "Plug-in failed with non-zero return code.\n" +\ 739 gp.sprint_var(rc, 1) 740 BuiltIn().fail(gp.sprint_error(error_message)) 741 742 if test_mode: 743 # In test mode, we'll pretend the boot worked by assigning its 744 # required end state to the default state value. 745 state = st.strip_anchor_state(boot_table[boot]['end']) 746 else: 747 # Assertion: We trust that the state data was made fresh by the 748 # caller. 749 750 gp.qprintn() 751 752 if boot_table[boot]['method_type'] == "keyword": 753 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 754 boot_table[boot]['method'], 755 quiet=quiet) 756 757 if boot_table[boot]['bmc_reboot']: 758 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 759 plug_in_setup() 760 rc, shell_rc, failed_plug_in_name = \ 761 grpi.rprocess_plug_in_packages(call_point="post_reboot") 762 if rc != 0: 763 error_message = "Plug-in failed with non-zero return code.\n" 764 error_message += gp.sprint_var(rc, 1) 765 BuiltIn().fail(gp.sprint_error(error_message)) 766 else: 767 match_state = st.anchor_state(state) 768 del match_state['epoch_seconds'] 769 # Wait for the state to change in any way. 770 st.wait_state(match_state, wait_time=state_change_timeout, 771 interval="10 seconds", invert=1) 772 773 gp.qprintn() 774 if boot_table[boot]['end']['chassis'] == "Off": 775 boot_timeout = power_off_timeout 776 else: 777 boot_timeout = power_on_timeout 778 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 779 interval="10 seconds") 780 781 plug_in_setup() 782 rc, shell_rc, failed_plug_in_name = \ 783 grpi.rprocess_plug_in_packages(call_point="post_boot") 784 if rc != 0: 785 error_message = "Plug-in failed with non-zero return code.\n" +\ 786 gp.sprint_var(rc, 1) 787 BuiltIn().fail(gp.sprint_error(error_message)) 788 789 790def test_loop_body(): 791 r""" 792 The main loop body for the loop in main_py. 793 794 Description of arguments: 795 boot_count The iteration number (starts at 1). 796 """ 797 798 global boot_count 799 global state 800 global next_boot 801 global boot_success 802 global boot_end_time 803 804 gp.qprintn() 805 806 next_boot = select_boot() 807 if next_boot == "": 808 return True 809 810 boot_count += 1 811 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 812 813 pre_boot_plug_in_setup() 814 815 cmd_buf = ["run_boot", next_boot] 816 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 817 if boot_status == "FAIL": 818 gp.qprint(msg) 819 820 gp.qprintn() 821 if boot_status == "PASS": 822 boot_success = 1 823 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot + 824 "\" succeeded.") 825 else: 826 boot_success = 0 827 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot + 828 "\" failed.") 829 830 # Set boot_end_time for use by plug-ins. 831 boot_end_time = completion_msg[1:33] 832 gp.qprint_var(boot_end_time) 833 834 gp.qprint(completion_msg) 835 836 boot_results.update(next_boot, boot_status) 837 838 plug_in_setup() 839 # NOTE: A post_test_case call point failure is NOT counted as a boot 840 # failure. 841 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 842 call_point='post_test_case', stop_on_plug_in_failure=0) 843 844 plug_in_setup() 845 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 846 call_point='ffdc_check', shell_rc=0x00000200, 847 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 848 if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200: 849 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 850 if status != 'PASS': 851 gp.qprint_error("Call to my_ffdc failed.\n") 852 853 if delete_errlogs: 854 # We need to purge error logs between boots or they build up. 855 grk.run_key("Delete Error logs", ignore=1) 856 857 boot_results.print_report() 858 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 859 860 plug_in_setup() 861 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 862 call_point='stop_check', shell_rc=0x00000200, stop_on_non_zero_rc=1) 863 if shell_rc == 0x00000200: 864 message = "Stopping as requested by user.\n" 865 gp.print_time(message) 866 BuiltIn().fail(message) 867 868 # This should help prevent ConnectionErrors. 869 grk.run_key_u("Close All Connections") 870 871 return True 872 873 874def obmc_boot_test_teardown(): 875 r""" 876 Clean up after the Main keyword. 877 """ 878 879 if cp_setup_called: 880 plug_in_setup() 881 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 882 call_point='cleanup', stop_on_plug_in_failure=0) 883 884 if 'boot_results_file_path' in globals(): 885 # Save boot_results object to a file in case it is needed again. 886 gp.qprint_timen("Saving boot_results to the following path.") 887 gp.qprint_var(boot_results_file_path) 888 pickle.dump(boot_results, open(boot_results_file_path, 'wb'), 889 pickle.HIGHEST_PROTOCOL) 890 891 global save_stack 892 # Restore any global values saved on the save_stack. 893 for parm_name in main_func_parm_list: 894 # Get the parm_value if it was saved on the stack. 895 try: 896 parm_value = save_stack.pop(parm_name) 897 except: 898 # If it was not saved, no further action is required. 899 continue 900 901 # Restore the saved value. 902 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 903 "}\", parm_value)" 904 gp.dpissuing(cmd_buf) 905 exec(cmd_buf) 906 907 gp.dprintn(save_stack.sprint_obj()) 908 909 910def test_teardown(): 911 r""" 912 Clean up after this test case. 913 """ 914 915 gp.qprintn() 916 cmd_buf = ["Print Error", 917 "A keyword timeout occurred ending this program.\n"] 918 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 919 920 grp.rqprint_pgm_footer() 921 922 923def obmc_boot_test_py(loc_boot_stack=None, 924 loc_stack_mode=None, 925 loc_quiet=None): 926 r""" 927 Do main program processing. 928 """ 929 930 global save_stack 931 932 # Process function parms. 933 for parm_name in main_func_parm_list: 934 # Get parm's value. 935 cmd_buf = "parm_value = loc_" + parm_name 936 exec(cmd_buf) 937 gp.dpvar(parm_name) 938 gp.dpvar(parm_value) 939 940 if parm_value is None: 941 # Parm was not specified by the calling function so set it to its 942 # corresponding global value. 943 cmd_buf = "loc_" + parm_name + " = BuiltIn().get_variable_value" +\ 944 "(\"${" + parm_name + "}\")" 945 gp.dpissuing(cmd_buf) 946 exec(cmd_buf) 947 else: 948 # Save the global value on a stack. 949 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 950 parm_name + "}\"), \"" + parm_name + "\")" 951 gp.dpissuing(cmd_buf) 952 exec(cmd_buf) 953 954 # Set the global value to the passed value. 955 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 956 "}\", loc_" + parm_name + ")" 957 gp.dpissuing(cmd_buf) 958 exec(cmd_buf) 959 960 gp.dprintn(save_stack.sprint_obj()) 961 962 setup() 963 964 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 965 966 if ffdc_only: 967 gp.qprint_timen("Caller requested ffdc_only.") 968 pre_boot_plug_in_setup() 969 grk.run_key_u("my_ffdc") 970 return 971 972 # Process caller's boot_stack. 973 while (len(boot_stack) > 0): 974 test_loop_body() 975 976 gp.qprint_timen("Finished processing stack.") 977 978 # Process caller's boot_list. 979 if len(boot_list) > 0: 980 for ix in range(1, max_num_tests + 1): 981 test_loop_body() 982 983 gp.qprint_timen("Completed all requested boot tests.") 984 985 boot_pass, boot_fail = boot_results.return_total_pass_fail() 986 new_fail = boot_fail - init_boot_fail 987 if new_fail > boot_fail_threshold: 988 error_message = "Boot failures exceed the boot failure" +\ 989 " threshold:\n" +\ 990 gp.sprint_var(new_fail) +\ 991 gp.sprint_var(boot_fail_threshold) 992 BuiltIn().fail(gp.sprint_error(error_message)) 993