1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import cPickle as pickle 14import socket 15 16from robot.utils import DotDict 17from robot.libraries.BuiltIn import BuiltIn 18 19from boot_data import * 20import gen_print as gp 21import gen_robot_print as grp 22import gen_robot_plug_in as grpi 23import gen_robot_valid as grv 24import gen_misc as gm 25import gen_cmd as gc 26import gen_robot_keyword as grk 27import state as st 28 29base_path = os.path.dirname(os.path.dirname( 30 imp.find_module("gen_robot_print")[1])) +\ 31 os.sep 32sys.path.append(base_path + "extended/") 33import run_keyword as rk 34 35# Setting master_pid correctly influences the behavior of plug-ins like 36# DB_Logging 37program_pid = os.getpid() 38master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 39 40# Set up boot data structures. 41boot_table = create_boot_table() 42valid_boot_types = create_valid_boot_list(boot_table) 43 44boot_lists = read_boot_lists() 45last_ten = [] 46 47state = st.return_default_state() 48cp_setup_called = 0 49next_boot = "" 50base_tool_dir_path = os.path.normpath(os.environ.get( 51 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 52 53ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 54boot_success = 0 55status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 56if status_dir_path != "": 57 status_dir_path = os.path.normpath(status_dir_path) + os.sep 58default_power_on = "REST Power On" 59default_power_off = "REST Power Off" 60boot_count = 0 61 62LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 63 64 65############################################################################### 66def process_host(host, 67 host_var_name=""): 68 69 r""" 70 Process a host by getting the associated host name and IP address and 71 setting them in global variables. 72 73 If the caller does not pass the host_var_name, this function will try to 74 figure out the name of the variable used by the caller for the host parm. 75 Callers are advised to explicitly specify the host_var_name when calling 76 with an exec command. In such cases, the get_arg_name cannot figure out 77 the host variable name. 78 79 This function will then create similar global variable names by 80 removing "_host" and appending "_host_name" or "_ip" to the host variable 81 name. 82 83 Example: 84 85 If a call is made like this: 86 process_host(openbmc_host) 87 88 Global variables openbmc_host_name and openbmc_ip will be set. 89 90 Description of argument(s): 91 host A host name or IP. The name of the variable used should 92 have a suffix of "_host". 93 host_var_name The name of the variable being used as the host parm. 94 """ 95 96 if host_var_name == "": 97 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 98 99 host_name_var_name = re.sub("host", "host_name", host_var_name) 100 ip_var_name = re.sub("host", "ip", host_var_name) 101 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 102 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 103 host + "')" 104 exec(cmd_buf) 105 106############################################################################### 107 108 109############################################################################### 110def process_pgm_parms(): 111 112 r""" 113 Process the program parameters by assigning them all to corresponding 114 globals. Also, set some global values that depend on program parameters. 115 """ 116 117 # Program parameter processing. 118 # Assign all program parms to python variables which are global to this 119 # module. 120 121 global parm_list 122 parm_list = BuiltIn().get_variable_value("${parm_list}") 123 # The following subset of parms should be processed as integers. 124 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 125 'boot_fail_threshold', 'quiet', 'test_mode', 'debug'] 126 for parm in parm_list: 127 if parm in int_list: 128 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 129 "}\", \"0\"))" 130 else: 131 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 132 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 133 exec(cmd_buf) 134 if re.match(r".*_host$", parm): 135 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 136 exec(cmd_buf) 137 if re.match(r".*_password$", parm): 138 # Register the value of any parm whose name ends in _password. 139 # This will cause the print functions to replace passwords with 140 # asterisks in the output. 141 cmd_buf = "gp.register_passwords(" + parm + ")" 142 exec(cmd_buf) 143 144 global ffdc_dir_path_style 145 global boot_list 146 global boot_stack 147 global boot_results_file_path 148 global boot_results 149 global ffdc_list_file_path 150 global ffdc_report_list_path 151 152 if ffdc_dir_path_style == "": 153 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 154 155 # Convert these program parms to lists for easier processing.. 156 boot_list = filter(None, boot_list.split(":")) 157 boot_stack = filter(None, boot_stack.split(":")) 158 159 boot_results_file_path = "/tmp/" + openbmc_nickname + ":pid_" +\ 160 str(master_pid) + ":boot_results" 161 162 if os.path.isfile(boot_results_file_path): 163 # We've been called before in this run so we'll load the saved 164 # boot_results object. 165 boot_results = pickle.load(open(boot_results_file_path, 'rb')) 166 else: 167 boot_results = boot_results(boot_table, boot_pass, boot_fail) 168 169 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 170 "/FFDC_FILE_LIST" 171 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 172 "/FFDC_REPORT_FILE_LIST" 173 174############################################################################### 175 176 177############################################################################### 178def initial_plug_in_setup(): 179 180 r""" 181 Initialize all plug-in environment variables which do not change for the 182 duration of the program. 183 184 """ 185 186 global LOG_LEVEL 187 BuiltIn().set_log_level("NONE") 188 189 BuiltIn().set_global_variable("${master_pid}", master_pid) 190 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 191 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 192 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 193 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 194 ffdc_list_file_path) 195 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 196 ffdc_report_list_path) 197 198 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 199 ffdc_dir_path_style) 200 BuiltIn().set_global_variable("${FFDC_CHECK}", 201 ffdc_check) 202 203 # For each program parameter, set the corresponding AUTOBOOT_ environment 204 # variable value. Also, set an AUTOBOOT_ environment variable for every 205 # element in additional_values. 206 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 207 "status_dir_path", "base_tool_dir_path", 208 "ffdc_list_file_path", "ffdc_report_list_path"] 209 210 plug_in_vars = parm_list + additional_values 211 212 for var_name in plug_in_vars: 213 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 214 var_name = var_name.upper() 215 if var_value is None: 216 var_value = "" 217 os.environ["AUTOBOOT_" + var_name] = str(var_value) 218 219 BuiltIn().set_log_level(LOG_LEVEL) 220 221 # Make sure the ffdc list directory exists. 222 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 223 if not os.path.exists(ffdc_list_dir_path): 224 os.makedirs(ffdc_list_dir_path) 225 226############################################################################### 227 228 229############################################################################### 230def plug_in_setup(): 231 232 r""" 233 Initialize all changing plug-in environment variables for use by the 234 plug-in programs. 235 """ 236 237 global LOG_LEVEL 238 global test_really_running 239 240 BuiltIn().set_log_level("NONE") 241 242 boot_pass, boot_fail = boot_results.return_total_pass_fail() 243 if boot_pass > 1: 244 test_really_running = 1 245 else: 246 test_really_running = 0 247 248 seconds = time.time() 249 loc_time = time.localtime(seconds) 250 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 251 252 ffdc_prefix = openbmc_nickname + "." + time_string 253 254 BuiltIn().set_global_variable("${test_really_running}", 255 test_really_running) 256 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 257 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 258 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 259 BuiltIn().set_global_variable("${boot_success}", boot_success) 260 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 261 262 # For each program parameter, set the corresponding AUTOBOOT_ environment 263 # variable value. Also, set an AUTOBOOT_ environment variable for every 264 # element in additional_values. 265 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 266 "boot_fail", "test_really_running", "ffdc_prefix"] 267 268 plug_in_vars = additional_values 269 270 for var_name in plug_in_vars: 271 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 272 var_name = var_name.upper() 273 if var_value is None: 274 var_value = "" 275 os.environ["AUTOBOOT_" + var_name] = str(var_value) 276 277 if debug: 278 shell_rc, out_buf = \ 279 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 280 281 BuiltIn().set_log_level(LOG_LEVEL) 282 283############################################################################### 284 285 286############################################################################### 287def pre_boot_plug_in_setup(): 288 289 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 290 try: 291 os.remove(ffdc_list_file_path) 292 except OSError: 293 pass 294 295 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 296 try: 297 os.remove(ffdc_report_list_path) 298 except OSError: 299 pass 300 301############################################################################### 302 303 304############################################################################### 305def setup(): 306 307 r""" 308 Do general program setup tasks. 309 """ 310 311 global cp_setup_called 312 313 gp.qprintn() 314 315 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 316 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 317 # If we can't find ssh_pw, then we don't have our repo bin in PATH. 318 shell_rc, out_buf = gc.cmd_fnc_u("which ssh_pw", quiet=1, print_output=0, 319 show_err=0) 320 if shell_rc != 0: 321 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 322 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 323 if robot_pgm_dir_path not in sys.path: 324 sys.path.append(robot_pgm_dir_path) 325 PYTHONPATH = os.environ.get("PYTHONPATH", "") 326 if PYTHONPATH == "": 327 os.environ['PYTHONPATH'] = robot_pgm_dir_path 328 else: 329 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 330 331 validate_parms() 332 333 grp.rqprint_pgm_header() 334 335 grk.run_key("Set BMC Power Policy RESTORE_LAST_STATE") 336 337 initial_plug_in_setup() 338 339 plug_in_setup() 340 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 341 call_point='setup') 342 if rc != 0: 343 error_message = "Plug-in setup failed.\n" 344 grp.rprint_error_report(error_message) 345 BuiltIn().fail(error_message) 346 # Setting cp_setup_called lets our Teardown know that it needs to call 347 # the cleanup plug-in call point. 348 cp_setup_called = 1 349 350 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 351 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 352 # FFDC_LOG_PATH is used by "FFDC" keyword. 353 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 354 355 # Also printed by FFDC. 356 global host_name 357 global host_ip 358 host = socket.gethostname() 359 host_name, host_ip = gm.get_host_name_ip(host) 360 361 gp.dprint_var(boot_table, 1) 362 gp.dprint_var(boot_lists) 363 364############################################################################### 365 366 367############################################################################### 368def validate_parms(): 369 370 r""" 371 Validate all program parameters. 372 """ 373 374 process_pgm_parms() 375 376 gp.qprintn() 377 378 global openbmc_model 379 grv.rvalid_value("openbmc_host") 380 grv.rvalid_value("openbmc_username") 381 grv.rvalid_value("openbmc_password") 382 if os_host != "": 383 grv.rvalid_value("os_username") 384 grv.rvalid_value("os_password") 385 386 if pdu_host != "": 387 grv.rvalid_value("pdu_username") 388 grv.rvalid_value("pdu_password") 389 grv.rvalid_integer("pdu_slot_no") 390 if openbmc_serial_host != "": 391 grv.rvalid_integer("openbmc_serial_port") 392 if openbmc_model == "": 393 status, ret_values =\ 394 grk.run_key_u("Get BMC System Model") 395 openbmc_model = ret_values 396 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 397 grv.rvalid_value("openbmc_model") 398 grv.rvalid_integer("max_num_tests") 399 grv.rvalid_integer("boot_pass") 400 grv.rvalid_integer("boot_fail") 401 402 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 403 BuiltIn().set_global_variable("${plug_in_packages_list}", 404 plug_in_packages_list) 405 406 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) 407 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 408 error_message = "You must provide either a value for either the" +\ 409 " boot_list or the boot_stack parm.\n" 410 BuiltIn().fail(gp.sprint_error(error_message)) 411 412 valid_boot_list(boot_list, valid_boot_types) 413 valid_boot_list(boot_stack, valid_boot_types) 414 415 selected_PDU_boots = list(set(boot_list + boot_stack) & 416 set(boot_lists['PDU_reboot'])) 417 418 if len(selected_PDU_boots) > 0 and pdu_host == "": 419 error_message = "You have selected the following boots which" +\ 420 " require a PDU host but no value for pdu_host:\n" 421 error_message += gp.sprint_var(selected_PDU_boots) 422 error_message += gp.sprint_var(pdu_host, 2) 423 BuiltIn().fail(gp.sprint_error(error_message)) 424 425 return 426 427############################################################################### 428 429 430############################################################################### 431def my_get_state(): 432 433 r""" 434 Get the system state plus a little bit of wrapping. 435 """ 436 437 global state 438 439 req_states = ['epoch_seconds'] + st.default_req_states 440 441 gp.qprint_timen("Getting system state.") 442 if test_mode: 443 state['epoch_seconds'] = int(time.time()) 444 else: 445 state = st.get_state(req_states=req_states, quiet=quiet) 446 gp.qprint_var(state) 447 448############################################################################### 449 450 451############################################################################### 452def select_boot(): 453 454 r""" 455 Select a boot test to be run based on our current state and return the 456 chosen boot type. 457 458 Description of arguments: 459 state The state of the machine. 460 """ 461 462 global boot_stack 463 464 gp.qprint_timen("Selecting a boot test.") 465 466 my_get_state() 467 468 stack_popped = 0 469 if len(boot_stack) > 0: 470 stack_popped = 1 471 gp.qprint_dashes() 472 gp.qprint_var(boot_stack) 473 gp.qprint_dashes() 474 skip_boot_printed = 0 475 while len(boot_stack) > 0: 476 boot_candidate = boot_stack.pop() 477 if stack_mode == 'normal': 478 break 479 else: 480 if st.compare_states(state, boot_table[boot_candidate]['end']): 481 if not skip_boot_printed: 482 gp.print_var(stack_mode) 483 gp.printn() 484 gp.print_timen("Skipping the following boot tests" + 485 " which are unnecessary since their" + 486 " required end states match the" + 487 " current machine state:") 488 skip_boot_printed = 1 489 gp.print_var(boot_candidate) 490 boot_candidate = "" 491 if boot_candidate == "": 492 gp.qprint_dashes() 493 gp.qprint_var(boot_stack) 494 gp.qprint_dashes() 495 return boot_candidate 496 if st.compare_states(state, boot_table[boot_candidate]['start']): 497 gp.qprint_timen("The machine state is valid for a '" + 498 boot_candidate + "' boot test.") 499 gp.qprint_dashes() 500 gp.qprint_var(boot_stack) 501 gp.qprint_dashes() 502 return boot_candidate 503 else: 504 gp.qprint_timen("The machine state does not match the required" + 505 " starting state for a '" + boot_candidate + 506 "' boot test:") 507 gp.print_varx("boot_table[" + boot_candidate + "][start]", 508 boot_table[boot_candidate]['start'], 1) 509 boot_stack.append(boot_candidate) 510 popped_boot = boot_candidate 511 512 # Loop through your list selecting a boot_candidates 513 boot_candidates = [] 514 for boot_candidate in boot_list: 515 if st.compare_states(state, boot_table[boot_candidate]['start']): 516 if stack_popped: 517 if st.compare_states(boot_table[boot_candidate]['end'], 518 boot_table[popped_boot]['start']): 519 boot_candidates.append(boot_candidate) 520 else: 521 boot_candidates.append(boot_candidate) 522 523 if len(boot_candidates) == 0: 524 gp.qprint_timen("The user's boot list contained no boot tests" + 525 " which are valid for the current machine state.") 526 boot_candidate = default_power_on 527 if not st.compare_states(state, boot_table[default_power_on]['start']): 528 boot_candidate = default_power_off 529 boot_candidates.append(boot_candidate) 530 gp.qprint_timen("Using default '" + boot_candidate + 531 "' boot type to transition to valid state.") 532 533 gp.dprint_var(boot_candidates) 534 535 # Randomly select a boot from the candidate list. 536 boot = random.choice(boot_candidates) 537 538 return boot 539 540############################################################################### 541 542 543############################################################################### 544def print_last_boots(): 545 546 r""" 547 Print the last ten boots done with their time stamps. 548 """ 549 550 # indent 0, 90 chars wide, linefeed, char is "=" 551 gp.qprint_dashes(0, 90) 552 gp.qprintn("Last 10 boots:\n") 553 554 for boot_entry in last_ten: 555 grp.rqprint(boot_entry) 556 gp.qprint_dashes(0, 90) 557 558############################################################################### 559 560 561############################################################################### 562def print_defect_report(): 563 564 r""" 565 Print a defect report. 566 """ 567 568 # At some point I'd like to have the 'Call FFDC Methods' return a list 569 # of files it has collected. In that case, the following "ls" command 570 # would no longer be needed. For now, however, glob shows the files 571 # named in FFDC_LIST_FILE_PATH so I will refrain from printing those 572 # out (so we don't see duplicates in the list). 573 574 # Get additional header data which may have been created by ffdc plug-ins. 575 # Also, delete the individual header files to cleanup. 576 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 577 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 578 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 579 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 580 show_err=0) 581 582 LOG_PREFIX = BuiltIn().get_variable_value("${LOG_PREFIX}") 583 584 output = '\n'.join(sorted(glob.glob(LOG_PREFIX + '*'))) 585 try: 586 ffdc_list = open(ffdc_list_file_path, 'r') 587 except IOError: 588 ffdc_list = "" 589 590 # Open ffdc_file_list for writing. We will write a complete list of 591 # FFDC files to it for possible use by plug-ins like cp_stop_check. 592 ffdc_list_file = open(ffdc_list_file_path, 'w') 593 594 gp.qprintn() 595 # indent=0, width=90, linefeed=1, char="=" 596 gp.qprint_dashes(0, 90, 1, "=") 597 gp.qprintn("Copy this data to the defect:\n") 598 599 if len(more_header_info) > 0: 600 gp.printn(more_header_info) 601 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 602 openbmc_host_name, openbmc_ip, openbmc_username, 603 openbmc_password, os_host, os_host_name, os_ip, os_username, 604 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 605 pdu_password, pdu_slot_no, openbmc_serial_host, 606 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 607 608 gp.qprintn() 609 610 print_last_boots() 611 gp.qprintn() 612 gp.qprint_var(state) 613 614 gp.qprintn() 615 gp.qprintn("FFDC data files:") 616 if status_file_path != "": 617 gp.qprintn(status_file_path) 618 ffdc_list_file.write(status_file_path + "\n") 619 620 gp.qprintn(output) 621 # gp.qprintn(ffdc_list) 622 gp.qprintn() 623 624 gp.qprint_dashes(0, 90, 1, "=") 625 626 ffdc_list_file.write(output + "\n") 627 ffdc_list_file.close() 628 629############################################################################### 630 631 632############################################################################### 633def my_ffdc(): 634 635 r""" 636 Collect FFDC data. 637 """ 638 639 global state 640 641 plug_in_setup() 642 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 643 call_point='ffdc', stop_on_plug_in_failure=1) 644 645 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 646 status, ret_values = grk.run_key_u("FFDC ffdc_prefix=" + 647 AUTOBOOT_FFDC_PREFIX + 648 " ffdc_function_list=" + 649 ffdc_function_list, ignore=1) 650 if status != 'PASS': 651 gp.print_error("Call to ffdc failed.\n") 652 653 my_get_state() 654 655 print_defect_report() 656 657############################################################################### 658 659 660############################################################################### 661def print_test_start_message(boot_keyword): 662 663 r""" 664 Print a message indicating what boot test is about to run. 665 666 Description of arguments: 667 boot_keyword The name of the boot which is to be run 668 (e.g. "BMC Power On"). 669 """ 670 671 global last_ten 672 673 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 674 gp.qprint(doing_msg) 675 676 last_ten.append(doing_msg) 677 678 if len(last_ten) > 10: 679 del last_ten[0] 680 681############################################################################### 682 683 684############################################################################### 685def run_boot(boot): 686 687 r""" 688 Run the specified boot. 689 690 Description of arguments: 691 boot The name of the boot test to be performed. 692 """ 693 694 global state 695 696 print_test_start_message(boot) 697 698 plug_in_setup() 699 rc, shell_rc, failed_plug_in_name = \ 700 grpi.rprocess_plug_in_packages(call_point="pre_boot") 701 if rc != 0: 702 error_message = "Plug-in failed with non-zero return code.\n" +\ 703 gp.sprint_var(rc, 1) 704 BuiltIn().fail(gp.sprint_error(error_message)) 705 706 if test_mode: 707 # In test mode, we'll pretend the boot worked by assigning its 708 # required end state to the default state value. 709 state = st.strip_anchor_state(boot_table[boot]['end']) 710 else: 711 # Assertion: We trust that the state data was made fresh by the 712 # caller. 713 714 gp.qprintn() 715 716 if boot_table[boot]['method_type'] == "keyword": 717 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 718 boot_table[boot]['method'], 719 quiet=quiet) 720 721 if boot_table[boot]['bmc_reboot']: 722 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 723 plug_in_setup() 724 rc, shell_rc, failed_plug_in_name = \ 725 grpi.rprocess_plug_in_packages(call_point="post_reboot") 726 if rc != 0: 727 error_message = "Plug-in failed with non-zero return code.\n" 728 error_message += gp.sprint_var(rc, 1) 729 BuiltIn().fail(gp.sprint_error(error_message)) 730 else: 731 match_state = st.anchor_state(state) 732 del match_state['epoch_seconds'] 733 # Wait for the state to change in any way. 734 st.wait_state(match_state, wait_time=state_change_timeout, 735 interval="3 seconds", invert=1) 736 737 gp.qprintn() 738 if boot_table[boot]['end']['chassis'] == "Off": 739 boot_timeout = power_off_timeout 740 else: 741 boot_timeout = power_on_timeout 742 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 743 interval="3 seconds") 744 745 plug_in_setup() 746 rc, shell_rc, failed_plug_in_name = \ 747 grpi.rprocess_plug_in_packages(call_point="post_boot") 748 if rc != 0: 749 error_message = "Plug-in failed with non-zero return code.\n" +\ 750 gp.sprint_var(rc, 1) 751 BuiltIn().fail(gp.sprint_error(error_message)) 752 753############################################################################### 754 755 756############################################################################### 757def test_loop_body(): 758 759 r""" 760 The main loop body for the loop in main_py. 761 762 Description of arguments: 763 boot_count The iteration number (starts at 1). 764 """ 765 766 global boot_count 767 global state 768 global next_boot 769 global boot_success 770 771 gp.qprintn() 772 773 next_boot = select_boot() 774 if next_boot == "": 775 return True 776 777 boot_count += 1 778 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 779 780 pre_boot_plug_in_setup() 781 782 cmd_buf = ["run_boot", next_boot] 783 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 784 if boot_status == "FAIL": 785 gp.qprint(msg) 786 787 gp.qprintn() 788 if boot_status == "PASS": 789 boot_success = 1 790 gp.qprint_timen("BOOT_SUCCESS: \"" + next_boot + "\" succeeded.") 791 else: 792 boot_success = 0 793 gp.qprint_timen("BOOT_FAILED: \"" + next_boot + "\" failed.") 794 795 boot_results.update(next_boot, boot_status) 796 797 plug_in_setup() 798 # NOTE: A post_test_case call point failure is NOT counted as a boot 799 # failure. 800 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 801 call_point='post_test_case', stop_on_plug_in_failure=1) 802 803 plug_in_setup() 804 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 805 call_point='ffdc_check', shell_rc=0x00000200, 806 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 807 if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200: 808 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 809 if status != 'PASS': 810 gp.print_error("Call to my_ffdc failed.\n") 811 812 # We need to purge error logs between boots or they build up. 813 grk.run_key("Delete Error logs", ignore=1) 814 815 boot_results.print_report() 816 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 817 818 plug_in_setup() 819 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 820 call_point='stop_check') 821 if rc != 0: 822 error_message = "Stopping as requested by user.\n" 823 grp.rprint_error_report(error_message) 824 BuiltIn().fail(error_message) 825 826 # This should help prevent ConnectionErrors. 827 grk.run_key_u("Delete All Sessions") 828 829 return True 830 831############################################################################### 832 833 834############################################################################### 835def obmc_boot_test_teardown(): 836 837 r""" 838 Clean up after the Main keyword. 839 """ 840 841 if cp_setup_called: 842 plug_in_setup() 843 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 844 call_point='cleanup', stop_on_plug_in_failure=1) 845 846 # Save boot_results object to a file in case it is needed again. 847 gp.qprint_timen("Saving boot_results to the following path.") 848 gp.qprint_var(boot_results_file_path) 849 pickle.dump(boot_results, open(boot_results_file_path, 'wb'), 850 pickle.HIGHEST_PROTOCOL) 851 852############################################################################### 853 854 855############################################################################### 856def test_teardown(): 857 858 r""" 859 Clean up after this test case. 860 """ 861 862 gp.qprintn() 863 cmd_buf = ["Print Error", 864 "A keyword timeout occurred ending this program.\n"] 865 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 866 867 grp.rqprint_pgm_footer() 868 869############################################################################### 870 871 872############################################################################### 873def obmc_boot_test_py(alt_boot_stack=None): 874 875 r""" 876 Do main program processing. 877 """ 878 879 if alt_boot_stack is not None: 880 BuiltIn().set_global_variable("${boot_stack}", alt_boot_stack) 881 882 setup() 883 884 if ffdc_only: 885 gp.qprint_timen("Caller requested ffdc_only.") 886 pre_boot_plug_in_setup() 887 grk.run_key_u("my_ffdc") 888 return 889 890 # Process caller's boot_stack. 891 while (len(boot_stack) > 0): 892 test_loop_body() 893 894 gp.qprint_timen("Finished processing stack.") 895 896 # Process caller's boot_list. 897 if len(boot_list) > 0: 898 for ix in range(1, max_num_tests + 1): 899 test_loop_body() 900 901 gp.qprint_timen("Completed all requested boot tests.") 902 903 boot_pass, boot_fail = boot_results.return_total_pass_fail() 904 if boot_fail > boot_fail_threshold: 905 error_message = "Boot failures exceed the boot failure" +\ 906 " threshold:\n" +\ 907 gp.sprint_var(boot_fail) +\ 908 gp.sprint_var(boot_fail_threshold) 909 BuiltIn().fail(gp.sprint_error(error_message)) 910 911############################################################################### 912