1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13try: 14 import cPickle as pickle 15except ImportError: 16 import pickle 17import socket 18 19from robot.utils import DotDict 20from robot.libraries.BuiltIn import BuiltIn 21 22from boot_data import * 23import gen_print as gp 24import gen_robot_print as grp 25import gen_robot_plug_in as grpi 26import gen_robot_valid as grv 27import gen_misc as gm 28import gen_cmd as gc 29import gen_robot_keyword as grk 30import state as st 31import var_stack as vs 32 33base_path = os.path.dirname(os.path.dirname( 34 imp.find_module("gen_robot_print")[1])) +\ 35 os.sep 36sys.path.append(base_path + "extended/") 37import run_keyword as rk 38 39# Setting master_pid correctly influences the behavior of plug-ins like 40# DB_Logging 41program_pid = os.getpid() 42master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 43pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 44 45# Set up boot data structures. 46boot_table = create_boot_table() 47valid_boot_types = create_valid_boot_list(boot_table) 48 49boot_lists = read_boot_lists() 50last_ten = [] 51 52state = st.return_state_constant('default_state') 53cp_setup_called = 0 54next_boot = "" 55base_tool_dir_path = os.path.normpath(os.environ.get( 56 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 57 58ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 59boot_success = 0 60status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 61if status_dir_path != "": 62 status_dir_path = os.path.normpath(status_dir_path) + os.sep 63default_power_on = "REST Power On" 64default_power_off = "REST Power Off" 65boot_count = 0 66 67LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 68ffdc_prefix = "" 69boot_start_time = "" 70boot_end_time = "" 71save_stack = vs.var_stack('save_stack') 72main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 73 74 75def process_host(host, 76 host_var_name=""): 77 r""" 78 Process a host by getting the associated host name and IP address and 79 setting them in global variables. 80 81 If the caller does not pass the host_var_name, this function will try to 82 figure out the name of the variable used by the caller for the host parm. 83 Callers are advised to explicitly specify the host_var_name when calling 84 with an exec command. In such cases, the get_arg_name cannot figure out 85 the host variable name. 86 87 This function will then create similar global variable names by 88 removing "_host" and appending "_host_name" or "_ip" to the host variable 89 name. 90 91 Example: 92 93 If a call is made like this: 94 process_host(openbmc_host) 95 96 Global variables openbmc_host_name and openbmc_ip will be set. 97 98 Description of argument(s): 99 host A host name or IP. The name of the variable used should 100 have a suffix of "_host". 101 host_var_name The name of the variable being used as the host parm. 102 """ 103 104 if host_var_name == "": 105 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 106 107 host_name_var_name = re.sub("host", "host_name", host_var_name) 108 ip_var_name = re.sub("host", "ip", host_var_name) 109 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 110 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 111 host + "')" 112 exec(cmd_buf) 113 114 115def process_pgm_parms(): 116 r""" 117 Process the program parameters by assigning them all to corresponding 118 globals. Also, set some global values that depend on program parameters. 119 """ 120 121 # Program parameter processing. 122 # Assign all program parms to python variables which are global to this 123 # module. 124 125 global parm_list 126 parm_list = BuiltIn().get_variable_value("${parm_list}") 127 # The following subset of parms should be processed as integers. 128 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 129 'boot_fail_threshold', 'delete_errlogs', 'quiet', 'test_mode', 130 'debug'] 131 for parm in parm_list: 132 if parm in int_list: 133 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 134 "}\", \"0\"))" 135 else: 136 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 137 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 138 gp.dpissuing(cmd_buf) 139 exec(cmd_buf) 140 if re.match(r".*_host$", parm): 141 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 142 exec(cmd_buf) 143 if re.match(r".*_password$", parm): 144 # Register the value of any parm whose name ends in _password. 145 # This will cause the print functions to replace passwords with 146 # asterisks in the output. 147 cmd_buf = "gp.register_passwords(" + parm + ")" 148 exec(cmd_buf) 149 150 global ffdc_dir_path_style 151 global boot_list 152 global boot_stack 153 global boot_results_file_path 154 global boot_results 155 global ffdc_list_file_path 156 global ffdc_report_list_path 157 global ffdc_summary_list_path 158 159 if ffdc_dir_path_style == "": 160 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 161 162 # Convert these program parms to lists for easier processing.. 163 boot_list = filter(None, boot_list.split(":")) 164 boot_stack = filter(None, boot_stack.split(":")) 165 166 cleanup_boot_results_file() 167 boot_results_file_path = create_boot_results_file_path(pgm_name, 168 openbmc_nickname, 169 master_pid) 170 171 if os.path.isfile(boot_results_file_path): 172 # We've been called before in this run so we'll load the saved 173 # boot_results object. 174 boot_results = pickle.load(open(boot_results_file_path, 'rb')) 175 else: 176 boot_results = boot_results(boot_table, boot_pass, boot_fail) 177 178 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 179 "/FFDC_FILE_LIST" 180 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 181 "/FFDC_REPORT_FILE_LIST" 182 183 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 184 "/FFDC_SUMMARY_FILE_LIST" 185 186 187def initial_plug_in_setup(): 188 r""" 189 Initialize all plug-in environment variables which do not change for the 190 duration of the program. 191 192 """ 193 194 global LOG_LEVEL 195 BuiltIn().set_log_level("NONE") 196 197 BuiltIn().set_global_variable("${master_pid}", master_pid) 198 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 199 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 200 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 201 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 202 ffdc_list_file_path) 203 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 204 ffdc_report_list_path) 205 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 206 ffdc_summary_list_path) 207 208 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 209 ffdc_dir_path_style) 210 BuiltIn().set_global_variable("${FFDC_CHECK}", 211 ffdc_check) 212 213 # For each program parameter, set the corresponding AUTOBOOT_ environment 214 # variable value. Also, set an AUTOBOOT_ environment variable for every 215 # element in additional_values. 216 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 217 "status_dir_path", "base_tool_dir_path", 218 "ffdc_list_file_path", "ffdc_report_list_path", 219 "ffdc_summary_list_path"] 220 221 plug_in_vars = parm_list + additional_values 222 223 for var_name in plug_in_vars: 224 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 225 var_name = var_name.upper() 226 if var_value is None: 227 var_value = "" 228 os.environ["AUTOBOOT_" + var_name] = str(var_value) 229 230 BuiltIn().set_log_level(LOG_LEVEL) 231 232 # Make sure the ffdc list directory exists. 233 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 234 if not os.path.exists(ffdc_list_dir_path): 235 os.makedirs(ffdc_list_dir_path) 236 237 238def plug_in_setup(): 239 r""" 240 Initialize all changing plug-in environment variables for use by the 241 plug-in programs. 242 """ 243 244 global LOG_LEVEL 245 global test_really_running 246 247 BuiltIn().set_log_level("NONE") 248 249 boot_pass, boot_fail = boot_results.return_total_pass_fail() 250 if boot_pass > 1: 251 test_really_running = 1 252 else: 253 test_really_running = 0 254 255 BuiltIn().set_global_variable("${test_really_running}", 256 test_really_running) 257 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 258 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 259 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 260 BuiltIn().set_global_variable("${boot_success}", boot_success) 261 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 262 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 263 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 264 265 # For each program parameter, set the corresponding AUTOBOOT_ environment 266 # variable value. Also, set an AUTOBOOT_ environment variable for every 267 # element in additional_values. 268 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 269 "boot_fail", "test_really_running", "ffdc_prefix", 270 "boot_start_time", "boot_end_time"] 271 272 plug_in_vars = additional_values 273 274 for var_name in plug_in_vars: 275 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 276 var_name = var_name.upper() 277 if var_value is None: 278 var_value = "" 279 os.environ["AUTOBOOT_" + var_name] = str(var_value) 280 281 if debug: 282 shell_rc, out_buf = \ 283 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 284 285 BuiltIn().set_log_level(LOG_LEVEL) 286 287 288def pre_boot_plug_in_setup(): 289 290 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 291 try: 292 os.remove(ffdc_list_file_path) 293 except OSError: 294 pass 295 296 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 297 try: 298 os.remove(ffdc_report_list_path) 299 except OSError: 300 pass 301 302 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 303 try: 304 os.remove(ffdc_summary_list_path) 305 except OSError: 306 pass 307 308 global ffdc_prefix 309 310 seconds = time.time() 311 loc_time = time.localtime(seconds) 312 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 313 314 ffdc_prefix = openbmc_nickname + "." + time_string 315 316 317def setup(): 318 r""" 319 Do general program setup tasks. 320 """ 321 322 global cp_setup_called 323 global transitional_boot_selected 324 325 gp.qprintn() 326 327 transitional_boot_selected = False 328 329 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 330 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 331 # If we can't find process_plug_in_packages.py, ssh_pw or 332 # validate_plug_ins.py, then we don't have our repo bin in PATH. 333 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 334 + " ssh_pw validate_plug_ins.py", quiet=1, 335 print_output=0, show_err=0) 336 if shell_rc != 0: 337 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 338 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 339 if robot_pgm_dir_path not in sys.path: 340 sys.path.append(robot_pgm_dir_path) 341 PYTHONPATH = os.environ.get("PYTHONPATH", "") 342 if PYTHONPATH == "": 343 os.environ['PYTHONPATH'] = robot_pgm_dir_path 344 else: 345 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 346 347 validate_parms() 348 349 grp.rqprint_pgm_header() 350 351 grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF") 352 353 initial_plug_in_setup() 354 355 plug_in_setup() 356 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 357 call_point='setup') 358 if rc != 0: 359 error_message = "Plug-in setup failed.\n" 360 grp.rprint_error_report(error_message) 361 BuiltIn().fail(error_message) 362 # Setting cp_setup_called lets our Teardown know that it needs to call 363 # the cleanup plug-in call point. 364 cp_setup_called = 1 365 366 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 367 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 368 # FFDC_LOG_PATH is used by "FFDC" keyword. 369 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 370 371 # Also printed by FFDC. 372 global host_name 373 global host_ip 374 host = socket.gethostname() 375 host_name, host_ip = gm.get_host_name_ip(host) 376 377 gp.dprint_var(boot_table, 1) 378 gp.dprint_var(boot_lists) 379 380 381def validate_parms(): 382 r""" 383 Validate all program parameters. 384 """ 385 386 process_pgm_parms() 387 388 gp.qprintn() 389 390 global openbmc_model 391 grv.rvalid_value("openbmc_host") 392 grv.rvalid_value("openbmc_username") 393 grv.rvalid_value("openbmc_password") 394 if os_host != "": 395 grv.rvalid_value("os_username") 396 grv.rvalid_value("os_password") 397 398 if pdu_host != "": 399 grv.rvalid_value("pdu_username") 400 grv.rvalid_value("pdu_password") 401 grv.rvalid_integer("pdu_slot_no") 402 if openbmc_serial_host != "": 403 grv.rvalid_integer("openbmc_serial_port") 404 if openbmc_model == "": 405 status, ret_values =\ 406 grk.run_key_u("Get BMC System Model") 407 openbmc_model = ret_values 408 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 409 grv.rvalid_value("openbmc_model") 410 grv.rvalid_integer("max_num_tests") 411 grv.rvalid_integer("boot_pass") 412 grv.rvalid_integer("boot_fail") 413 414 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 415 BuiltIn().set_global_variable("${plug_in_packages_list}", 416 plug_in_packages_list) 417 418 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) 419 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 420 error_message = "You must provide either a value for either the" +\ 421 " boot_list or the boot_stack parm.\n" 422 BuiltIn().fail(gp.sprint_error(error_message)) 423 424 valid_boot_list(boot_list, valid_boot_types) 425 valid_boot_list(boot_stack, valid_boot_types) 426 427 selected_PDU_boots = list(set(boot_list + boot_stack) 428 & set(boot_lists['PDU_reboot'])) 429 430 if len(selected_PDU_boots) > 0 and pdu_host == "": 431 error_message = "You have selected the following boots which" +\ 432 " require a PDU host but no value for pdu_host:\n" 433 error_message += gp.sprint_var(selected_PDU_boots) 434 error_message += gp.sprint_var(pdu_host, 2) 435 BuiltIn().fail(gp.sprint_error(error_message)) 436 437 return 438 439 440def my_get_state(): 441 r""" 442 Get the system state plus a little bit of wrapping. 443 """ 444 445 global state 446 447 req_states = ['epoch_seconds'] + st.default_req_states 448 449 gp.qprint_timen("Getting system state.") 450 if test_mode: 451 state['epoch_seconds'] = int(time.time()) 452 else: 453 state = st.get_state(req_states=req_states, quiet=quiet) 454 gp.qprint_var(state) 455 456 457def valid_state(): 458 r""" 459 Verify that our state dictionary contains no blank values. If we don't get 460 valid state data, we cannot continue to work. 461 """ 462 463 if st.compare_states(state, st.invalid_state_match, 'or'): 464 error_message = "The state dictionary contains blank fields which" +\ 465 " is illegal.\n" + gp.sprint_var(state) 466 BuiltIn().fail(gp.sprint_error(error_message)) 467 468 469def select_boot(): 470 r""" 471 Select a boot test to be run based on our current state and return the 472 chosen boot type. 473 474 Description of arguments: 475 state The state of the machine. 476 """ 477 478 global transitional_boot_selected 479 global boot_stack 480 481 gp.qprint_timen("Selecting a boot test.") 482 483 if transitional_boot_selected and not boot_success: 484 prior_boot = next_boot 485 boot_candidate = boot_stack.pop() 486 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 487 + " transition to a valid state for '" + boot_candidate 488 + "' which was at the top of the boot_stack. Since" 489 + " the '" + next_boot + "' failed, the '" 490 + boot_candidate + "' has been removed from the stack" 491 + " to avoid and endless failure loop.") 492 if len(boot_stack) == 0: 493 return "" 494 495 my_get_state() 496 valid_state() 497 498 transitional_boot_selected = False 499 stack_popped = 0 500 if len(boot_stack) > 0: 501 stack_popped = 1 502 gp.qprint_dashes() 503 gp.qprint_var(boot_stack) 504 gp.qprint_dashes() 505 skip_boot_printed = 0 506 while len(boot_stack) > 0: 507 boot_candidate = boot_stack.pop() 508 if stack_mode == 'normal': 509 break 510 else: 511 if st.compare_states(state, boot_table[boot_candidate]['end']): 512 if not skip_boot_printed: 513 gp.qprint_var(stack_mode) 514 gp.qprintn() 515 gp.qprint_timen("Skipping the following boot tests" 516 + " which are unnecessary since their" 517 + " required end states match the" 518 + " current machine state:") 519 skip_boot_printed = 1 520 gp.qprint_var(boot_candidate) 521 boot_candidate = "" 522 if boot_candidate == "": 523 gp.qprint_dashes() 524 gp.qprint_var(boot_stack) 525 gp.qprint_dashes() 526 return boot_candidate 527 if st.compare_states(state, boot_table[boot_candidate]['start']): 528 gp.qprint_timen("The machine state is valid for a '" 529 + boot_candidate + "' boot test.") 530 gp.qprint_dashes() 531 gp.qprint_var(boot_stack) 532 gp.qprint_dashes() 533 return boot_candidate 534 else: 535 gp.qprint_timen("The machine state does not match the required" 536 + " starting state for a '" + boot_candidate 537 + "' boot test:") 538 gp.qprint_varx("boot_table[" + boot_candidate + "][start]", 539 boot_table[boot_candidate]['start'], 1) 540 boot_stack.append(boot_candidate) 541 transitional_boot_selected = True 542 popped_boot = boot_candidate 543 544 # Loop through your list selecting a boot_candidates 545 boot_candidates = [] 546 for boot_candidate in boot_list: 547 if st.compare_states(state, boot_table[boot_candidate]['start']): 548 if stack_popped: 549 if st.compare_states(boot_table[boot_candidate]['end'], 550 boot_table[popped_boot]['start']): 551 boot_candidates.append(boot_candidate) 552 else: 553 boot_candidates.append(boot_candidate) 554 555 if len(boot_candidates) == 0: 556 gp.qprint_timen("The user's boot list contained no boot tests" 557 + " which are valid for the current machine state.") 558 boot_candidate = default_power_on 559 if not st.compare_states(state, boot_table[default_power_on]['start']): 560 boot_candidate = default_power_off 561 boot_candidates.append(boot_candidate) 562 gp.qprint_timen("Using default '" + boot_candidate 563 + "' boot type to transition to valid state.") 564 565 gp.dprint_var(boot_candidates) 566 567 # Randomly select a boot from the candidate list. 568 boot = random.choice(boot_candidates) 569 570 return boot 571 572 573def print_last_boots(): 574 r""" 575 Print the last ten boots done with their time stamps. 576 """ 577 578 # indent 0, 90 chars wide, linefeed, char is "=" 579 gp.qprint_dashes(0, 90) 580 gp.qprintn("Last 10 boots:\n") 581 582 for boot_entry in last_ten: 583 grp.rqprint(boot_entry) 584 gp.qprint_dashes(0, 90) 585 586 587def print_defect_report(ffdc_file_list): 588 r""" 589 Print a defect report. 590 591 Description of argument(s): 592 ffdc_file_list A list of files which were collected by our ffdc functions. 593 """ 594 595 # Making deliberate choice to NOT run plug_in_setup(). We don't want 596 # ffdc_prefix updated. 597 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 598 call_point='ffdc_report', stop_on_plug_in_failure=0) 599 600 # Get additional header data which may have been created by ffdc plug-ins. 601 # Also, delete the individual header files to cleanup. 602 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 603 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 604 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 605 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 606 show_err=0) 607 608 # Get additional summary data which may have been created by ffdc plug-ins. 609 # Also, delete the individual header files to cleanup. 610 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 611 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 612 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 613 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 614 show_err=0) 615 616 # ffdc_list_file_path contains a list of any ffdc files created by plug- 617 # ins, etc. Read that data into a list. 618 try: 619 plug_in_ffdc_list = \ 620 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 621 plug_in_ffdc_list = filter(None, plug_in_ffdc_list) 622 except IOError: 623 plug_in_ffdc_list = [] 624 625 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 626 # in. Eliminate duplicates and sort the list. 627 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 628 629 if status_file_path != "": 630 ffdc_file_list.insert(0, status_file_path) 631 632 # Convert the list to a printable list. 633 printable_ffdc_file_list = "\n".join(ffdc_file_list) 634 635 # Open ffdc_file_list for writing. We will write a complete list of 636 # FFDC files to it for possible use by plug-ins like cp_stop_check. 637 ffdc_list_file = open(ffdc_list_file_path, 'w') 638 ffdc_list_file.write(printable_ffdc_file_list + "\n") 639 ffdc_list_file.close() 640 641 indent = 0 642 width = 90 643 linefeed = 1 644 char = "=" 645 646 gp.qprintn() 647 gp.qprint_dashes(indent, width, linefeed, char) 648 gp.qprintn("Copy this data to the defect:\n") 649 650 if len(more_header_info) > 0: 651 gp.qprintn(more_header_info) 652 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 653 openbmc_host_name, openbmc_ip, openbmc_username, 654 openbmc_password, os_host, os_host_name, os_ip, os_username, 655 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 656 pdu_password, pdu_slot_no, openbmc_serial_host, 657 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 658 659 gp.qprintn() 660 print_last_boots() 661 gp.qprintn() 662 gp.qprint_var(state) 663 gp.qprintn() 664 gp.qprintn("FFDC data files:") 665 gp.qprintn(printable_ffdc_file_list) 666 gp.qprintn() 667 668 if len(ffdc_summary_info) > 0: 669 gp.qprintn(ffdc_summary_info) 670 671 gp.qprint_dashes(indent, width, linefeed, char) 672 673 674def my_ffdc(): 675 r""" 676 Collect FFDC data. 677 """ 678 679 global state 680 681 plug_in_setup() 682 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 683 call_point='ffdc', stop_on_plug_in_failure=0) 684 685 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 686 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 687 + AUTOBOOT_FFDC_PREFIX 688 + " ffdc_function_list=" 689 + ffdc_function_list, ignore=1) 690 if status != 'PASS': 691 gp.qprint_error("Call to ffdc failed.\n") 692 693 my_get_state() 694 695 print_defect_report(ffdc_file_list) 696 697 698def print_test_start_message(boot_keyword): 699 r""" 700 Print a message indicating what boot test is about to run. 701 702 Description of arguments: 703 boot_keyword The name of the boot which is to be run 704 (e.g. "BMC Power On"). 705 """ 706 707 global last_ten 708 global boot_start_time 709 710 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 711 712 # Set boot_start_time for use by plug-ins. 713 boot_start_time = doing_msg[1:33] 714 gp.qprint_var(boot_start_time) 715 716 gp.qprint(doing_msg) 717 718 last_ten.append(doing_msg) 719 720 if len(last_ten) > 10: 721 del last_ten[0] 722 723 724def run_boot(boot): 725 r""" 726 Run the specified boot. 727 728 Description of arguments: 729 boot The name of the boot test to be performed. 730 """ 731 732 global state 733 734 print_test_start_message(boot) 735 736 plug_in_setup() 737 rc, shell_rc, failed_plug_in_name = \ 738 grpi.rprocess_plug_in_packages(call_point="pre_boot") 739 if rc != 0: 740 error_message = "Plug-in failed with non-zero return code.\n" +\ 741 gp.sprint_var(rc, 1) 742 BuiltIn().fail(gp.sprint_error(error_message)) 743 744 if test_mode: 745 # In test mode, we'll pretend the boot worked by assigning its 746 # required end state to the default state value. 747 state = st.strip_anchor_state(boot_table[boot]['end']) 748 else: 749 # Assertion: We trust that the state data was made fresh by the 750 # caller. 751 752 gp.qprintn() 753 754 if boot_table[boot]['method_type'] == "keyword": 755 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 756 boot_table[boot]['method'], 757 quiet=quiet) 758 759 if boot_table[boot]['bmc_reboot']: 760 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 761 plug_in_setup() 762 rc, shell_rc, failed_plug_in_name = \ 763 grpi.rprocess_plug_in_packages(call_point="post_reboot") 764 if rc != 0: 765 error_message = "Plug-in failed with non-zero return code.\n" 766 error_message += gp.sprint_var(rc, 1) 767 BuiltIn().fail(gp.sprint_error(error_message)) 768 else: 769 match_state = st.anchor_state(state) 770 del match_state['epoch_seconds'] 771 # Wait for the state to change in any way. 772 st.wait_state(match_state, wait_time=state_change_timeout, 773 interval="10 seconds", invert=1) 774 775 gp.qprintn() 776 if boot_table[boot]['end']['chassis'] == "Off": 777 boot_timeout = power_off_timeout 778 else: 779 boot_timeout = power_on_timeout 780 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 781 interval="10 seconds") 782 783 plug_in_setup() 784 rc, shell_rc, failed_plug_in_name = \ 785 grpi.rprocess_plug_in_packages(call_point="post_boot") 786 if rc != 0: 787 error_message = "Plug-in failed with non-zero return code.\n" +\ 788 gp.sprint_var(rc, 1) 789 BuiltIn().fail(gp.sprint_error(error_message)) 790 791 792def test_loop_body(): 793 r""" 794 The main loop body for the loop in main_py. 795 796 Description of arguments: 797 boot_count The iteration number (starts at 1). 798 """ 799 800 global boot_count 801 global state 802 global next_boot 803 global boot_success 804 global boot_end_time 805 806 gp.qprintn() 807 808 next_boot = select_boot() 809 if next_boot == "": 810 return True 811 812 boot_count += 1 813 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 814 815 pre_boot_plug_in_setup() 816 817 cmd_buf = ["run_boot", next_boot] 818 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 819 if boot_status == "FAIL": 820 gp.qprint(msg) 821 822 gp.qprintn() 823 if boot_status == "PASS": 824 boot_success = 1 825 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 826 + "\" succeeded.") 827 else: 828 boot_success = 0 829 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 830 + "\" failed.") 831 832 # Set boot_end_time for use by plug-ins. 833 boot_end_time = completion_msg[1:33] 834 gp.qprint_var(boot_end_time) 835 836 gp.qprint(completion_msg) 837 838 boot_results.update(next_boot, boot_status) 839 840 plug_in_setup() 841 # NOTE: A post_test_case call point failure is NOT counted as a boot 842 # failure. 843 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 844 call_point='post_test_case', stop_on_plug_in_failure=0) 845 846 plug_in_setup() 847 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 848 call_point='ffdc_check', shell_rc=0x00000200, 849 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 850 if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200: 851 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 852 if status != 'PASS': 853 gp.qprint_error("Call to my_ffdc failed.\n") 854 855 if delete_errlogs: 856 # We need to purge error logs between boots or they build up. 857 grk.run_key("Delete Error logs", ignore=1) 858 859 boot_results.print_report() 860 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 861 862 plug_in_setup() 863 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 864 call_point='stop_check', shell_rc=0x00000200, stop_on_non_zero_rc=1) 865 if shell_rc == 0x00000200: 866 message = "Stopping as requested by user.\n" 867 gp.print_time(message) 868 BuiltIn().fail(message) 869 870 # This should help prevent ConnectionErrors. 871 grk.run_key_u("Close All Connections") 872 873 return True 874 875 876def obmc_boot_test_teardown(): 877 r""" 878 Clean up after the Main keyword. 879 """ 880 881 if cp_setup_called: 882 plug_in_setup() 883 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 884 call_point='cleanup', stop_on_plug_in_failure=0) 885 886 if 'boot_results_file_path' in globals(): 887 # Save boot_results object to a file in case it is needed again. 888 gp.qprint_timen("Saving boot_results to the following path.") 889 gp.qprint_var(boot_results_file_path) 890 pickle.dump(boot_results, open(boot_results_file_path, 'wb'), 891 pickle.HIGHEST_PROTOCOL) 892 893 global save_stack 894 # Restore any global values saved on the save_stack. 895 for parm_name in main_func_parm_list: 896 # Get the parm_value if it was saved on the stack. 897 try: 898 parm_value = save_stack.pop(parm_name) 899 except BaseException: 900 # If it was not saved, no further action is required. 901 continue 902 903 # Restore the saved value. 904 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 905 "}\", parm_value)" 906 gp.dpissuing(cmd_buf) 907 exec(cmd_buf) 908 909 gp.dprintn(save_stack.sprint_obj()) 910 911 912def test_teardown(): 913 r""" 914 Clean up after this test case. 915 """ 916 917 gp.qprintn() 918 cmd_buf = ["Print Error", 919 "A keyword timeout occurred ending this program.\n"] 920 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 921 922 grp.rqprint_pgm_footer() 923 924 925def obmc_boot_test_py(loc_boot_stack=None, 926 loc_stack_mode=None, 927 loc_quiet=None): 928 r""" 929 Do main program processing. 930 """ 931 932 global save_stack 933 934 # Process function parms. 935 for parm_name in main_func_parm_list: 936 # Get parm's value. 937 cmd_buf = "parm_value = loc_" + parm_name 938 exec(cmd_buf) 939 gp.dpvar(parm_name) 940 gp.dpvar(parm_value) 941 942 if parm_value is None: 943 # Parm was not specified by the calling function so set it to its 944 # corresponding global value. 945 cmd_buf = "loc_" + parm_name + " = BuiltIn().get_variable_value" +\ 946 "(\"${" + parm_name + "}\")" 947 gp.dpissuing(cmd_buf) 948 exec(cmd_buf) 949 else: 950 # Save the global value on a stack. 951 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 952 parm_name + "}\"), \"" + parm_name + "\")" 953 gp.dpissuing(cmd_buf) 954 exec(cmd_buf) 955 956 # Set the global value to the passed value. 957 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 958 "}\", loc_" + parm_name + ")" 959 gp.dpissuing(cmd_buf) 960 exec(cmd_buf) 961 962 gp.dprintn(save_stack.sprint_obj()) 963 964 setup() 965 966 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 967 968 if ffdc_only: 969 gp.qprint_timen("Caller requested ffdc_only.") 970 pre_boot_plug_in_setup() 971 grk.run_key_u("my_ffdc") 972 return 973 974 # Process caller's boot_stack. 975 while (len(boot_stack) > 0): 976 test_loop_body() 977 978 gp.qprint_timen("Finished processing stack.") 979 980 # Process caller's boot_list. 981 if len(boot_list) > 0: 982 for ix in range(1, max_num_tests + 1): 983 test_loop_body() 984 985 gp.qprint_timen("Completed all requested boot tests.") 986 987 boot_pass, boot_fail = boot_results.return_total_pass_fail() 988 new_fail = boot_fail - init_boot_fail 989 if new_fail > boot_fail_threshold: 990 error_message = "Boot failures exceed the boot failure" +\ 991 " threshold:\n" +\ 992 gp.sprint_var(new_fail) +\ 993 gp.sprint_var(boot_fail_threshold) 994 BuiltIn().fail(gp.sprint_error(error_message)) 995