1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_arg as ga 27import gen_valid as gv 28import gen_misc as gm 29import gen_cmd as gc 30import gen_robot_keyword as grk 31import state as st 32import var_stack as vs 33import gen_plug_in_utils as gpu 34import pel_utils as pel 35 36base_path = os.path.dirname(os.path.dirname( 37 imp.find_module("gen_robot_print")[1])) +\ 38 os.sep 39sys.path.append(base_path + "extended/") 40import run_keyword as rk 41 42# Setting master_pid correctly influences the behavior of plug-ins like 43# DB_Logging 44program_pid = os.getpid() 45master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 46pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 47 48# Set up boot data structures. 49os_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 50 51boot_lists = read_boot_lists() 52 53# The maximum number of entries that can be in the boot_history global variable. 54max_boot_history = 10 55boot_history = [] 56 57state = st.return_state_constant('default_state') 58cp_setup_called = 0 59next_boot = "" 60base_tool_dir_path = os.path.normpath(os.environ.get( 61 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 62 63ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 64boot_success = 0 65status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 66if status_dir_path != "": 67 status_dir_path = os.path.normpath(status_dir_path) + os.sep 68redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False) 69redfish_rest_supported = BuiltIn().get_variable_value("${REDFISH_REST_SUPPORTED}", default=False) 70if redfish_supported: 71 redfish = BuiltIn().get_library_instance('redfish') 72 default_power_on = "Redfish Power On" 73 default_power_off = "Redfish Power Off" 74 if redfish_rest_supported: 75 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 76 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 77 else: 78 delete_errlogs_cmd = "Redfish Purge Event Log" 79 default_set_power_policy = "Redfish Set Power Restore Policy AlwaysOff" 80else: 81 default_power_on = "REST Power On" 82 default_power_off = "REST Power Off" 83 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 84 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 85boot_count = 0 86 87LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 88AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 89ffdc_prefix = AUTOBOOT_FFDC_PREFIX 90boot_start_time = "" 91boot_end_time = "" 92save_stack = vs.var_stack('save_stack') 93main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 94 95 96def dump_ffdc_rc(): 97 r""" 98 Return the constant dump ffdc test return code value. 99 100 When a plug-in call point program returns this value, it indicates that 101 this program should collect FFDC. 102 """ 103 104 return 0x00000200 105 106 107def stop_test_rc(): 108 r""" 109 Return the constant stop test return code value. 110 111 When a plug-in call point program returns this value, it indicates that 112 this program should stop running. 113 """ 114 115 return 0x00000200 116 117 118def process_host(host, 119 host_var_name=""): 120 r""" 121 Process a host by getting the associated host name and IP address and 122 setting them in global variables. 123 124 If the caller does not pass the host_var_name, this function will try to 125 figure out the name of the variable used by the caller for the host parm. 126 Callers are advised to explicitly specify the host_var_name when calling 127 with an exec command. In such cases, the get_arg_name cannot figure out 128 the host variable name. 129 130 This function will then create similar global variable names by 131 removing "_host" and appending "_host_name" or "_ip" to the host variable 132 name. 133 134 Example: 135 136 If a call is made like this: 137 process_host(openbmc_host) 138 139 Global variables openbmc_host_name and openbmc_ip will be set. 140 141 Description of argument(s): 142 host A host name or IP. The name of the variable used should 143 have a suffix of "_host". 144 host_var_name The name of the variable being used as the host parm. 145 """ 146 147 if host_var_name == "": 148 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 149 150 host_name_var_name = re.sub("host", "host_name", host_var_name) 151 ip_var_name = re.sub("host", "ip", host_var_name) 152 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 153 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 154 host + "')" 155 exec(cmd_buf) 156 157 158def process_pgm_parms(): 159 r""" 160 Process the program parameters by assigning them all to corresponding 161 globals. Also, set some global values that depend on program parameters. 162 """ 163 164 # Program parameter processing. 165 # Assign all program parms to python variables which are global to this 166 # module. 167 168 global parm_list 169 parm_list = BuiltIn().get_variable_value("${parm_list}") 170 # The following subset of parms should be processed as integers. 171 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 172 'boot_fail_threshold', 'delete_errlogs', 173 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 174 'test_mode', 'debug'] 175 for parm in parm_list: 176 if parm in int_list: 177 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 178 "}\", \"0\"))" 179 else: 180 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 181 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 182 gp.dpissuing(cmd_buf) 183 exec(cmd_buf) 184 if re.match(r".*_host$", parm): 185 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 186 exec(cmd_buf) 187 if re.match(r".*_password$", parm): 188 # Register the value of any parm whose name ends in _password. 189 # This will cause the print functions to replace passwords with 190 # asterisks in the output. 191 cmd_buf = "gp.register_passwords(" + parm + ")" 192 exec(cmd_buf) 193 194 global ffdc_dir_path_style 195 global boot_list 196 global boot_stack 197 global boot_results_file_path 198 global boot_results 199 global boot_history 200 global ffdc_list_file_path 201 global ffdc_report_list_path 202 global ffdc_summary_list_path 203 global boot_table 204 global valid_boot_types 205 206 if ffdc_dir_path_style == "": 207 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 208 209 # Convert these program parms to lists for easier processing.. 210 boot_list = list(filter(None, boot_list.split(":"))) 211 boot_stack = list(filter(None, boot_stack.split(":"))) 212 213 boot_table = create_boot_table(boot_table_path, os_host=os_host) 214 valid_boot_types = create_valid_boot_list(boot_table) 215 216 cleanup_boot_results_file() 217 boot_results_file_path = create_boot_results_file_path(pgm_name, 218 openbmc_nickname, 219 master_pid) 220 221 if os.path.isfile(boot_results_file_path): 222 # We've been called before in this run so we'll load the saved 223 # boot_results and boot_history objects. 224 boot_results, boot_history =\ 225 pickle.load(open(boot_results_file_path, 'rb')) 226 else: 227 boot_results = boot_results(boot_table, boot_pass, boot_fail) 228 229 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 230 "/FFDC_FILE_LIST" 231 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 232 "/FFDC_REPORT_FILE_LIST" 233 234 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 235 "/FFDC_SUMMARY_FILE_LIST" 236 237 238def initial_plug_in_setup(): 239 r""" 240 Initialize all plug-in environment variables which do not change for the 241 duration of the program. 242 243 """ 244 245 global LOG_LEVEL 246 BuiltIn().set_log_level("NONE") 247 248 BuiltIn().set_global_variable("${master_pid}", master_pid) 249 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 250 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 251 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 252 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 253 ffdc_list_file_path) 254 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 255 ffdc_report_list_path) 256 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 257 ffdc_summary_list_path) 258 259 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 260 ffdc_dir_path_style) 261 BuiltIn().set_global_variable("${FFDC_CHECK}", 262 ffdc_check) 263 264 # For each program parameter, set the corresponding AUTOBOOT_ environment 265 # variable value. Also, set an AUTOBOOT_ environment variable for every 266 # element in additional_values. 267 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 268 "status_dir_path", "base_tool_dir_path", 269 "ffdc_list_file_path", "ffdc_report_list_path", 270 "ffdc_summary_list_path", "execdir", "redfish_supported"] 271 272 plug_in_vars = parm_list + additional_values 273 274 for var_name in plug_in_vars: 275 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 276 var_name = var_name.upper() 277 if var_value is None: 278 var_value = "" 279 os.environ["AUTOBOOT_" + var_name] = str(var_value) 280 281 BuiltIn().set_log_level(LOG_LEVEL) 282 283 # Make sure the ffdc list directory exists. 284 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 285 if not os.path.exists(ffdc_list_dir_path): 286 os.makedirs(ffdc_list_dir_path) 287 288 289def plug_in_setup(): 290 r""" 291 Initialize all changing plug-in environment variables for use by the 292 plug-in programs. 293 """ 294 295 global LOG_LEVEL 296 global test_really_running 297 298 BuiltIn().set_log_level("NONE") 299 300 boot_pass, boot_fail = boot_results.return_total_pass_fail() 301 if boot_pass > 1: 302 test_really_running = 1 303 else: 304 test_really_running = 0 305 306 BuiltIn().set_global_variable("${test_really_running}", 307 test_really_running) 308 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 309 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 310 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 311 BuiltIn().set_global_variable("${boot_success}", boot_success) 312 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 313 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 314 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 315 316 # For each program parameter, set the corresponding AUTOBOOT_ environment 317 # variable value. Also, set an AUTOBOOT_ environment variable for every 318 # element in additional_values. 319 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 320 "boot_fail", "test_really_running", "ffdc_prefix", 321 "boot_start_time", "boot_end_time"] 322 323 plug_in_vars = additional_values 324 325 for var_name in plug_in_vars: 326 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 327 var_name = var_name.upper() 328 if var_value is None: 329 var_value = "" 330 os.environ["AUTOBOOT_" + var_name] = str(var_value) 331 332 if debug: 333 shell_rc, out_buf = \ 334 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 335 336 BuiltIn().set_log_level(LOG_LEVEL) 337 338 339def pre_boot_plug_in_setup(): 340 341 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 342 try: 343 os.remove(ffdc_list_file_path) 344 except OSError: 345 pass 346 347 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 348 try: 349 os.remove(ffdc_report_list_path) 350 except OSError: 351 pass 352 353 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 354 try: 355 os.remove(ffdc_summary_list_path) 356 except OSError: 357 pass 358 359 global ffdc_prefix 360 361 seconds = time.time() 362 loc_time = time.localtime(seconds) 363 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 364 365 ffdc_prefix = openbmc_nickname + "." + time_string 366 367 368def default_sigusr1(signal_number=0, 369 frame=None): 370 r""" 371 Handle SIGUSR1 by doing nothing. 372 373 This function assists in debugging SIGUSR1 processing by printing messages 374 to stdout and to the log.html file. 375 376 Description of argument(s): 377 signal_number The signal number (should always be 10 for SIGUSR1). 378 frame The frame data. 379 """ 380 381 gp.qprintn() 382 gp.qprint_executing() 383 gp.lprint_executing() 384 385 386def set_default_siguser1(): 387 r""" 388 Set the default_sigusr1 function to be the SIGUSR1 handler. 389 """ 390 391 gp.qprintn() 392 gp.qprint_executing() 393 gp.lprint_executing() 394 signal.signal(signal.SIGUSR1, default_sigusr1) 395 396 397def setup(): 398 r""" 399 Do general program setup tasks. 400 """ 401 402 global cp_setup_called 403 global transitional_boot_selected 404 405 gp.qprintn() 406 407 if redfish_supported: 408 redfish.login() 409 410 set_default_siguser1() 411 transitional_boot_selected = False 412 413 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 414 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 415 # If we can't find process_plug_in_packages.py, ssh_pw or 416 # validate_plug_ins.py, then we don't have our repo bin in PATH. 417 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 418 + " ssh_pw validate_plug_ins.py", quiet=1, 419 print_output=0, show_err=0) 420 if shell_rc != 0: 421 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 422 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 423 if robot_pgm_dir_path not in sys.path: 424 sys.path.append(robot_pgm_dir_path) 425 PYTHONPATH = os.environ.get("PYTHONPATH", "") 426 if PYTHONPATH == "": 427 os.environ['PYTHONPATH'] = robot_pgm_dir_path 428 else: 429 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 430 431 validate_parms() 432 433 gp.qprint_pgm_header() 434 435 grk.run_key_u(default_set_power_policy) 436 437 initial_plug_in_setup() 438 439 plug_in_setup() 440 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 441 call_point='setup') 442 if rc != 0: 443 error_message = "Plug-in setup failed.\n" 444 gp.print_error_report(error_message) 445 BuiltIn().fail(error_message) 446 # Setting cp_setup_called lets our Teardown know that it needs to call 447 # the cleanup plug-in call point. 448 cp_setup_called = 1 449 450 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 451 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 452 # FFDC_LOG_PATH is used by "FFDC" keyword. 453 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 454 455 # Also printed by FFDC. 456 global host_name 457 global host_ip 458 host = socket.gethostname() 459 host_name, host_ip = gm.get_host_name_ip(host) 460 461 gp.dprint_var(boot_table) 462 gp.dprint_var(boot_lists) 463 464 465def validate_parms(): 466 r""" 467 Validate all program parameters. 468 """ 469 470 process_pgm_parms() 471 472 gp.qprintn() 473 474 global openbmc_model 475 if openbmc_model == "": 476 status, ret_values =\ 477 grk.run_key_u("Get BMC System Model") 478 openbmc_model = ret_values 479 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 480 gv.set_exit_on_error(True) 481 gv.valid_value(openbmc_host) 482 gv.valid_value(openbmc_username) 483 gv.valid_value(openbmc_password) 484 gv.valid_value(rest_username) 485 gv.valid_value(rest_password) 486 gv.valid_value(ipmi_username) 487 gv.valid_value(ipmi_password) 488 if os_host != "": 489 gv.valid_value(os_username) 490 gv.valid_value(os_password) 491 if pdu_host != "": 492 gv.valid_value(pdu_username) 493 gv.valid_value(pdu_password) 494 gv.valid_integer(pdu_slot_no) 495 if openbmc_serial_host != "": 496 gv.valid_integer(openbmc_serial_port) 497 gv.valid_value(openbmc_model) 498 gv.valid_integer(max_num_tests) 499 gv.valid_integer(boot_pass) 500 gv.valid_integer(boot_fail) 501 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 502 BuiltIn().set_global_variable("${plug_in_packages_list}", 503 plug_in_packages_list) 504 gv.valid_value(stack_mode, valid_values=['normal', 'skip']) 505 gv.set_exit_on_error(False) 506 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 507 error_message = "You must provide either a value for either the" +\ 508 " boot_list or the boot_stack parm.\n" 509 BuiltIn().fail(gp.sprint_error(error_message)) 510 valid_boot_list(boot_list, valid_boot_types) 511 valid_boot_list(boot_stack, valid_boot_types) 512 selected_PDU_boots = list(set(boot_list + boot_stack) 513 & set(boot_lists['PDU_reboot'])) 514 if len(selected_PDU_boots) > 0 and pdu_host == "": 515 error_message = "You have selected the following boots which" +\ 516 " require a PDU host but no value for pdu_host:\n" 517 error_message += gp.sprint_var(selected_PDU_boots) 518 error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 519 BuiltIn().fail(gp.sprint_error(error_message)) 520 521 return 522 523 524def my_get_state(): 525 r""" 526 Get the system state plus a little bit of wrapping. 527 """ 528 529 global state 530 531 req_states = ['epoch_seconds'] + st.default_req_states 532 533 gp.qprint_timen("Getting system state.") 534 if test_mode: 535 state['epoch_seconds'] = int(time.time()) 536 else: 537 state = st.get_state(req_states=req_states, quiet=quiet) 538 gp.qprint_var(state) 539 540 541def valid_state(): 542 r""" 543 Verify that our state dictionary contains no blank values. If we don't get 544 valid state data, we cannot continue to work. 545 """ 546 547 if st.compare_states(state, st.invalid_state_match, 'or'): 548 error_message = "The state dictionary contains blank fields which" +\ 549 " is illegal.\n" + gp.sprint_var(state) 550 BuiltIn().fail(gp.sprint_error(error_message)) 551 552 553def select_boot(): 554 r""" 555 Select a boot test to be run based on our current state and return the 556 chosen boot type. 557 558 Description of arguments: 559 state The state of the machine. 560 """ 561 562 global transitional_boot_selected 563 global boot_stack 564 565 gp.qprint_timen("Selecting a boot test.") 566 567 if transitional_boot_selected and not boot_success: 568 prior_boot = next_boot 569 boot_candidate = boot_stack.pop() 570 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 571 + " transition to a valid state for '" + boot_candidate 572 + "' which was at the top of the boot_stack. Since" 573 + " the '" + next_boot + "' failed, the '" 574 + boot_candidate + "' has been removed from the stack" 575 + " to avoid and endless failure loop.") 576 if len(boot_stack) == 0: 577 return "" 578 579 my_get_state() 580 valid_state() 581 582 transitional_boot_selected = False 583 stack_popped = 0 584 if len(boot_stack) > 0: 585 stack_popped = 1 586 gp.qprint_dashes() 587 gp.qprint_var(boot_stack) 588 gp.qprint_dashes() 589 skip_boot_printed = 0 590 while len(boot_stack) > 0: 591 boot_candidate = boot_stack.pop() 592 if stack_mode == 'normal': 593 break 594 else: 595 if st.compare_states(state, boot_table[boot_candidate]['end']): 596 if not skip_boot_printed: 597 gp.qprint_var(stack_mode) 598 gp.qprintn() 599 gp.qprint_timen("Skipping the following boot tests" 600 + " which are unnecessary since their" 601 + " required end states match the" 602 + " current machine state:") 603 skip_boot_printed = 1 604 gp.qprint_var(boot_candidate) 605 boot_candidate = "" 606 if boot_candidate == "": 607 gp.qprint_dashes() 608 gp.qprint_var(boot_stack) 609 gp.qprint_dashes() 610 return boot_candidate 611 if st.compare_states(state, boot_table[boot_candidate]['start']): 612 gp.qprint_timen("The machine state is valid for a '" 613 + boot_candidate + "' boot test.") 614 gp.qprint_dashes() 615 gp.qprint_var(boot_stack) 616 gp.qprint_dashes() 617 return boot_candidate 618 else: 619 gp.qprint_timen("The machine state does not match the required" 620 + " starting state for a '" + boot_candidate 621 + "' boot test:") 622 gp.qprint_varx("boot_table_start_entry", 623 boot_table[boot_candidate]['start']) 624 boot_stack.append(boot_candidate) 625 transitional_boot_selected = True 626 popped_boot = boot_candidate 627 628 # Loop through your list selecting a boot_candidates 629 boot_candidates = [] 630 for boot_candidate in boot_list: 631 if st.compare_states(state, boot_table[boot_candidate]['start']): 632 if stack_popped: 633 if st.compare_states(boot_table[boot_candidate]['end'], 634 boot_table[popped_boot]['start']): 635 boot_candidates.append(boot_candidate) 636 else: 637 boot_candidates.append(boot_candidate) 638 639 if len(boot_candidates) == 0: 640 gp.qprint_timen("The user's boot list contained no boot tests" 641 + " which are valid for the current machine state.") 642 boot_candidate = default_power_on 643 if not st.compare_states(state, boot_table[default_power_on]['start']): 644 boot_candidate = default_power_off 645 boot_candidates.append(boot_candidate) 646 gp.qprint_timen("Using default '" + boot_candidate 647 + "' boot type to transition to valid state.") 648 649 gp.dprint_var(boot_candidates) 650 651 # Randomly select a boot from the candidate list. 652 boot = random.choice(boot_candidates) 653 654 return boot 655 656 657def print_defect_report(ffdc_file_list): 658 r""" 659 Print a defect report. 660 661 Description of argument(s): 662 ffdc_file_list A list of files which were collected by our ffdc functions. 663 """ 664 665 # Making deliberate choice to NOT run plug_in_setup(). We don't want 666 # ffdc_prefix updated. 667 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 668 call_point='ffdc_report', stop_on_plug_in_failure=0) 669 670 # Get additional header data which may have been created by ffdc plug-ins. 671 # Also, delete the individual header files to cleanup. 672 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 673 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 674 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 675 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 676 show_err=0) 677 678 # Get additional summary data which may have been created by ffdc plug-ins. 679 # Also, delete the individual header files to cleanup. 680 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 681 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 682 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 683 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 684 show_err=0) 685 686 # ffdc_list_file_path contains a list of any ffdc files created by plug- 687 # ins, etc. Read that data into a list. 688 try: 689 plug_in_ffdc_list = \ 690 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 691 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 692 except IOError: 693 plug_in_ffdc_list = [] 694 695 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 696 # in. Eliminate duplicates and sort the list. 697 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 698 699 if status_file_path != "": 700 ffdc_file_list.insert(0, status_file_path) 701 702 # Convert the list to a printable list. 703 printable_ffdc_file_list = "\n".join(ffdc_file_list) 704 705 # Open ffdc_file_list for writing. We will write a complete list of 706 # FFDC files to it for possible use by plug-ins like cp_stop_check. 707 ffdc_list_file = open(ffdc_list_file_path, 'w') 708 ffdc_list_file.write(printable_ffdc_file_list + "\n") 709 ffdc_list_file.close() 710 711 indent = 0 712 width = 90 713 linefeed = 1 714 char = "=" 715 716 gp.qprintn() 717 gp.qprint_dashes(indent, width, linefeed, char) 718 gp.qprintn("Copy this data to the defect:\n") 719 720 if len(more_header_info) > 0: 721 gp.qprintn(more_header_info) 722 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 723 openbmc_host_name, openbmc_ip, openbmc_username, 724 openbmc_password, rest_username, rest_password, ipmi_username, 725 ipmi_password, os_host, os_host_name, os_ip, os_username, 726 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 727 pdu_password, pdu_slot_no, openbmc_serial_host, 728 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 729 730 gp.qprintn() 731 print_boot_history(boot_history) 732 gp.qprintn() 733 gp.qprint_var(state) 734 gp.qprintn() 735 gp.qprintn("FFDC data files:") 736 gp.qprintn(printable_ffdc_file_list) 737 gp.qprintn() 738 739 if len(ffdc_summary_info) > 0: 740 gp.qprintn(ffdc_summary_info) 741 742 gp.qprint_dashes(indent, width, linefeed, char) 743 744 745def my_ffdc(): 746 r""" 747 Collect FFDC data. 748 """ 749 750 global state 751 752 plug_in_setup() 753 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 754 call_point='ffdc', stop_on_plug_in_failure=0) 755 756 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 757 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 758 + AUTOBOOT_FFDC_PREFIX 759 + " ffdc_function_list=" 760 + ffdc_function_list, ignore=1) 761 if status != 'PASS': 762 gp.qprint_error("Call to ffdc failed.\n") 763 if type(ffdc_file_list) is not list: 764 ffdc_file_list = [] 765 # Leave a record for caller that "soft" errors occurred. 766 soft_errors = 1 767 gpu.save_plug_in_value(soft_errors, pgm_name) 768 769 my_get_state() 770 771 print_defect_report(ffdc_file_list) 772 773 774def print_test_start_message(boot_keyword): 775 r""" 776 Print a message indicating what boot test is about to run. 777 778 Description of arguments: 779 boot_keyword The name of the boot which is to be run 780 (e.g. "BMC Power On"). 781 """ 782 783 global boot_history 784 global boot_start_time 785 786 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 787 788 # Set boot_start_time for use by plug-ins. 789 boot_start_time = doing_msg[1:33] 790 gp.qprint_var(boot_start_time) 791 792 gp.qprint(doing_msg) 793 794 update_boot_history(boot_history, doing_msg, max_boot_history) 795 796 797def stop_boot_test(signal_number=0, 798 frame=None): 799 r""" 800 Handle SIGUSR1 by aborting the boot test that is running. 801 802 Description of argument(s): 803 signal_number The signal number (should always be 10 for SIGUSR1). 804 frame The frame data. 805 """ 806 807 gp.qprintn() 808 gp.qprint_executing() 809 gp.lprint_executing() 810 811 # Restore original sigusr1 handler. 812 set_default_siguser1() 813 814 message = "The caller has asked that the boot test be stopped and marked" 815 message += " as a failure." 816 817 function_stack = gm.get_function_stack() 818 if "wait_state" in function_stack: 819 st.set_exit_wait_early_message(message) 820 else: 821 BuiltIn().fail(gp.sprint_error(message)) 822 823 824def run_boot(boot): 825 r""" 826 Run the specified boot. 827 828 Description of arguments: 829 boot The name of the boot test to be performed. 830 """ 831 832 global state 833 834 signal.signal(signal.SIGUSR1, stop_boot_test) 835 gp.qprint_timen("stop_boot_test is armed.") 836 837 print_test_start_message(boot) 838 839 plug_in_setup() 840 rc, shell_rc, failed_plug_in_name = \ 841 grpi.rprocess_plug_in_packages(call_point="pre_boot") 842 if rc != 0: 843 error_message = "Plug-in failed with non-zero return code.\n" +\ 844 gp.sprint_var(rc, fmt=gp.hexa()) 845 set_default_siguser1() 846 BuiltIn().fail(gp.sprint_error(error_message)) 847 848 if test_mode: 849 # In test mode, we'll pretend the boot worked by assigning its 850 # required end state to the default state value. 851 state = st.strip_anchor_state(boot_table[boot]['end']) 852 else: 853 # Assertion: We trust that the state data was made fresh by the 854 # caller. 855 856 gp.qprintn() 857 858 if boot_table[boot]['method_type'] == "keyword": 859 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 860 boot_table[boot]['method'], 861 quiet=quiet) 862 863 if boot_table[boot]['bmc_reboot']: 864 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 865 plug_in_setup() 866 rc, shell_rc, failed_plug_in_name = \ 867 grpi.rprocess_plug_in_packages(call_point="post_reboot") 868 if rc != 0: 869 error_message = "Plug-in failed with non-zero return code.\n" 870 error_message += gp.sprint_var(rc, fmt=gp.hexa()) 871 set_default_siguser1() 872 BuiltIn().fail(gp.sprint_error(error_message)) 873 else: 874 match_state = st.anchor_state(state) 875 del match_state['epoch_seconds'] 876 # Wait for the state to change in any way. 877 st.wait_state(match_state, wait_time=state_change_timeout, 878 interval="10 seconds", invert=1) 879 880 gp.qprintn() 881 if boot_table[boot]['end']['chassis'] == "Off": 882 boot_timeout = power_off_timeout 883 else: 884 boot_timeout = power_on_timeout 885 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 886 interval="10 seconds") 887 888 plug_in_setup() 889 rc, shell_rc, failed_plug_in_name = \ 890 grpi.rprocess_plug_in_packages(call_point="post_boot") 891 if rc != 0: 892 error_message = "Plug-in failed with non-zero return code.\n" +\ 893 gp.sprint_var(rc, fmt=gp.hexa()) 894 set_default_siguser1() 895 BuiltIn().fail(gp.sprint_error(error_message)) 896 897 # Restore original sigusr1 handler. 898 set_default_siguser1() 899 900 901def test_loop_body(): 902 r""" 903 The main loop body for the loop in main_py. 904 905 Description of arguments: 906 boot_count The iteration number (starts at 1). 907 """ 908 909 global boot_count 910 global state 911 global next_boot 912 global boot_success 913 global boot_end_time 914 915 gp.qprintn() 916 917 next_boot = select_boot() 918 if next_boot == "": 919 return True 920 921 boot_count += 1 922 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 923 924 pre_boot_plug_in_setup() 925 926 cmd_buf = ["run_boot", next_boot] 927 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 928 if boot_status == "FAIL": 929 gp.qprint(msg) 930 931 gp.qprintn() 932 if boot_status == "PASS": 933 boot_success = 1 934 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 935 + "\" succeeded.") 936 else: 937 boot_success = 0 938 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 939 + "\" failed.") 940 941 # Set boot_end_time for use by plug-ins. 942 boot_end_time = completion_msg[1:33] 943 gp.qprint_var(boot_end_time) 944 945 gp.qprint(completion_msg) 946 947 boot_results.update(next_boot, boot_status) 948 949 plug_in_setup() 950 # NOTE: A post_test_case call point failure is NOT counted as a boot 951 # failure. 952 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 953 call_point='post_test_case', stop_on_plug_in_failure=0) 954 955 plug_in_setup() 956 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 957 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 958 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 959 if ffdc_check == "All" or\ 960 shell_rc == dump_ffdc_rc(): 961 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 962 if status != 'PASS': 963 gp.qprint_error("Call to my_ffdc failed.\n") 964 # Leave a record for caller that "soft" errors occurred. 965 soft_errors = 1 966 gpu.save_plug_in_value(soft_errors, pgm_name) 967 968 if delete_errlogs: 969 # print error logs before delete 970 status, error_logs = grk.run_key_u("Get Error Logs") 971 pels = pel.peltool("-l", ignore_err=1) 972 gp.qprint_vars(error_logs, pels) 973 974 # We need to purge error logs between boots or they build up. 975 grk.run_key(delete_errlogs_cmd, ignore=1) 976 977 boot_results.print_report() 978 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 979 980 plug_in_setup() 981 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 982 call_point='stop_check', shell_rc=stop_test_rc(), 983 stop_on_non_zero_rc=1) 984 if shell_rc == stop_test_rc(): 985 message = "Stopping as requested by user.\n" 986 gp.qprint_time(message) 987 BuiltIn().fail(message) 988 989 # This should help prevent ConnectionErrors. 990 grk.run_key_u("Close All Connections") 991 992 return True 993 994 995def obmc_boot_test_teardown(): 996 r""" 997 Clean up after the main keyword. 998 """ 999 gp.qprint_executing() 1000 1001 if ga.psutil_imported: 1002 ga.terminate_descendants() 1003 1004 if cp_setup_called: 1005 plug_in_setup() 1006 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1007 call_point='cleanup', stop_on_plug_in_failure=0) 1008 1009 if 'boot_results_file_path' in globals(): 1010 # Save boot_results and boot_history objects to a file in case they are 1011 # needed again. 1012 gp.qprint_timen("Saving boot_results to the following path.") 1013 gp.qprint_var(boot_results_file_path) 1014 pickle.dump((boot_results, boot_history), 1015 open(boot_results_file_path, 'wb'), 1016 pickle.HIGHEST_PROTOCOL) 1017 1018 global save_stack 1019 # Restore any global values saved on the save_stack. 1020 for parm_name in main_func_parm_list: 1021 # Get the parm_value if it was saved on the stack. 1022 try: 1023 parm_value = save_stack.pop(parm_name) 1024 except BaseException: 1025 # If it was not saved, no further action is required. 1026 continue 1027 1028 # Restore the saved value. 1029 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1030 "}\", parm_value)" 1031 gp.dpissuing(cmd_buf) 1032 exec(cmd_buf) 1033 1034 gp.dprintn(save_stack.sprint_obj()) 1035 1036 1037def test_teardown(): 1038 r""" 1039 Clean up after this test case. 1040 """ 1041 1042 gp.qprintn() 1043 gp.qprint_executing() 1044 1045 if ga.psutil_imported: 1046 ga.terminate_descendants() 1047 1048 cmd_buf = ["Print Error", 1049 "A keyword timeout occurred ending this program.\n"] 1050 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1051 1052 if redfish_supported: 1053 redfish.logout() 1054 1055 gp.qprint_pgm_footer() 1056 1057 1058def post_stack(): 1059 r""" 1060 Process post_stack plug-in programs. 1061 """ 1062 1063 if not call_post_stack_plug: 1064 # The caller does not wish to have post_stack plug-in processing done. 1065 return 1066 1067 global boot_success 1068 1069 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1070 pre_boot_plug_in_setup() 1071 # For the purposes of the following plug-ins, mark the "boot" as a success. 1072 boot_success = 1 1073 plug_in_setup() 1074 rc, shell_rc, failed_plug_in_name, history =\ 1075 grpi.rprocess_plug_in_packages(call_point='post_stack', 1076 stop_on_plug_in_failure=0, 1077 return_history=True) 1078 for doing_msg in history: 1079 update_boot_history(boot_history, doing_msg, max_boot_history) 1080 if rc != 0: 1081 boot_success = 0 1082 1083 plug_in_setup() 1084 rc, shell_rc, failed_plug_in_name =\ 1085 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1086 shell_rc=dump_ffdc_rc(), 1087 stop_on_plug_in_failure=1, 1088 stop_on_non_zero_rc=1) 1089 if shell_rc == dump_ffdc_rc(): 1090 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1091 if status != 'PASS': 1092 gp.qprint_error("Call to my_ffdc failed.\n") 1093 # Leave a record for caller that "soft" errors occurred. 1094 soft_errors = 1 1095 gpu.save_plug_in_value(soft_errors, pgm_name) 1096 1097 plug_in_setup() 1098 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1099 call_point='stop_check', shell_rc=stop_test_rc(), 1100 stop_on_non_zero_rc=1) 1101 if shell_rc == stop_test_rc(): 1102 message = "Stopping as requested by user.\n" 1103 gp.qprint_time(message) 1104 BuiltIn().fail(message) 1105 1106 1107def obmc_boot_test_py(loc_boot_stack=None, 1108 loc_stack_mode=None, 1109 loc_quiet=None): 1110 r""" 1111 Do main program processing. 1112 """ 1113 1114 global save_stack 1115 1116 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']}) 1117 1118 gp.dprintn() 1119 # Process function parms. 1120 for parm_name in main_func_parm_list: 1121 # Get parm's value. 1122 parm_value = eval("loc_" + parm_name) 1123 gp.dpvars(parm_name, parm_value) 1124 1125 if parm_value is not None: 1126 # Save the global value on a stack. 1127 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1128 parm_name + "}\"), \"" + parm_name + "\")" 1129 gp.dpissuing(cmd_buf) 1130 exec(cmd_buf) 1131 1132 # Set the global value to the passed value. 1133 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1134 "}\", loc_" + parm_name + ")" 1135 gp.dpissuing(cmd_buf) 1136 exec(cmd_buf) 1137 1138 gp.dprintn(save_stack.sprint_obj()) 1139 1140 setup() 1141 1142 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1143 1144 if ffdc_only: 1145 gp.qprint_timen("Caller requested ffdc_only.") 1146 if do_pre_boot_plug_in_setup: 1147 pre_boot_plug_in_setup() 1148 grk.run_key_u("my_ffdc") 1149 return 1150 1151 if delete_errlogs: 1152 # print error logs before delete 1153 status, error_logs = grk.run_key_u("Get Error Logs") 1154 pels = pel.peltool("-l", ignore_err=1) 1155 gp.qprint_vars(error_logs, pels) 1156 1157 # Delete errlogs prior to doing any boot tests. 1158 grk.run_key(delete_errlogs_cmd, ignore=1) 1159 1160 # Process caller's boot_stack. 1161 while (len(boot_stack) > 0): 1162 test_loop_body() 1163 1164 gp.qprint_timen("Finished processing stack.") 1165 1166 post_stack() 1167 1168 # Process caller's boot_list. 1169 if len(boot_list) > 0: 1170 for ix in range(1, max_num_tests + 1): 1171 test_loop_body() 1172 1173 gp.qprint_timen("Completed all requested boot tests.") 1174 1175 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1176 new_fail = boot_fail - init_boot_fail 1177 if new_fail > boot_fail_threshold: 1178 error_message = "Boot failures exceed the boot failure" +\ 1179 " threshold:\n" +\ 1180 gp.sprint_var(new_fail) +\ 1181 gp.sprint_var(boot_fail_threshold) 1182 BuiltIn().fail(gp.sprint_error(error_message)) 1183