1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_arg as ga 27import gen_valid as gv 28import gen_misc as gm 29import gen_cmd as gc 30import gen_robot_keyword as grk 31import state as st 32import var_stack as vs 33import gen_plug_in_utils as gpu 34import pel_utils as pel 35 36base_path = os.path.dirname(os.path.dirname( 37 imp.find_module("gen_robot_print")[1])) +\ 38 os.sep 39sys.path.append(base_path + "extended/") 40import run_keyword as rk 41 42# Setting master_pid correctly influences the behavior of plug-ins like 43# DB_Logging 44program_pid = os.getpid() 45master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 46pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 47 48# Set up boot data structures. 49os_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 50 51boot_lists = read_boot_lists() 52 53# The maximum number of entries that can be in the boot_history global variable. 54max_boot_history = 10 55boot_history = [] 56 57state = st.return_state_constant('default_state') 58cp_setup_called = 0 59next_boot = "" 60base_tool_dir_path = os.path.normpath(os.environ.get( 61 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 62 63ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 64boot_success = 0 65status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 66if status_dir_path != "": 67 status_dir_path = os.path.normpath(status_dir_path) + os.sep 68redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False) 69redfish_rest_supported = BuiltIn().get_variable_value("${REDFISH_REST_SUPPORTED}", default=False) 70if redfish_supported: 71 redfish = BuiltIn().get_library_instance('redfish') 72 default_power_on = "Redfish Power On" 73 default_power_off = "Redfish Power Off" 74 if redfish_rest_supported: 75 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 76 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 77 else: 78 delete_errlogs_cmd = "Redfish Purge Event Log" 79 default_set_power_policy = "Redfish Set Power Restore Policy AlwaysOff" 80else: 81 default_power_on = "REST Power On" 82 default_power_off = "REST Power Off" 83 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 84 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 85boot_count = 0 86 87LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 88AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 89ffdc_prefix = AUTOBOOT_FFDC_PREFIX 90boot_start_time = "" 91boot_end_time = "" 92save_stack = vs.var_stack('save_stack') 93main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 94 95 96def dump_ffdc_rc(): 97 r""" 98 Return the constant dump ffdc test return code value. 99 100 When a plug-in call point program returns this value, it indicates that 101 this program should collect FFDC. 102 """ 103 104 return 0x00000200 105 106 107def stop_test_rc(): 108 r""" 109 Return the constant stop test return code value. 110 111 When a plug-in call point program returns this value, it indicates that 112 this program should stop running. 113 """ 114 115 return 0x00000200 116 117 118def process_host(host, 119 host_var_name=""): 120 r""" 121 Process a host by getting the associated host name and IP address and 122 setting them in global variables. 123 124 If the caller does not pass the host_var_name, this function will try to 125 figure out the name of the variable used by the caller for the host parm. 126 Callers are advised to explicitly specify the host_var_name when calling 127 with an exec command. In such cases, the get_arg_name cannot figure out 128 the host variable name. 129 130 This function will then create similar global variable names by 131 removing "_host" and appending "_host_name" or "_ip" to the host variable 132 name. 133 134 Example: 135 136 If a call is made like this: 137 process_host(openbmc_host) 138 139 Global variables openbmc_host_name and openbmc_ip will be set. 140 141 Description of argument(s): 142 host A host name or IP. The name of the variable used should 143 have a suffix of "_host". 144 host_var_name The name of the variable being used as the host parm. 145 """ 146 147 if host_var_name == "": 148 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 149 150 host_name_var_name = re.sub("host", "host_name", host_var_name) 151 ip_var_name = re.sub("host", "ip", host_var_name) 152 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 153 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 154 host + "')" 155 exec(cmd_buf) 156 157 158def process_pgm_parms(): 159 r""" 160 Process the program parameters by assigning them all to corresponding 161 globals. Also, set some global values that depend on program parameters. 162 """ 163 164 # Program parameter processing. 165 # Assign all program parms to python variables which are global to this 166 # module. 167 168 global parm_list 169 parm_list = BuiltIn().get_variable_value("${parm_list}") 170 # The following subset of parms should be processed as integers. 171 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 172 'boot_fail_threshold', 'delete_errlogs', 173 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 174 'test_mode', 'debug'] 175 for parm in parm_list: 176 if parm in int_list: 177 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 178 "}\", \"0\"))" 179 else: 180 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 181 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 182 gp.dpissuing(cmd_buf) 183 exec(cmd_buf) 184 if re.match(r".*_host$", parm): 185 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 186 exec(cmd_buf) 187 if re.match(r".*_password$", parm): 188 # Register the value of any parm whose name ends in _password. 189 # This will cause the print functions to replace passwords with 190 # asterisks in the output. 191 cmd_buf = "gp.register_passwords(" + parm + ")" 192 exec(cmd_buf) 193 194 global ffdc_dir_path_style 195 global boot_list 196 global boot_stack 197 global boot_results_file_path 198 global boot_results 199 global boot_history 200 global ffdc_list_file_path 201 global ffdc_report_list_path 202 global ffdc_summary_list_path 203 global boot_table 204 global valid_boot_types 205 206 if ffdc_dir_path_style == "": 207 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 208 209 # Convert these program parms to lists for easier processing.. 210 boot_list = list(filter(None, boot_list.split(":"))) 211 boot_stack = list(filter(None, boot_stack.split(":"))) 212 213 boot_table = create_boot_table(boot_table_path, os_host=os_host) 214 valid_boot_types = create_valid_boot_list(boot_table) 215 216 cleanup_boot_results_file() 217 boot_results_file_path = create_boot_results_file_path(pgm_name, 218 openbmc_nickname, 219 master_pid) 220 221 if os.path.isfile(boot_results_file_path): 222 # We've been called before in this run so we'll load the saved 223 # boot_results and boot_history objects. 224 boot_results, boot_history =\ 225 pickle.load(open(boot_results_file_path, 'rb')) 226 else: 227 boot_results = boot_results(boot_table, boot_pass, boot_fail) 228 229 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 230 "/FFDC_FILE_LIST" 231 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 232 "/FFDC_REPORT_FILE_LIST" 233 234 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 235 "/FFDC_SUMMARY_FILE_LIST" 236 237 238def initial_plug_in_setup(): 239 r""" 240 Initialize all plug-in environment variables which do not change for the 241 duration of the program. 242 243 """ 244 245 global LOG_LEVEL 246 BuiltIn().set_log_level("NONE") 247 248 BuiltIn().set_global_variable("${master_pid}", master_pid) 249 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 250 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 251 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 252 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 253 ffdc_list_file_path) 254 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 255 ffdc_report_list_path) 256 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 257 ffdc_summary_list_path) 258 259 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 260 ffdc_dir_path_style) 261 BuiltIn().set_global_variable("${FFDC_CHECK}", 262 ffdc_check) 263 264 # For each program parameter, set the corresponding AUTOBOOT_ environment 265 # variable value. Also, set an AUTOBOOT_ environment variable for every 266 # element in additional_values. 267 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 268 "status_dir_path", "base_tool_dir_path", 269 "ffdc_list_file_path", "ffdc_report_list_path", 270 "ffdc_summary_list_path", "execdir", "redfish_supported", 271 "redfish_rest_supported"] 272 273 plug_in_vars = parm_list + additional_values 274 275 for var_name in plug_in_vars: 276 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 277 var_name = var_name.upper() 278 if var_value is None: 279 var_value = "" 280 os.environ["AUTOBOOT_" + var_name] = str(var_value) 281 282 BuiltIn().set_log_level(LOG_LEVEL) 283 284 # Make sure the ffdc list directory exists. 285 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 286 if not os.path.exists(ffdc_list_dir_path): 287 os.makedirs(ffdc_list_dir_path) 288 289 290def plug_in_setup(): 291 r""" 292 Initialize all changing plug-in environment variables for use by the 293 plug-in programs. 294 """ 295 296 global LOG_LEVEL 297 global test_really_running 298 299 BuiltIn().set_log_level("NONE") 300 301 boot_pass, boot_fail = boot_results.return_total_pass_fail() 302 if boot_pass > 1: 303 test_really_running = 1 304 else: 305 test_really_running = 0 306 307 BuiltIn().set_global_variable("${test_really_running}", 308 test_really_running) 309 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 310 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 311 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 312 BuiltIn().set_global_variable("${boot_success}", boot_success) 313 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 314 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 315 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 316 317 # For each program parameter, set the corresponding AUTOBOOT_ environment 318 # variable value. Also, set an AUTOBOOT_ environment variable for every 319 # element in additional_values. 320 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 321 "boot_fail", "test_really_running", "ffdc_prefix", 322 "boot_start_time", "boot_end_time"] 323 324 plug_in_vars = additional_values 325 326 for var_name in plug_in_vars: 327 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 328 var_name = var_name.upper() 329 if var_value is None: 330 var_value = "" 331 os.environ["AUTOBOOT_" + var_name] = str(var_value) 332 333 if debug: 334 shell_rc, out_buf = \ 335 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 336 337 BuiltIn().set_log_level(LOG_LEVEL) 338 339 340def pre_boot_plug_in_setup(): 341 342 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 343 try: 344 os.remove(ffdc_list_file_path) 345 except OSError: 346 pass 347 348 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 349 try: 350 os.remove(ffdc_report_list_path) 351 except OSError: 352 pass 353 354 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 355 try: 356 os.remove(ffdc_summary_list_path) 357 except OSError: 358 pass 359 360 global ffdc_prefix 361 362 seconds = time.time() 363 loc_time = time.localtime(seconds) 364 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 365 366 ffdc_prefix = openbmc_nickname + "." + time_string 367 368 369def default_sigusr1(signal_number=0, 370 frame=None): 371 r""" 372 Handle SIGUSR1 by doing nothing. 373 374 This function assists in debugging SIGUSR1 processing by printing messages 375 to stdout and to the log.html file. 376 377 Description of argument(s): 378 signal_number The signal number (should always be 10 for SIGUSR1). 379 frame The frame data. 380 """ 381 382 gp.qprintn() 383 gp.qprint_executing() 384 gp.lprint_executing() 385 386 387def set_default_siguser1(): 388 r""" 389 Set the default_sigusr1 function to be the SIGUSR1 handler. 390 """ 391 392 gp.qprintn() 393 gp.qprint_executing() 394 gp.lprint_executing() 395 signal.signal(signal.SIGUSR1, default_sigusr1) 396 397 398def setup(): 399 r""" 400 Do general program setup tasks. 401 """ 402 403 global cp_setup_called 404 global transitional_boot_selected 405 406 gp.qprintn() 407 408 if redfish_supported: 409 redfish.login() 410 411 set_default_siguser1() 412 transitional_boot_selected = False 413 414 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 415 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 416 # If we can't find process_plug_in_packages.py, ssh_pw or 417 # validate_plug_ins.py, then we don't have our repo bin in PATH. 418 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 419 + " ssh_pw validate_plug_ins.py", quiet=1, 420 print_output=0, show_err=0) 421 if shell_rc != 0: 422 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 423 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 424 if robot_pgm_dir_path not in sys.path: 425 sys.path.append(robot_pgm_dir_path) 426 PYTHONPATH = os.environ.get("PYTHONPATH", "") 427 if PYTHONPATH == "": 428 os.environ['PYTHONPATH'] = robot_pgm_dir_path 429 else: 430 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 431 432 validate_parms() 433 434 gp.qprint_pgm_header() 435 436 grk.run_key_u(default_set_power_policy) 437 438 initial_plug_in_setup() 439 440 plug_in_setup() 441 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 442 call_point='setup') 443 if rc != 0: 444 error_message = "Plug-in setup failed.\n" 445 gp.print_error_report(error_message) 446 BuiltIn().fail(error_message) 447 # Setting cp_setup_called lets our Teardown know that it needs to call 448 # the cleanup plug-in call point. 449 cp_setup_called = 1 450 451 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 452 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 453 # FFDC_LOG_PATH is used by "FFDC" keyword. 454 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 455 456 # Also printed by FFDC. 457 global host_name 458 global host_ip 459 host = socket.gethostname() 460 host_name, host_ip = gm.get_host_name_ip(host) 461 462 gp.dprint_var(boot_table) 463 gp.dprint_var(boot_lists) 464 465 466def validate_parms(): 467 r""" 468 Validate all program parameters. 469 """ 470 471 process_pgm_parms() 472 473 gp.qprintn() 474 475 global openbmc_model 476 if openbmc_model == "": 477 status, ret_values =\ 478 grk.run_key_u("Get BMC System Model") 479 openbmc_model = ret_values 480 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 481 gv.set_exit_on_error(True) 482 gv.valid_value(openbmc_host) 483 gv.valid_value(openbmc_username) 484 gv.valid_value(openbmc_password) 485 gv.valid_value(rest_username) 486 gv.valid_value(rest_password) 487 gv.valid_value(ipmi_username) 488 gv.valid_value(ipmi_password) 489 if os_host != "": 490 gv.valid_value(os_username) 491 gv.valid_value(os_password) 492 if pdu_host != "": 493 gv.valid_value(pdu_username) 494 gv.valid_value(pdu_password) 495 gv.valid_integer(pdu_slot_no) 496 if openbmc_serial_host != "": 497 gv.valid_integer(openbmc_serial_port) 498 gv.valid_value(openbmc_model) 499 gv.valid_integer(max_num_tests) 500 gv.valid_integer(boot_pass) 501 gv.valid_integer(boot_fail) 502 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 503 BuiltIn().set_global_variable("${plug_in_packages_list}", 504 plug_in_packages_list) 505 gv.valid_value(stack_mode, valid_values=['normal', 'skip']) 506 gv.set_exit_on_error(False) 507 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 508 error_message = "You must provide either a value for either the" +\ 509 " boot_list or the boot_stack parm.\n" 510 BuiltIn().fail(gp.sprint_error(error_message)) 511 valid_boot_list(boot_list, valid_boot_types) 512 valid_boot_list(boot_stack, valid_boot_types) 513 selected_PDU_boots = list(set(boot_list + boot_stack) 514 & set(boot_lists['PDU_reboot'])) 515 if len(selected_PDU_boots) > 0 and pdu_host == "": 516 error_message = "You have selected the following boots which" +\ 517 " require a PDU host but no value for pdu_host:\n" 518 error_message += gp.sprint_var(selected_PDU_boots) 519 error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 520 BuiltIn().fail(gp.sprint_error(error_message)) 521 522 return 523 524 525def my_get_state(): 526 r""" 527 Get the system state plus a little bit of wrapping. 528 """ 529 530 global state 531 532 req_states = ['epoch_seconds'] + st.default_req_states 533 534 gp.qprint_timen("Getting system state.") 535 if test_mode: 536 state['epoch_seconds'] = int(time.time()) 537 else: 538 state = st.get_state(req_states=req_states, quiet=quiet) 539 gp.qprint_var(state) 540 541 542def valid_state(): 543 r""" 544 Verify that our state dictionary contains no blank values. If we don't get 545 valid state data, we cannot continue to work. 546 """ 547 548 if st.compare_states(state, st.invalid_state_match, 'or'): 549 error_message = "The state dictionary contains blank fields which" +\ 550 " is illegal.\n" + gp.sprint_var(state) 551 BuiltIn().fail(gp.sprint_error(error_message)) 552 553 554def select_boot(): 555 r""" 556 Select a boot test to be run based on our current state and return the 557 chosen boot type. 558 559 Description of arguments: 560 state The state of the machine. 561 """ 562 563 global transitional_boot_selected 564 global boot_stack 565 566 gp.qprint_timen("Selecting a boot test.") 567 568 if transitional_boot_selected and not boot_success: 569 prior_boot = next_boot 570 boot_candidate = boot_stack.pop() 571 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 572 + " transition to a valid state for '" + boot_candidate 573 + "' which was at the top of the boot_stack. Since" 574 + " the '" + next_boot + "' failed, the '" 575 + boot_candidate + "' has been removed from the stack" 576 + " to avoid and endless failure loop.") 577 if len(boot_stack) == 0: 578 return "" 579 580 my_get_state() 581 valid_state() 582 583 transitional_boot_selected = False 584 stack_popped = 0 585 if len(boot_stack) > 0: 586 stack_popped = 1 587 gp.qprint_dashes() 588 gp.qprint_var(boot_stack) 589 gp.qprint_dashes() 590 skip_boot_printed = 0 591 while len(boot_stack) > 0: 592 boot_candidate = boot_stack.pop() 593 if stack_mode == 'normal': 594 break 595 else: 596 if st.compare_states(state, boot_table[boot_candidate]['end']): 597 if not skip_boot_printed: 598 gp.qprint_var(stack_mode) 599 gp.qprintn() 600 gp.qprint_timen("Skipping the following boot tests" 601 + " which are unnecessary since their" 602 + " required end states match the" 603 + " current machine state:") 604 skip_boot_printed = 1 605 gp.qprint_var(boot_candidate) 606 boot_candidate = "" 607 if boot_candidate == "": 608 gp.qprint_dashes() 609 gp.qprint_var(boot_stack) 610 gp.qprint_dashes() 611 return boot_candidate 612 if st.compare_states(state, boot_table[boot_candidate]['start']): 613 gp.qprint_timen("The machine state is valid for a '" 614 + boot_candidate + "' boot test.") 615 gp.qprint_dashes() 616 gp.qprint_var(boot_stack) 617 gp.qprint_dashes() 618 return boot_candidate 619 else: 620 gp.qprint_timen("The machine state does not match the required" 621 + " starting state for a '" + boot_candidate 622 + "' boot test:") 623 gp.qprint_varx("boot_table_start_entry", 624 boot_table[boot_candidate]['start']) 625 boot_stack.append(boot_candidate) 626 transitional_boot_selected = True 627 popped_boot = boot_candidate 628 629 # Loop through your list selecting a boot_candidates 630 boot_candidates = [] 631 for boot_candidate in boot_list: 632 if st.compare_states(state, boot_table[boot_candidate]['start']): 633 if stack_popped: 634 if st.compare_states(boot_table[boot_candidate]['end'], 635 boot_table[popped_boot]['start']): 636 boot_candidates.append(boot_candidate) 637 else: 638 boot_candidates.append(boot_candidate) 639 640 if len(boot_candidates) == 0: 641 gp.qprint_timen("The user's boot list contained no boot tests" 642 + " which are valid for the current machine state.") 643 boot_candidate = default_power_on 644 if not st.compare_states(state, boot_table[default_power_on]['start']): 645 boot_candidate = default_power_off 646 boot_candidates.append(boot_candidate) 647 gp.qprint_timen("Using default '" + boot_candidate 648 + "' boot type to transition to valid state.") 649 650 gp.dprint_var(boot_candidates) 651 652 # Randomly select a boot from the candidate list. 653 boot = random.choice(boot_candidates) 654 655 return boot 656 657 658def print_defect_report(ffdc_file_list): 659 r""" 660 Print a defect report. 661 662 Description of argument(s): 663 ffdc_file_list A list of files which were collected by our ffdc functions. 664 """ 665 666 # Making deliberate choice to NOT run plug_in_setup(). We don't want 667 # ffdc_prefix updated. 668 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 669 call_point='ffdc_report', stop_on_plug_in_failure=0) 670 671 # Get additional header data which may have been created by ffdc plug-ins. 672 # Also, delete the individual header files to cleanup. 673 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 674 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 675 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 676 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 677 show_err=0) 678 679 # Get additional summary data which may have been created by ffdc plug-ins. 680 # Also, delete the individual header files to cleanup. 681 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 682 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 683 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 684 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 685 show_err=0) 686 687 # ffdc_list_file_path contains a list of any ffdc files created by plug- 688 # ins, etc. Read that data into a list. 689 try: 690 plug_in_ffdc_list = \ 691 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 692 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 693 except IOError: 694 plug_in_ffdc_list = [] 695 696 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 697 # in. Eliminate duplicates and sort the list. 698 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 699 700 if status_file_path != "": 701 ffdc_file_list.insert(0, status_file_path) 702 703 # Convert the list to a printable list. 704 printable_ffdc_file_list = "\n".join(ffdc_file_list) 705 706 # Open ffdc_file_list for writing. We will write a complete list of 707 # FFDC files to it for possible use by plug-ins like cp_stop_check. 708 ffdc_list_file = open(ffdc_list_file_path, 'w') 709 ffdc_list_file.write(printable_ffdc_file_list + "\n") 710 ffdc_list_file.close() 711 712 indent = 0 713 width = 90 714 linefeed = 1 715 char = "=" 716 717 gp.qprintn() 718 gp.qprint_dashes(indent, width, linefeed, char) 719 gp.qprintn("Copy this data to the defect:\n") 720 721 if len(more_header_info) > 0: 722 gp.qprintn(more_header_info) 723 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 724 openbmc_host_name, openbmc_ip, openbmc_username, 725 openbmc_password, rest_username, rest_password, ipmi_username, 726 ipmi_password, os_host, os_host_name, os_ip, os_username, 727 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 728 pdu_password, pdu_slot_no, openbmc_serial_host, 729 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 730 731 gp.qprintn() 732 print_boot_history(boot_history) 733 gp.qprintn() 734 gp.qprint_var(state) 735 gp.qprintn() 736 gp.qprintn("FFDC data files:") 737 gp.qprintn(printable_ffdc_file_list) 738 gp.qprintn() 739 740 if len(ffdc_summary_info) > 0: 741 gp.qprintn(ffdc_summary_info) 742 743 gp.qprint_dashes(indent, width, linefeed, char) 744 745 746def my_ffdc(): 747 r""" 748 Collect FFDC data. 749 """ 750 751 global state 752 753 plug_in_setup() 754 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 755 call_point='ffdc', stop_on_plug_in_failure=0) 756 757 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 758 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 759 + AUTOBOOT_FFDC_PREFIX 760 + " ffdc_function_list=" 761 + ffdc_function_list, ignore=1) 762 if status != 'PASS': 763 gp.qprint_error("Call to ffdc failed.\n") 764 if type(ffdc_file_list) is not list: 765 ffdc_file_list = [] 766 # Leave a record for caller that "soft" errors occurred. 767 soft_errors = 1 768 gpu.save_plug_in_value(soft_errors, pgm_name) 769 770 my_get_state() 771 772 print_defect_report(ffdc_file_list) 773 774 775def print_test_start_message(boot_keyword): 776 r""" 777 Print a message indicating what boot test is about to run. 778 779 Description of arguments: 780 boot_keyword The name of the boot which is to be run 781 (e.g. "BMC Power On"). 782 """ 783 784 global boot_history 785 global boot_start_time 786 787 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 788 789 # Set boot_start_time for use by plug-ins. 790 boot_start_time = doing_msg[1:33] 791 gp.qprint_var(boot_start_time) 792 793 gp.qprint(doing_msg) 794 795 update_boot_history(boot_history, doing_msg, max_boot_history) 796 797 798def stop_boot_test(signal_number=0, 799 frame=None): 800 r""" 801 Handle SIGUSR1 by aborting the boot test that is running. 802 803 Description of argument(s): 804 signal_number The signal number (should always be 10 for SIGUSR1). 805 frame The frame data. 806 """ 807 808 gp.qprintn() 809 gp.qprint_executing() 810 gp.lprint_executing() 811 812 # Restore original sigusr1 handler. 813 set_default_siguser1() 814 815 message = "The caller has asked that the boot test be stopped and marked" 816 message += " as a failure." 817 818 function_stack = gm.get_function_stack() 819 if "wait_state" in function_stack: 820 st.set_exit_wait_early_message(message) 821 else: 822 BuiltIn().fail(gp.sprint_error(message)) 823 824 825def run_boot(boot): 826 r""" 827 Run the specified boot. 828 829 Description of arguments: 830 boot The name of the boot test to be performed. 831 """ 832 833 global state 834 835 signal.signal(signal.SIGUSR1, stop_boot_test) 836 gp.qprint_timen("stop_boot_test is armed.") 837 838 print_test_start_message(boot) 839 840 plug_in_setup() 841 rc, shell_rc, failed_plug_in_name = \ 842 grpi.rprocess_plug_in_packages(call_point="pre_boot") 843 if rc != 0: 844 error_message = "Plug-in failed with non-zero return code.\n" +\ 845 gp.sprint_var(rc, fmt=gp.hexa()) 846 set_default_siguser1() 847 BuiltIn().fail(gp.sprint_error(error_message)) 848 849 if test_mode: 850 # In test mode, we'll pretend the boot worked by assigning its 851 # required end state to the default state value. 852 state = st.strip_anchor_state(boot_table[boot]['end']) 853 else: 854 # Assertion: We trust that the state data was made fresh by the 855 # caller. 856 857 gp.qprintn() 858 859 if boot_table[boot]['method_type'] == "keyword": 860 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 861 boot_table[boot]['method'], 862 quiet=quiet) 863 864 if boot_table[boot]['bmc_reboot']: 865 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 866 plug_in_setup() 867 rc, shell_rc, failed_plug_in_name = \ 868 grpi.rprocess_plug_in_packages(call_point="post_reboot") 869 if rc != 0: 870 error_message = "Plug-in failed with non-zero return code.\n" 871 error_message += gp.sprint_var(rc, fmt=gp.hexa()) 872 set_default_siguser1() 873 BuiltIn().fail(gp.sprint_error(error_message)) 874 else: 875 match_state = st.anchor_state(state) 876 del match_state['epoch_seconds'] 877 # Wait for the state to change in any way. 878 st.wait_state(match_state, wait_time=state_change_timeout, 879 interval="10 seconds", invert=1) 880 881 gp.qprintn() 882 if boot_table[boot]['end']['chassis'] == "Off": 883 boot_timeout = power_off_timeout 884 else: 885 boot_timeout = power_on_timeout 886 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 887 interval="10 seconds") 888 889 plug_in_setup() 890 rc, shell_rc, failed_plug_in_name = \ 891 grpi.rprocess_plug_in_packages(call_point="post_boot") 892 if rc != 0: 893 error_message = "Plug-in failed with non-zero return code.\n" +\ 894 gp.sprint_var(rc, fmt=gp.hexa()) 895 set_default_siguser1() 896 BuiltIn().fail(gp.sprint_error(error_message)) 897 898 # Restore original sigusr1 handler. 899 set_default_siguser1() 900 901 902def test_loop_body(): 903 r""" 904 The main loop body for the loop in main_py. 905 906 Description of arguments: 907 boot_count The iteration number (starts at 1). 908 """ 909 910 global boot_count 911 global state 912 global next_boot 913 global boot_success 914 global boot_end_time 915 916 gp.qprintn() 917 918 next_boot = select_boot() 919 if next_boot == "": 920 return True 921 922 boot_count += 1 923 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 924 925 pre_boot_plug_in_setup() 926 927 cmd_buf = ["run_boot", next_boot] 928 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 929 if boot_status == "FAIL": 930 gp.qprint(msg) 931 932 gp.qprintn() 933 if boot_status == "PASS": 934 boot_success = 1 935 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 936 + "\" succeeded.") 937 else: 938 boot_success = 0 939 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 940 + "\" failed.") 941 942 # Set boot_end_time for use by plug-ins. 943 boot_end_time = completion_msg[1:33] 944 gp.qprint_var(boot_end_time) 945 946 gp.qprint(completion_msg) 947 948 boot_results.update(next_boot, boot_status) 949 950 plug_in_setup() 951 # NOTE: A post_test_case call point failure is NOT counted as a boot 952 # failure. 953 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 954 call_point='post_test_case', stop_on_plug_in_failure=0) 955 956 plug_in_setup() 957 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 958 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 959 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 960 if ffdc_check == "All" or\ 961 shell_rc == dump_ffdc_rc(): 962 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 963 if status != 'PASS': 964 gp.qprint_error("Call to my_ffdc failed.\n") 965 # Leave a record for caller that "soft" errors occurred. 966 soft_errors = 1 967 gpu.save_plug_in_value(soft_errors, pgm_name) 968 969 if delete_errlogs: 970 # print error logs before delete 971 status, error_logs = grk.run_key_u("Get Error Logs") 972 pels = pel.peltool("-l", ignore_err=1) 973 gp.qprint_vars(error_logs, pels) 974 975 # We need to purge error logs between boots or they build up. 976 grk.run_key(delete_errlogs_cmd, ignore=1) 977 978 boot_results.print_report() 979 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 980 981 plug_in_setup() 982 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 983 call_point='stop_check', shell_rc=stop_test_rc(), 984 stop_on_non_zero_rc=1) 985 if shell_rc == stop_test_rc(): 986 message = "Stopping as requested by user.\n" 987 gp.qprint_time(message) 988 BuiltIn().fail(message) 989 990 # This should help prevent ConnectionErrors. 991 grk.run_key_u("Close All Connections") 992 993 return True 994 995 996def obmc_boot_test_teardown(): 997 r""" 998 Clean up after the main keyword. 999 """ 1000 gp.qprint_executing() 1001 1002 if ga.psutil_imported: 1003 ga.terminate_descendants() 1004 1005 if cp_setup_called: 1006 plug_in_setup() 1007 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1008 call_point='cleanup', stop_on_plug_in_failure=0) 1009 1010 if 'boot_results_file_path' in globals(): 1011 # Save boot_results and boot_history objects to a file in case they are 1012 # needed again. 1013 gp.qprint_timen("Saving boot_results to the following path.") 1014 gp.qprint_var(boot_results_file_path) 1015 pickle.dump((boot_results, boot_history), 1016 open(boot_results_file_path, 'wb'), 1017 pickle.HIGHEST_PROTOCOL) 1018 1019 global save_stack 1020 # Restore any global values saved on the save_stack. 1021 for parm_name in main_func_parm_list: 1022 # Get the parm_value if it was saved on the stack. 1023 try: 1024 parm_value = save_stack.pop(parm_name) 1025 except BaseException: 1026 # If it was not saved, no further action is required. 1027 continue 1028 1029 # Restore the saved value. 1030 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1031 "}\", parm_value)" 1032 gp.dpissuing(cmd_buf) 1033 exec(cmd_buf) 1034 1035 gp.dprintn(save_stack.sprint_obj()) 1036 1037 1038def test_teardown(): 1039 r""" 1040 Clean up after this test case. 1041 """ 1042 1043 gp.qprintn() 1044 gp.qprint_executing() 1045 1046 if ga.psutil_imported: 1047 ga.terminate_descendants() 1048 1049 cmd_buf = ["Print Error", 1050 "A keyword timeout occurred ending this program.\n"] 1051 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1052 1053 if redfish_supported: 1054 redfish.logout() 1055 1056 gp.qprint_pgm_footer() 1057 1058 1059def post_stack(): 1060 r""" 1061 Process post_stack plug-in programs. 1062 """ 1063 1064 if not call_post_stack_plug: 1065 # The caller does not wish to have post_stack plug-in processing done. 1066 return 1067 1068 global boot_success 1069 1070 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1071 pre_boot_plug_in_setup() 1072 # For the purposes of the following plug-ins, mark the "boot" as a success. 1073 boot_success = 1 1074 plug_in_setup() 1075 rc, shell_rc, failed_plug_in_name, history =\ 1076 grpi.rprocess_plug_in_packages(call_point='post_stack', 1077 stop_on_plug_in_failure=0, 1078 return_history=True) 1079 for doing_msg in history: 1080 update_boot_history(boot_history, doing_msg, max_boot_history) 1081 if rc != 0: 1082 boot_success = 0 1083 1084 plug_in_setup() 1085 rc, shell_rc, failed_plug_in_name =\ 1086 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1087 shell_rc=dump_ffdc_rc(), 1088 stop_on_plug_in_failure=1, 1089 stop_on_non_zero_rc=1) 1090 if shell_rc == dump_ffdc_rc(): 1091 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1092 if status != 'PASS': 1093 gp.qprint_error("Call to my_ffdc failed.\n") 1094 # Leave a record for caller that "soft" errors occurred. 1095 soft_errors = 1 1096 gpu.save_plug_in_value(soft_errors, pgm_name) 1097 1098 plug_in_setup() 1099 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1100 call_point='stop_check', shell_rc=stop_test_rc(), 1101 stop_on_non_zero_rc=1) 1102 if shell_rc == stop_test_rc(): 1103 message = "Stopping as requested by user.\n" 1104 gp.qprint_time(message) 1105 BuiltIn().fail(message) 1106 1107 1108def obmc_boot_test_py(loc_boot_stack=None, 1109 loc_stack_mode=None, 1110 loc_quiet=None): 1111 r""" 1112 Do main program processing. 1113 """ 1114 1115 global save_stack 1116 1117 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']}) 1118 1119 gp.dprintn() 1120 # Process function parms. 1121 for parm_name in main_func_parm_list: 1122 # Get parm's value. 1123 parm_value = eval("loc_" + parm_name) 1124 gp.dpvars(parm_name, parm_value) 1125 1126 if parm_value is not None: 1127 # Save the global value on a stack. 1128 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1129 parm_name + "}\"), \"" + parm_name + "\")" 1130 gp.dpissuing(cmd_buf) 1131 exec(cmd_buf) 1132 1133 # Set the global value to the passed value. 1134 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1135 "}\", loc_" + parm_name + ")" 1136 gp.dpissuing(cmd_buf) 1137 exec(cmd_buf) 1138 1139 gp.dprintn(save_stack.sprint_obj()) 1140 1141 setup() 1142 1143 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1144 1145 if ffdc_only: 1146 gp.qprint_timen("Caller requested ffdc_only.") 1147 if do_pre_boot_plug_in_setup: 1148 pre_boot_plug_in_setup() 1149 grk.run_key_u("my_ffdc") 1150 return 1151 1152 if delete_errlogs: 1153 # print error logs before delete 1154 status, error_logs = grk.run_key_u("Get Error Logs") 1155 pels = pel.peltool("-l", ignore_err=1) 1156 gp.qprint_vars(error_logs, pels) 1157 1158 # Delete errlogs prior to doing any boot tests. 1159 grk.run_key(delete_errlogs_cmd, ignore=1) 1160 1161 # Process caller's boot_stack. 1162 while (len(boot_stack) > 0): 1163 test_loop_body() 1164 1165 gp.qprint_timen("Finished processing stack.") 1166 1167 post_stack() 1168 1169 # Process caller's boot_list. 1170 if len(boot_list) > 0: 1171 for ix in range(1, max_num_tests + 1): 1172 test_loop_body() 1173 1174 gp.qprint_timen("Completed all requested boot tests.") 1175 1176 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1177 new_fail = boot_fail - init_boot_fail 1178 if new_fail > boot_fail_threshold: 1179 error_message = "Boot failures exceed the boot failure" +\ 1180 " threshold:\n" +\ 1181 gp.sprint_var(new_fail) +\ 1182 gp.sprint_var(boot_fail_threshold) 1183 BuiltIn().fail(gp.sprint_error(error_message)) 1184