1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_arg as ga 27import gen_valid as gv 28import gen_misc as gm 29import gen_cmd as gc 30import gen_robot_keyword as grk 31import state as st 32import var_stack as vs 33import gen_plug_in_utils as gpu 34import pel_utils as pel 35import logging_utils as log 36 37base_path = os.path.dirname(os.path.dirname( 38 imp.find_module("gen_robot_print")[1])) +\ 39 os.sep 40sys.path.append(base_path + "extended/") 41import run_keyword as rk 42 43# Setting master_pid correctly influences the behavior of plug-ins like 44# DB_Logging 45program_pid = os.getpid() 46master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 47pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 48 49# Set up boot data structures. 50os_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 51 52boot_lists = read_boot_lists() 53 54# The maximum number of entries that can be in the boot_history global variable. 55max_boot_history = 10 56boot_history = [] 57 58state = st.return_state_constant('default_state') 59cp_setup_called = 0 60next_boot = "" 61base_tool_dir_path = os.path.normpath(os.environ.get( 62 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 63 64ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 65boot_success = 0 66status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 67if status_dir_path != "": 68 status_dir_path = os.path.normpath(status_dir_path) + os.sep 69redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False) 70redfish_rest_supported = BuiltIn().get_variable_value("${REDFISH_REST_SUPPORTED}", default=False) 71if redfish_supported: 72 redfish = BuiltIn().get_library_instance('redfish') 73 default_power_on = "Redfish Power On" 74 default_power_off = "Redfish Power Off" 75 if redfish_rest_supported: 76 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 77 delete_bmcdump_cmd = "Delete All BMC Dump" 78 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 79 else: 80 delete_errlogs_cmd = "Redfish Purge Event Log" 81 delete_bmcdump_cmd = "Redfish Delete All BMC Dumps" 82 default_set_power_policy = "Redfish Set Power Restore Policy AlwaysOff" 83else: 84 default_power_on = "REST Power On" 85 default_power_off = "REST Power Off" 86 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 87 delete_bmcdump_cmd = "Delete All BMC Dump" 88 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 89boot_count = 0 90 91LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 92AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 93ffdc_prefix = AUTOBOOT_FFDC_PREFIX 94boot_start_time = "" 95boot_end_time = "" 96save_stack = vs.var_stack('save_stack') 97main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 98 99 100def dump_ffdc_rc(): 101 r""" 102 Return the constant dump ffdc test return code value. 103 104 When a plug-in call point program returns this value, it indicates that 105 this program should collect FFDC. 106 """ 107 108 return 0x00000200 109 110 111def stop_test_rc(): 112 r""" 113 Return the constant stop test return code value. 114 115 When a plug-in call point program returns this value, it indicates that 116 this program should stop running. 117 """ 118 119 return 0x00000200 120 121 122def process_host(host, 123 host_var_name=""): 124 r""" 125 Process a host by getting the associated host name and IP address and 126 setting them in global variables. 127 128 If the caller does not pass the host_var_name, this function will try to 129 figure out the name of the variable used by the caller for the host parm. 130 Callers are advised to explicitly specify the host_var_name when calling 131 with an exec command. In such cases, the get_arg_name cannot figure out 132 the host variable name. 133 134 This function will then create similar global variable names by 135 removing "_host" and appending "_host_name" or "_ip" to the host variable 136 name. 137 138 Example: 139 140 If a call is made like this: 141 process_host(openbmc_host) 142 143 Global variables openbmc_host_name and openbmc_ip will be set. 144 145 Description of argument(s): 146 host A host name or IP. The name of the variable used should 147 have a suffix of "_host". 148 host_var_name The name of the variable being used as the host parm. 149 """ 150 151 if host_var_name == "": 152 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 153 154 host_name_var_name = re.sub("host", "host_name", host_var_name) 155 ip_var_name = re.sub("host", "ip", host_var_name) 156 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 157 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 158 host + "')" 159 exec(cmd_buf) 160 161 162def process_pgm_parms(): 163 r""" 164 Process the program parameters by assigning them all to corresponding 165 globals. Also, set some global values that depend on program parameters. 166 """ 167 168 # Program parameter processing. 169 # Assign all program parms to python variables which are global to this 170 # module. 171 172 global parm_list 173 parm_list = BuiltIn().get_variable_value("${parm_list}") 174 # The following subset of parms should be processed as integers. 175 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 176 'boot_fail_threshold', 'delete_errlogs', 177 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 178 'test_mode', 'debug'] 179 for parm in parm_list: 180 if parm in int_list: 181 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 182 "}\", \"0\"))" 183 else: 184 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 185 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 186 gp.dpissuing(cmd_buf) 187 exec(cmd_buf) 188 if re.match(r".*_host$", parm): 189 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 190 exec(cmd_buf) 191 if re.match(r".*_password$", parm): 192 # Register the value of any parm whose name ends in _password. 193 # This will cause the print functions to replace passwords with 194 # asterisks in the output. 195 cmd_buf = "gp.register_passwords(" + parm + ")" 196 exec(cmd_buf) 197 198 global ffdc_dir_path_style 199 global boot_list 200 global boot_stack 201 global boot_results_file_path 202 global boot_results 203 global boot_history 204 global ffdc_list_file_path 205 global ffdc_report_list_path 206 global ffdc_summary_list_path 207 global boot_table 208 global valid_boot_types 209 210 if ffdc_dir_path_style == "": 211 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 212 213 # Convert these program parms to lists for easier processing.. 214 boot_list = list(filter(None, boot_list.split(":"))) 215 boot_stack = list(filter(None, boot_stack.split(":"))) 216 217 boot_table = create_boot_table(boot_table_path, os_host=os_host) 218 valid_boot_types = create_valid_boot_list(boot_table) 219 220 cleanup_boot_results_file() 221 boot_results_file_path = create_boot_results_file_path(pgm_name, 222 openbmc_nickname, 223 master_pid) 224 225 if os.path.isfile(boot_results_file_path): 226 # We've been called before in this run so we'll load the saved 227 # boot_results and boot_history objects. 228 boot_results, boot_history =\ 229 pickle.load(open(boot_results_file_path, 'rb')) 230 else: 231 boot_results = boot_results(boot_table, boot_pass, boot_fail) 232 233 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 234 "/FFDC_FILE_LIST" 235 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 236 "/FFDC_REPORT_FILE_LIST" 237 238 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 239 "/FFDC_SUMMARY_FILE_LIST" 240 241 242def initial_plug_in_setup(): 243 r""" 244 Initialize all plug-in environment variables which do not change for the 245 duration of the program. 246 247 """ 248 249 global LOG_LEVEL 250 BuiltIn().set_log_level("NONE") 251 252 BuiltIn().set_global_variable("${master_pid}", master_pid) 253 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 254 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 255 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 256 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 257 ffdc_list_file_path) 258 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 259 ffdc_report_list_path) 260 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 261 ffdc_summary_list_path) 262 263 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 264 ffdc_dir_path_style) 265 BuiltIn().set_global_variable("${FFDC_CHECK}", 266 ffdc_check) 267 268 # For each program parameter, set the corresponding AUTOBOOT_ environment 269 # variable value. Also, set an AUTOBOOT_ environment variable for every 270 # element in additional_values. 271 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 272 "status_dir_path", "base_tool_dir_path", 273 "ffdc_list_file_path", "ffdc_report_list_path", 274 "ffdc_summary_list_path", "execdir", "redfish_supported", 275 "redfish_rest_supported"] 276 277 plug_in_vars = parm_list + additional_values 278 279 for var_name in plug_in_vars: 280 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 281 var_name = var_name.upper() 282 if var_value is None: 283 var_value = "" 284 os.environ["AUTOBOOT_" + var_name] = str(var_value) 285 286 BuiltIn().set_log_level(LOG_LEVEL) 287 288 # Make sure the ffdc list directory exists. 289 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 290 if not os.path.exists(ffdc_list_dir_path): 291 os.makedirs(ffdc_list_dir_path) 292 293 294def plug_in_setup(): 295 r""" 296 Initialize all changing plug-in environment variables for use by the 297 plug-in programs. 298 """ 299 300 global LOG_LEVEL 301 global test_really_running 302 303 BuiltIn().set_log_level("NONE") 304 305 boot_pass, boot_fail = boot_results.return_total_pass_fail() 306 if boot_pass > 1: 307 test_really_running = 1 308 else: 309 test_really_running = 0 310 311 BuiltIn().set_global_variable("${test_really_running}", 312 test_really_running) 313 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 314 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 315 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 316 BuiltIn().set_global_variable("${boot_success}", boot_success) 317 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 318 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 319 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 320 321 # For each program parameter, set the corresponding AUTOBOOT_ environment 322 # variable value. Also, set an AUTOBOOT_ environment variable for every 323 # element in additional_values. 324 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 325 "boot_fail", "test_really_running", "ffdc_prefix", 326 "boot_start_time", "boot_end_time"] 327 328 plug_in_vars = additional_values 329 330 for var_name in plug_in_vars: 331 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 332 var_name = var_name.upper() 333 if var_value is None: 334 var_value = "" 335 os.environ["AUTOBOOT_" + var_name] = str(var_value) 336 337 if debug: 338 shell_rc, out_buf = \ 339 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 340 341 BuiltIn().set_log_level(LOG_LEVEL) 342 343 344def pre_boot_plug_in_setup(): 345 346 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 347 try: 348 os.remove(ffdc_list_file_path) 349 except OSError: 350 pass 351 352 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 353 try: 354 os.remove(ffdc_report_list_path) 355 except OSError: 356 pass 357 358 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 359 try: 360 os.remove(ffdc_summary_list_path) 361 except OSError: 362 pass 363 364 global ffdc_prefix 365 366 seconds = time.time() 367 loc_time = time.localtime(seconds) 368 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 369 370 ffdc_prefix = openbmc_nickname + "." + time_string 371 372 373def default_sigusr1(signal_number=0, 374 frame=None): 375 r""" 376 Handle SIGUSR1 by doing nothing. 377 378 This function assists in debugging SIGUSR1 processing by printing messages 379 to stdout and to the log.html file. 380 381 Description of argument(s): 382 signal_number The signal number (should always be 10 for SIGUSR1). 383 frame The frame data. 384 """ 385 386 gp.qprintn() 387 gp.qprint_executing() 388 gp.lprint_executing() 389 390 391def set_default_siguser1(): 392 r""" 393 Set the default_sigusr1 function to be the SIGUSR1 handler. 394 """ 395 396 gp.qprintn() 397 gp.qprint_executing() 398 gp.lprint_executing() 399 signal.signal(signal.SIGUSR1, default_sigusr1) 400 401 402def setup(): 403 r""" 404 Do general program setup tasks. 405 """ 406 407 global cp_setup_called 408 global transitional_boot_selected 409 410 gp.qprintn() 411 412 if redfish_supported: 413 redfish.login() 414 415 set_default_siguser1() 416 transitional_boot_selected = False 417 418 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 419 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 420 # If we can't find process_plug_in_packages.py, ssh_pw or 421 # validate_plug_ins.py, then we don't have our repo bin in PATH. 422 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 423 + " ssh_pw validate_plug_ins.py", quiet=1, 424 print_output=0, show_err=0) 425 if shell_rc != 0: 426 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 427 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 428 if robot_pgm_dir_path not in sys.path: 429 sys.path.append(robot_pgm_dir_path) 430 PYTHONPATH = os.environ.get("PYTHONPATH", "") 431 if PYTHONPATH == "": 432 os.environ['PYTHONPATH'] = robot_pgm_dir_path 433 else: 434 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 435 436 validate_parms() 437 438 gp.qprint_pgm_header() 439 440 grk.run_key_u(default_set_power_policy) 441 442 initial_plug_in_setup() 443 444 plug_in_setup() 445 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 446 call_point='setup') 447 if rc != 0: 448 error_message = "Plug-in setup failed.\n" 449 gp.print_error_report(error_message) 450 BuiltIn().fail(error_message) 451 # Setting cp_setup_called lets our Teardown know that it needs to call 452 # the cleanup plug-in call point. 453 cp_setup_called = 1 454 455 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 456 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 457 # FFDC_LOG_PATH is used by "FFDC" keyword. 458 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 459 460 # Also printed by FFDC. 461 global host_name 462 global host_ip 463 host = socket.gethostname() 464 host_name, host_ip = gm.get_host_name_ip(host) 465 466 gp.dprint_var(boot_table) 467 gp.dprint_var(boot_lists) 468 469 470def validate_parms(): 471 r""" 472 Validate all program parameters. 473 """ 474 475 process_pgm_parms() 476 477 gp.qprintn() 478 479 global openbmc_model 480 if openbmc_model == "": 481 status, ret_values =\ 482 grk.run_key_u("Get BMC System Model") 483 openbmc_model = ret_values 484 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 485 gv.set_exit_on_error(True) 486 gv.valid_value(openbmc_host) 487 gv.valid_value(openbmc_username) 488 gv.valid_value(openbmc_password) 489 gv.valid_value(rest_username) 490 gv.valid_value(rest_password) 491 gv.valid_value(ipmi_username) 492 gv.valid_value(ipmi_password) 493 if os_host != "": 494 gv.valid_value(os_username) 495 gv.valid_value(os_password) 496 if pdu_host != "": 497 gv.valid_value(pdu_username) 498 gv.valid_value(pdu_password) 499 gv.valid_integer(pdu_slot_no) 500 if openbmc_serial_host != "": 501 gv.valid_integer(openbmc_serial_port) 502 gv.valid_value(openbmc_model) 503 gv.valid_integer(max_num_tests) 504 gv.valid_integer(boot_pass) 505 gv.valid_integer(boot_fail) 506 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 507 BuiltIn().set_global_variable("${plug_in_packages_list}", 508 plug_in_packages_list) 509 gv.valid_value(stack_mode, valid_values=['normal', 'skip']) 510 gv.set_exit_on_error(False) 511 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 512 error_message = "You must provide either a value for either the" +\ 513 " boot_list or the boot_stack parm.\n" 514 BuiltIn().fail(gp.sprint_error(error_message)) 515 valid_boot_list(boot_list, valid_boot_types) 516 valid_boot_list(boot_stack, valid_boot_types) 517 selected_PDU_boots = list(set(boot_list + boot_stack) 518 & set(boot_lists['PDU_reboot'])) 519 if len(selected_PDU_boots) > 0 and pdu_host == "": 520 error_message = "You have selected the following boots which" +\ 521 " require a PDU host but no value for pdu_host:\n" 522 error_message += gp.sprint_var(selected_PDU_boots) 523 error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 524 BuiltIn().fail(gp.sprint_error(error_message)) 525 526 return 527 528 529def my_get_state(): 530 r""" 531 Get the system state plus a little bit of wrapping. 532 """ 533 534 global state 535 536 req_states = ['epoch_seconds'] + st.default_req_states 537 538 gp.qprint_timen("Getting system state.") 539 if test_mode: 540 state['epoch_seconds'] = int(time.time()) 541 else: 542 state = st.get_state(req_states=req_states, quiet=quiet) 543 gp.qprint_var(state) 544 545 546def valid_state(): 547 r""" 548 Verify that our state dictionary contains no blank values. If we don't get 549 valid state data, we cannot continue to work. 550 """ 551 552 if st.compare_states(state, st.invalid_state_match, 'or'): 553 error_message = "The state dictionary contains blank fields which" +\ 554 " is illegal.\n" + gp.sprint_var(state) 555 BuiltIn().fail(gp.sprint_error(error_message)) 556 557 558def select_boot(): 559 r""" 560 Select a boot test to be run based on our current state and return the 561 chosen boot type. 562 563 Description of arguments: 564 state The state of the machine. 565 """ 566 567 global transitional_boot_selected 568 global boot_stack 569 570 gp.qprint_timen("Selecting a boot test.") 571 572 if transitional_boot_selected and not boot_success: 573 prior_boot = next_boot 574 boot_candidate = boot_stack.pop() 575 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 576 + " transition to a valid state for '" + boot_candidate 577 + "' which was at the top of the boot_stack. Since" 578 + " the '" + next_boot + "' failed, the '" 579 + boot_candidate + "' has been removed from the stack" 580 + " to avoid and endless failure loop.") 581 if len(boot_stack) == 0: 582 return "" 583 584 my_get_state() 585 valid_state() 586 587 transitional_boot_selected = False 588 stack_popped = 0 589 if len(boot_stack) > 0: 590 stack_popped = 1 591 gp.qprint_dashes() 592 gp.qprint_var(boot_stack) 593 gp.qprint_dashes() 594 skip_boot_printed = 0 595 while len(boot_stack) > 0: 596 boot_candidate = boot_stack.pop() 597 if stack_mode == 'normal': 598 break 599 else: 600 if st.compare_states(state, boot_table[boot_candidate]['end']): 601 if not skip_boot_printed: 602 gp.qprint_var(stack_mode) 603 gp.qprintn() 604 gp.qprint_timen("Skipping the following boot tests" 605 + " which are unnecessary since their" 606 + " required end states match the" 607 + " current machine state:") 608 skip_boot_printed = 1 609 gp.qprint_var(boot_candidate) 610 boot_candidate = "" 611 if boot_candidate == "": 612 gp.qprint_dashes() 613 gp.qprint_var(boot_stack) 614 gp.qprint_dashes() 615 return boot_candidate 616 if st.compare_states(state, boot_table[boot_candidate]['start']): 617 gp.qprint_timen("The machine state is valid for a '" 618 + boot_candidate + "' boot test.") 619 gp.qprint_dashes() 620 gp.qprint_var(boot_stack) 621 gp.qprint_dashes() 622 return boot_candidate 623 else: 624 gp.qprint_timen("The machine state does not match the required" 625 + " starting state for a '" + boot_candidate 626 + "' boot test:") 627 gp.qprint_varx("boot_table_start_entry", 628 boot_table[boot_candidate]['start']) 629 boot_stack.append(boot_candidate) 630 transitional_boot_selected = True 631 popped_boot = boot_candidate 632 633 # Loop through your list selecting a boot_candidates 634 boot_candidates = [] 635 for boot_candidate in boot_list: 636 if st.compare_states(state, boot_table[boot_candidate]['start']): 637 if stack_popped: 638 if st.compare_states(boot_table[boot_candidate]['end'], 639 boot_table[popped_boot]['start']): 640 boot_candidates.append(boot_candidate) 641 else: 642 boot_candidates.append(boot_candidate) 643 644 if len(boot_candidates) == 0: 645 gp.qprint_timen("The user's boot list contained no boot tests" 646 + " which are valid for the current machine state.") 647 boot_candidate = default_power_on 648 if not st.compare_states(state, boot_table[default_power_on]['start']): 649 boot_candidate = default_power_off 650 boot_candidates.append(boot_candidate) 651 gp.qprint_timen("Using default '" + boot_candidate 652 + "' boot type to transition to valid state.") 653 654 gp.dprint_var(boot_candidates) 655 656 # Randomly select a boot from the candidate list. 657 boot = random.choice(boot_candidates) 658 659 return boot 660 661 662def print_defect_report(ffdc_file_list): 663 r""" 664 Print a defect report. 665 666 Description of argument(s): 667 ffdc_file_list A list of files which were collected by our ffdc functions. 668 """ 669 670 # Making deliberate choice to NOT run plug_in_setup(). We don't want 671 # ffdc_prefix updated. 672 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 673 call_point='ffdc_report', stop_on_plug_in_failure=0) 674 675 # Get additional header data which may have been created by ffdc plug-ins. 676 # Also, delete the individual header files to cleanup. 677 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 678 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 679 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 680 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 681 show_err=0) 682 683 # Get additional summary data which may have been created by ffdc plug-ins. 684 # Also, delete the individual header files to cleanup. 685 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 686 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 687 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 688 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 689 show_err=0) 690 691 # ffdc_list_file_path contains a list of any ffdc files created by plug- 692 # ins, etc. Read that data into a list. 693 try: 694 plug_in_ffdc_list = \ 695 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 696 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 697 except IOError: 698 plug_in_ffdc_list = [] 699 700 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 701 # in. Eliminate duplicates and sort the list. 702 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 703 704 if status_file_path != "": 705 ffdc_file_list.insert(0, status_file_path) 706 707 # Convert the list to a printable list. 708 printable_ffdc_file_list = "\n".join(ffdc_file_list) 709 710 # Open ffdc_file_list for writing. We will write a complete list of 711 # FFDC files to it for possible use by plug-ins like cp_stop_check. 712 ffdc_list_file = open(ffdc_list_file_path, 'w') 713 ffdc_list_file.write(printable_ffdc_file_list + "\n") 714 ffdc_list_file.close() 715 716 indent = 0 717 width = 90 718 linefeed = 1 719 char = "=" 720 721 gp.qprintn() 722 gp.qprint_dashes(indent, width, linefeed, char) 723 gp.qprintn("Copy this data to the defect:\n") 724 725 if len(more_header_info) > 0: 726 gp.qprintn(more_header_info) 727 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 728 openbmc_host_name, openbmc_ip, openbmc_username, 729 openbmc_password, rest_username, rest_password, ipmi_username, 730 ipmi_password, os_host, os_host_name, os_ip, os_username, 731 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 732 pdu_password, pdu_slot_no, openbmc_serial_host, 733 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 734 735 gp.qprintn() 736 print_boot_history(boot_history) 737 gp.qprintn() 738 gp.qprint_var(state) 739 gp.qprintn() 740 gp.qprintn("FFDC data files:") 741 gp.qprintn(printable_ffdc_file_list) 742 gp.qprintn() 743 744 if len(ffdc_summary_info) > 0: 745 gp.qprintn(ffdc_summary_info) 746 747 gp.qprint_dashes(indent, width, linefeed, char) 748 749 750def my_ffdc(): 751 r""" 752 Collect FFDC data. 753 """ 754 755 global state 756 757 plug_in_setup() 758 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 759 call_point='ffdc', stop_on_plug_in_failure=0) 760 761 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 762 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 763 + AUTOBOOT_FFDC_PREFIX 764 + " ffdc_function_list=" 765 + ffdc_function_list, ignore=1) 766 if status != 'PASS': 767 gp.qprint_error("Call to ffdc failed.\n") 768 if type(ffdc_file_list) is not list: 769 ffdc_file_list = [] 770 # Leave a record for caller that "soft" errors occurred. 771 soft_errors = 1 772 gpu.save_plug_in_value(soft_errors, pgm_name) 773 774 my_get_state() 775 776 print_defect_report(ffdc_file_list) 777 778 779def print_test_start_message(boot_keyword): 780 r""" 781 Print a message indicating what boot test is about to run. 782 783 Description of arguments: 784 boot_keyword The name of the boot which is to be run 785 (e.g. "BMC Power On"). 786 """ 787 788 global boot_history 789 global boot_start_time 790 791 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 792 793 # Set boot_start_time for use by plug-ins. 794 boot_start_time = doing_msg[1:33] 795 gp.qprint_var(boot_start_time) 796 797 gp.qprint(doing_msg) 798 799 update_boot_history(boot_history, doing_msg, max_boot_history) 800 801 802def stop_boot_test(signal_number=0, 803 frame=None): 804 r""" 805 Handle SIGUSR1 by aborting the boot test that is running. 806 807 Description of argument(s): 808 signal_number The signal number (should always be 10 for SIGUSR1). 809 frame The frame data. 810 """ 811 812 gp.qprintn() 813 gp.qprint_executing() 814 gp.lprint_executing() 815 816 # Restore original sigusr1 handler. 817 set_default_siguser1() 818 819 message = "The caller has asked that the boot test be stopped and marked" 820 message += " as a failure." 821 822 function_stack = gm.get_function_stack() 823 if "wait_state" in function_stack: 824 st.set_exit_wait_early_message(message) 825 else: 826 BuiltIn().fail(gp.sprint_error(message)) 827 828 829def run_boot(boot): 830 r""" 831 Run the specified boot. 832 833 Description of arguments: 834 boot The name of the boot test to be performed. 835 """ 836 837 global state 838 839 signal.signal(signal.SIGUSR1, stop_boot_test) 840 gp.qprint_timen("stop_boot_test is armed.") 841 842 print_test_start_message(boot) 843 844 plug_in_setup() 845 rc, shell_rc, failed_plug_in_name = \ 846 grpi.rprocess_plug_in_packages(call_point="pre_boot") 847 if rc != 0: 848 error_message = "Plug-in failed with non-zero return code.\n" +\ 849 gp.sprint_var(rc, fmt=gp.hexa()) 850 set_default_siguser1() 851 BuiltIn().fail(gp.sprint_error(error_message)) 852 853 if test_mode: 854 # In test mode, we'll pretend the boot worked by assigning its 855 # required end state to the default state value. 856 state = st.strip_anchor_state(boot_table[boot]['end']) 857 else: 858 # Assertion: We trust that the state data was made fresh by the 859 # caller. 860 861 gp.qprintn() 862 863 if boot_table[boot]['method_type'] == "keyword": 864 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 865 boot_table[boot]['method'], 866 quiet=quiet) 867 868 if boot_table[boot]['bmc_reboot']: 869 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 870 plug_in_setup() 871 rc, shell_rc, failed_plug_in_name = \ 872 grpi.rprocess_plug_in_packages(call_point="post_reboot") 873 if rc != 0: 874 error_message = "Plug-in failed with non-zero return code.\n" 875 error_message += gp.sprint_var(rc, fmt=gp.hexa()) 876 set_default_siguser1() 877 BuiltIn().fail(gp.sprint_error(error_message)) 878 else: 879 match_state = st.anchor_state(state) 880 del match_state['epoch_seconds'] 881 # Wait for the state to change in any way. 882 st.wait_state(match_state, wait_time=state_change_timeout, 883 interval="10 seconds", invert=1) 884 885 gp.qprintn() 886 if boot_table[boot]['end']['chassis'] == "Off": 887 boot_timeout = power_off_timeout 888 else: 889 boot_timeout = power_on_timeout 890 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 891 interval="10 seconds") 892 893 plug_in_setup() 894 rc, shell_rc, failed_plug_in_name = \ 895 grpi.rprocess_plug_in_packages(call_point="post_boot") 896 if rc != 0: 897 error_message = "Plug-in failed with non-zero return code.\n" +\ 898 gp.sprint_var(rc, fmt=gp.hexa()) 899 set_default_siguser1() 900 BuiltIn().fail(gp.sprint_error(error_message)) 901 902 # Restore original sigusr1 handler. 903 set_default_siguser1() 904 905 906def test_loop_body(): 907 r""" 908 The main loop body for the loop in main_py. 909 910 Description of arguments: 911 boot_count The iteration number (starts at 1). 912 """ 913 914 global boot_count 915 global state 916 global next_boot 917 global boot_success 918 global boot_end_time 919 920 gp.qprintn() 921 922 next_boot = select_boot() 923 if next_boot == "": 924 return True 925 926 boot_count += 1 927 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 928 929 pre_boot_plug_in_setup() 930 931 cmd_buf = ["run_boot", next_boot] 932 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 933 if boot_status == "FAIL": 934 gp.qprint(msg) 935 936 gp.qprintn() 937 if boot_status == "PASS": 938 boot_success = 1 939 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 940 + "\" succeeded.") 941 else: 942 boot_success = 0 943 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 944 + "\" failed.") 945 946 # Set boot_end_time for use by plug-ins. 947 boot_end_time = completion_msg[1:33] 948 gp.qprint_var(boot_end_time) 949 950 gp.qprint(completion_msg) 951 952 boot_results.update(next_boot, boot_status) 953 954 plug_in_setup() 955 # NOTE: A post_test_case call point failure is NOT counted as a boot 956 # failure. 957 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 958 call_point='post_test_case', stop_on_plug_in_failure=0) 959 960 plug_in_setup() 961 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 962 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 963 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 964 if ffdc_check == "All" or\ 965 shell_rc == dump_ffdc_rc(): 966 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 967 if status != 'PASS': 968 gp.qprint_error("Call to my_ffdc failed.\n") 969 # Leave a record for caller that "soft" errors occurred. 970 soft_errors = 1 971 gpu.save_plug_in_value(soft_errors, pgm_name) 972 973 if delete_errlogs: 974 # print error logs before delete 975 status, error_logs = grk.run_key_u("Get Error Logs") 976 pels = pel.peltool("-l", ignore_err=1) 977 log.print_error_logs(error_logs, "AdditionalData Message Severity") 978 gp.qprint_var(pels) 979 980 # We need to purge error logs between boots or they build up. 981 grk.run_key(delete_errlogs_cmd, ignore=1) 982 grk.run_key(delete_bmcdump_cmd, ignore=1) 983 984 boot_results.print_report() 985 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 986 987 plug_in_setup() 988 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 989 call_point='stop_check', shell_rc=stop_test_rc(), 990 stop_on_non_zero_rc=1) 991 if shell_rc == stop_test_rc(): 992 message = "Stopping as requested by user.\n" 993 gp.qprint_time(message) 994 BuiltIn().fail(message) 995 996 # This should help prevent ConnectionErrors. 997 grk.run_key_u("Close All Connections") 998 999 return True 1000 1001 1002def obmc_boot_test_teardown(): 1003 r""" 1004 Clean up after the main keyword. 1005 """ 1006 gp.qprint_executing() 1007 1008 if ga.psutil_imported: 1009 ga.terminate_descendants() 1010 1011 if cp_setup_called: 1012 plug_in_setup() 1013 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1014 call_point='cleanup', stop_on_plug_in_failure=0) 1015 1016 if 'boot_results_file_path' in globals(): 1017 # Save boot_results and boot_history objects to a file in case they are 1018 # needed again. 1019 gp.qprint_timen("Saving boot_results to the following path.") 1020 gp.qprint_var(boot_results_file_path) 1021 pickle.dump((boot_results, boot_history), 1022 open(boot_results_file_path, 'wb'), 1023 pickle.HIGHEST_PROTOCOL) 1024 1025 global save_stack 1026 # Restore any global values saved on the save_stack. 1027 for parm_name in main_func_parm_list: 1028 # Get the parm_value if it was saved on the stack. 1029 try: 1030 parm_value = save_stack.pop(parm_name) 1031 except BaseException: 1032 # If it was not saved, no further action is required. 1033 continue 1034 1035 # Restore the saved value. 1036 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1037 "}\", parm_value)" 1038 gp.dpissuing(cmd_buf) 1039 exec(cmd_buf) 1040 1041 gp.dprintn(save_stack.sprint_obj()) 1042 1043 1044def test_teardown(): 1045 r""" 1046 Clean up after this test case. 1047 """ 1048 1049 gp.qprintn() 1050 gp.qprint_executing() 1051 1052 if ga.psutil_imported: 1053 ga.terminate_descendants() 1054 1055 cmd_buf = ["Print Error", 1056 "A keyword timeout occurred ending this program.\n"] 1057 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1058 1059 if redfish_supported: 1060 redfish.logout() 1061 1062 gp.qprint_pgm_footer() 1063 1064 1065def post_stack(): 1066 r""" 1067 Process post_stack plug-in programs. 1068 """ 1069 1070 if not call_post_stack_plug: 1071 # The caller does not wish to have post_stack plug-in processing done. 1072 return 1073 1074 global boot_success 1075 1076 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1077 pre_boot_plug_in_setup() 1078 # For the purposes of the following plug-ins, mark the "boot" as a success. 1079 boot_success = 1 1080 plug_in_setup() 1081 rc, shell_rc, failed_plug_in_name, history =\ 1082 grpi.rprocess_plug_in_packages(call_point='post_stack', 1083 stop_on_plug_in_failure=0, 1084 return_history=True) 1085 for doing_msg in history: 1086 update_boot_history(boot_history, doing_msg, max_boot_history) 1087 if rc != 0: 1088 boot_success = 0 1089 1090 plug_in_setup() 1091 rc, shell_rc, failed_plug_in_name =\ 1092 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1093 shell_rc=dump_ffdc_rc(), 1094 stop_on_plug_in_failure=1, 1095 stop_on_non_zero_rc=1) 1096 if shell_rc == dump_ffdc_rc(): 1097 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1098 if status != 'PASS': 1099 gp.qprint_error("Call to my_ffdc failed.\n") 1100 # Leave a record for caller that "soft" errors occurred. 1101 soft_errors = 1 1102 gpu.save_plug_in_value(soft_errors, pgm_name) 1103 1104 plug_in_setup() 1105 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1106 call_point='stop_check', shell_rc=stop_test_rc(), 1107 stop_on_non_zero_rc=1) 1108 if shell_rc == stop_test_rc(): 1109 message = "Stopping as requested by user.\n" 1110 gp.qprint_time(message) 1111 BuiltIn().fail(message) 1112 1113 1114def obmc_boot_test_py(loc_boot_stack=None, 1115 loc_stack_mode=None, 1116 loc_quiet=None): 1117 r""" 1118 Do main program processing. 1119 """ 1120 1121 global save_stack 1122 1123 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']}) 1124 1125 gp.dprintn() 1126 # Process function parms. 1127 for parm_name in main_func_parm_list: 1128 # Get parm's value. 1129 parm_value = eval("loc_" + parm_name) 1130 gp.dpvars(parm_name, parm_value) 1131 1132 if parm_value is not None: 1133 # Save the global value on a stack. 1134 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1135 parm_name + "}\"), \"" + parm_name + "\")" 1136 gp.dpissuing(cmd_buf) 1137 exec(cmd_buf) 1138 1139 # Set the global value to the passed value. 1140 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1141 "}\", loc_" + parm_name + ")" 1142 gp.dpissuing(cmd_buf) 1143 exec(cmd_buf) 1144 1145 gp.dprintn(save_stack.sprint_obj()) 1146 1147 setup() 1148 1149 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1150 1151 if ffdc_only: 1152 gp.qprint_timen("Caller requested ffdc_only.") 1153 if do_pre_boot_plug_in_setup: 1154 pre_boot_plug_in_setup() 1155 grk.run_key_u("my_ffdc") 1156 return 1157 1158 if delete_errlogs: 1159 # print error logs before delete 1160 status, error_logs = grk.run_key_u("Get Error Logs") 1161 pels = pel.peltool("-l", ignore_err=1) 1162 log.print_error_logs(error_logs, "AdditionalData Message Severity") 1163 gp.qprint_var(pels) 1164 1165 # Delete errlogs prior to doing any boot tests. 1166 grk.run_key(delete_errlogs_cmd, ignore=1) 1167 grk.run_key(delete_bmcdump_cmd, ignore=1) 1168 1169 # Process caller's boot_stack. 1170 while (len(boot_stack) > 0): 1171 test_loop_body() 1172 1173 gp.qprint_timen("Finished processing stack.") 1174 1175 post_stack() 1176 1177 # Process caller's boot_list. 1178 if len(boot_list) > 0: 1179 for ix in range(1, max_num_tests + 1): 1180 test_loop_body() 1181 1182 gp.qprint_timen("Completed all requested boot tests.") 1183 1184 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1185 new_fail = boot_fail - init_boot_fail 1186 if new_fail > boot_fail_threshold: 1187 error_message = "Boot failures exceed the boot failure" +\ 1188 " threshold:\n" +\ 1189 gp.sprint_var(new_fail) +\ 1190 gp.sprint_var(boot_fail_threshold) 1191 BuiltIn().fail(gp.sprint_error(error_message)) 1192