1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_arg as ga 27import gen_valid as gv 28import gen_misc as gm 29import gen_cmd as gc 30import gen_robot_keyword as grk 31import state as st 32import var_stack as vs 33import gen_plug_in_utils as gpu 34import pel_utils as pel 35import logging_utils as log 36 37base_path = os.path.dirname(os.path.dirname( 38 imp.find_module("gen_robot_print")[1])) +\ 39 os.sep 40sys.path.append(base_path + "extended/") 41import run_keyword as rk 42 43# Setting master_pid correctly influences the behavior of plug-ins like 44# DB_Logging 45program_pid = os.getpid() 46master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 47pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 48 49# Set up boot data structures. 50os_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 51 52boot_lists = read_boot_lists() 53 54# The maximum number of entries that can be in the boot_history global variable. 55max_boot_history = 10 56boot_history = [] 57 58state = st.return_state_constant('default_state') 59cp_setup_called = 0 60next_boot = "" 61base_tool_dir_path = os.path.normpath(os.environ.get( 62 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 63 64ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 65boot_success = 0 66status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 67if status_dir_path != "": 68 status_dir_path = os.path.normpath(status_dir_path) + os.sep 69redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False) 70redfish_rest_supported = BuiltIn().get_variable_value("${REDFISH_REST_SUPPORTED}", default=False) 71redfish_delete_sessions = int(BuiltIn().get_variable_value("${REDFISH_DELETE_SESSIONS}", default=1)) 72if redfish_supported: 73 redfish = BuiltIn().get_library_instance('redfish') 74 default_power_on = "Redfish Power On" 75 default_power_off = "Redfish Power Off" 76 if redfish_rest_supported: 77 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 78 delete_bmcdump_cmd = "Delete All BMC Dump" 79 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 80 else: 81 delete_errlogs_cmd = "Redfish Purge Event Log" 82 delete_bmcdump_cmd = "Redfish Delete All BMC Dumps" 83 default_set_power_policy = "Redfish Set Power Restore Policy AlwaysOff" 84else: 85 default_power_on = "REST Power On" 86 default_power_off = "REST Power Off" 87 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 88 delete_bmcdump_cmd = "Delete All BMC Dump" 89 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 90boot_count = 0 91 92LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 93AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 94ffdc_prefix = AUTOBOOT_FFDC_PREFIX 95boot_start_time = "" 96boot_end_time = "" 97save_stack = vs.var_stack('save_stack') 98main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 99 100 101def dump_ffdc_rc(): 102 r""" 103 Return the constant dump ffdc test return code value. 104 105 When a plug-in call point program returns this value, it indicates that 106 this program should collect FFDC. 107 """ 108 109 return 0x00000200 110 111 112def stop_test_rc(): 113 r""" 114 Return the constant stop test return code value. 115 116 When a plug-in call point program returns this value, it indicates that 117 this program should stop running. 118 """ 119 120 return 0x00000200 121 122 123def process_host(host, 124 host_var_name=""): 125 r""" 126 Process a host by getting the associated host name and IP address and 127 setting them in global variables. 128 129 If the caller does not pass the host_var_name, this function will try to 130 figure out the name of the variable used by the caller for the host parm. 131 Callers are advised to explicitly specify the host_var_name when calling 132 with an exec command. In such cases, the get_arg_name cannot figure out 133 the host variable name. 134 135 This function will then create similar global variable names by 136 removing "_host" and appending "_host_name" or "_ip" to the host variable 137 name. 138 139 Example: 140 141 If a call is made like this: 142 process_host(openbmc_host) 143 144 Global variables openbmc_host_name and openbmc_ip will be set. 145 146 Description of argument(s): 147 host A host name or IP. The name of the variable used should 148 have a suffix of "_host". 149 host_var_name The name of the variable being used as the host parm. 150 """ 151 152 if host_var_name == "": 153 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 154 155 host_name_var_name = re.sub("host", "host_name", host_var_name) 156 ip_var_name = re.sub("host", "ip", host_var_name) 157 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 158 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 159 host + "')" 160 exec(cmd_buf) 161 162 163def process_pgm_parms(): 164 r""" 165 Process the program parameters by assigning them all to corresponding 166 globals. Also, set some global values that depend on program parameters. 167 """ 168 169 # Program parameter processing. 170 # Assign all program parms to python variables which are global to this 171 # module. 172 173 global parm_list 174 parm_list = BuiltIn().get_variable_value("${parm_list}") 175 # The following subset of parms should be processed as integers. 176 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 177 'boot_fail_threshold', 'delete_errlogs', 178 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 179 'test_mode', 'debug'] 180 for parm in parm_list: 181 if parm in int_list: 182 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 183 "}\", \"0\"))" 184 else: 185 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 186 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 187 gp.dpissuing(cmd_buf) 188 exec(cmd_buf) 189 if re.match(r".*_host$", parm): 190 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 191 exec(cmd_buf) 192 if re.match(r".*_password$", parm): 193 # Register the value of any parm whose name ends in _password. 194 # This will cause the print functions to replace passwords with 195 # asterisks in the output. 196 cmd_buf = "gp.register_passwords(" + parm + ")" 197 exec(cmd_buf) 198 199 global ffdc_dir_path_style 200 global boot_list 201 global boot_stack 202 global boot_results_file_path 203 global boot_results 204 global boot_history 205 global ffdc_list_file_path 206 global ffdc_report_list_path 207 global ffdc_summary_list_path 208 global boot_table 209 global valid_boot_types 210 211 if ffdc_dir_path_style == "": 212 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 213 214 # Convert these program parms to lists for easier processing.. 215 boot_list = list(filter(None, boot_list.split(":"))) 216 boot_stack = list(filter(None, boot_stack.split(":"))) 217 218 boot_table = create_boot_table(boot_table_path, os_host=os_host) 219 valid_boot_types = create_valid_boot_list(boot_table) 220 221 cleanup_boot_results_file() 222 boot_results_file_path = create_boot_results_file_path(pgm_name, 223 openbmc_nickname, 224 master_pid) 225 226 if os.path.isfile(boot_results_file_path): 227 # We've been called before in this run so we'll load the saved 228 # boot_results and boot_history objects. 229 boot_results, boot_history =\ 230 pickle.load(open(boot_results_file_path, 'rb')) 231 else: 232 boot_results = boot_results(boot_table, boot_pass, boot_fail) 233 234 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 235 "/FFDC_FILE_LIST" 236 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 237 "/FFDC_REPORT_FILE_LIST" 238 239 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 240 "/FFDC_SUMMARY_FILE_LIST" 241 242 243def initial_plug_in_setup(): 244 r""" 245 Initialize all plug-in environment variables which do not change for the 246 duration of the program. 247 248 """ 249 250 global LOG_LEVEL 251 BuiltIn().set_log_level("NONE") 252 253 BuiltIn().set_global_variable("${master_pid}", master_pid) 254 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 255 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 256 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 257 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 258 ffdc_list_file_path) 259 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 260 ffdc_report_list_path) 261 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 262 ffdc_summary_list_path) 263 264 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 265 ffdc_dir_path_style) 266 BuiltIn().set_global_variable("${FFDC_CHECK}", 267 ffdc_check) 268 269 # For each program parameter, set the corresponding AUTOBOOT_ environment 270 # variable value. Also, set an AUTOBOOT_ environment variable for every 271 # element in additional_values. 272 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 273 "status_dir_path", "base_tool_dir_path", 274 "ffdc_list_file_path", "ffdc_report_list_path", 275 "ffdc_summary_list_path", "execdir", "redfish_supported", 276 "redfish_rest_supported"] 277 278 plug_in_vars = parm_list + additional_values 279 280 for var_name in plug_in_vars: 281 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 282 var_name = var_name.upper() 283 if var_value is None: 284 var_value = "" 285 os.environ["AUTOBOOT_" + var_name] = str(var_value) 286 287 BuiltIn().set_log_level(LOG_LEVEL) 288 289 # Make sure the ffdc list directory exists. 290 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 291 if not os.path.exists(ffdc_list_dir_path): 292 os.makedirs(ffdc_list_dir_path) 293 294 295def plug_in_setup(): 296 r""" 297 Initialize all changing plug-in environment variables for use by the 298 plug-in programs. 299 """ 300 301 global LOG_LEVEL 302 global test_really_running 303 304 BuiltIn().set_log_level("NONE") 305 306 boot_pass, boot_fail = boot_results.return_total_pass_fail() 307 if boot_pass > 1: 308 test_really_running = 1 309 else: 310 test_really_running = 0 311 312 BuiltIn().set_global_variable("${test_really_running}", 313 test_really_running) 314 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 315 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 316 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 317 BuiltIn().set_global_variable("${boot_success}", boot_success) 318 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 319 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 320 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 321 322 # For each program parameter, set the corresponding AUTOBOOT_ environment 323 # variable value. Also, set an AUTOBOOT_ environment variable for every 324 # element in additional_values. 325 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 326 "boot_fail", "test_really_running", "ffdc_prefix", 327 "boot_start_time", "boot_end_time"] 328 329 plug_in_vars = additional_values 330 331 for var_name in plug_in_vars: 332 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 333 var_name = var_name.upper() 334 if var_value is None: 335 var_value = "" 336 os.environ["AUTOBOOT_" + var_name] = str(var_value) 337 338 if debug: 339 shell_rc, out_buf = \ 340 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 341 342 BuiltIn().set_log_level(LOG_LEVEL) 343 344 345def pre_boot_plug_in_setup(): 346 347 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 348 try: 349 os.remove(ffdc_list_file_path) 350 except OSError: 351 pass 352 353 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 354 try: 355 os.remove(ffdc_report_list_path) 356 except OSError: 357 pass 358 359 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 360 try: 361 os.remove(ffdc_summary_list_path) 362 except OSError: 363 pass 364 365 global ffdc_prefix 366 367 seconds = time.time() 368 loc_time = time.localtime(seconds) 369 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 370 371 ffdc_prefix = openbmc_nickname + "." + time_string 372 373 374def default_sigusr1(signal_number=0, 375 frame=None): 376 r""" 377 Handle SIGUSR1 by doing nothing. 378 379 This function assists in debugging SIGUSR1 processing by printing messages 380 to stdout and to the log.html file. 381 382 Description of argument(s): 383 signal_number The signal number (should always be 10 for SIGUSR1). 384 frame The frame data. 385 """ 386 387 gp.qprintn() 388 gp.qprint_executing() 389 gp.lprint_executing() 390 391 392def set_default_siguser1(): 393 r""" 394 Set the default_sigusr1 function to be the SIGUSR1 handler. 395 """ 396 397 gp.qprintn() 398 gp.qprint_executing() 399 gp.lprint_executing() 400 signal.signal(signal.SIGUSR1, default_sigusr1) 401 402 403def setup(): 404 r""" 405 Do general program setup tasks. 406 """ 407 408 global cp_setup_called 409 global transitional_boot_selected 410 411 gp.qprintn() 412 413 if redfish_supported: 414 redfish.login() 415 416 set_default_siguser1() 417 transitional_boot_selected = False 418 419 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 420 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 421 # If we can't find process_plug_in_packages.py, ssh_pw or 422 # validate_plug_ins.py, then we don't have our repo bin in PATH. 423 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 424 + " ssh_pw validate_plug_ins.py", quiet=1, 425 print_output=0, show_err=0) 426 if shell_rc != 0: 427 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 428 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 429 if robot_pgm_dir_path not in sys.path: 430 sys.path.append(robot_pgm_dir_path) 431 PYTHONPATH = os.environ.get("PYTHONPATH", "") 432 if PYTHONPATH == "": 433 os.environ['PYTHONPATH'] = robot_pgm_dir_path 434 else: 435 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 436 437 validate_parms() 438 439 gp.qprint_pgm_header() 440 441 grk.run_key_u(default_set_power_policy) 442 443 initial_plug_in_setup() 444 445 plug_in_setup() 446 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 447 call_point='setup') 448 if rc != 0: 449 error_message = "Plug-in setup failed.\n" 450 gp.print_error_report(error_message) 451 BuiltIn().fail(error_message) 452 # Setting cp_setup_called lets our Teardown know that it needs to call 453 # the cleanup plug-in call point. 454 cp_setup_called = 1 455 456 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 457 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 458 # FFDC_LOG_PATH is used by "FFDC" keyword. 459 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 460 461 # Also printed by FFDC. 462 global host_name 463 global host_ip 464 host = socket.gethostname() 465 host_name, host_ip = gm.get_host_name_ip(host) 466 467 gp.dprint_var(boot_table) 468 gp.dprint_var(boot_lists) 469 470 471def validate_parms(): 472 r""" 473 Validate all program parameters. 474 """ 475 476 process_pgm_parms() 477 478 gp.qprintn() 479 480 global openbmc_model 481 if openbmc_model == "": 482 status, ret_values =\ 483 grk.run_key_u("Get BMC System Model") 484 openbmc_model = ret_values 485 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 486 gv.set_exit_on_error(True) 487 gv.valid_value(openbmc_host) 488 gv.valid_value(openbmc_username) 489 gv.valid_value(openbmc_password) 490 gv.valid_value(rest_username) 491 gv.valid_value(rest_password) 492 gv.valid_value(ipmi_username) 493 gv.valid_value(ipmi_password) 494 if os_host != "": 495 gv.valid_value(os_username) 496 gv.valid_value(os_password) 497 if pdu_host != "": 498 gv.valid_value(pdu_username) 499 gv.valid_value(pdu_password) 500 gv.valid_integer(pdu_slot_no) 501 if openbmc_serial_host != "": 502 gv.valid_integer(openbmc_serial_port) 503 gv.valid_value(openbmc_model) 504 gv.valid_integer(max_num_tests) 505 gv.valid_integer(boot_pass) 506 gv.valid_integer(boot_fail) 507 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 508 BuiltIn().set_global_variable("${plug_in_packages_list}", 509 plug_in_packages_list) 510 gv.valid_value(stack_mode, valid_values=['normal', 'skip']) 511 gv.set_exit_on_error(False) 512 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 513 error_message = "You must provide either a value for either the" +\ 514 " boot_list or the boot_stack parm.\n" 515 BuiltIn().fail(gp.sprint_error(error_message)) 516 valid_boot_list(boot_list, valid_boot_types) 517 valid_boot_list(boot_stack, valid_boot_types) 518 selected_PDU_boots = list(set(boot_list + boot_stack) 519 & set(boot_lists['PDU_reboot'])) 520 if len(selected_PDU_boots) > 0 and pdu_host == "": 521 error_message = "You have selected the following boots which" +\ 522 " require a PDU host but no value for pdu_host:\n" 523 error_message += gp.sprint_var(selected_PDU_boots) 524 error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 525 BuiltIn().fail(gp.sprint_error(error_message)) 526 527 return 528 529 530def my_get_state(): 531 r""" 532 Get the system state plus a little bit of wrapping. 533 """ 534 535 global state 536 537 req_states = ['epoch_seconds'] + st.default_req_states 538 539 gp.qprint_timen("Getting system state.") 540 if test_mode: 541 state['epoch_seconds'] = int(time.time()) 542 else: 543 state = st.get_state(req_states=req_states, quiet=quiet) 544 gp.qprint_var(state) 545 546 547def valid_state(): 548 r""" 549 Verify that our state dictionary contains no blank values. If we don't get 550 valid state data, we cannot continue to work. 551 """ 552 553 if st.compare_states(state, st.invalid_state_match, 'or'): 554 error_message = "The state dictionary contains blank fields which" +\ 555 " is illegal.\n" + gp.sprint_var(state) 556 BuiltIn().fail(gp.sprint_error(error_message)) 557 558 559def select_boot(): 560 r""" 561 Select a boot test to be run based on our current state and return the 562 chosen boot type. 563 564 Description of arguments: 565 state The state of the machine. 566 """ 567 568 global transitional_boot_selected 569 global boot_stack 570 571 gp.qprint_timen("Selecting a boot test.") 572 573 if transitional_boot_selected and not boot_success: 574 prior_boot = next_boot 575 boot_candidate = boot_stack.pop() 576 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 577 + " transition to a valid state for '" + boot_candidate 578 + "' which was at the top of the boot_stack. Since" 579 + " the '" + next_boot + "' failed, the '" 580 + boot_candidate + "' has been removed from the stack" 581 + " to avoid and endless failure loop.") 582 if len(boot_stack) == 0: 583 return "" 584 585 my_get_state() 586 valid_state() 587 588 transitional_boot_selected = False 589 stack_popped = 0 590 if len(boot_stack) > 0: 591 stack_popped = 1 592 gp.qprint_dashes() 593 gp.qprint_var(boot_stack) 594 gp.qprint_dashes() 595 skip_boot_printed = 0 596 while len(boot_stack) > 0: 597 boot_candidate = boot_stack.pop() 598 if stack_mode == 'normal': 599 break 600 else: 601 if st.compare_states(state, boot_table[boot_candidate]['end']): 602 if not skip_boot_printed: 603 gp.qprint_var(stack_mode) 604 gp.qprintn() 605 gp.qprint_timen("Skipping the following boot tests" 606 + " which are unnecessary since their" 607 + " required end states match the" 608 + " current machine state:") 609 skip_boot_printed = 1 610 gp.qprint_var(boot_candidate) 611 boot_candidate = "" 612 if boot_candidate == "": 613 gp.qprint_dashes() 614 gp.qprint_var(boot_stack) 615 gp.qprint_dashes() 616 return boot_candidate 617 if st.compare_states(state, boot_table[boot_candidate]['start']): 618 gp.qprint_timen("The machine state is valid for a '" 619 + boot_candidate + "' boot test.") 620 gp.qprint_dashes() 621 gp.qprint_var(boot_stack) 622 gp.qprint_dashes() 623 return boot_candidate 624 else: 625 gp.qprint_timen("The machine state does not match the required" 626 + " starting state for a '" + boot_candidate 627 + "' boot test:") 628 gp.qprint_varx("boot_table_start_entry", 629 boot_table[boot_candidate]['start']) 630 boot_stack.append(boot_candidate) 631 transitional_boot_selected = True 632 popped_boot = boot_candidate 633 634 # Loop through your list selecting a boot_candidates 635 boot_candidates = [] 636 for boot_candidate in boot_list: 637 if st.compare_states(state, boot_table[boot_candidate]['start']): 638 if stack_popped: 639 if st.compare_states(boot_table[boot_candidate]['end'], 640 boot_table[popped_boot]['start']): 641 boot_candidates.append(boot_candidate) 642 else: 643 boot_candidates.append(boot_candidate) 644 645 if len(boot_candidates) == 0: 646 gp.qprint_timen("The user's boot list contained no boot tests" 647 + " which are valid for the current machine state.") 648 boot_candidate = default_power_on 649 if not st.compare_states(state, boot_table[default_power_on]['start']): 650 boot_candidate = default_power_off 651 boot_candidates.append(boot_candidate) 652 gp.qprint_timen("Using default '" + boot_candidate 653 + "' boot type to transition to valid state.") 654 655 gp.dprint_var(boot_candidates) 656 657 # Randomly select a boot from the candidate list. 658 boot = random.choice(boot_candidates) 659 660 return boot 661 662 663def print_defect_report(ffdc_file_list): 664 r""" 665 Print a defect report. 666 667 Description of argument(s): 668 ffdc_file_list A list of files which were collected by our ffdc functions. 669 """ 670 671 # Making deliberate choice to NOT run plug_in_setup(). We don't want 672 # ffdc_prefix updated. 673 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 674 call_point='ffdc_report', stop_on_plug_in_failure=0) 675 676 # Get additional header data which may have been created by ffdc plug-ins. 677 # Also, delete the individual header files to cleanup. 678 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 679 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 680 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 681 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 682 show_err=0) 683 684 # Get additional summary data which may have been created by ffdc plug-ins. 685 # Also, delete the individual header files to cleanup. 686 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 687 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 688 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 689 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 690 show_err=0) 691 692 # ffdc_list_file_path contains a list of any ffdc files created by plug- 693 # ins, etc. Read that data into a list. 694 try: 695 plug_in_ffdc_list = \ 696 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 697 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 698 except IOError: 699 plug_in_ffdc_list = [] 700 701 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 702 # in. Eliminate duplicates and sort the list. 703 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 704 705 if status_file_path != "": 706 ffdc_file_list.insert(0, status_file_path) 707 708 # Convert the list to a printable list. 709 printable_ffdc_file_list = "\n".join(ffdc_file_list) 710 711 # Open ffdc_file_list for writing. We will write a complete list of 712 # FFDC files to it for possible use by plug-ins like cp_stop_check. 713 ffdc_list_file = open(ffdc_list_file_path, 'w') 714 ffdc_list_file.write(printable_ffdc_file_list + "\n") 715 ffdc_list_file.close() 716 717 indent = 0 718 width = 90 719 linefeed = 1 720 char = "=" 721 722 gp.qprintn() 723 gp.qprint_dashes(indent, width, linefeed, char) 724 gp.qprintn("Copy this data to the defect:\n") 725 726 if len(more_header_info) > 0: 727 gp.qprintn(more_header_info) 728 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 729 openbmc_host_name, openbmc_ip, openbmc_username, 730 openbmc_password, rest_username, rest_password, ipmi_username, 731 ipmi_password, os_host, os_host_name, os_ip, os_username, 732 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 733 pdu_password, pdu_slot_no, openbmc_serial_host, 734 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 735 736 gp.qprintn() 737 print_boot_history(boot_history) 738 gp.qprintn() 739 gp.qprint_var(state) 740 gp.qprintn() 741 gp.qprintn("FFDC data files:") 742 gp.qprintn(printable_ffdc_file_list) 743 gp.qprintn() 744 745 if len(ffdc_summary_info) > 0: 746 gp.qprintn(ffdc_summary_info) 747 748 gp.qprint_dashes(indent, width, linefeed, char) 749 750 751def my_ffdc(): 752 r""" 753 Collect FFDC data. 754 """ 755 756 global state 757 758 plug_in_setup() 759 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 760 call_point='ffdc', stop_on_plug_in_failure=0) 761 762 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 763 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 764 + AUTOBOOT_FFDC_PREFIX 765 + " ffdc_function_list=" 766 + ffdc_function_list, ignore=1) 767 if status != 'PASS': 768 gp.qprint_error("Call to ffdc failed.\n") 769 if type(ffdc_file_list) is not list: 770 ffdc_file_list = [] 771 # Leave a record for caller that "soft" errors occurred. 772 soft_errors = 1 773 gpu.save_plug_in_value(soft_errors, pgm_name) 774 775 my_get_state() 776 777 print_defect_report(ffdc_file_list) 778 779 780def print_test_start_message(boot_keyword): 781 r""" 782 Print a message indicating what boot test is about to run. 783 784 Description of arguments: 785 boot_keyword The name of the boot which is to be run 786 (e.g. "BMC Power On"). 787 """ 788 789 global boot_history 790 global boot_start_time 791 792 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 793 794 # Set boot_start_time for use by plug-ins. 795 boot_start_time = doing_msg[1:33] 796 gp.qprint_var(boot_start_time) 797 798 gp.qprint(doing_msg) 799 800 update_boot_history(boot_history, doing_msg, max_boot_history) 801 802 803def stop_boot_test(signal_number=0, 804 frame=None): 805 r""" 806 Handle SIGUSR1 by aborting the boot test that is running. 807 808 Description of argument(s): 809 signal_number The signal number (should always be 10 for SIGUSR1). 810 frame The frame data. 811 """ 812 813 gp.qprintn() 814 gp.qprint_executing() 815 gp.lprint_executing() 816 817 # Restore original sigusr1 handler. 818 set_default_siguser1() 819 820 message = "The caller has asked that the boot test be stopped and marked" 821 message += " as a failure." 822 823 function_stack = gm.get_function_stack() 824 if "wait_state" in function_stack: 825 st.set_exit_wait_early_message(message) 826 else: 827 BuiltIn().fail(gp.sprint_error(message)) 828 829 830def run_boot(boot): 831 r""" 832 Run the specified boot. 833 834 Description of arguments: 835 boot The name of the boot test to be performed. 836 """ 837 838 global state 839 840 signal.signal(signal.SIGUSR1, stop_boot_test) 841 gp.qprint_timen("stop_boot_test is armed.") 842 843 print_test_start_message(boot) 844 845 plug_in_setup() 846 rc, shell_rc, failed_plug_in_name = \ 847 grpi.rprocess_plug_in_packages(call_point="pre_boot") 848 if rc != 0: 849 error_message = "Plug-in failed with non-zero return code.\n" +\ 850 gp.sprint_var(rc, fmt=gp.hexa()) 851 set_default_siguser1() 852 BuiltIn().fail(gp.sprint_error(error_message)) 853 854 if test_mode: 855 # In test mode, we'll pretend the boot worked by assigning its 856 # required end state to the default state value. 857 state = st.strip_anchor_state(boot_table[boot]['end']) 858 else: 859 # Assertion: We trust that the state data was made fresh by the 860 # caller. 861 862 gp.qprintn() 863 864 if boot_table[boot]['method_type'] == "keyword": 865 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 866 boot_table[boot]['method'], 867 quiet=quiet) 868 869 if boot_table[boot]['bmc_reboot']: 870 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 871 plug_in_setup() 872 rc, shell_rc, failed_plug_in_name = \ 873 grpi.rprocess_plug_in_packages(call_point="post_reboot") 874 if rc != 0: 875 error_message = "Plug-in failed with non-zero return code.\n" 876 error_message += gp.sprint_var(rc, fmt=gp.hexa()) 877 set_default_siguser1() 878 BuiltIn().fail(gp.sprint_error(error_message)) 879 else: 880 match_state = st.anchor_state(state) 881 del match_state['epoch_seconds'] 882 # Wait for the state to change in any way. 883 st.wait_state(match_state, wait_time=state_change_timeout, 884 interval="10 seconds", invert=1) 885 886 gp.qprintn() 887 if boot_table[boot]['end']['chassis'] == "Off": 888 boot_timeout = power_off_timeout 889 else: 890 boot_timeout = power_on_timeout 891 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 892 interval="10 seconds") 893 894 plug_in_setup() 895 rc, shell_rc, failed_plug_in_name = \ 896 grpi.rprocess_plug_in_packages(call_point="post_boot") 897 if rc != 0: 898 error_message = "Plug-in failed with non-zero return code.\n" +\ 899 gp.sprint_var(rc, fmt=gp.hexa()) 900 set_default_siguser1() 901 BuiltIn().fail(gp.sprint_error(error_message)) 902 903 # Restore original sigusr1 handler. 904 set_default_siguser1() 905 906 907def test_loop_body(): 908 r""" 909 The main loop body for the loop in main_py. 910 911 Description of arguments: 912 boot_count The iteration number (starts at 1). 913 """ 914 915 global boot_count 916 global state 917 global next_boot 918 global boot_success 919 global boot_end_time 920 921 gp.qprintn() 922 923 next_boot = select_boot() 924 if next_boot == "": 925 return True 926 927 boot_count += 1 928 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 929 930 pre_boot_plug_in_setup() 931 932 cmd_buf = ["run_boot", next_boot] 933 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 934 if boot_status == "FAIL": 935 gp.qprint(msg) 936 937 gp.qprintn() 938 if boot_status == "PASS": 939 boot_success = 1 940 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 941 + "\" succeeded.") 942 else: 943 boot_success = 0 944 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 945 + "\" failed.") 946 947 # Set boot_end_time for use by plug-ins. 948 boot_end_time = completion_msg[1:33] 949 gp.qprint_var(boot_end_time) 950 951 gp.qprint(completion_msg) 952 953 boot_results.update(next_boot, boot_status) 954 955 plug_in_setup() 956 # NOTE: A post_test_case call point failure is NOT counted as a boot 957 # failure. 958 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 959 call_point='post_test_case', stop_on_plug_in_failure=0) 960 961 plug_in_setup() 962 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 963 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 964 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 965 if ffdc_check == "All" or\ 966 shell_rc == dump_ffdc_rc(): 967 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 968 if status != 'PASS': 969 gp.qprint_error("Call to my_ffdc failed.\n") 970 # Leave a record for caller that "soft" errors occurred. 971 soft_errors = 1 972 gpu.save_plug_in_value(soft_errors, pgm_name) 973 974 if delete_errlogs: 975 # print error logs before delete 976 status, error_logs = grk.run_key_u("Get Error Logs") 977 pels = pel.peltool("-l", ignore_err=1) 978 log.print_error_logs(error_logs, "AdditionalData Message Severity") 979 gp.qprint_var(pels) 980 981 # We need to purge error logs between boots or they build up. 982 grk.run_key(delete_errlogs_cmd, ignore=1) 983 grk.run_key(delete_bmcdump_cmd, ignore=1) 984 985 boot_results.print_report() 986 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 987 988 plug_in_setup() 989 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 990 call_point='stop_check', shell_rc=stop_test_rc(), 991 stop_on_non_zero_rc=1) 992 if shell_rc == stop_test_rc(): 993 message = "Stopping as requested by user.\n" 994 gp.qprint_time(message) 995 BuiltIn().fail(message) 996 997 # This should help prevent ConnectionErrors. 998 # Purge all redfish and REST connection sessions. 999 if redfish_delete_sessions: 1000 grk.run_key_u("Close All Connections", ignore=1) 1001 grk.run_key_u("Delete All Redfish Sessions", ignore=1) 1002 1003 return True 1004 1005 1006def obmc_boot_test_teardown(): 1007 r""" 1008 Clean up after the main keyword. 1009 """ 1010 gp.qprint_executing() 1011 1012 if ga.psutil_imported: 1013 ga.terminate_descendants() 1014 1015 if cp_setup_called: 1016 plug_in_setup() 1017 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1018 call_point='cleanup', stop_on_plug_in_failure=0) 1019 1020 if 'boot_results_file_path' in globals(): 1021 # Save boot_results and boot_history objects to a file in case they are 1022 # needed again. 1023 gp.qprint_timen("Saving boot_results to the following path.") 1024 gp.qprint_var(boot_results_file_path) 1025 pickle.dump((boot_results, boot_history), 1026 open(boot_results_file_path, 'wb'), 1027 pickle.HIGHEST_PROTOCOL) 1028 1029 global save_stack 1030 # Restore any global values saved on the save_stack. 1031 for parm_name in main_func_parm_list: 1032 # Get the parm_value if it was saved on the stack. 1033 try: 1034 parm_value = save_stack.pop(parm_name) 1035 except BaseException: 1036 # If it was not saved, no further action is required. 1037 continue 1038 1039 # Restore the saved value. 1040 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1041 "}\", parm_value)" 1042 gp.dpissuing(cmd_buf) 1043 exec(cmd_buf) 1044 1045 gp.dprintn(save_stack.sprint_obj()) 1046 1047 1048def test_teardown(): 1049 r""" 1050 Clean up after this test case. 1051 """ 1052 1053 gp.qprintn() 1054 gp.qprint_executing() 1055 1056 if ga.psutil_imported: 1057 ga.terminate_descendants() 1058 1059 cmd_buf = ["Print Error", 1060 "A keyword timeout occurred ending this program.\n"] 1061 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1062 1063 if redfish_supported: 1064 redfish.logout() 1065 1066 gp.qprint_pgm_footer() 1067 1068 1069def post_stack(): 1070 r""" 1071 Process post_stack plug-in programs. 1072 """ 1073 1074 if not call_post_stack_plug: 1075 # The caller does not wish to have post_stack plug-in processing done. 1076 return 1077 1078 global boot_success 1079 1080 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1081 pre_boot_plug_in_setup() 1082 # For the purposes of the following plug-ins, mark the "boot" as a success. 1083 boot_success = 1 1084 plug_in_setup() 1085 rc, shell_rc, failed_plug_in_name, history =\ 1086 grpi.rprocess_plug_in_packages(call_point='post_stack', 1087 stop_on_plug_in_failure=0, 1088 return_history=True) 1089 for doing_msg in history: 1090 update_boot_history(boot_history, doing_msg, max_boot_history) 1091 if rc != 0: 1092 boot_success = 0 1093 1094 plug_in_setup() 1095 rc, shell_rc, failed_plug_in_name =\ 1096 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1097 shell_rc=dump_ffdc_rc(), 1098 stop_on_plug_in_failure=1, 1099 stop_on_non_zero_rc=1) 1100 if shell_rc == dump_ffdc_rc(): 1101 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1102 if status != 'PASS': 1103 gp.qprint_error("Call to my_ffdc failed.\n") 1104 # Leave a record for caller that "soft" errors occurred. 1105 soft_errors = 1 1106 gpu.save_plug_in_value(soft_errors, pgm_name) 1107 1108 plug_in_setup() 1109 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1110 call_point='stop_check', shell_rc=stop_test_rc(), 1111 stop_on_non_zero_rc=1) 1112 if shell_rc == stop_test_rc(): 1113 message = "Stopping as requested by user.\n" 1114 gp.qprint_time(message) 1115 BuiltIn().fail(message) 1116 1117 1118def obmc_boot_test_py(loc_boot_stack=None, 1119 loc_stack_mode=None, 1120 loc_quiet=None): 1121 r""" 1122 Do main program processing. 1123 """ 1124 1125 global save_stack 1126 1127 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']}) 1128 1129 gp.dprintn() 1130 # Process function parms. 1131 for parm_name in main_func_parm_list: 1132 # Get parm's value. 1133 parm_value = eval("loc_" + parm_name) 1134 gp.dpvars(parm_name, parm_value) 1135 1136 if parm_value is not None: 1137 # Save the global value on a stack. 1138 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1139 parm_name + "}\"), \"" + parm_name + "\")" 1140 gp.dpissuing(cmd_buf) 1141 exec(cmd_buf) 1142 1143 # Set the global value to the passed value. 1144 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1145 "}\", loc_" + parm_name + ")" 1146 gp.dpissuing(cmd_buf) 1147 exec(cmd_buf) 1148 1149 gp.dprintn(save_stack.sprint_obj()) 1150 1151 setup() 1152 1153 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1154 1155 if ffdc_only: 1156 gp.qprint_timen("Caller requested ffdc_only.") 1157 if do_pre_boot_plug_in_setup: 1158 pre_boot_plug_in_setup() 1159 grk.run_key_u("my_ffdc") 1160 return 1161 1162 if delete_errlogs: 1163 # print error logs before delete 1164 status, error_logs = grk.run_key_u("Get Error Logs") 1165 pels = pel.peltool("-l", ignore_err=1) 1166 log.print_error_logs(error_logs, "AdditionalData Message Severity") 1167 gp.qprint_var(pels) 1168 1169 # Delete errlogs prior to doing any boot tests. 1170 grk.run_key(delete_errlogs_cmd, ignore=1) 1171 grk.run_key(delete_bmcdump_cmd, ignore=1) 1172 1173 # Process caller's boot_stack. 1174 while (len(boot_stack) > 0): 1175 test_loop_body() 1176 1177 gp.qprint_timen("Finished processing stack.") 1178 1179 post_stack() 1180 1181 # Process caller's boot_list. 1182 if len(boot_list) > 0: 1183 for ix in range(1, max_num_tests + 1): 1184 test_loop_body() 1185 1186 gp.qprint_timen("Completed all requested boot tests.") 1187 1188 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1189 new_fail = boot_fail - init_boot_fail 1190 if new_fail > boot_fail_threshold: 1191 error_message = "Boot failures exceed the boot failure" +\ 1192 " threshold:\n" +\ 1193 gp.sprint_var(new_fail) +\ 1194 gp.sprint_var(boot_fail_threshold) 1195 BuiltIn().fail(gp.sprint_error(error_message)) 1196