1#!/usr/bin/env python3 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_arg as ga 27import gen_valid as gv 28import gen_misc as gm 29import gen_cmd as gc 30import gen_robot_keyword as grk 31import state as st 32import var_stack as vs 33import gen_plug_in_utils as gpu 34import pel_utils as pel 35import logging_utils as log 36import run_keyword as rk 37 38base_path = os.path.dirname(os.path.dirname( 39 imp.find_module("gen_robot_print")[1])) +\ 40 os.sep 41sys.path.append(base_path + "extended/") 42 43# Setting master_pid correctly influences the behavior of plug-ins like 44# DB_Logging 45program_pid = os.getpid() 46master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 47pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 48 49# Set up boot data structures. 50os_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 51 52boot_lists = read_boot_lists() 53 54# The maximum number of entries that can be in the boot_history global variable. 55max_boot_history = 10 56boot_history = [] 57 58state = st.return_state_constant('default_state') 59cp_setup_called = 0 60next_boot = "" 61base_tool_dir_path = os.path.normpath(os.environ.get( 62 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 63 64ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 65boot_success = 0 66 67status_dir_path = os.environ.get('STATUS_DIR_PATH', "") or \ 68 BuiltIn().get_variable_value("${STATUS_DIR_PATH}", default="") 69if status_dir_path != "": 70 status_dir_path = os.path.normpath(status_dir_path) + os.sep 71 # For plugin expecting env gen_call_robot.py 72 os.environ['STATUS_DIR_PATH'] = status_dir_path 73 74redfish_support_trans_state = int(os.environ.get('REDFISH_SUPPORT_TRANS_STATE', 0)) or \ 75 int(BuiltIn().get_variable_value("${REDFISH_SUPPORT_TRANS_STATE}", default=0)) 76redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False) 77redfish_rest_supported = BuiltIn().get_variable_value("${REDFISH_REST_SUPPORTED}", default=False) 78redfish_delete_sessions = int(BuiltIn().get_variable_value("${REDFISH_DELETE_SESSIONS}", default=1)) 79if redfish_supported: 80 redfish = BuiltIn().get_library_instance('redfish') 81 default_power_on = "Redfish Power On" 82 default_power_off = "Redfish Power Off" 83 if not redfish_support_trans_state: 84 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 85 delete_bmcdump_cmd = "Delete All BMC Dump" 86 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 87 else: 88 delete_errlogs_cmd = "Redfish Purge Event Log" 89 delete_bmcdump_cmd = "Redfish Delete All BMC Dumps" 90 delete_sysdump_cmd = "Redfish Delete All System Dumps" 91 default_set_power_policy = "Redfish Set Power Restore Policy AlwaysOff" 92else: 93 default_power_on = "REST Power On" 94 default_power_off = "REST Power Off" 95 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 96 delete_bmcdump_cmd = "Delete All BMC Dump" 97 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 98boot_count = 0 99 100LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 101AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 102ffdc_prefix = AUTOBOOT_FFDC_PREFIX 103boot_start_time = "" 104boot_end_time = "" 105save_stack = vs.var_stack('save_stack') 106main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 107 108 109def dump_ffdc_rc(): 110 r""" 111 Return the constant dump ffdc test return code value. 112 113 When a plug-in call point program returns this value, it indicates that 114 this program should collect FFDC. 115 """ 116 117 return 0x00000200 118 119 120def stop_test_rc(): 121 r""" 122 Return the constant stop test return code value. 123 124 When a plug-in call point program returns this value, it indicates that 125 this program should stop running. 126 """ 127 128 return 0x00000200 129 130 131def process_host(host, 132 host_var_name=""): 133 r""" 134 Process a host by getting the associated host name and IP address and 135 setting them in global variables. 136 137 If the caller does not pass the host_var_name, this function will try to 138 figure out the name of the variable used by the caller for the host parm. 139 Callers are advised to explicitly specify the host_var_name when calling 140 with an exec command. In such cases, the get_arg_name cannot figure out 141 the host variable name. 142 143 This function will then create similar global variable names by 144 removing "_host" and appending "_host_name" or "_ip" to the host variable 145 name. 146 147 Example: 148 149 If a call is made like this: 150 process_host(openbmc_host) 151 152 Global variables openbmc_host_name and openbmc_ip will be set. 153 154 Description of argument(s): 155 host A host name or IP. The name of the variable used should 156 have a suffix of "_host". 157 host_var_name The name of the variable being used as the host parm. 158 """ 159 160 if host_var_name == "": 161 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 162 163 host_name_var_name = re.sub("host", "host_name", host_var_name) 164 ip_var_name = re.sub("host", "ip", host_var_name) 165 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 166 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 167 host + "')" 168 exec(cmd_buf) 169 170 171def process_pgm_parms(): 172 r""" 173 Process the program parameters by assigning them all to corresponding 174 globals. Also, set some global values that depend on program parameters. 175 """ 176 177 # Program parameter processing. 178 # Assign all program parms to python variables which are global to this 179 # module. 180 181 global parm_list 182 parm_list = BuiltIn().get_variable_value("${parm_list}") 183 # The following subset of parms should be processed as integers. 184 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 185 'boot_fail_threshold', 'delete_errlogs', 186 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 187 'test_mode', 'debug'] 188 for parm in parm_list: 189 if parm in int_list: 190 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 191 "}\", \"0\"))" 192 else: 193 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 194 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 195 gp.dpissuing(cmd_buf) 196 exec(cmd_buf) 197 if re.match(r".*_host$", parm): 198 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 199 exec(cmd_buf) 200 if re.match(r".*_password$", parm): 201 # Register the value of any parm whose name ends in _password. 202 # This will cause the print functions to replace passwords with 203 # asterisks in the output. 204 cmd_buf = "gp.register_passwords(" + parm + ")" 205 exec(cmd_buf) 206 207 global ffdc_dir_path_style 208 global boot_list 209 global boot_stack 210 global boot_results_file_path 211 global boot_results 212 global boot_history 213 global ffdc_list_file_path 214 global ffdc_report_list_path 215 global ffdc_summary_list_path 216 global boot_table 217 global valid_boot_types 218 219 if ffdc_dir_path_style == "": 220 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 221 222 # Convert these program parms to lists for easier processing.. 223 boot_list = list(filter(None, boot_list.split(":"))) 224 boot_stack = list(filter(None, boot_stack.split(":"))) 225 226 boot_table = create_boot_table(boot_table_path, os_host=os_host) 227 valid_boot_types = create_valid_boot_list(boot_table) 228 229 cleanup_boot_results_file() 230 boot_results_file_path = create_boot_results_file_path(pgm_name, 231 openbmc_nickname, 232 master_pid) 233 234 if os.path.isfile(boot_results_file_path): 235 # We've been called before in this run so we'll load the saved 236 # boot_results and boot_history objects. 237 boot_results, boot_history =\ 238 pickle.load(open(boot_results_file_path, 'rb')) 239 else: 240 boot_results = boot_results(boot_table, boot_pass, boot_fail) 241 242 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 243 "/FFDC_FILE_LIST" 244 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 245 "/FFDC_REPORT_FILE_LIST" 246 247 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 248 "/FFDC_SUMMARY_FILE_LIST" 249 250 251def initial_plug_in_setup(): 252 r""" 253 Initialize all plug-in environment variables which do not change for the 254 duration of the program. 255 256 """ 257 258 global LOG_LEVEL 259 BuiltIn().set_log_level("NONE") 260 261 BuiltIn().set_global_variable("${master_pid}", master_pid) 262 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 263 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 264 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 265 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 266 ffdc_list_file_path) 267 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 268 ffdc_report_list_path) 269 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 270 ffdc_summary_list_path) 271 272 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 273 ffdc_dir_path_style) 274 BuiltIn().set_global_variable("${FFDC_CHECK}", 275 ffdc_check) 276 277 # For each program parameter, set the corresponding AUTOBOOT_ environment 278 # variable value. Also, set an AUTOBOOT_ environment variable for every 279 # element in additional_values. 280 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 281 "status_dir_path", "base_tool_dir_path", 282 "ffdc_list_file_path", "ffdc_report_list_path", 283 "ffdc_summary_list_path", "execdir", "redfish_supported", 284 "redfish_rest_supported", "redfish_support_trans_state"] 285 286 plug_in_vars = parm_list + additional_values 287 288 for var_name in plug_in_vars: 289 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 290 var_name = var_name.upper() 291 if var_value is None: 292 var_value = "" 293 os.environ["AUTOBOOT_" + var_name] = str(var_value) 294 295 BuiltIn().set_log_level(LOG_LEVEL) 296 297 # Make sure the ffdc list directory exists. 298 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 299 if not os.path.exists(ffdc_list_dir_path): 300 os.makedirs(ffdc_list_dir_path) 301 302 303def plug_in_setup(): 304 r""" 305 Initialize all changing plug-in environment variables for use by the 306 plug-in programs. 307 """ 308 309 global LOG_LEVEL 310 global test_really_running 311 312 BuiltIn().set_log_level("NONE") 313 314 boot_pass, boot_fail = boot_results.return_total_pass_fail() 315 if boot_pass > 1: 316 test_really_running = 1 317 else: 318 test_really_running = 0 319 320 BuiltIn().set_global_variable("${test_really_running}", 321 test_really_running) 322 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 323 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 324 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 325 BuiltIn().set_global_variable("${boot_success}", boot_success) 326 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 327 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 328 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 329 330 # For each program parameter, set the corresponding AUTOBOOT_ environment 331 # variable value. Also, set an AUTOBOOT_ environment variable for every 332 # element in additional_values. 333 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 334 "boot_fail", "test_really_running", "ffdc_prefix", 335 "boot_start_time", "boot_end_time"] 336 337 plug_in_vars = additional_values 338 339 for var_name in plug_in_vars: 340 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 341 var_name = var_name.upper() 342 if var_value is None: 343 var_value = "" 344 os.environ["AUTOBOOT_" + var_name] = str(var_value) 345 346 if debug: 347 shell_rc, out_buf = \ 348 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 349 350 BuiltIn().set_log_level(LOG_LEVEL) 351 352 353def pre_boot_plug_in_setup(): 354 355 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 356 try: 357 os.remove(ffdc_list_file_path) 358 except OSError: 359 pass 360 361 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 362 try: 363 os.remove(ffdc_report_list_path) 364 except OSError: 365 pass 366 367 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 368 try: 369 os.remove(ffdc_summary_list_path) 370 except OSError: 371 pass 372 373 global ffdc_prefix 374 375 seconds = time.time() 376 loc_time = time.localtime(seconds) 377 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 378 379 ffdc_prefix = openbmc_nickname + "." + time_string 380 381 382def default_sigusr1(signal_number=0, 383 frame=None): 384 r""" 385 Handle SIGUSR1 by doing nothing. 386 387 This function assists in debugging SIGUSR1 processing by printing messages 388 to stdout and to the log.html file. 389 390 Description of argument(s): 391 signal_number The signal number (should always be 10 for SIGUSR1). 392 frame The frame data. 393 """ 394 395 gp.qprintn() 396 gp.qprint_executing() 397 gp.lprint_executing() 398 399 400def set_default_siguser1(): 401 r""" 402 Set the default_sigusr1 function to be the SIGUSR1 handler. 403 """ 404 405 gp.qprintn() 406 gp.qprint_executing() 407 gp.lprint_executing() 408 signal.signal(signal.SIGUSR1, default_sigusr1) 409 410 411def setup(): 412 r""" 413 Do general program setup tasks. 414 """ 415 416 global cp_setup_called 417 global transitional_boot_selected 418 419 gp.qprintn() 420 421 if redfish_supported: 422 redfish.login() 423 424 set_default_siguser1() 425 transitional_boot_selected = False 426 427 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 428 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 429 # If we can't find process_plug_in_packages.py, ssh_pw or 430 # validate_plug_ins.py, then we don't have our repo bin in PATH. 431 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 432 + " ssh_pw validate_plug_ins.py", quiet=1, 433 print_output=0, show_err=0) 434 if shell_rc != 0: 435 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 436 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 437 if robot_pgm_dir_path not in sys.path: 438 sys.path.append(robot_pgm_dir_path) 439 PYTHONPATH = os.environ.get("PYTHONPATH", "") 440 if PYTHONPATH == "": 441 os.environ['PYTHONPATH'] = robot_pgm_dir_path 442 else: 443 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 444 445 validate_parms() 446 447 gp.qprint_pgm_header() 448 449 grk.run_key_u(default_set_power_policy, ignore=1) 450 451 initial_plug_in_setup() 452 453 plug_in_setup() 454 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 455 call_point='setup') 456 if rc != 0: 457 error_message = "Plug-in setup failed.\n" 458 gp.print_error_report(error_message) 459 BuiltIn().fail(error_message) 460 # Setting cp_setup_called lets our Teardown know that it needs to call 461 # the cleanup plug-in call point. 462 cp_setup_called = 1 463 464 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 465 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 466 # FFDC_LOG_PATH is used by "FFDC" keyword. 467 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 468 469 # Also printed by FFDC. 470 global host_name 471 global host_ip 472 host = socket.gethostname() 473 host_name, host_ip = gm.get_host_name_ip(host) 474 475 gp.dprint_var(boot_table) 476 gp.dprint_var(boot_lists) 477 478 479def validate_parms(): 480 r""" 481 Validate all program parameters. 482 """ 483 484 process_pgm_parms() 485 486 gp.qprintn() 487 488 global openbmc_model 489 if openbmc_model == "": 490 status, ret_values =\ 491 grk.run_key_u("Get BMC System Model", ignore=1) 492 # Set the model to default "OPENBMC" if getting it from BMC fails. 493 if status == 'FAIL': 494 openbmc_model = 'OPENBMC' 495 else: 496 openbmc_model = ret_values 497 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 498 gv.set_exit_on_error(True) 499 gv.valid_value(openbmc_host) 500 gv.valid_value(openbmc_username) 501 gv.valid_value(openbmc_password) 502 gv.valid_value(rest_username) 503 gv.valid_value(rest_password) 504 gv.valid_value(ipmi_username) 505 gv.valid_value(ipmi_password) 506 if os_host != "": 507 gv.valid_value(os_username) 508 gv.valid_value(os_password) 509 if pdu_host != "": 510 gv.valid_value(pdu_username) 511 gv.valid_value(pdu_password) 512 gv.valid_integer(pdu_slot_no) 513 if openbmc_serial_host != "": 514 gv.valid_integer(openbmc_serial_port) 515 gv.valid_value(openbmc_model) 516 gv.valid_integer(max_num_tests) 517 gv.valid_integer(boot_pass) 518 gv.valid_integer(boot_fail) 519 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 520 BuiltIn().set_global_variable("${plug_in_packages_list}", 521 plug_in_packages_list) 522 gv.valid_value(stack_mode, valid_values=['normal', 'skip']) 523 gv.set_exit_on_error(False) 524 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 525 error_message = "You must provide either a value for either the" +\ 526 " boot_list or the boot_stack parm.\n" 527 BuiltIn().fail(gp.sprint_error(error_message)) 528 valid_boot_list(boot_list, valid_boot_types) 529 valid_boot_list(boot_stack, valid_boot_types) 530 selected_PDU_boots = list(set(boot_list + boot_stack) 531 & set(boot_lists['PDU_reboot'])) 532 if len(selected_PDU_boots) > 0 and pdu_host == "": 533 error_message = "You have selected the following boots which" +\ 534 " require a PDU host but no value for pdu_host:\n" 535 error_message += gp.sprint_var(selected_PDU_boots) 536 error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 537 BuiltIn().fail(gp.sprint_error(error_message)) 538 539 return 540 541 542def my_get_state(): 543 r""" 544 Get the system state plus a little bit of wrapping. 545 """ 546 547 global state 548 549 req_states = ['epoch_seconds'] + st.default_req_states 550 551 gp.qprint_timen("Getting system state.") 552 if test_mode: 553 state['epoch_seconds'] = int(time.time()) 554 else: 555 state = st.get_state(req_states=req_states, quiet=quiet) 556 gp.qprint_var(state) 557 558 559def valid_state(): 560 r""" 561 Verify that our state dictionary contains no blank values. If we don't get 562 valid state data, we cannot continue to work. 563 """ 564 565 if st.compare_states(state, st.invalid_state_match, 'or'): 566 error_message = "The state dictionary contains blank fields which" +\ 567 " is illegal.\n" + gp.sprint_var(state) 568 BuiltIn().fail(gp.sprint_error(error_message)) 569 570 571def select_boot(): 572 r""" 573 Select a boot test to be run based on our current state and return the 574 chosen boot type. 575 576 Description of arguments: 577 state The state of the machine. 578 """ 579 580 global transitional_boot_selected 581 global boot_stack 582 583 gp.qprint_timen("Selecting a boot test.") 584 585 if transitional_boot_selected and not boot_success: 586 prior_boot = next_boot 587 boot_candidate = boot_stack.pop() 588 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 589 + " transition to a valid state for '" + boot_candidate 590 + "' which was at the top of the boot_stack. Since" 591 + " the '" + next_boot + "' failed, the '" 592 + boot_candidate + "' has been removed from the stack" 593 + " to avoid and endless failure loop.") 594 if len(boot_stack) == 0: 595 return "" 596 597 my_get_state() 598 valid_state() 599 600 transitional_boot_selected = False 601 stack_popped = 0 602 if len(boot_stack) > 0: 603 stack_popped = 1 604 gp.qprint_dashes() 605 gp.qprint_var(boot_stack) 606 gp.qprint_dashes() 607 skip_boot_printed = 0 608 while len(boot_stack) > 0: 609 boot_candidate = boot_stack.pop() 610 if stack_mode == 'normal': 611 break 612 else: 613 if st.compare_states(state, boot_table[boot_candidate]['end']): 614 if not skip_boot_printed: 615 gp.qprint_var(stack_mode) 616 gp.qprintn() 617 gp.qprint_timen("Skipping the following boot tests" 618 + " which are unnecessary since their" 619 + " required end states match the" 620 + " current machine state:") 621 skip_boot_printed = 1 622 gp.qprint_var(boot_candidate) 623 boot_candidate = "" 624 if boot_candidate == "": 625 gp.qprint_dashes() 626 gp.qprint_var(boot_stack) 627 gp.qprint_dashes() 628 return boot_candidate 629 if st.compare_states(state, boot_table[boot_candidate]['start']): 630 gp.qprint_timen("The machine state is valid for a '" 631 + boot_candidate + "' boot test.") 632 gp.qprint_dashes() 633 gp.qprint_var(boot_stack) 634 gp.qprint_dashes() 635 return boot_candidate 636 else: 637 gp.qprint_timen("The machine state does not match the required" 638 + " starting state for a '" + boot_candidate 639 + "' boot test:") 640 gp.qprint_varx("boot_table_start_entry", 641 boot_table[boot_candidate]['start']) 642 boot_stack.append(boot_candidate) 643 transitional_boot_selected = True 644 popped_boot = boot_candidate 645 646 # Loop through your list selecting a boot_candidates 647 boot_candidates = [] 648 for boot_candidate in boot_list: 649 if st.compare_states(state, boot_table[boot_candidate]['start']): 650 if stack_popped: 651 if st.compare_states(boot_table[boot_candidate]['end'], 652 boot_table[popped_boot]['start']): 653 boot_candidates.append(boot_candidate) 654 else: 655 boot_candidates.append(boot_candidate) 656 657 if len(boot_candidates) == 0: 658 gp.qprint_timen("The user's boot list contained no boot tests" 659 + " which are valid for the current machine state.") 660 boot_candidate = default_power_on 661 if not st.compare_states(state, boot_table[default_power_on]['start']): 662 boot_candidate = default_power_off 663 boot_candidates.append(boot_candidate) 664 gp.qprint_timen("Using default '" + boot_candidate 665 + "' boot type to transition to valid state.") 666 667 gp.dprint_var(boot_candidates) 668 669 # Randomly select a boot from the candidate list. 670 boot = random.choice(boot_candidates) 671 672 return boot 673 674 675def print_defect_report(ffdc_file_list): 676 r""" 677 Print a defect report. 678 679 Description of argument(s): 680 ffdc_file_list A list of files which were collected by our ffdc functions. 681 """ 682 683 # Making deliberate choice to NOT run plug_in_setup(). We don't want 684 # ffdc_prefix updated. 685 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 686 call_point='ffdc_report', stop_on_plug_in_failure=0) 687 688 # Get additional header data which may have been created by ffdc plug-ins. 689 # Also, delete the individual header files to cleanup. 690 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 691 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 692 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 693 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 694 show_err=0) 695 696 # Get additional summary data which may have been created by ffdc plug-ins. 697 # Also, delete the individual header files to cleanup. 698 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 699 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 700 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 701 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 702 show_err=0) 703 704 # ffdc_list_file_path contains a list of any ffdc files created by plug- 705 # ins, etc. Read that data into a list. 706 try: 707 plug_in_ffdc_list = \ 708 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 709 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 710 except IOError: 711 plug_in_ffdc_list = [] 712 713 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 714 # in. Eliminate duplicates and sort the list. 715 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 716 717 if status_file_path != "": 718 ffdc_file_list.insert(0, status_file_path) 719 720 # Convert the list to a printable list. 721 printable_ffdc_file_list = "\n".join(ffdc_file_list) 722 723 # Open ffdc_file_list for writing. We will write a complete list of 724 # FFDC files to it for possible use by plug-ins like cp_stop_check. 725 ffdc_list_file = open(ffdc_list_file_path, 'w') 726 ffdc_list_file.write(printable_ffdc_file_list + "\n") 727 ffdc_list_file.close() 728 729 indent = 0 730 width = 90 731 linefeed = 1 732 char = "=" 733 734 gp.qprintn() 735 gp.qprint_dashes(indent, width, linefeed, char) 736 gp.qprintn("Copy this data to the defect:\n") 737 738 if len(more_header_info) > 0: 739 gp.qprintn(more_header_info) 740 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 741 openbmc_host_name, openbmc_ip, openbmc_username, 742 openbmc_password, rest_username, rest_password, ipmi_username, 743 ipmi_password, os_host, os_host_name, os_ip, os_username, 744 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 745 pdu_password, pdu_slot_no, openbmc_serial_host, 746 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 747 748 gp.qprintn() 749 print_boot_history(boot_history) 750 gp.qprintn() 751 gp.qprint_var(state) 752 gp.qprintn() 753 gp.qprintn("FFDC data files:") 754 gp.qprintn(printable_ffdc_file_list) 755 gp.qprintn() 756 757 if len(ffdc_summary_info) > 0: 758 gp.qprintn(ffdc_summary_info) 759 760 gp.qprint_dashes(indent, width, linefeed, char) 761 762 763def my_ffdc(): 764 r""" 765 Collect FFDC data. 766 """ 767 768 global state 769 770 plug_in_setup() 771 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 772 call_point='ffdc', stop_on_plug_in_failure=0) 773 774 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 775 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 776 + AUTOBOOT_FFDC_PREFIX 777 + " ffdc_function_list=" 778 + ffdc_function_list, ignore=1) 779 if status != 'PASS': 780 gp.qprint_error("Call to ffdc failed.\n") 781 if type(ffdc_file_list) is not list: 782 ffdc_file_list = [] 783 # Leave a record for caller that "soft" errors occurred. 784 soft_errors = 1 785 gpu.save_plug_in_value(soft_errors, pgm_name) 786 787 my_get_state() 788 789 print_defect_report(ffdc_file_list) 790 791 792def print_test_start_message(boot_keyword): 793 r""" 794 Print a message indicating what boot test is about to run. 795 796 Description of arguments: 797 boot_keyword The name of the boot which is to be run 798 (e.g. "BMC Power On"). 799 """ 800 801 global boot_history 802 global boot_start_time 803 804 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 805 806 # Set boot_start_time for use by plug-ins. 807 boot_start_time = doing_msg[1:33] 808 gp.qprint_var(boot_start_time) 809 810 gp.qprint(doing_msg) 811 812 update_boot_history(boot_history, doing_msg, max_boot_history) 813 814 815def stop_boot_test(signal_number=0, 816 frame=None): 817 r""" 818 Handle SIGUSR1 by aborting the boot test that is running. 819 820 Description of argument(s): 821 signal_number The signal number (should always be 10 for SIGUSR1). 822 frame The frame data. 823 """ 824 825 gp.qprintn() 826 gp.qprint_executing() 827 gp.lprint_executing() 828 829 # Restore original sigusr1 handler. 830 set_default_siguser1() 831 832 message = "The caller has asked that the boot test be stopped and marked" 833 message += " as a failure." 834 835 function_stack = gm.get_function_stack() 836 if "wait_state" in function_stack: 837 st.set_exit_wait_early_message(message) 838 else: 839 BuiltIn().fail(gp.sprint_error(message)) 840 841 842def run_boot(boot): 843 r""" 844 Run the specified boot. 845 846 Description of arguments: 847 boot The name of the boot test to be performed. 848 """ 849 850 global state 851 852 signal.signal(signal.SIGUSR1, stop_boot_test) 853 gp.qprint_timen("stop_boot_test is armed.") 854 855 print_test_start_message(boot) 856 857 plug_in_setup() 858 rc, shell_rc, failed_plug_in_name = \ 859 grpi.rprocess_plug_in_packages(call_point="pre_boot") 860 if rc != 0: 861 error_message = "Plug-in failed with non-zero return code.\n" +\ 862 gp.sprint_var(rc, fmt=gp.hexa()) 863 set_default_siguser1() 864 BuiltIn().fail(gp.sprint_error(error_message)) 865 866 if test_mode: 867 # In test mode, we'll pretend the boot worked by assigning its 868 # required end state to the default state value. 869 state = st.strip_anchor_state(boot_table[boot]['end']) 870 else: 871 # Assertion: We trust that the state data was made fresh by the 872 # caller. 873 874 gp.qprintn() 875 876 if boot_table[boot]['method_type'] == "keyword": 877 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 878 boot_table[boot]['method'], 879 quiet=quiet) 880 881 if boot_table[boot]['bmc_reboot']: 882 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 883 plug_in_setup() 884 rc, shell_rc, failed_plug_in_name = \ 885 grpi.rprocess_plug_in_packages(call_point="post_reboot") 886 if rc != 0: 887 error_message = "Plug-in failed with non-zero return code.\n" 888 error_message += gp.sprint_var(rc, fmt=gp.hexa()) 889 set_default_siguser1() 890 BuiltIn().fail(gp.sprint_error(error_message)) 891 else: 892 match_state = st.anchor_state(state) 893 del match_state['epoch_seconds'] 894 # Wait for the state to change in any way. 895 st.wait_state(match_state, wait_time=state_change_timeout, 896 interval="10 seconds", invert=1) 897 898 gp.qprintn() 899 if boot_table[boot]['end']['chassis'] == "Off": 900 boot_timeout = power_off_timeout 901 else: 902 boot_timeout = power_on_timeout 903 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 904 interval="10 seconds") 905 906 plug_in_setup() 907 rc, shell_rc, failed_plug_in_name = \ 908 grpi.rprocess_plug_in_packages(call_point="post_boot") 909 if rc != 0: 910 error_message = "Plug-in failed with non-zero return code.\n" +\ 911 gp.sprint_var(rc, fmt=gp.hexa()) 912 set_default_siguser1() 913 BuiltIn().fail(gp.sprint_error(error_message)) 914 915 # Restore original sigusr1 handler. 916 set_default_siguser1() 917 918 919def test_loop_body(): 920 r""" 921 The main loop body for the loop in main_py. 922 923 Description of arguments: 924 boot_count The iteration number (starts at 1). 925 """ 926 927 global boot_count 928 global state 929 global next_boot 930 global boot_success 931 global boot_end_time 932 933 gp.qprintn() 934 935 next_boot = select_boot() 936 if next_boot == "": 937 return True 938 939 boot_count += 1 940 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 941 942 pre_boot_plug_in_setup() 943 944 cmd_buf = ["run_boot", next_boot] 945 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 946 if boot_status == "FAIL": 947 gp.qprint(msg) 948 949 gp.qprintn() 950 if boot_status == "PASS": 951 boot_success = 1 952 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 953 + "\" succeeded.") 954 else: 955 boot_success = 0 956 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 957 + "\" failed.") 958 959 # Set boot_end_time for use by plug-ins. 960 boot_end_time = completion_msg[1:33] 961 gp.qprint_var(boot_end_time) 962 963 gp.qprint(completion_msg) 964 965 boot_results.update(next_boot, boot_status) 966 967 plug_in_setup() 968 # NOTE: A post_test_case call point failure is NOT counted as a boot 969 # failure. 970 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 971 call_point='post_test_case', stop_on_plug_in_failure=0) 972 973 plug_in_setup() 974 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 975 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 976 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 977 if ffdc_check == "All" or\ 978 shell_rc == dump_ffdc_rc(): 979 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 980 if status != 'PASS': 981 gp.qprint_error("Call to my_ffdc failed.\n") 982 # Leave a record for caller that "soft" errors occurred. 983 soft_errors = 1 984 gpu.save_plug_in_value(soft_errors, pgm_name) 985 986 if delete_errlogs: 987 # print error logs before delete 988 if redfish_support_trans_state: 989 status, error_logs = grk.run_key_u("Get Redfish Event Logs") 990 log.print_error_logs(error_logs, "AdditionalDataURI Message Severity") 991 else: 992 status, error_logs = grk.run_key_u("Get Error Logs") 993 log.print_error_logs(error_logs, "AdditionalData Message Severity") 994 pels = pel.peltool("-l", ignore_err=1) 995 gp.qprint_var(pels) 996 997 # We need to purge error logs between boots or they build up. 998 grk.run_key(delete_errlogs_cmd, ignore=1) 999 grk.run_key(delete_bmcdump_cmd, ignore=1) 1000 if redfish_support_trans_state: 1001 grk.run_key(delete_sysdump_cmd, ignore=1) 1002 1003 boot_results.print_report() 1004 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 1005 1006 plug_in_setup() 1007 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1008 call_point='stop_check', shell_rc=stop_test_rc(), 1009 stop_on_non_zero_rc=1) 1010 if shell_rc == stop_test_rc(): 1011 message = "Stopping as requested by user.\n" 1012 gp.qprint_time(message) 1013 BuiltIn().fail(message) 1014 1015 # This should help prevent ConnectionErrors. 1016 # Purge all redfish and REST connection sessions. 1017 if redfish_delete_sessions: 1018 grk.run_key_u("Close All Connections", ignore=1) 1019 grk.run_key_u("Delete All Redfish Sessions", ignore=1) 1020 1021 return True 1022 1023 1024def obmc_boot_test_teardown(): 1025 r""" 1026 Clean up after the main keyword. 1027 """ 1028 gp.qprint_executing() 1029 1030 if ga.psutil_imported: 1031 ga.terminate_descendants() 1032 1033 if cp_setup_called: 1034 plug_in_setup() 1035 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1036 call_point='cleanup', stop_on_plug_in_failure=0) 1037 1038 if 'boot_results_file_path' in globals(): 1039 # Save boot_results and boot_history objects to a file in case they are 1040 # needed again. 1041 gp.qprint_timen("Saving boot_results to the following path.") 1042 gp.qprint_var(boot_results_file_path) 1043 pickle.dump((boot_results, boot_history), 1044 open(boot_results_file_path, 'wb'), 1045 pickle.HIGHEST_PROTOCOL) 1046 1047 global save_stack 1048 # Restore any global values saved on the save_stack. 1049 for parm_name in main_func_parm_list: 1050 # Get the parm_value if it was saved on the stack. 1051 try: 1052 parm_value = save_stack.pop(parm_name) 1053 except BaseException: 1054 # If it was not saved, no further action is required. 1055 continue 1056 1057 # Restore the saved value. 1058 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1059 "}\", parm_value)" 1060 gp.dpissuing(cmd_buf) 1061 exec(cmd_buf) 1062 1063 gp.dprintn(save_stack.sprint_obj()) 1064 1065 1066def test_teardown(): 1067 r""" 1068 Clean up after this test case. 1069 """ 1070 1071 gp.qprintn() 1072 gp.qprint_executing() 1073 1074 if ga.psutil_imported: 1075 ga.terminate_descendants() 1076 1077 cmd_buf = ["Print Error", 1078 "A keyword timeout occurred ending this program.\n"] 1079 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1080 1081 if redfish_supported: 1082 redfish.logout() 1083 1084 gp.qprint_pgm_footer() 1085 1086 1087def post_stack(): 1088 r""" 1089 Process post_stack plug-in programs. 1090 """ 1091 1092 if not call_post_stack_plug: 1093 # The caller does not wish to have post_stack plug-in processing done. 1094 return 1095 1096 global boot_success 1097 1098 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1099 pre_boot_plug_in_setup() 1100 # For the purposes of the following plug-ins, mark the "boot" as a success. 1101 boot_success = 1 1102 plug_in_setup() 1103 rc, shell_rc, failed_plug_in_name, history =\ 1104 grpi.rprocess_plug_in_packages(call_point='post_stack', 1105 stop_on_plug_in_failure=0, 1106 return_history=True) 1107 for doing_msg in history: 1108 update_boot_history(boot_history, doing_msg, max_boot_history) 1109 if rc != 0: 1110 boot_success = 0 1111 1112 plug_in_setup() 1113 rc, shell_rc, failed_plug_in_name =\ 1114 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1115 shell_rc=dump_ffdc_rc(), 1116 stop_on_plug_in_failure=1, 1117 stop_on_non_zero_rc=1) 1118 if shell_rc == dump_ffdc_rc(): 1119 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1120 if status != 'PASS': 1121 gp.qprint_error("Call to my_ffdc failed.\n") 1122 # Leave a record for caller that "soft" errors occurred. 1123 soft_errors = 1 1124 gpu.save_plug_in_value(soft_errors, pgm_name) 1125 1126 plug_in_setup() 1127 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1128 call_point='stop_check', shell_rc=stop_test_rc(), 1129 stop_on_non_zero_rc=1) 1130 if shell_rc == stop_test_rc(): 1131 message = "Stopping as requested by user.\n" 1132 gp.qprint_time(message) 1133 BuiltIn().fail(message) 1134 1135 1136def obmc_boot_test_py(loc_boot_stack=None, 1137 loc_stack_mode=None, 1138 loc_quiet=None): 1139 r""" 1140 Do main program processing. 1141 """ 1142 1143 global save_stack 1144 1145 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']}) 1146 1147 gp.dprintn() 1148 # Process function parms. 1149 for parm_name in main_func_parm_list: 1150 # Get parm's value. 1151 parm_value = eval("loc_" + parm_name) 1152 gp.dpvars(parm_name, parm_value) 1153 1154 if parm_value is not None: 1155 # Save the global value on a stack. 1156 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1157 parm_name + "}\"), \"" + parm_name + "\")" 1158 gp.dpissuing(cmd_buf) 1159 exec(cmd_buf) 1160 1161 # Set the global value to the passed value. 1162 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1163 "}\", loc_" + parm_name + ")" 1164 gp.dpissuing(cmd_buf) 1165 exec(cmd_buf) 1166 1167 gp.dprintn(save_stack.sprint_obj()) 1168 1169 setup() 1170 1171 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1172 1173 if ffdc_only: 1174 gp.qprint_timen("Caller requested ffdc_only.") 1175 if do_pre_boot_plug_in_setup: 1176 pre_boot_plug_in_setup() 1177 grk.run_key_u("my_ffdc") 1178 return 1179 1180 if delete_errlogs: 1181 # print error logs before delete 1182 if redfish_support_trans_state: 1183 status, error_logs = grk.run_key_u("Get Redfish Event Logs") 1184 log.print_error_logs(error_logs, "AdditionalDataURI Message Severity") 1185 else: 1186 status, error_logs = grk.run_key_u("Get Error Logs") 1187 log.print_error_logs(error_logs, "AdditionalData Message Severity") 1188 pels = pel.peltool("-l", ignore_err=1) 1189 gp.qprint_var(pels) 1190 1191 # Delete errlogs prior to doing any boot tests. 1192 grk.run_key(delete_errlogs_cmd, ignore=1) 1193 grk.run_key(delete_bmcdump_cmd, ignore=1) 1194 if redfish_support_trans_state: 1195 grk.run_key(delete_sysdump_cmd, ignore=1) 1196 1197 # Process caller's boot_stack. 1198 while (len(boot_stack) > 0): 1199 test_loop_body() 1200 1201 gp.qprint_timen("Finished processing stack.") 1202 1203 post_stack() 1204 1205 # Process caller's boot_list. 1206 if len(boot_list) > 0: 1207 for ix in range(1, max_num_tests + 1): 1208 test_loop_body() 1209 1210 gp.qprint_timen("Completed all requested boot tests.") 1211 1212 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1213 new_fail = boot_fail - init_boot_fail 1214 if new_fail > boot_fail_threshold: 1215 error_message = "Boot failures exceed the boot failure" +\ 1216 " threshold:\n" +\ 1217 gp.sprint_var(new_fail) +\ 1218 gp.sprint_var(boot_fail_threshold) 1219 BuiltIn().fail(gp.sprint_error(error_message)) 1220