1#!/usr/bin/env python3 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_arg as ga 27import gen_valid as gv 28import gen_misc as gm 29import gen_cmd as gc 30import gen_robot_keyword as grk 31import state as st 32import var_stack as vs 33import gen_plug_in_utils as gpu 34import pel_utils as pel 35import logging_utils as log 36 37base_path = os.path.dirname(os.path.dirname( 38 imp.find_module("gen_robot_print")[1])) +\ 39 os.sep 40sys.path.append(base_path + "extended/") 41import run_keyword as rk 42 43# Setting master_pid correctly influences the behavior of plug-ins like 44# DB_Logging 45program_pid = os.getpid() 46master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 47pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 48 49# Set up boot data structures. 50os_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 51 52boot_lists = read_boot_lists() 53 54# The maximum number of entries that can be in the boot_history global variable. 55max_boot_history = 10 56boot_history = [] 57 58state = st.return_state_constant('default_state') 59cp_setup_called = 0 60next_boot = "" 61base_tool_dir_path = os.path.normpath(os.environ.get( 62 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 63 64ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 65boot_success = 0 66 67status_dir_path = os.environ.get('STATUS_DIR_PATH', "") or \ 68 BuiltIn().get_variable_value("${STATUS_DIR_PATH}", default="") 69if status_dir_path != "": 70 status_dir_path = os.path.normpath(status_dir_path) + os.sep 71 # For plugin expecting env gen_call_robot.py 72 os.environ['STATUS_DIR_PATH'] = status_dir_path 73 74redfish_support_trans_state = int(os.environ.get('REDFISH_SUPPORT_TRANS_STATE', 0)) or \ 75 int(BuiltIn().get_variable_value("${REDFISH_SUPPORT_TRANS_STATE}", default=0)) 76redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False) 77redfish_rest_supported = BuiltIn().get_variable_value("${REDFISH_REST_SUPPORTED}", default=False) 78redfish_delete_sessions = int(BuiltIn().get_variable_value("${REDFISH_DELETE_SESSIONS}", default=1)) 79if redfish_supported: 80 redfish = BuiltIn().get_library_instance('redfish') 81 default_power_on = "Redfish Power On" 82 default_power_off = "Redfish Power Off" 83 if not redfish_support_trans_state: 84 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 85 delete_bmcdump_cmd = "Delete All BMC Dump" 86 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 87 else: 88 delete_errlogs_cmd = "Redfish Purge Event Log" 89 delete_bmcdump_cmd = "Redfish Delete All BMC Dumps" 90 delete_sysdump_cmd = "Redfish Delete All System Dumps" 91 default_set_power_policy = "Redfish Set Power Restore Policy AlwaysOff" 92else: 93 default_power_on = "REST Power On" 94 default_power_off = "REST Power Off" 95 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 96 delete_bmcdump_cmd = "Delete All BMC Dump" 97 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 98boot_count = 0 99 100LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 101AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 102ffdc_prefix = AUTOBOOT_FFDC_PREFIX 103boot_start_time = "" 104boot_end_time = "" 105save_stack = vs.var_stack('save_stack') 106main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 107 108 109def dump_ffdc_rc(): 110 r""" 111 Return the constant dump ffdc test return code value. 112 113 When a plug-in call point program returns this value, it indicates that 114 this program should collect FFDC. 115 """ 116 117 return 0x00000200 118 119 120def stop_test_rc(): 121 r""" 122 Return the constant stop test return code value. 123 124 When a plug-in call point program returns this value, it indicates that 125 this program should stop running. 126 """ 127 128 return 0x00000200 129 130 131def process_host(host, 132 host_var_name=""): 133 r""" 134 Process a host by getting the associated host name and IP address and 135 setting them in global variables. 136 137 If the caller does not pass the host_var_name, this function will try to 138 figure out the name of the variable used by the caller for the host parm. 139 Callers are advised to explicitly specify the host_var_name when calling 140 with an exec command. In such cases, the get_arg_name cannot figure out 141 the host variable name. 142 143 This function will then create similar global variable names by 144 removing "_host" and appending "_host_name" or "_ip" to the host variable 145 name. 146 147 Example: 148 149 If a call is made like this: 150 process_host(openbmc_host) 151 152 Global variables openbmc_host_name and openbmc_ip will be set. 153 154 Description of argument(s): 155 host A host name or IP. The name of the variable used should 156 have a suffix of "_host". 157 host_var_name The name of the variable being used as the host parm. 158 """ 159 160 if host_var_name == "": 161 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 162 163 host_name_var_name = re.sub("host", "host_name", host_var_name) 164 ip_var_name = re.sub("host", "ip", host_var_name) 165 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 166 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 167 host + "')" 168 exec(cmd_buf) 169 170 171def process_pgm_parms(): 172 r""" 173 Process the program parameters by assigning them all to corresponding 174 globals. Also, set some global values that depend on program parameters. 175 """ 176 177 # Program parameter processing. 178 # Assign all program parms to python variables which are global to this 179 # module. 180 181 global parm_list 182 parm_list = BuiltIn().get_variable_value("${parm_list}") 183 # The following subset of parms should be processed as integers. 184 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 185 'boot_fail_threshold', 'delete_errlogs', 186 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 187 'test_mode', 'debug'] 188 for parm in parm_list: 189 if parm in int_list: 190 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 191 "}\", \"0\"))" 192 else: 193 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 194 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 195 gp.dpissuing(cmd_buf) 196 exec(cmd_buf) 197 if re.match(r".*_host$", parm): 198 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 199 exec(cmd_buf) 200 if re.match(r".*_password$", parm): 201 # Register the value of any parm whose name ends in _password. 202 # This will cause the print functions to replace passwords with 203 # asterisks in the output. 204 cmd_buf = "gp.register_passwords(" + parm + ")" 205 exec(cmd_buf) 206 207 global ffdc_dir_path_style 208 global boot_list 209 global boot_stack 210 global boot_results_file_path 211 global boot_results 212 global boot_history 213 global ffdc_list_file_path 214 global ffdc_report_list_path 215 global ffdc_summary_list_path 216 global boot_table 217 global valid_boot_types 218 219 if ffdc_dir_path_style == "": 220 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 221 222 # Convert these program parms to lists for easier processing.. 223 boot_list = list(filter(None, boot_list.split(":"))) 224 boot_stack = list(filter(None, boot_stack.split(":"))) 225 226 boot_table = create_boot_table(boot_table_path, os_host=os_host) 227 valid_boot_types = create_valid_boot_list(boot_table) 228 229 cleanup_boot_results_file() 230 boot_results_file_path = create_boot_results_file_path(pgm_name, 231 openbmc_nickname, 232 master_pid) 233 234 if os.path.isfile(boot_results_file_path): 235 # We've been called before in this run so we'll load the saved 236 # boot_results and boot_history objects. 237 boot_results, boot_history =\ 238 pickle.load(open(boot_results_file_path, 'rb')) 239 else: 240 boot_results = boot_results(boot_table, boot_pass, boot_fail) 241 242 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 243 "/FFDC_FILE_LIST" 244 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 245 "/FFDC_REPORT_FILE_LIST" 246 247 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 248 "/FFDC_SUMMARY_FILE_LIST" 249 250 251def initial_plug_in_setup(): 252 r""" 253 Initialize all plug-in environment variables which do not change for the 254 duration of the program. 255 256 """ 257 258 global LOG_LEVEL 259 BuiltIn().set_log_level("NONE") 260 261 BuiltIn().set_global_variable("${master_pid}", master_pid) 262 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 263 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 264 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 265 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 266 ffdc_list_file_path) 267 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 268 ffdc_report_list_path) 269 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 270 ffdc_summary_list_path) 271 272 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 273 ffdc_dir_path_style) 274 BuiltIn().set_global_variable("${FFDC_CHECK}", 275 ffdc_check) 276 277 # For each program parameter, set the corresponding AUTOBOOT_ environment 278 # variable value. Also, set an AUTOBOOT_ environment variable for every 279 # element in additional_values. 280 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 281 "status_dir_path", "base_tool_dir_path", 282 "ffdc_list_file_path", "ffdc_report_list_path", 283 "ffdc_summary_list_path", "execdir", "redfish_supported", 284 "redfish_rest_supported", "redfish_support_trans_state"] 285 286 plug_in_vars = parm_list + additional_values 287 288 for var_name in plug_in_vars: 289 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 290 var_name = var_name.upper() 291 if var_value is None: 292 var_value = "" 293 os.environ["AUTOBOOT_" + var_name] = str(var_value) 294 295 BuiltIn().set_log_level(LOG_LEVEL) 296 297 # Make sure the ffdc list directory exists. 298 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 299 if not os.path.exists(ffdc_list_dir_path): 300 os.makedirs(ffdc_list_dir_path) 301 302 303def plug_in_setup(): 304 r""" 305 Initialize all changing plug-in environment variables for use by the 306 plug-in programs. 307 """ 308 309 global LOG_LEVEL 310 global test_really_running 311 312 BuiltIn().set_log_level("NONE") 313 314 boot_pass, boot_fail = boot_results.return_total_pass_fail() 315 if boot_pass > 1: 316 test_really_running = 1 317 else: 318 test_really_running = 0 319 320 BuiltIn().set_global_variable("${test_really_running}", 321 test_really_running) 322 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 323 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 324 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 325 BuiltIn().set_global_variable("${boot_success}", boot_success) 326 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 327 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 328 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 329 330 # For each program parameter, set the corresponding AUTOBOOT_ environment 331 # variable value. Also, set an AUTOBOOT_ environment variable for every 332 # element in additional_values. 333 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 334 "boot_fail", "test_really_running", "ffdc_prefix", 335 "boot_start_time", "boot_end_time"] 336 337 plug_in_vars = additional_values 338 339 for var_name in plug_in_vars: 340 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 341 var_name = var_name.upper() 342 if var_value is None: 343 var_value = "" 344 os.environ["AUTOBOOT_" + var_name] = str(var_value) 345 346 if debug: 347 shell_rc, out_buf = \ 348 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 349 350 BuiltIn().set_log_level(LOG_LEVEL) 351 352 353def pre_boot_plug_in_setup(): 354 355 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 356 try: 357 os.remove(ffdc_list_file_path) 358 except OSError: 359 pass 360 361 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 362 try: 363 os.remove(ffdc_report_list_path) 364 except OSError: 365 pass 366 367 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 368 try: 369 os.remove(ffdc_summary_list_path) 370 except OSError: 371 pass 372 373 global ffdc_prefix 374 375 seconds = time.time() 376 loc_time = time.localtime(seconds) 377 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 378 379 ffdc_prefix = openbmc_nickname + "." + time_string 380 381 382def default_sigusr1(signal_number=0, 383 frame=None): 384 r""" 385 Handle SIGUSR1 by doing nothing. 386 387 This function assists in debugging SIGUSR1 processing by printing messages 388 to stdout and to the log.html file. 389 390 Description of argument(s): 391 signal_number The signal number (should always be 10 for SIGUSR1). 392 frame The frame data. 393 """ 394 395 gp.qprintn() 396 gp.qprint_executing() 397 gp.lprint_executing() 398 399 400def set_default_siguser1(): 401 r""" 402 Set the default_sigusr1 function to be the SIGUSR1 handler. 403 """ 404 405 gp.qprintn() 406 gp.qprint_executing() 407 gp.lprint_executing() 408 signal.signal(signal.SIGUSR1, default_sigusr1) 409 410 411def setup(): 412 r""" 413 Do general program setup tasks. 414 """ 415 416 global cp_setup_called 417 global transitional_boot_selected 418 419 gp.qprintn() 420 421 if redfish_supported: 422 redfish.login() 423 424 set_default_siguser1() 425 transitional_boot_selected = False 426 427 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 428 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 429 # If we can't find process_plug_in_packages.py, ssh_pw or 430 # validate_plug_ins.py, then we don't have our repo bin in PATH. 431 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 432 + " ssh_pw validate_plug_ins.py", quiet=1, 433 print_output=0, show_err=0) 434 if shell_rc != 0: 435 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 436 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 437 if robot_pgm_dir_path not in sys.path: 438 sys.path.append(robot_pgm_dir_path) 439 PYTHONPATH = os.environ.get("PYTHONPATH", "") 440 if PYTHONPATH == "": 441 os.environ['PYTHONPATH'] = robot_pgm_dir_path 442 else: 443 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 444 445 validate_parms() 446 447 gp.qprint_pgm_header() 448 449 grk.run_key_u(default_set_power_policy) 450 451 initial_plug_in_setup() 452 453 plug_in_setup() 454 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 455 call_point='setup') 456 if rc != 0: 457 error_message = "Plug-in setup failed.\n" 458 gp.print_error_report(error_message) 459 BuiltIn().fail(error_message) 460 # Setting cp_setup_called lets our Teardown know that it needs to call 461 # the cleanup plug-in call point. 462 cp_setup_called = 1 463 464 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 465 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 466 # FFDC_LOG_PATH is used by "FFDC" keyword. 467 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 468 469 # Also printed by FFDC. 470 global host_name 471 global host_ip 472 host = socket.gethostname() 473 host_name, host_ip = gm.get_host_name_ip(host) 474 475 gp.dprint_var(boot_table) 476 gp.dprint_var(boot_lists) 477 478 479def validate_parms(): 480 r""" 481 Validate all program parameters. 482 """ 483 484 process_pgm_parms() 485 486 gp.qprintn() 487 488 global openbmc_model 489 if openbmc_model == "": 490 status, ret_values =\ 491 grk.run_key_u("Get BMC System Model") 492 openbmc_model = ret_values 493 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 494 gv.set_exit_on_error(True) 495 gv.valid_value(openbmc_host) 496 gv.valid_value(openbmc_username) 497 gv.valid_value(openbmc_password) 498 gv.valid_value(rest_username) 499 gv.valid_value(rest_password) 500 gv.valid_value(ipmi_username) 501 gv.valid_value(ipmi_password) 502 if os_host != "": 503 gv.valid_value(os_username) 504 gv.valid_value(os_password) 505 if pdu_host != "": 506 gv.valid_value(pdu_username) 507 gv.valid_value(pdu_password) 508 gv.valid_integer(pdu_slot_no) 509 if openbmc_serial_host != "": 510 gv.valid_integer(openbmc_serial_port) 511 gv.valid_value(openbmc_model) 512 gv.valid_integer(max_num_tests) 513 gv.valid_integer(boot_pass) 514 gv.valid_integer(boot_fail) 515 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 516 BuiltIn().set_global_variable("${plug_in_packages_list}", 517 plug_in_packages_list) 518 gv.valid_value(stack_mode, valid_values=['normal', 'skip']) 519 gv.set_exit_on_error(False) 520 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 521 error_message = "You must provide either a value for either the" +\ 522 " boot_list or the boot_stack parm.\n" 523 BuiltIn().fail(gp.sprint_error(error_message)) 524 valid_boot_list(boot_list, valid_boot_types) 525 valid_boot_list(boot_stack, valid_boot_types) 526 selected_PDU_boots = list(set(boot_list + boot_stack) 527 & set(boot_lists['PDU_reboot'])) 528 if len(selected_PDU_boots) > 0 and pdu_host == "": 529 error_message = "You have selected the following boots which" +\ 530 " require a PDU host but no value for pdu_host:\n" 531 error_message += gp.sprint_var(selected_PDU_boots) 532 error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 533 BuiltIn().fail(gp.sprint_error(error_message)) 534 535 return 536 537 538def my_get_state(): 539 r""" 540 Get the system state plus a little bit of wrapping. 541 """ 542 543 global state 544 545 req_states = ['epoch_seconds'] + st.default_req_states 546 547 gp.qprint_timen("Getting system state.") 548 if test_mode: 549 state['epoch_seconds'] = int(time.time()) 550 else: 551 state = st.get_state(req_states=req_states, quiet=quiet) 552 gp.qprint_var(state) 553 554 555def valid_state(): 556 r""" 557 Verify that our state dictionary contains no blank values. If we don't get 558 valid state data, we cannot continue to work. 559 """ 560 561 if st.compare_states(state, st.invalid_state_match, 'or'): 562 error_message = "The state dictionary contains blank fields which" +\ 563 " is illegal.\n" + gp.sprint_var(state) 564 BuiltIn().fail(gp.sprint_error(error_message)) 565 566 567def select_boot(): 568 r""" 569 Select a boot test to be run based on our current state and return the 570 chosen boot type. 571 572 Description of arguments: 573 state The state of the machine. 574 """ 575 576 global transitional_boot_selected 577 global boot_stack 578 579 gp.qprint_timen("Selecting a boot test.") 580 581 if transitional_boot_selected and not boot_success: 582 prior_boot = next_boot 583 boot_candidate = boot_stack.pop() 584 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 585 + " transition to a valid state for '" + boot_candidate 586 + "' which was at the top of the boot_stack. Since" 587 + " the '" + next_boot + "' failed, the '" 588 + boot_candidate + "' has been removed from the stack" 589 + " to avoid and endless failure loop.") 590 if len(boot_stack) == 0: 591 return "" 592 593 my_get_state() 594 valid_state() 595 596 transitional_boot_selected = False 597 stack_popped = 0 598 if len(boot_stack) > 0: 599 stack_popped = 1 600 gp.qprint_dashes() 601 gp.qprint_var(boot_stack) 602 gp.qprint_dashes() 603 skip_boot_printed = 0 604 while len(boot_stack) > 0: 605 boot_candidate = boot_stack.pop() 606 if stack_mode == 'normal': 607 break 608 else: 609 if st.compare_states(state, boot_table[boot_candidate]['end']): 610 if not skip_boot_printed: 611 gp.qprint_var(stack_mode) 612 gp.qprintn() 613 gp.qprint_timen("Skipping the following boot tests" 614 + " which are unnecessary since their" 615 + " required end states match the" 616 + " current machine state:") 617 skip_boot_printed = 1 618 gp.qprint_var(boot_candidate) 619 boot_candidate = "" 620 if boot_candidate == "": 621 gp.qprint_dashes() 622 gp.qprint_var(boot_stack) 623 gp.qprint_dashes() 624 return boot_candidate 625 if st.compare_states(state, boot_table[boot_candidate]['start']): 626 gp.qprint_timen("The machine state is valid for a '" 627 + boot_candidate + "' boot test.") 628 gp.qprint_dashes() 629 gp.qprint_var(boot_stack) 630 gp.qprint_dashes() 631 return boot_candidate 632 else: 633 gp.qprint_timen("The machine state does not match the required" 634 + " starting state for a '" + boot_candidate 635 + "' boot test:") 636 gp.qprint_varx("boot_table_start_entry", 637 boot_table[boot_candidate]['start']) 638 boot_stack.append(boot_candidate) 639 transitional_boot_selected = True 640 popped_boot = boot_candidate 641 642 # Loop through your list selecting a boot_candidates 643 boot_candidates = [] 644 for boot_candidate in boot_list: 645 if st.compare_states(state, boot_table[boot_candidate]['start']): 646 if stack_popped: 647 if st.compare_states(boot_table[boot_candidate]['end'], 648 boot_table[popped_boot]['start']): 649 boot_candidates.append(boot_candidate) 650 else: 651 boot_candidates.append(boot_candidate) 652 653 if len(boot_candidates) == 0: 654 gp.qprint_timen("The user's boot list contained no boot tests" 655 + " which are valid for the current machine state.") 656 boot_candidate = default_power_on 657 if not st.compare_states(state, boot_table[default_power_on]['start']): 658 boot_candidate = default_power_off 659 boot_candidates.append(boot_candidate) 660 gp.qprint_timen("Using default '" + boot_candidate 661 + "' boot type to transition to valid state.") 662 663 gp.dprint_var(boot_candidates) 664 665 # Randomly select a boot from the candidate list. 666 boot = random.choice(boot_candidates) 667 668 return boot 669 670 671def print_defect_report(ffdc_file_list): 672 r""" 673 Print a defect report. 674 675 Description of argument(s): 676 ffdc_file_list A list of files which were collected by our ffdc functions. 677 """ 678 679 # Making deliberate choice to NOT run plug_in_setup(). We don't want 680 # ffdc_prefix updated. 681 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 682 call_point='ffdc_report', stop_on_plug_in_failure=0) 683 684 # Get additional header data which may have been created by ffdc plug-ins. 685 # Also, delete the individual header files to cleanup. 686 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 687 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 688 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 689 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 690 show_err=0) 691 692 # Get additional summary data which may have been created by ffdc plug-ins. 693 # Also, delete the individual header files to cleanup. 694 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 695 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 696 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 697 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 698 show_err=0) 699 700 # ffdc_list_file_path contains a list of any ffdc files created by plug- 701 # ins, etc. Read that data into a list. 702 try: 703 plug_in_ffdc_list = \ 704 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 705 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 706 except IOError: 707 plug_in_ffdc_list = [] 708 709 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 710 # in. Eliminate duplicates and sort the list. 711 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 712 713 if status_file_path != "": 714 ffdc_file_list.insert(0, status_file_path) 715 716 # Convert the list to a printable list. 717 printable_ffdc_file_list = "\n".join(ffdc_file_list) 718 719 # Open ffdc_file_list for writing. We will write a complete list of 720 # FFDC files to it for possible use by plug-ins like cp_stop_check. 721 ffdc_list_file = open(ffdc_list_file_path, 'w') 722 ffdc_list_file.write(printable_ffdc_file_list + "\n") 723 ffdc_list_file.close() 724 725 indent = 0 726 width = 90 727 linefeed = 1 728 char = "=" 729 730 gp.qprintn() 731 gp.qprint_dashes(indent, width, linefeed, char) 732 gp.qprintn("Copy this data to the defect:\n") 733 734 if len(more_header_info) > 0: 735 gp.qprintn(more_header_info) 736 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 737 openbmc_host_name, openbmc_ip, openbmc_username, 738 openbmc_password, rest_username, rest_password, ipmi_username, 739 ipmi_password, os_host, os_host_name, os_ip, os_username, 740 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 741 pdu_password, pdu_slot_no, openbmc_serial_host, 742 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 743 744 gp.qprintn() 745 print_boot_history(boot_history) 746 gp.qprintn() 747 gp.qprint_var(state) 748 gp.qprintn() 749 gp.qprintn("FFDC data files:") 750 gp.qprintn(printable_ffdc_file_list) 751 gp.qprintn() 752 753 if len(ffdc_summary_info) > 0: 754 gp.qprintn(ffdc_summary_info) 755 756 gp.qprint_dashes(indent, width, linefeed, char) 757 758 759def my_ffdc(): 760 r""" 761 Collect FFDC data. 762 """ 763 764 global state 765 766 plug_in_setup() 767 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 768 call_point='ffdc', stop_on_plug_in_failure=0) 769 770 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 771 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 772 + AUTOBOOT_FFDC_PREFIX 773 + " ffdc_function_list=" 774 + ffdc_function_list, ignore=1) 775 if status != 'PASS': 776 gp.qprint_error("Call to ffdc failed.\n") 777 if type(ffdc_file_list) is not list: 778 ffdc_file_list = [] 779 # Leave a record for caller that "soft" errors occurred. 780 soft_errors = 1 781 gpu.save_plug_in_value(soft_errors, pgm_name) 782 783 my_get_state() 784 785 print_defect_report(ffdc_file_list) 786 787 788def print_test_start_message(boot_keyword): 789 r""" 790 Print a message indicating what boot test is about to run. 791 792 Description of arguments: 793 boot_keyword The name of the boot which is to be run 794 (e.g. "BMC Power On"). 795 """ 796 797 global boot_history 798 global boot_start_time 799 800 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 801 802 # Set boot_start_time for use by plug-ins. 803 boot_start_time = doing_msg[1:33] 804 gp.qprint_var(boot_start_time) 805 806 gp.qprint(doing_msg) 807 808 update_boot_history(boot_history, doing_msg, max_boot_history) 809 810 811def stop_boot_test(signal_number=0, 812 frame=None): 813 r""" 814 Handle SIGUSR1 by aborting the boot test that is running. 815 816 Description of argument(s): 817 signal_number The signal number (should always be 10 for SIGUSR1). 818 frame The frame data. 819 """ 820 821 gp.qprintn() 822 gp.qprint_executing() 823 gp.lprint_executing() 824 825 # Restore original sigusr1 handler. 826 set_default_siguser1() 827 828 message = "The caller has asked that the boot test be stopped and marked" 829 message += " as a failure." 830 831 function_stack = gm.get_function_stack() 832 if "wait_state" in function_stack: 833 st.set_exit_wait_early_message(message) 834 else: 835 BuiltIn().fail(gp.sprint_error(message)) 836 837 838def run_boot(boot): 839 r""" 840 Run the specified boot. 841 842 Description of arguments: 843 boot The name of the boot test to be performed. 844 """ 845 846 global state 847 848 signal.signal(signal.SIGUSR1, stop_boot_test) 849 gp.qprint_timen("stop_boot_test is armed.") 850 851 print_test_start_message(boot) 852 853 plug_in_setup() 854 rc, shell_rc, failed_plug_in_name = \ 855 grpi.rprocess_plug_in_packages(call_point="pre_boot") 856 if rc != 0: 857 error_message = "Plug-in failed with non-zero return code.\n" +\ 858 gp.sprint_var(rc, fmt=gp.hexa()) 859 set_default_siguser1() 860 BuiltIn().fail(gp.sprint_error(error_message)) 861 862 if test_mode: 863 # In test mode, we'll pretend the boot worked by assigning its 864 # required end state to the default state value. 865 state = st.strip_anchor_state(boot_table[boot]['end']) 866 else: 867 # Assertion: We trust that the state data was made fresh by the 868 # caller. 869 870 gp.qprintn() 871 872 if boot_table[boot]['method_type'] == "keyword": 873 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 874 boot_table[boot]['method'], 875 quiet=quiet) 876 877 if boot_table[boot]['bmc_reboot']: 878 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 879 plug_in_setup() 880 rc, shell_rc, failed_plug_in_name = \ 881 grpi.rprocess_plug_in_packages(call_point="post_reboot") 882 if rc != 0: 883 error_message = "Plug-in failed with non-zero return code.\n" 884 error_message += gp.sprint_var(rc, fmt=gp.hexa()) 885 set_default_siguser1() 886 BuiltIn().fail(gp.sprint_error(error_message)) 887 else: 888 match_state = st.anchor_state(state) 889 del match_state['epoch_seconds'] 890 # Wait for the state to change in any way. 891 st.wait_state(match_state, wait_time=state_change_timeout, 892 interval="10 seconds", invert=1) 893 894 gp.qprintn() 895 if boot_table[boot]['end']['chassis'] == "Off": 896 boot_timeout = power_off_timeout 897 else: 898 boot_timeout = power_on_timeout 899 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 900 interval="10 seconds") 901 902 plug_in_setup() 903 rc, shell_rc, failed_plug_in_name = \ 904 grpi.rprocess_plug_in_packages(call_point="post_boot") 905 if rc != 0: 906 error_message = "Plug-in failed with non-zero return code.\n" +\ 907 gp.sprint_var(rc, fmt=gp.hexa()) 908 set_default_siguser1() 909 BuiltIn().fail(gp.sprint_error(error_message)) 910 911 # Restore original sigusr1 handler. 912 set_default_siguser1() 913 914 915def test_loop_body(): 916 r""" 917 The main loop body for the loop in main_py. 918 919 Description of arguments: 920 boot_count The iteration number (starts at 1). 921 """ 922 923 global boot_count 924 global state 925 global next_boot 926 global boot_success 927 global boot_end_time 928 929 gp.qprintn() 930 931 next_boot = select_boot() 932 if next_boot == "": 933 return True 934 935 boot_count += 1 936 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 937 938 pre_boot_plug_in_setup() 939 940 cmd_buf = ["run_boot", next_boot] 941 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 942 if boot_status == "FAIL": 943 gp.qprint(msg) 944 945 gp.qprintn() 946 if boot_status == "PASS": 947 boot_success = 1 948 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 949 + "\" succeeded.") 950 else: 951 boot_success = 0 952 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 953 + "\" failed.") 954 955 # Set boot_end_time for use by plug-ins. 956 boot_end_time = completion_msg[1:33] 957 gp.qprint_var(boot_end_time) 958 959 gp.qprint(completion_msg) 960 961 boot_results.update(next_boot, boot_status) 962 963 plug_in_setup() 964 # NOTE: A post_test_case call point failure is NOT counted as a boot 965 # failure. 966 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 967 call_point='post_test_case', stop_on_plug_in_failure=0) 968 969 plug_in_setup() 970 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 971 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 972 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 973 if ffdc_check == "All" or\ 974 shell_rc == dump_ffdc_rc(): 975 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 976 if status != 'PASS': 977 gp.qprint_error("Call to my_ffdc failed.\n") 978 # Leave a record for caller that "soft" errors occurred. 979 soft_errors = 1 980 gpu.save_plug_in_value(soft_errors, pgm_name) 981 982 if delete_errlogs: 983 # print error logs before delete 984 if redfish_support_trans_state: 985 status, error_logs = grk.run_key_u("Get Redfish Event Logs") 986 log.print_error_logs(error_logs, "AdditionalDataURI Message Severity") 987 else: 988 status, error_logs = grk.run_key_u("Get Error Logs") 989 log.print_error_logs(error_logs, "AdditionalData Message Severity") 990 pels = pel.peltool("-l", ignore_err=1) 991 gp.qprint_var(pels) 992 993 # We need to purge error logs between boots or they build up. 994 grk.run_key(delete_errlogs_cmd, ignore=1) 995 grk.run_key(delete_bmcdump_cmd, ignore=1) 996 if redfish_support_trans_state: 997 grk.run_key(delete_sysdump_cmd, ignore=1) 998 999 boot_results.print_report() 1000 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 1001 1002 plug_in_setup() 1003 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1004 call_point='stop_check', shell_rc=stop_test_rc(), 1005 stop_on_non_zero_rc=1) 1006 if shell_rc == stop_test_rc(): 1007 message = "Stopping as requested by user.\n" 1008 gp.qprint_time(message) 1009 BuiltIn().fail(message) 1010 1011 # This should help prevent ConnectionErrors. 1012 # Purge all redfish and REST connection sessions. 1013 if redfish_delete_sessions: 1014 grk.run_key_u("Close All Connections", ignore=1) 1015 grk.run_key_u("Delete All Redfish Sessions", ignore=1) 1016 1017 return True 1018 1019 1020def obmc_boot_test_teardown(): 1021 r""" 1022 Clean up after the main keyword. 1023 """ 1024 gp.qprint_executing() 1025 1026 if ga.psutil_imported: 1027 ga.terminate_descendants() 1028 1029 if cp_setup_called: 1030 plug_in_setup() 1031 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1032 call_point='cleanup', stop_on_plug_in_failure=0) 1033 1034 if 'boot_results_file_path' in globals(): 1035 # Save boot_results and boot_history objects to a file in case they are 1036 # needed again. 1037 gp.qprint_timen("Saving boot_results to the following path.") 1038 gp.qprint_var(boot_results_file_path) 1039 pickle.dump((boot_results, boot_history), 1040 open(boot_results_file_path, 'wb'), 1041 pickle.HIGHEST_PROTOCOL) 1042 1043 global save_stack 1044 # Restore any global values saved on the save_stack. 1045 for parm_name in main_func_parm_list: 1046 # Get the parm_value if it was saved on the stack. 1047 try: 1048 parm_value = save_stack.pop(parm_name) 1049 except BaseException: 1050 # If it was not saved, no further action is required. 1051 continue 1052 1053 # Restore the saved value. 1054 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1055 "}\", parm_value)" 1056 gp.dpissuing(cmd_buf) 1057 exec(cmd_buf) 1058 1059 gp.dprintn(save_stack.sprint_obj()) 1060 1061 1062def test_teardown(): 1063 r""" 1064 Clean up after this test case. 1065 """ 1066 1067 gp.qprintn() 1068 gp.qprint_executing() 1069 1070 if ga.psutil_imported: 1071 ga.terminate_descendants() 1072 1073 cmd_buf = ["Print Error", 1074 "A keyword timeout occurred ending this program.\n"] 1075 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1076 1077 if redfish_supported: 1078 redfish.logout() 1079 1080 gp.qprint_pgm_footer() 1081 1082 1083def post_stack(): 1084 r""" 1085 Process post_stack plug-in programs. 1086 """ 1087 1088 if not call_post_stack_plug: 1089 # The caller does not wish to have post_stack plug-in processing done. 1090 return 1091 1092 global boot_success 1093 1094 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1095 pre_boot_plug_in_setup() 1096 # For the purposes of the following plug-ins, mark the "boot" as a success. 1097 boot_success = 1 1098 plug_in_setup() 1099 rc, shell_rc, failed_plug_in_name, history =\ 1100 grpi.rprocess_plug_in_packages(call_point='post_stack', 1101 stop_on_plug_in_failure=0, 1102 return_history=True) 1103 for doing_msg in history: 1104 update_boot_history(boot_history, doing_msg, max_boot_history) 1105 if rc != 0: 1106 boot_success = 0 1107 1108 plug_in_setup() 1109 rc, shell_rc, failed_plug_in_name =\ 1110 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1111 shell_rc=dump_ffdc_rc(), 1112 stop_on_plug_in_failure=1, 1113 stop_on_non_zero_rc=1) 1114 if shell_rc == dump_ffdc_rc(): 1115 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1116 if status != 'PASS': 1117 gp.qprint_error("Call to my_ffdc failed.\n") 1118 # Leave a record for caller that "soft" errors occurred. 1119 soft_errors = 1 1120 gpu.save_plug_in_value(soft_errors, pgm_name) 1121 1122 plug_in_setup() 1123 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1124 call_point='stop_check', shell_rc=stop_test_rc(), 1125 stop_on_non_zero_rc=1) 1126 if shell_rc == stop_test_rc(): 1127 message = "Stopping as requested by user.\n" 1128 gp.qprint_time(message) 1129 BuiltIn().fail(message) 1130 1131 1132def obmc_boot_test_py(loc_boot_stack=None, 1133 loc_stack_mode=None, 1134 loc_quiet=None): 1135 r""" 1136 Do main program processing. 1137 """ 1138 1139 global save_stack 1140 1141 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']}) 1142 1143 gp.dprintn() 1144 # Process function parms. 1145 for parm_name in main_func_parm_list: 1146 # Get parm's value. 1147 parm_value = eval("loc_" + parm_name) 1148 gp.dpvars(parm_name, parm_value) 1149 1150 if parm_value is not None: 1151 # Save the global value on a stack. 1152 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1153 parm_name + "}\"), \"" + parm_name + "\")" 1154 gp.dpissuing(cmd_buf) 1155 exec(cmd_buf) 1156 1157 # Set the global value to the passed value. 1158 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1159 "}\", loc_" + parm_name + ")" 1160 gp.dpissuing(cmd_buf) 1161 exec(cmd_buf) 1162 1163 gp.dprintn(save_stack.sprint_obj()) 1164 1165 setup() 1166 1167 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1168 1169 if ffdc_only: 1170 gp.qprint_timen("Caller requested ffdc_only.") 1171 if do_pre_boot_plug_in_setup: 1172 pre_boot_plug_in_setup() 1173 grk.run_key_u("my_ffdc") 1174 return 1175 1176 if delete_errlogs: 1177 # print error logs before delete 1178 if redfish_support_trans_state: 1179 status, error_logs = grk.run_key_u("Get Redfish Event Logs") 1180 log.print_error_logs(error_logs, "AdditionalDataURI Message Severity") 1181 else: 1182 status, error_logs = grk.run_key_u("Get Error Logs") 1183 log.print_error_logs(error_logs, "AdditionalData Message Severity") 1184 pels = pel.peltool("-l", ignore_err=1) 1185 gp.qprint_var(pels) 1186 1187 # Delete errlogs prior to doing any boot tests. 1188 grk.run_key(delete_errlogs_cmd, ignore=1) 1189 grk.run_key(delete_bmcdump_cmd, ignore=1) 1190 if redfish_support_trans_state: 1191 grk.run_key(delete_sysdump_cmd, ignore=1) 1192 1193 # Process caller's boot_stack. 1194 while (len(boot_stack) > 0): 1195 test_loop_body() 1196 1197 gp.qprint_timen("Finished processing stack.") 1198 1199 post_stack() 1200 1201 # Process caller's boot_list. 1202 if len(boot_list) > 0: 1203 for ix in range(1, max_num_tests + 1): 1204 test_loop_body() 1205 1206 gp.qprint_timen("Completed all requested boot tests.") 1207 1208 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1209 new_fail = boot_fail - init_boot_fail 1210 if new_fail > boot_fail_threshold: 1211 error_message = "Boot failures exceed the boot failure" +\ 1212 " threshold:\n" +\ 1213 gp.sprint_var(new_fail) +\ 1214 gp.sprint_var(boot_fail_threshold) 1215 BuiltIn().fail(gp.sprint_error(error_message)) 1216