1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_arg as ga 27import gen_valid as gv 28import gen_misc as gm 29import gen_cmd as gc 30import gen_robot_keyword as grk 31import state as st 32import var_stack as vs 33import gen_plug_in_utils as gpu 34import pel_utils as pel 35import logging_utils as log 36 37base_path = os.path.dirname(os.path.dirname( 38 imp.find_module("gen_robot_print")[1])) +\ 39 os.sep 40sys.path.append(base_path + "extended/") 41import run_keyword as rk 42 43# Setting master_pid correctly influences the behavior of plug-ins like 44# DB_Logging 45program_pid = os.getpid() 46master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 47pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 48 49# Set up boot data structures. 50os_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 51 52boot_lists = read_boot_lists() 53 54# The maximum number of entries that can be in the boot_history global variable. 55max_boot_history = 10 56boot_history = [] 57 58state = st.return_state_constant('default_state') 59cp_setup_called = 0 60next_boot = "" 61base_tool_dir_path = os.path.normpath(os.environ.get( 62 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 63 64ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 65boot_success = 0 66status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 67if status_dir_path != "": 68 status_dir_path = os.path.normpath(status_dir_path) + os.sep 69redfish_support_trans_state = int(os.environ.get('REDFISH_SUPPORT_TRANS_STATE', 0)) or \ 70 int(BuiltIn().get_variable_value("${REDFISH_SUPPORT_TRANS_STATE}", default=0)) 71redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False) 72redfish_rest_supported = BuiltIn().get_variable_value("${REDFISH_REST_SUPPORTED}", default=False) 73redfish_delete_sessions = int(BuiltIn().get_variable_value("${REDFISH_DELETE_SESSIONS}", default=1)) 74if redfish_supported: 75 redfish = BuiltIn().get_library_instance('redfish') 76 default_power_on = "Redfish Power On" 77 default_power_off = "Redfish Power Off" 78 if not redfish_support_trans_state: 79 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 80 delete_bmcdump_cmd = "Delete All BMC Dump" 81 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 82 else: 83 delete_errlogs_cmd = "Redfish Purge Event Log" 84 delete_bmcdump_cmd = "Redfish Delete All BMC Dumps" 85 delete_sysdump_cmd = "Redfish Delete All System Dumps" 86 default_set_power_policy = "Redfish Set Power Restore Policy AlwaysOff" 87else: 88 default_power_on = "REST Power On" 89 default_power_off = "REST Power Off" 90 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 91 delete_bmcdump_cmd = "Delete All BMC Dump" 92 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 93boot_count = 0 94 95LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 96AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 97ffdc_prefix = AUTOBOOT_FFDC_PREFIX 98boot_start_time = "" 99boot_end_time = "" 100save_stack = vs.var_stack('save_stack') 101main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 102 103 104def dump_ffdc_rc(): 105 r""" 106 Return the constant dump ffdc test return code value. 107 108 When a plug-in call point program returns this value, it indicates that 109 this program should collect FFDC. 110 """ 111 112 return 0x00000200 113 114 115def stop_test_rc(): 116 r""" 117 Return the constant stop test return code value. 118 119 When a plug-in call point program returns this value, it indicates that 120 this program should stop running. 121 """ 122 123 return 0x00000200 124 125 126def process_host(host, 127 host_var_name=""): 128 r""" 129 Process a host by getting the associated host name and IP address and 130 setting them in global variables. 131 132 If the caller does not pass the host_var_name, this function will try to 133 figure out the name of the variable used by the caller for the host parm. 134 Callers are advised to explicitly specify the host_var_name when calling 135 with an exec command. In such cases, the get_arg_name cannot figure out 136 the host variable name. 137 138 This function will then create similar global variable names by 139 removing "_host" and appending "_host_name" or "_ip" to the host variable 140 name. 141 142 Example: 143 144 If a call is made like this: 145 process_host(openbmc_host) 146 147 Global variables openbmc_host_name and openbmc_ip will be set. 148 149 Description of argument(s): 150 host A host name or IP. The name of the variable used should 151 have a suffix of "_host". 152 host_var_name The name of the variable being used as the host parm. 153 """ 154 155 if host_var_name == "": 156 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 157 158 host_name_var_name = re.sub("host", "host_name", host_var_name) 159 ip_var_name = re.sub("host", "ip", host_var_name) 160 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 161 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 162 host + "')" 163 exec(cmd_buf) 164 165 166def process_pgm_parms(): 167 r""" 168 Process the program parameters by assigning them all to corresponding 169 globals. Also, set some global values that depend on program parameters. 170 """ 171 172 # Program parameter processing. 173 # Assign all program parms to python variables which are global to this 174 # module. 175 176 global parm_list 177 parm_list = BuiltIn().get_variable_value("${parm_list}") 178 # The following subset of parms should be processed as integers. 179 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 180 'boot_fail_threshold', 'delete_errlogs', 181 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 182 'test_mode', 'debug'] 183 for parm in parm_list: 184 if parm in int_list: 185 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 186 "}\", \"0\"))" 187 else: 188 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 189 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 190 gp.dpissuing(cmd_buf) 191 exec(cmd_buf) 192 if re.match(r".*_host$", parm): 193 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 194 exec(cmd_buf) 195 if re.match(r".*_password$", parm): 196 # Register the value of any parm whose name ends in _password. 197 # This will cause the print functions to replace passwords with 198 # asterisks in the output. 199 cmd_buf = "gp.register_passwords(" + parm + ")" 200 exec(cmd_buf) 201 202 global ffdc_dir_path_style 203 global boot_list 204 global boot_stack 205 global boot_results_file_path 206 global boot_results 207 global boot_history 208 global ffdc_list_file_path 209 global ffdc_report_list_path 210 global ffdc_summary_list_path 211 global boot_table 212 global valid_boot_types 213 214 if ffdc_dir_path_style == "": 215 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 216 217 # Convert these program parms to lists for easier processing.. 218 boot_list = list(filter(None, boot_list.split(":"))) 219 boot_stack = list(filter(None, boot_stack.split(":"))) 220 221 boot_table = create_boot_table(boot_table_path, os_host=os_host) 222 valid_boot_types = create_valid_boot_list(boot_table) 223 224 cleanup_boot_results_file() 225 boot_results_file_path = create_boot_results_file_path(pgm_name, 226 openbmc_nickname, 227 master_pid) 228 229 if os.path.isfile(boot_results_file_path): 230 # We've been called before in this run so we'll load the saved 231 # boot_results and boot_history objects. 232 boot_results, boot_history =\ 233 pickle.load(open(boot_results_file_path, 'rb')) 234 else: 235 boot_results = boot_results(boot_table, boot_pass, boot_fail) 236 237 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 238 "/FFDC_FILE_LIST" 239 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 240 "/FFDC_REPORT_FILE_LIST" 241 242 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 243 "/FFDC_SUMMARY_FILE_LIST" 244 245 246def initial_plug_in_setup(): 247 r""" 248 Initialize all plug-in environment variables which do not change for the 249 duration of the program. 250 251 """ 252 253 global LOG_LEVEL 254 BuiltIn().set_log_level("NONE") 255 256 BuiltIn().set_global_variable("${master_pid}", master_pid) 257 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 258 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 259 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 260 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 261 ffdc_list_file_path) 262 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 263 ffdc_report_list_path) 264 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 265 ffdc_summary_list_path) 266 267 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 268 ffdc_dir_path_style) 269 BuiltIn().set_global_variable("${FFDC_CHECK}", 270 ffdc_check) 271 272 # For each program parameter, set the corresponding AUTOBOOT_ environment 273 # variable value. Also, set an AUTOBOOT_ environment variable for every 274 # element in additional_values. 275 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 276 "status_dir_path", "base_tool_dir_path", 277 "ffdc_list_file_path", "ffdc_report_list_path", 278 "ffdc_summary_list_path", "execdir", "redfish_supported", 279 "redfish_rest_supported", "redfish_support_trans_state"] 280 281 plug_in_vars = parm_list + additional_values 282 283 for var_name in plug_in_vars: 284 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 285 var_name = var_name.upper() 286 if var_value is None: 287 var_value = "" 288 os.environ["AUTOBOOT_" + var_name] = str(var_value) 289 290 BuiltIn().set_log_level(LOG_LEVEL) 291 292 # Make sure the ffdc list directory exists. 293 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 294 if not os.path.exists(ffdc_list_dir_path): 295 os.makedirs(ffdc_list_dir_path) 296 297 298def plug_in_setup(): 299 r""" 300 Initialize all changing plug-in environment variables for use by the 301 plug-in programs. 302 """ 303 304 global LOG_LEVEL 305 global test_really_running 306 307 BuiltIn().set_log_level("NONE") 308 309 boot_pass, boot_fail = boot_results.return_total_pass_fail() 310 if boot_pass > 1: 311 test_really_running = 1 312 else: 313 test_really_running = 0 314 315 BuiltIn().set_global_variable("${test_really_running}", 316 test_really_running) 317 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 318 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 319 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 320 BuiltIn().set_global_variable("${boot_success}", boot_success) 321 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 322 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 323 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 324 325 # For each program parameter, set the corresponding AUTOBOOT_ environment 326 # variable value. Also, set an AUTOBOOT_ environment variable for every 327 # element in additional_values. 328 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 329 "boot_fail", "test_really_running", "ffdc_prefix", 330 "boot_start_time", "boot_end_time"] 331 332 plug_in_vars = additional_values 333 334 for var_name in plug_in_vars: 335 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 336 var_name = var_name.upper() 337 if var_value is None: 338 var_value = "" 339 os.environ["AUTOBOOT_" + var_name] = str(var_value) 340 341 if debug: 342 shell_rc, out_buf = \ 343 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 344 345 BuiltIn().set_log_level(LOG_LEVEL) 346 347 348def pre_boot_plug_in_setup(): 349 350 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 351 try: 352 os.remove(ffdc_list_file_path) 353 except OSError: 354 pass 355 356 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 357 try: 358 os.remove(ffdc_report_list_path) 359 except OSError: 360 pass 361 362 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 363 try: 364 os.remove(ffdc_summary_list_path) 365 except OSError: 366 pass 367 368 global ffdc_prefix 369 370 seconds = time.time() 371 loc_time = time.localtime(seconds) 372 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 373 374 ffdc_prefix = openbmc_nickname + "." + time_string 375 376 377def default_sigusr1(signal_number=0, 378 frame=None): 379 r""" 380 Handle SIGUSR1 by doing nothing. 381 382 This function assists in debugging SIGUSR1 processing by printing messages 383 to stdout and to the log.html file. 384 385 Description of argument(s): 386 signal_number The signal number (should always be 10 for SIGUSR1). 387 frame The frame data. 388 """ 389 390 gp.qprintn() 391 gp.qprint_executing() 392 gp.lprint_executing() 393 394 395def set_default_siguser1(): 396 r""" 397 Set the default_sigusr1 function to be the SIGUSR1 handler. 398 """ 399 400 gp.qprintn() 401 gp.qprint_executing() 402 gp.lprint_executing() 403 signal.signal(signal.SIGUSR1, default_sigusr1) 404 405 406def setup(): 407 r""" 408 Do general program setup tasks. 409 """ 410 411 global cp_setup_called 412 global transitional_boot_selected 413 414 gp.qprintn() 415 416 if redfish_supported: 417 redfish.login() 418 419 set_default_siguser1() 420 transitional_boot_selected = False 421 422 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 423 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 424 # If we can't find process_plug_in_packages.py, ssh_pw or 425 # validate_plug_ins.py, then we don't have our repo bin in PATH. 426 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 427 + " ssh_pw validate_plug_ins.py", quiet=1, 428 print_output=0, show_err=0) 429 if shell_rc != 0: 430 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 431 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 432 if robot_pgm_dir_path not in sys.path: 433 sys.path.append(robot_pgm_dir_path) 434 PYTHONPATH = os.environ.get("PYTHONPATH", "") 435 if PYTHONPATH == "": 436 os.environ['PYTHONPATH'] = robot_pgm_dir_path 437 else: 438 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 439 440 validate_parms() 441 442 gp.qprint_pgm_header() 443 444 grk.run_key_u(default_set_power_policy) 445 446 initial_plug_in_setup() 447 448 plug_in_setup() 449 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 450 call_point='setup') 451 if rc != 0: 452 error_message = "Plug-in setup failed.\n" 453 gp.print_error_report(error_message) 454 BuiltIn().fail(error_message) 455 # Setting cp_setup_called lets our Teardown know that it needs to call 456 # the cleanup plug-in call point. 457 cp_setup_called = 1 458 459 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 460 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 461 # FFDC_LOG_PATH is used by "FFDC" keyword. 462 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 463 464 # Also printed by FFDC. 465 global host_name 466 global host_ip 467 host = socket.gethostname() 468 host_name, host_ip = gm.get_host_name_ip(host) 469 470 gp.dprint_var(boot_table) 471 gp.dprint_var(boot_lists) 472 473 474def validate_parms(): 475 r""" 476 Validate all program parameters. 477 """ 478 479 process_pgm_parms() 480 481 gp.qprintn() 482 483 global openbmc_model 484 if openbmc_model == "": 485 status, ret_values =\ 486 grk.run_key_u("Get BMC System Model") 487 openbmc_model = ret_values 488 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 489 gv.set_exit_on_error(True) 490 gv.valid_value(openbmc_host) 491 gv.valid_value(openbmc_username) 492 gv.valid_value(openbmc_password) 493 gv.valid_value(rest_username) 494 gv.valid_value(rest_password) 495 gv.valid_value(ipmi_username) 496 gv.valid_value(ipmi_password) 497 if os_host != "": 498 gv.valid_value(os_username) 499 gv.valid_value(os_password) 500 if pdu_host != "": 501 gv.valid_value(pdu_username) 502 gv.valid_value(pdu_password) 503 gv.valid_integer(pdu_slot_no) 504 if openbmc_serial_host != "": 505 gv.valid_integer(openbmc_serial_port) 506 gv.valid_value(openbmc_model) 507 gv.valid_integer(max_num_tests) 508 gv.valid_integer(boot_pass) 509 gv.valid_integer(boot_fail) 510 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 511 BuiltIn().set_global_variable("${plug_in_packages_list}", 512 plug_in_packages_list) 513 gv.valid_value(stack_mode, valid_values=['normal', 'skip']) 514 gv.set_exit_on_error(False) 515 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 516 error_message = "You must provide either a value for either the" +\ 517 " boot_list or the boot_stack parm.\n" 518 BuiltIn().fail(gp.sprint_error(error_message)) 519 valid_boot_list(boot_list, valid_boot_types) 520 valid_boot_list(boot_stack, valid_boot_types) 521 selected_PDU_boots = list(set(boot_list + boot_stack) 522 & set(boot_lists['PDU_reboot'])) 523 if len(selected_PDU_boots) > 0 and pdu_host == "": 524 error_message = "You have selected the following boots which" +\ 525 " require a PDU host but no value for pdu_host:\n" 526 error_message += gp.sprint_var(selected_PDU_boots) 527 error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 528 BuiltIn().fail(gp.sprint_error(error_message)) 529 530 return 531 532 533def my_get_state(): 534 r""" 535 Get the system state plus a little bit of wrapping. 536 """ 537 538 global state 539 540 req_states = ['epoch_seconds'] + st.default_req_states 541 542 gp.qprint_timen("Getting system state.") 543 if test_mode: 544 state['epoch_seconds'] = int(time.time()) 545 else: 546 state = st.get_state(req_states=req_states, quiet=quiet) 547 gp.qprint_var(state) 548 549 550def valid_state(): 551 r""" 552 Verify that our state dictionary contains no blank values. If we don't get 553 valid state data, we cannot continue to work. 554 """ 555 556 if st.compare_states(state, st.invalid_state_match, 'or'): 557 error_message = "The state dictionary contains blank fields which" +\ 558 " is illegal.\n" + gp.sprint_var(state) 559 BuiltIn().fail(gp.sprint_error(error_message)) 560 561 562def select_boot(): 563 r""" 564 Select a boot test to be run based on our current state and return the 565 chosen boot type. 566 567 Description of arguments: 568 state The state of the machine. 569 """ 570 571 global transitional_boot_selected 572 global boot_stack 573 574 gp.qprint_timen("Selecting a boot test.") 575 576 if transitional_boot_selected and not boot_success: 577 prior_boot = next_boot 578 boot_candidate = boot_stack.pop() 579 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 580 + " transition to a valid state for '" + boot_candidate 581 + "' which was at the top of the boot_stack. Since" 582 + " the '" + next_boot + "' failed, the '" 583 + boot_candidate + "' has been removed from the stack" 584 + " to avoid and endless failure loop.") 585 if len(boot_stack) == 0: 586 return "" 587 588 my_get_state() 589 valid_state() 590 591 transitional_boot_selected = False 592 stack_popped = 0 593 if len(boot_stack) > 0: 594 stack_popped = 1 595 gp.qprint_dashes() 596 gp.qprint_var(boot_stack) 597 gp.qprint_dashes() 598 skip_boot_printed = 0 599 while len(boot_stack) > 0: 600 boot_candidate = boot_stack.pop() 601 if stack_mode == 'normal': 602 break 603 else: 604 if st.compare_states(state, boot_table[boot_candidate]['end']): 605 if not skip_boot_printed: 606 gp.qprint_var(stack_mode) 607 gp.qprintn() 608 gp.qprint_timen("Skipping the following boot tests" 609 + " which are unnecessary since their" 610 + " required end states match the" 611 + " current machine state:") 612 skip_boot_printed = 1 613 gp.qprint_var(boot_candidate) 614 boot_candidate = "" 615 if boot_candidate == "": 616 gp.qprint_dashes() 617 gp.qprint_var(boot_stack) 618 gp.qprint_dashes() 619 return boot_candidate 620 if st.compare_states(state, boot_table[boot_candidate]['start']): 621 gp.qprint_timen("The machine state is valid for a '" 622 + boot_candidate + "' boot test.") 623 gp.qprint_dashes() 624 gp.qprint_var(boot_stack) 625 gp.qprint_dashes() 626 return boot_candidate 627 else: 628 gp.qprint_timen("The machine state does not match the required" 629 + " starting state for a '" + boot_candidate 630 + "' boot test:") 631 gp.qprint_varx("boot_table_start_entry", 632 boot_table[boot_candidate]['start']) 633 boot_stack.append(boot_candidate) 634 transitional_boot_selected = True 635 popped_boot = boot_candidate 636 637 # Loop through your list selecting a boot_candidates 638 boot_candidates = [] 639 for boot_candidate in boot_list: 640 if st.compare_states(state, boot_table[boot_candidate]['start']): 641 if stack_popped: 642 if st.compare_states(boot_table[boot_candidate]['end'], 643 boot_table[popped_boot]['start']): 644 boot_candidates.append(boot_candidate) 645 else: 646 boot_candidates.append(boot_candidate) 647 648 if len(boot_candidates) == 0: 649 gp.qprint_timen("The user's boot list contained no boot tests" 650 + " which are valid for the current machine state.") 651 boot_candidate = default_power_on 652 if not st.compare_states(state, boot_table[default_power_on]['start']): 653 boot_candidate = default_power_off 654 boot_candidates.append(boot_candidate) 655 gp.qprint_timen("Using default '" + boot_candidate 656 + "' boot type to transition to valid state.") 657 658 gp.dprint_var(boot_candidates) 659 660 # Randomly select a boot from the candidate list. 661 boot = random.choice(boot_candidates) 662 663 return boot 664 665 666def print_defect_report(ffdc_file_list): 667 r""" 668 Print a defect report. 669 670 Description of argument(s): 671 ffdc_file_list A list of files which were collected by our ffdc functions. 672 """ 673 674 # Making deliberate choice to NOT run plug_in_setup(). We don't want 675 # ffdc_prefix updated. 676 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 677 call_point='ffdc_report', stop_on_plug_in_failure=0) 678 679 # Get additional header data which may have been created by ffdc plug-ins. 680 # Also, delete the individual header files to cleanup. 681 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 682 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 683 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 684 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 685 show_err=0) 686 687 # Get additional summary data which may have been created by ffdc plug-ins. 688 # Also, delete the individual header files to cleanup. 689 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 690 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 691 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 692 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 693 show_err=0) 694 695 # ffdc_list_file_path contains a list of any ffdc files created by plug- 696 # ins, etc. Read that data into a list. 697 try: 698 plug_in_ffdc_list = \ 699 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 700 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 701 except IOError: 702 plug_in_ffdc_list = [] 703 704 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 705 # in. Eliminate duplicates and sort the list. 706 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 707 708 if status_file_path != "": 709 ffdc_file_list.insert(0, status_file_path) 710 711 # Convert the list to a printable list. 712 printable_ffdc_file_list = "\n".join(ffdc_file_list) 713 714 # Open ffdc_file_list for writing. We will write a complete list of 715 # FFDC files to it for possible use by plug-ins like cp_stop_check. 716 ffdc_list_file = open(ffdc_list_file_path, 'w') 717 ffdc_list_file.write(printable_ffdc_file_list + "\n") 718 ffdc_list_file.close() 719 720 indent = 0 721 width = 90 722 linefeed = 1 723 char = "=" 724 725 gp.qprintn() 726 gp.qprint_dashes(indent, width, linefeed, char) 727 gp.qprintn("Copy this data to the defect:\n") 728 729 if len(more_header_info) > 0: 730 gp.qprintn(more_header_info) 731 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 732 openbmc_host_name, openbmc_ip, openbmc_username, 733 openbmc_password, rest_username, rest_password, ipmi_username, 734 ipmi_password, os_host, os_host_name, os_ip, os_username, 735 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 736 pdu_password, pdu_slot_no, openbmc_serial_host, 737 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 738 739 gp.qprintn() 740 print_boot_history(boot_history) 741 gp.qprintn() 742 gp.qprint_var(state) 743 gp.qprintn() 744 gp.qprintn("FFDC data files:") 745 gp.qprintn(printable_ffdc_file_list) 746 gp.qprintn() 747 748 if len(ffdc_summary_info) > 0: 749 gp.qprintn(ffdc_summary_info) 750 751 gp.qprint_dashes(indent, width, linefeed, char) 752 753 754def my_ffdc(): 755 r""" 756 Collect FFDC data. 757 """ 758 759 global state 760 761 plug_in_setup() 762 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 763 call_point='ffdc', stop_on_plug_in_failure=0) 764 765 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 766 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 767 + AUTOBOOT_FFDC_PREFIX 768 + " ffdc_function_list=" 769 + ffdc_function_list, ignore=1) 770 if status != 'PASS': 771 gp.qprint_error("Call to ffdc failed.\n") 772 if type(ffdc_file_list) is not list: 773 ffdc_file_list = [] 774 # Leave a record for caller that "soft" errors occurred. 775 soft_errors = 1 776 gpu.save_plug_in_value(soft_errors, pgm_name) 777 778 my_get_state() 779 780 print_defect_report(ffdc_file_list) 781 782 783def print_test_start_message(boot_keyword): 784 r""" 785 Print a message indicating what boot test is about to run. 786 787 Description of arguments: 788 boot_keyword The name of the boot which is to be run 789 (e.g. "BMC Power On"). 790 """ 791 792 global boot_history 793 global boot_start_time 794 795 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 796 797 # Set boot_start_time for use by plug-ins. 798 boot_start_time = doing_msg[1:33] 799 gp.qprint_var(boot_start_time) 800 801 gp.qprint(doing_msg) 802 803 update_boot_history(boot_history, doing_msg, max_boot_history) 804 805 806def stop_boot_test(signal_number=0, 807 frame=None): 808 r""" 809 Handle SIGUSR1 by aborting the boot test that is running. 810 811 Description of argument(s): 812 signal_number The signal number (should always be 10 for SIGUSR1). 813 frame The frame data. 814 """ 815 816 gp.qprintn() 817 gp.qprint_executing() 818 gp.lprint_executing() 819 820 # Restore original sigusr1 handler. 821 set_default_siguser1() 822 823 message = "The caller has asked that the boot test be stopped and marked" 824 message += " as a failure." 825 826 function_stack = gm.get_function_stack() 827 if "wait_state" in function_stack: 828 st.set_exit_wait_early_message(message) 829 else: 830 BuiltIn().fail(gp.sprint_error(message)) 831 832 833def run_boot(boot): 834 r""" 835 Run the specified boot. 836 837 Description of arguments: 838 boot The name of the boot test to be performed. 839 """ 840 841 global state 842 843 signal.signal(signal.SIGUSR1, stop_boot_test) 844 gp.qprint_timen("stop_boot_test is armed.") 845 846 print_test_start_message(boot) 847 848 plug_in_setup() 849 rc, shell_rc, failed_plug_in_name = \ 850 grpi.rprocess_plug_in_packages(call_point="pre_boot") 851 if rc != 0: 852 error_message = "Plug-in failed with non-zero return code.\n" +\ 853 gp.sprint_var(rc, fmt=gp.hexa()) 854 set_default_siguser1() 855 BuiltIn().fail(gp.sprint_error(error_message)) 856 857 if test_mode: 858 # In test mode, we'll pretend the boot worked by assigning its 859 # required end state to the default state value. 860 state = st.strip_anchor_state(boot_table[boot]['end']) 861 else: 862 # Assertion: We trust that the state data was made fresh by the 863 # caller. 864 865 gp.qprintn() 866 867 if boot_table[boot]['method_type'] == "keyword": 868 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 869 boot_table[boot]['method'], 870 quiet=quiet) 871 872 if boot_table[boot]['bmc_reboot']: 873 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 874 plug_in_setup() 875 rc, shell_rc, failed_plug_in_name = \ 876 grpi.rprocess_plug_in_packages(call_point="post_reboot") 877 if rc != 0: 878 error_message = "Plug-in failed with non-zero return code.\n" 879 error_message += gp.sprint_var(rc, fmt=gp.hexa()) 880 set_default_siguser1() 881 BuiltIn().fail(gp.sprint_error(error_message)) 882 else: 883 match_state = st.anchor_state(state) 884 del match_state['epoch_seconds'] 885 # Wait for the state to change in any way. 886 st.wait_state(match_state, wait_time=state_change_timeout, 887 interval="10 seconds", invert=1) 888 889 gp.qprintn() 890 if boot_table[boot]['end']['chassis'] == "Off": 891 boot_timeout = power_off_timeout 892 else: 893 boot_timeout = power_on_timeout 894 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 895 interval="10 seconds") 896 897 plug_in_setup() 898 rc, shell_rc, failed_plug_in_name = \ 899 grpi.rprocess_plug_in_packages(call_point="post_boot") 900 if rc != 0: 901 error_message = "Plug-in failed with non-zero return code.\n" +\ 902 gp.sprint_var(rc, fmt=gp.hexa()) 903 set_default_siguser1() 904 BuiltIn().fail(gp.sprint_error(error_message)) 905 906 # Restore original sigusr1 handler. 907 set_default_siguser1() 908 909 910def test_loop_body(): 911 r""" 912 The main loop body for the loop in main_py. 913 914 Description of arguments: 915 boot_count The iteration number (starts at 1). 916 """ 917 918 global boot_count 919 global state 920 global next_boot 921 global boot_success 922 global boot_end_time 923 924 gp.qprintn() 925 926 next_boot = select_boot() 927 if next_boot == "": 928 return True 929 930 boot_count += 1 931 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 932 933 pre_boot_plug_in_setup() 934 935 cmd_buf = ["run_boot", next_boot] 936 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 937 if boot_status == "FAIL": 938 gp.qprint(msg) 939 940 gp.qprintn() 941 if boot_status == "PASS": 942 boot_success = 1 943 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 944 + "\" succeeded.") 945 else: 946 boot_success = 0 947 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 948 + "\" failed.") 949 950 # Set boot_end_time for use by plug-ins. 951 boot_end_time = completion_msg[1:33] 952 gp.qprint_var(boot_end_time) 953 954 gp.qprint(completion_msg) 955 956 boot_results.update(next_boot, boot_status) 957 958 plug_in_setup() 959 # NOTE: A post_test_case call point failure is NOT counted as a boot 960 # failure. 961 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 962 call_point='post_test_case', stop_on_plug_in_failure=0) 963 964 plug_in_setup() 965 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 966 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 967 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 968 if ffdc_check == "All" or\ 969 shell_rc == dump_ffdc_rc(): 970 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 971 if status != 'PASS': 972 gp.qprint_error("Call to my_ffdc failed.\n") 973 # Leave a record for caller that "soft" errors occurred. 974 soft_errors = 1 975 gpu.save_plug_in_value(soft_errors, pgm_name) 976 977 if delete_errlogs: 978 # print error logs before delete 979 status, error_logs = grk.run_key_u("Get Error Logs") 980 pels = pel.peltool("-l", ignore_err=1) 981 log.print_error_logs(error_logs, "AdditionalData Message Severity") 982 gp.qprint_var(pels) 983 984 # We need to purge error logs between boots or they build up. 985 grk.run_key(delete_errlogs_cmd, ignore=1) 986 grk.run_key(delete_bmcdump_cmd, ignore=1) 987 if redfish_support_trans_state: 988 grk.run_key(delete_sysdump_cmd, ignore=1) 989 990 boot_results.print_report() 991 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 992 993 plug_in_setup() 994 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 995 call_point='stop_check', shell_rc=stop_test_rc(), 996 stop_on_non_zero_rc=1) 997 if shell_rc == stop_test_rc(): 998 message = "Stopping as requested by user.\n" 999 gp.qprint_time(message) 1000 BuiltIn().fail(message) 1001 1002 # This should help prevent ConnectionErrors. 1003 # Purge all redfish and REST connection sessions. 1004 if redfish_delete_sessions: 1005 grk.run_key_u("Close All Connections", ignore=1) 1006 grk.run_key_u("Delete All Redfish Sessions", ignore=1) 1007 1008 return True 1009 1010 1011def obmc_boot_test_teardown(): 1012 r""" 1013 Clean up after the main keyword. 1014 """ 1015 gp.qprint_executing() 1016 1017 if ga.psutil_imported: 1018 ga.terminate_descendants() 1019 1020 if cp_setup_called: 1021 plug_in_setup() 1022 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1023 call_point='cleanup', stop_on_plug_in_failure=0) 1024 1025 if 'boot_results_file_path' in globals(): 1026 # Save boot_results and boot_history objects to a file in case they are 1027 # needed again. 1028 gp.qprint_timen("Saving boot_results to the following path.") 1029 gp.qprint_var(boot_results_file_path) 1030 pickle.dump((boot_results, boot_history), 1031 open(boot_results_file_path, 'wb'), 1032 pickle.HIGHEST_PROTOCOL) 1033 1034 global save_stack 1035 # Restore any global values saved on the save_stack. 1036 for parm_name in main_func_parm_list: 1037 # Get the parm_value if it was saved on the stack. 1038 try: 1039 parm_value = save_stack.pop(parm_name) 1040 except BaseException: 1041 # If it was not saved, no further action is required. 1042 continue 1043 1044 # Restore the saved value. 1045 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1046 "}\", parm_value)" 1047 gp.dpissuing(cmd_buf) 1048 exec(cmd_buf) 1049 1050 gp.dprintn(save_stack.sprint_obj()) 1051 1052 1053def test_teardown(): 1054 r""" 1055 Clean up after this test case. 1056 """ 1057 1058 gp.qprintn() 1059 gp.qprint_executing() 1060 1061 if ga.psutil_imported: 1062 ga.terminate_descendants() 1063 1064 cmd_buf = ["Print Error", 1065 "A keyword timeout occurred ending this program.\n"] 1066 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1067 1068 if redfish_supported: 1069 redfish.logout() 1070 1071 gp.qprint_pgm_footer() 1072 1073 1074def post_stack(): 1075 r""" 1076 Process post_stack plug-in programs. 1077 """ 1078 1079 if not call_post_stack_plug: 1080 # The caller does not wish to have post_stack plug-in processing done. 1081 return 1082 1083 global boot_success 1084 1085 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1086 pre_boot_plug_in_setup() 1087 # For the purposes of the following plug-ins, mark the "boot" as a success. 1088 boot_success = 1 1089 plug_in_setup() 1090 rc, shell_rc, failed_plug_in_name, history =\ 1091 grpi.rprocess_plug_in_packages(call_point='post_stack', 1092 stop_on_plug_in_failure=0, 1093 return_history=True) 1094 for doing_msg in history: 1095 update_boot_history(boot_history, doing_msg, max_boot_history) 1096 if rc != 0: 1097 boot_success = 0 1098 1099 plug_in_setup() 1100 rc, shell_rc, failed_plug_in_name =\ 1101 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1102 shell_rc=dump_ffdc_rc(), 1103 stop_on_plug_in_failure=1, 1104 stop_on_non_zero_rc=1) 1105 if shell_rc == dump_ffdc_rc(): 1106 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1107 if status != 'PASS': 1108 gp.qprint_error("Call to my_ffdc failed.\n") 1109 # Leave a record for caller that "soft" errors occurred. 1110 soft_errors = 1 1111 gpu.save_plug_in_value(soft_errors, pgm_name) 1112 1113 plug_in_setup() 1114 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1115 call_point='stop_check', shell_rc=stop_test_rc(), 1116 stop_on_non_zero_rc=1) 1117 if shell_rc == stop_test_rc(): 1118 message = "Stopping as requested by user.\n" 1119 gp.qprint_time(message) 1120 BuiltIn().fail(message) 1121 1122 1123def obmc_boot_test_py(loc_boot_stack=None, 1124 loc_stack_mode=None, 1125 loc_quiet=None): 1126 r""" 1127 Do main program processing. 1128 """ 1129 1130 global save_stack 1131 1132 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']}) 1133 1134 gp.dprintn() 1135 # Process function parms. 1136 for parm_name in main_func_parm_list: 1137 # Get parm's value. 1138 parm_value = eval("loc_" + parm_name) 1139 gp.dpvars(parm_name, parm_value) 1140 1141 if parm_value is not None: 1142 # Save the global value on a stack. 1143 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1144 parm_name + "}\"), \"" + parm_name + "\")" 1145 gp.dpissuing(cmd_buf) 1146 exec(cmd_buf) 1147 1148 # Set the global value to the passed value. 1149 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1150 "}\", loc_" + parm_name + ")" 1151 gp.dpissuing(cmd_buf) 1152 exec(cmd_buf) 1153 1154 gp.dprintn(save_stack.sprint_obj()) 1155 1156 setup() 1157 1158 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1159 1160 if ffdc_only: 1161 gp.qprint_timen("Caller requested ffdc_only.") 1162 if do_pre_boot_plug_in_setup: 1163 pre_boot_plug_in_setup() 1164 grk.run_key_u("my_ffdc") 1165 return 1166 1167 if delete_errlogs: 1168 # print error logs before delete 1169 status, error_logs = grk.run_key_u("Get Error Logs") 1170 pels = pel.peltool("-l", ignore_err=1) 1171 log.print_error_logs(error_logs, "AdditionalData Message Severity") 1172 gp.qprint_var(pels) 1173 1174 # Delete errlogs prior to doing any boot tests. 1175 grk.run_key(delete_errlogs_cmd, ignore=1) 1176 grk.run_key(delete_bmcdump_cmd, ignore=1) 1177 if redfish_support_trans_state: 1178 grk.run_key(delete_sysdump_cmd, ignore=1) 1179 1180 # Process caller's boot_stack. 1181 while (len(boot_stack) > 0): 1182 test_loop_body() 1183 1184 gp.qprint_timen("Finished processing stack.") 1185 1186 post_stack() 1187 1188 # Process caller's boot_list. 1189 if len(boot_list) > 0: 1190 for ix in range(1, max_num_tests + 1): 1191 test_loop_body() 1192 1193 gp.qprint_timen("Completed all requested boot tests.") 1194 1195 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1196 new_fail = boot_fail - init_boot_fail 1197 if new_fail > boot_fail_threshold: 1198 error_message = "Boot failures exceed the boot failure" +\ 1199 " threshold:\n" +\ 1200 gp.sprint_var(new_fail) +\ 1201 gp.sprint_var(boot_fail_threshold) 1202 BuiltIn().fail(gp.sprint_error(error_message)) 1203