1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_arg as ga 27import gen_valid as gv 28import gen_misc as gm 29import gen_cmd as gc 30import gen_robot_keyword as grk 31import state as st 32import var_stack as vs 33import gen_plug_in_utils as gpu 34import pel_utils as pel 35import logging_utils as log 36 37base_path = os.path.dirname(os.path.dirname( 38 imp.find_module("gen_robot_print")[1])) +\ 39 os.sep 40sys.path.append(base_path + "extended/") 41import run_keyword as rk 42 43# Setting master_pid correctly influences the behavior of plug-ins like 44# DB_Logging 45program_pid = os.getpid() 46master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 47pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 48 49# Set up boot data structures. 50os_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 51 52boot_lists = read_boot_lists() 53 54# The maximum number of entries that can be in the boot_history global variable. 55max_boot_history = 10 56boot_history = [] 57 58state = st.return_state_constant('default_state') 59cp_setup_called = 0 60next_boot = "" 61base_tool_dir_path = os.path.normpath(os.environ.get( 62 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 63 64ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 65boot_success = 0 66status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 67if status_dir_path != "": 68 status_dir_path = os.path.normpath(status_dir_path) + os.sep 69redfish_support_trans_state = int(os.environ.get('REDFISH_SUPPORT_TRANS_STATE', 0)) or \ 70 int(BuiltIn().get_variable_value("${REDFISH_SUPPORT_TRANS_STATE}", default=0)) 71redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False) 72redfish_rest_supported = BuiltIn().get_variable_value("${REDFISH_REST_SUPPORTED}", default=False) 73redfish_delete_sessions = int(BuiltIn().get_variable_value("${REDFISH_DELETE_SESSIONS}", default=1)) 74if redfish_supported: 75 redfish = BuiltIn().get_library_instance('redfish') 76 default_power_on = "Redfish Power On" 77 default_power_off = "Redfish Power Off" 78 if not redfish_support_trans_state: 79 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 80 delete_bmcdump_cmd = "Delete All BMC Dump" 81 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 82 else: 83 delete_errlogs_cmd = "Redfish Purge Event Log" 84 delete_bmcdump_cmd = "Redfish Delete All BMC Dumps" 85 delete_sysdump_cmd = "Redfish Delete All System Dumps" 86 default_set_power_policy = "Redfish Set Power Restore Policy AlwaysOff" 87else: 88 default_power_on = "REST Power On" 89 default_power_off = "REST Power Off" 90 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 91 delete_bmcdump_cmd = "Delete All BMC Dump" 92 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 93boot_count = 0 94 95LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 96AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 97ffdc_prefix = AUTOBOOT_FFDC_PREFIX 98boot_start_time = "" 99boot_end_time = "" 100save_stack = vs.var_stack('save_stack') 101main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 102 103 104def dump_ffdc_rc(): 105 r""" 106 Return the constant dump ffdc test return code value. 107 108 When a plug-in call point program returns this value, it indicates that 109 this program should collect FFDC. 110 """ 111 112 return 0x00000200 113 114 115def stop_test_rc(): 116 r""" 117 Return the constant stop test return code value. 118 119 When a plug-in call point program returns this value, it indicates that 120 this program should stop running. 121 """ 122 123 return 0x00000200 124 125 126def process_host(host, 127 host_var_name=""): 128 r""" 129 Process a host by getting the associated host name and IP address and 130 setting them in global variables. 131 132 If the caller does not pass the host_var_name, this function will try to 133 figure out the name of the variable used by the caller for the host parm. 134 Callers are advised to explicitly specify the host_var_name when calling 135 with an exec command. In such cases, the get_arg_name cannot figure out 136 the host variable name. 137 138 This function will then create similar global variable names by 139 removing "_host" and appending "_host_name" or "_ip" to the host variable 140 name. 141 142 Example: 143 144 If a call is made like this: 145 process_host(openbmc_host) 146 147 Global variables openbmc_host_name and openbmc_ip will be set. 148 149 Description of argument(s): 150 host A host name or IP. The name of the variable used should 151 have a suffix of "_host". 152 host_var_name The name of the variable being used as the host parm. 153 """ 154 155 if host_var_name == "": 156 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 157 158 host_name_var_name = re.sub("host", "host_name", host_var_name) 159 ip_var_name = re.sub("host", "ip", host_var_name) 160 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 161 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 162 host + "')" 163 exec(cmd_buf) 164 165 166def process_pgm_parms(): 167 r""" 168 Process the program parameters by assigning them all to corresponding 169 globals. Also, set some global values that depend on program parameters. 170 """ 171 172 # Program parameter processing. 173 # Assign all program parms to python variables which are global to this 174 # module. 175 176 global parm_list 177 parm_list = BuiltIn().get_variable_value("${parm_list}") 178 # The following subset of parms should be processed as integers. 179 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 180 'boot_fail_threshold', 'delete_errlogs', 181 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 182 'test_mode', 'debug'] 183 for parm in parm_list: 184 if parm in int_list: 185 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 186 "}\", \"0\"))" 187 else: 188 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 189 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 190 gp.dpissuing(cmd_buf) 191 exec(cmd_buf) 192 if re.match(r".*_host$", parm): 193 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 194 exec(cmd_buf) 195 if re.match(r".*_password$", parm): 196 # Register the value of any parm whose name ends in _password. 197 # This will cause the print functions to replace passwords with 198 # asterisks in the output. 199 cmd_buf = "gp.register_passwords(" + parm + ")" 200 exec(cmd_buf) 201 202 global ffdc_dir_path_style 203 global boot_list 204 global boot_stack 205 global boot_results_file_path 206 global boot_results 207 global boot_history 208 global ffdc_list_file_path 209 global ffdc_report_list_path 210 global ffdc_summary_list_path 211 global boot_table 212 global valid_boot_types 213 214 if ffdc_dir_path_style == "": 215 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 216 217 # Convert these program parms to lists for easier processing.. 218 boot_list = list(filter(None, boot_list.split(":"))) 219 boot_stack = list(filter(None, boot_stack.split(":"))) 220 221 boot_table = create_boot_table(boot_table_path, os_host=os_host) 222 valid_boot_types = create_valid_boot_list(boot_table) 223 224 cleanup_boot_results_file() 225 boot_results_file_path = create_boot_results_file_path(pgm_name, 226 openbmc_nickname, 227 master_pid) 228 229 if os.path.isfile(boot_results_file_path): 230 # We've been called before in this run so we'll load the saved 231 # boot_results and boot_history objects. 232 boot_results, boot_history =\ 233 pickle.load(open(boot_results_file_path, 'rb')) 234 else: 235 boot_results = boot_results(boot_table, boot_pass, boot_fail) 236 237 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 238 "/FFDC_FILE_LIST" 239 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 240 "/FFDC_REPORT_FILE_LIST" 241 242 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 243 "/FFDC_SUMMARY_FILE_LIST" 244 245 246def initial_plug_in_setup(): 247 r""" 248 Initialize all plug-in environment variables which do not change for the 249 duration of the program. 250 251 """ 252 253 global LOG_LEVEL 254 BuiltIn().set_log_level("NONE") 255 256 BuiltIn().set_global_variable("${master_pid}", master_pid) 257 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 258 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 259 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 260 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 261 ffdc_list_file_path) 262 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 263 ffdc_report_list_path) 264 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 265 ffdc_summary_list_path) 266 267 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 268 ffdc_dir_path_style) 269 BuiltIn().set_global_variable("${FFDC_CHECK}", 270 ffdc_check) 271 272 # For each program parameter, set the corresponding AUTOBOOT_ environment 273 # variable value. Also, set an AUTOBOOT_ environment variable for every 274 # element in additional_values. 275 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 276 "status_dir_path", "base_tool_dir_path", 277 "ffdc_list_file_path", "ffdc_report_list_path", 278 "ffdc_summary_list_path", "execdir", "redfish_supported", 279 "redfish_rest_supported", "redfish_support_trans_state"] 280 281 plug_in_vars = parm_list + additional_values 282 283 for var_name in plug_in_vars: 284 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 285 var_name = var_name.upper() 286 if var_value is None: 287 var_value = "" 288 os.environ["AUTOBOOT_" + var_name] = str(var_value) 289 290 BuiltIn().set_log_level(LOG_LEVEL) 291 292 # Make sure the ffdc list directory exists. 293 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 294 if not os.path.exists(ffdc_list_dir_path): 295 os.makedirs(ffdc_list_dir_path) 296 297 298def plug_in_setup(): 299 r""" 300 Initialize all changing plug-in environment variables for use by the 301 plug-in programs. 302 """ 303 304 global LOG_LEVEL 305 global test_really_running 306 307 BuiltIn().set_log_level("NONE") 308 309 boot_pass, boot_fail = boot_results.return_total_pass_fail() 310 if boot_pass > 1: 311 test_really_running = 1 312 else: 313 test_really_running = 0 314 315 BuiltIn().set_global_variable("${test_really_running}", 316 test_really_running) 317 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 318 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 319 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 320 BuiltIn().set_global_variable("${boot_success}", boot_success) 321 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 322 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 323 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 324 325 # For each program parameter, set the corresponding AUTOBOOT_ environment 326 # variable value. Also, set an AUTOBOOT_ environment variable for every 327 # element in additional_values. 328 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 329 "boot_fail", "test_really_running", "ffdc_prefix", 330 "boot_start_time", "boot_end_time"] 331 332 plug_in_vars = additional_values 333 334 for var_name in plug_in_vars: 335 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 336 var_name = var_name.upper() 337 if var_value is None: 338 var_value = "" 339 os.environ["AUTOBOOT_" + var_name] = str(var_value) 340 341 if debug: 342 shell_rc, out_buf = \ 343 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 344 345 BuiltIn().set_log_level(LOG_LEVEL) 346 347 348def pre_boot_plug_in_setup(): 349 350 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 351 try: 352 os.remove(ffdc_list_file_path) 353 except OSError: 354 pass 355 356 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 357 try: 358 os.remove(ffdc_report_list_path) 359 except OSError: 360 pass 361 362 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 363 try: 364 os.remove(ffdc_summary_list_path) 365 except OSError: 366 pass 367 368 global ffdc_prefix 369 370 seconds = time.time() 371 loc_time = time.localtime(seconds) 372 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 373 374 ffdc_prefix = openbmc_nickname + "." + time_string 375 376 377def default_sigusr1(signal_number=0, 378 frame=None): 379 r""" 380 Handle SIGUSR1 by doing nothing. 381 382 This function assists in debugging SIGUSR1 processing by printing messages 383 to stdout and to the log.html file. 384 385 Description of argument(s): 386 signal_number The signal number (should always be 10 for SIGUSR1). 387 frame The frame data. 388 """ 389 390 gp.qprintn() 391 gp.qprint_executing() 392 gp.lprint_executing() 393 394 395def set_default_siguser1(): 396 r""" 397 Set the default_sigusr1 function to be the SIGUSR1 handler. 398 """ 399 400 gp.qprintn() 401 gp.qprint_executing() 402 gp.lprint_executing() 403 signal.signal(signal.SIGUSR1, default_sigusr1) 404 405 406def setup(): 407 r""" 408 Do general program setup tasks. 409 """ 410 411 global cp_setup_called 412 global transitional_boot_selected 413 414 gp.qprintn() 415 416 if redfish_supported: 417 redfish.login() 418 419 set_default_siguser1() 420 transitional_boot_selected = False 421 422 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 423 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 424 # If we can't find process_plug_in_packages.py, ssh_pw or 425 # validate_plug_ins.py, then we don't have our repo bin in PATH. 426 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 427 + " ssh_pw validate_plug_ins.py", quiet=1, 428 print_output=0, show_err=0) 429 if shell_rc != 0: 430 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 431 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 432 if robot_pgm_dir_path not in sys.path: 433 sys.path.append(robot_pgm_dir_path) 434 PYTHONPATH = os.environ.get("PYTHONPATH", "") 435 if PYTHONPATH == "": 436 os.environ['PYTHONPATH'] = robot_pgm_dir_path 437 else: 438 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 439 440 validate_parms() 441 442 gp.qprint_pgm_header() 443 444 grk.run_key_u(default_set_power_policy) 445 446 initial_plug_in_setup() 447 448 plug_in_setup() 449 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 450 call_point='setup') 451 if rc != 0: 452 error_message = "Plug-in setup failed.\n" 453 gp.print_error_report(error_message) 454 BuiltIn().fail(error_message) 455 # Setting cp_setup_called lets our Teardown know that it needs to call 456 # the cleanup plug-in call point. 457 cp_setup_called = 1 458 459 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 460 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 461 # FFDC_LOG_PATH is used by "FFDC" keyword. 462 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 463 464 # Also printed by FFDC. 465 global host_name 466 global host_ip 467 host = socket.gethostname() 468 host_name, host_ip = gm.get_host_name_ip(host) 469 470 gp.dprint_var(boot_table) 471 gp.dprint_var(boot_lists) 472 473 474def validate_parms(): 475 r""" 476 Validate all program parameters. 477 """ 478 479 process_pgm_parms() 480 481 gp.qprintn() 482 483 global openbmc_model 484 if openbmc_model == "": 485 status, ret_values =\ 486 grk.run_key_u("Get BMC System Model") 487 openbmc_model = ret_values 488 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 489 gv.set_exit_on_error(True) 490 gv.valid_value(openbmc_host) 491 gv.valid_value(openbmc_username) 492 gv.valid_value(openbmc_password) 493 gv.valid_value(rest_username) 494 gv.valid_value(rest_password) 495 gv.valid_value(ipmi_username) 496 gv.valid_value(ipmi_password) 497 if os_host != "": 498 gv.valid_value(os_username) 499 gv.valid_value(os_password) 500 if pdu_host != "": 501 gv.valid_value(pdu_username) 502 gv.valid_value(pdu_password) 503 gv.valid_integer(pdu_slot_no) 504 if openbmc_serial_host != "": 505 gv.valid_integer(openbmc_serial_port) 506 gv.valid_value(openbmc_model) 507 gv.valid_integer(max_num_tests) 508 gv.valid_integer(boot_pass) 509 gv.valid_integer(boot_fail) 510 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 511 BuiltIn().set_global_variable("${plug_in_packages_list}", 512 plug_in_packages_list) 513 gv.valid_value(stack_mode, valid_values=['normal', 'skip']) 514 gv.set_exit_on_error(False) 515 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 516 error_message = "You must provide either a value for either the" +\ 517 " boot_list or the boot_stack parm.\n" 518 BuiltIn().fail(gp.sprint_error(error_message)) 519 valid_boot_list(boot_list, valid_boot_types) 520 valid_boot_list(boot_stack, valid_boot_types) 521 selected_PDU_boots = list(set(boot_list + boot_stack) 522 & set(boot_lists['PDU_reboot'])) 523 if len(selected_PDU_boots) > 0 and pdu_host == "": 524 error_message = "You have selected the following boots which" +\ 525 " require a PDU host but no value for pdu_host:\n" 526 error_message += gp.sprint_var(selected_PDU_boots) 527 error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 528 BuiltIn().fail(gp.sprint_error(error_message)) 529 530 return 531 532 533def my_get_state(): 534 r""" 535 Get the system state plus a little bit of wrapping. 536 """ 537 538 global state 539 540 req_states = ['epoch_seconds'] + st.default_req_states 541 542 gp.qprint_timen("Getting system state.") 543 if test_mode: 544 state['epoch_seconds'] = int(time.time()) 545 else: 546 state = st.get_state(req_states=req_states, quiet=quiet) 547 gp.qprint_var(state) 548 549 550def valid_state(): 551 r""" 552 Verify that our state dictionary contains no blank values. If we don't get 553 valid state data, we cannot continue to work. 554 """ 555 556 if st.compare_states(state, st.invalid_state_match, 'or'): 557 error_message = "The state dictionary contains blank fields which" +\ 558 " is illegal.\n" + gp.sprint_var(state) 559 BuiltIn().fail(gp.sprint_error(error_message)) 560 561 562def select_boot(): 563 r""" 564 Select a boot test to be run based on our current state and return the 565 chosen boot type. 566 567 Description of arguments: 568 state The state of the machine. 569 """ 570 571 global transitional_boot_selected 572 global boot_stack 573 574 gp.qprint_timen("Selecting a boot test.") 575 576 if transitional_boot_selected and not boot_success: 577 prior_boot = next_boot 578 boot_candidate = boot_stack.pop() 579 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 580 + " transition to a valid state for '" + boot_candidate 581 + "' which was at the top of the boot_stack. Since" 582 + " the '" + next_boot + "' failed, the '" 583 + boot_candidate + "' has been removed from the stack" 584 + " to avoid and endless failure loop.") 585 if len(boot_stack) == 0: 586 return "" 587 588 my_get_state() 589 valid_state() 590 591 transitional_boot_selected = False 592 stack_popped = 0 593 if len(boot_stack) > 0: 594 stack_popped = 1 595 gp.qprint_dashes() 596 gp.qprint_var(boot_stack) 597 gp.qprint_dashes() 598 skip_boot_printed = 0 599 while len(boot_stack) > 0: 600 boot_candidate = boot_stack.pop() 601 if stack_mode == 'normal': 602 break 603 else: 604 if st.compare_states(state, boot_table[boot_candidate]['end']): 605 if not skip_boot_printed: 606 gp.qprint_var(stack_mode) 607 gp.qprintn() 608 gp.qprint_timen("Skipping the following boot tests" 609 + " which are unnecessary since their" 610 + " required end states match the" 611 + " current machine state:") 612 skip_boot_printed = 1 613 gp.qprint_var(boot_candidate) 614 boot_candidate = "" 615 if boot_candidate == "": 616 gp.qprint_dashes() 617 gp.qprint_var(boot_stack) 618 gp.qprint_dashes() 619 return boot_candidate 620 if st.compare_states(state, boot_table[boot_candidate]['start']): 621 gp.qprint_timen("The machine state is valid for a '" 622 + boot_candidate + "' boot test.") 623 gp.qprint_dashes() 624 gp.qprint_var(boot_stack) 625 gp.qprint_dashes() 626 return boot_candidate 627 else: 628 gp.qprint_timen("The machine state does not match the required" 629 + " starting state for a '" + boot_candidate 630 + "' boot test:") 631 gp.qprint_varx("boot_table_start_entry", 632 boot_table[boot_candidate]['start']) 633 boot_stack.append(boot_candidate) 634 transitional_boot_selected = True 635 popped_boot = boot_candidate 636 637 # Loop through your list selecting a boot_candidates 638 boot_candidates = [] 639 for boot_candidate in boot_list: 640 if st.compare_states(state, boot_table[boot_candidate]['start']): 641 if stack_popped: 642 if st.compare_states(boot_table[boot_candidate]['end'], 643 boot_table[popped_boot]['start']): 644 boot_candidates.append(boot_candidate) 645 else: 646 boot_candidates.append(boot_candidate) 647 648 if len(boot_candidates) == 0: 649 gp.qprint_timen("The user's boot list contained no boot tests" 650 + " which are valid for the current machine state.") 651 boot_candidate = default_power_on 652 if not st.compare_states(state, boot_table[default_power_on]['start']): 653 boot_candidate = default_power_off 654 boot_candidates.append(boot_candidate) 655 gp.qprint_timen("Using default '" + boot_candidate 656 + "' boot type to transition to valid state.") 657 658 gp.dprint_var(boot_candidates) 659 660 # Randomly select a boot from the candidate list. 661 boot = random.choice(boot_candidates) 662 663 return boot 664 665 666def print_defect_report(ffdc_file_list): 667 r""" 668 Print a defect report. 669 670 Description of argument(s): 671 ffdc_file_list A list of files which were collected by our ffdc functions. 672 """ 673 674 # Making deliberate choice to NOT run plug_in_setup(). We don't want 675 # ffdc_prefix updated. 676 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 677 call_point='ffdc_report', stop_on_plug_in_failure=0) 678 679 # Get additional header data which may have been created by ffdc plug-ins. 680 # Also, delete the individual header files to cleanup. 681 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 682 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 683 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 684 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 685 show_err=0) 686 687 # Get additional summary data which may have been created by ffdc plug-ins. 688 # Also, delete the individual header files to cleanup. 689 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 690 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 691 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 692 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 693 show_err=0) 694 695 # ffdc_list_file_path contains a list of any ffdc files created by plug- 696 # ins, etc. Read that data into a list. 697 try: 698 plug_in_ffdc_list = \ 699 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 700 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 701 except IOError: 702 plug_in_ffdc_list = [] 703 704 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 705 # in. Eliminate duplicates and sort the list. 706 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 707 708 if status_file_path != "": 709 ffdc_file_list.insert(0, status_file_path) 710 711 # Convert the list to a printable list. 712 printable_ffdc_file_list = "\n".join(ffdc_file_list) 713 714 # Open ffdc_file_list for writing. We will write a complete list of 715 # FFDC files to it for possible use by plug-ins like cp_stop_check. 716 ffdc_list_file = open(ffdc_list_file_path, 'w') 717 ffdc_list_file.write(printable_ffdc_file_list + "\n") 718 ffdc_list_file.close() 719 720 indent = 0 721 width = 90 722 linefeed = 1 723 char = "=" 724 725 gp.qprintn() 726 gp.qprint_dashes(indent, width, linefeed, char) 727 gp.qprintn("Copy this data to the defect:\n") 728 729 if len(more_header_info) > 0: 730 gp.qprintn(more_header_info) 731 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 732 openbmc_host_name, openbmc_ip, openbmc_username, 733 openbmc_password, rest_username, rest_password, ipmi_username, 734 ipmi_password, os_host, os_host_name, os_ip, os_username, 735 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 736 pdu_password, pdu_slot_no, openbmc_serial_host, 737 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 738 739 gp.qprintn() 740 print_boot_history(boot_history) 741 gp.qprintn() 742 gp.qprint_var(state) 743 gp.qprintn() 744 gp.qprintn("FFDC data files:") 745 gp.qprintn(printable_ffdc_file_list) 746 gp.qprintn() 747 748 if len(ffdc_summary_info) > 0: 749 gp.qprintn(ffdc_summary_info) 750 751 gp.qprint_dashes(indent, width, linefeed, char) 752 753 754def my_ffdc(): 755 r""" 756 Collect FFDC data. 757 """ 758 759 global state 760 761 plug_in_setup() 762 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 763 call_point='ffdc', stop_on_plug_in_failure=0) 764 765 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 766 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 767 + AUTOBOOT_FFDC_PREFIX 768 + " ffdc_function_list=" 769 + ffdc_function_list, ignore=1) 770 if status != 'PASS': 771 gp.qprint_error("Call to ffdc failed.\n") 772 if type(ffdc_file_list) is not list: 773 ffdc_file_list = [] 774 # Leave a record for caller that "soft" errors occurred. 775 soft_errors = 1 776 gpu.save_plug_in_value(soft_errors, pgm_name) 777 778 my_get_state() 779 780 print_defect_report(ffdc_file_list) 781 782 783def print_test_start_message(boot_keyword): 784 r""" 785 Print a message indicating what boot test is about to run. 786 787 Description of arguments: 788 boot_keyword The name of the boot which is to be run 789 (e.g. "BMC Power On"). 790 """ 791 792 global boot_history 793 global boot_start_time 794 795 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 796 797 # Set boot_start_time for use by plug-ins. 798 boot_start_time = doing_msg[1:33] 799 gp.qprint_var(boot_start_time) 800 801 gp.qprint(doing_msg) 802 803 update_boot_history(boot_history, doing_msg, max_boot_history) 804 805 806def stop_boot_test(signal_number=0, 807 frame=None): 808 r""" 809 Handle SIGUSR1 by aborting the boot test that is running. 810 811 Description of argument(s): 812 signal_number The signal number (should always be 10 for SIGUSR1). 813 frame The frame data. 814 """ 815 816 gp.qprintn() 817 gp.qprint_executing() 818 gp.lprint_executing() 819 820 # Restore original sigusr1 handler. 821 set_default_siguser1() 822 823 message = "The caller has asked that the boot test be stopped and marked" 824 message += " as a failure." 825 826 function_stack = gm.get_function_stack() 827 if "wait_state" in function_stack: 828 st.set_exit_wait_early_message(message) 829 else: 830 BuiltIn().fail(gp.sprint_error(message)) 831 832 833def run_boot(boot): 834 r""" 835 Run the specified boot. 836 837 Description of arguments: 838 boot The name of the boot test to be performed. 839 """ 840 841 global state 842 843 signal.signal(signal.SIGUSR1, stop_boot_test) 844 gp.qprint_timen("stop_boot_test is armed.") 845 846 print_test_start_message(boot) 847 848 plug_in_setup() 849 rc, shell_rc, failed_plug_in_name = \ 850 grpi.rprocess_plug_in_packages(call_point="pre_boot") 851 if rc != 0: 852 error_message = "Plug-in failed with non-zero return code.\n" +\ 853 gp.sprint_var(rc, fmt=gp.hexa()) 854 set_default_siguser1() 855 BuiltIn().fail(gp.sprint_error(error_message)) 856 857 if test_mode: 858 # In test mode, we'll pretend the boot worked by assigning its 859 # required end state to the default state value. 860 state = st.strip_anchor_state(boot_table[boot]['end']) 861 else: 862 # Assertion: We trust that the state data was made fresh by the 863 # caller. 864 865 gp.qprintn() 866 867 if boot_table[boot]['method_type'] == "keyword": 868 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 869 boot_table[boot]['method'], 870 quiet=quiet) 871 872 if boot_table[boot]['bmc_reboot']: 873 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 874 plug_in_setup() 875 rc, shell_rc, failed_plug_in_name = \ 876 grpi.rprocess_plug_in_packages(call_point="post_reboot") 877 if rc != 0: 878 error_message = "Plug-in failed with non-zero return code.\n" 879 error_message += gp.sprint_var(rc, fmt=gp.hexa()) 880 set_default_siguser1() 881 BuiltIn().fail(gp.sprint_error(error_message)) 882 else: 883 match_state = st.anchor_state(state) 884 del match_state['epoch_seconds'] 885 # Wait for the state to change in any way. 886 st.wait_state(match_state, wait_time=state_change_timeout, 887 interval="10 seconds", invert=1) 888 889 gp.qprintn() 890 if boot_table[boot]['end']['chassis'] == "Off": 891 boot_timeout = power_off_timeout 892 else: 893 boot_timeout = power_on_timeout 894 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 895 interval="10 seconds") 896 897 plug_in_setup() 898 rc, shell_rc, failed_plug_in_name = \ 899 grpi.rprocess_plug_in_packages(call_point="post_boot") 900 if rc != 0: 901 error_message = "Plug-in failed with non-zero return code.\n" +\ 902 gp.sprint_var(rc, fmt=gp.hexa()) 903 set_default_siguser1() 904 BuiltIn().fail(gp.sprint_error(error_message)) 905 906 # Restore original sigusr1 handler. 907 set_default_siguser1() 908 909 910def test_loop_body(): 911 r""" 912 The main loop body for the loop in main_py. 913 914 Description of arguments: 915 boot_count The iteration number (starts at 1). 916 """ 917 918 global boot_count 919 global state 920 global next_boot 921 global boot_success 922 global boot_end_time 923 924 gp.qprintn() 925 926 next_boot = select_boot() 927 if next_boot == "": 928 return True 929 930 boot_count += 1 931 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 932 933 pre_boot_plug_in_setup() 934 935 cmd_buf = ["run_boot", next_boot] 936 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 937 if boot_status == "FAIL": 938 gp.qprint(msg) 939 940 gp.qprintn() 941 if boot_status == "PASS": 942 boot_success = 1 943 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 944 + "\" succeeded.") 945 else: 946 boot_success = 0 947 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 948 + "\" failed.") 949 950 # Set boot_end_time for use by plug-ins. 951 boot_end_time = completion_msg[1:33] 952 gp.qprint_var(boot_end_time) 953 954 gp.qprint(completion_msg) 955 956 boot_results.update(next_boot, boot_status) 957 958 plug_in_setup() 959 # NOTE: A post_test_case call point failure is NOT counted as a boot 960 # failure. 961 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 962 call_point='post_test_case', stop_on_plug_in_failure=0) 963 964 plug_in_setup() 965 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 966 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 967 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 968 if ffdc_check == "All" or\ 969 shell_rc == dump_ffdc_rc(): 970 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 971 if status != 'PASS': 972 gp.qprint_error("Call to my_ffdc failed.\n") 973 # Leave a record for caller that "soft" errors occurred. 974 soft_errors = 1 975 gpu.save_plug_in_value(soft_errors, pgm_name) 976 977 if delete_errlogs: 978 # print error logs before delete 979 if redfish_support_trans_state: 980 status, error_logs = grk.run_key_u("Get Redfish Event Logs") 981 log.print_error_logs(error_logs, "AdditionalDataURI Message Severity") 982 else: 983 status, error_logs = grk.run_key_u("Get Error Logs") 984 log.print_error_logs(error_logs, "AdditionalData Message Severity") 985 pels = pel.peltool("-l", ignore_err=1) 986 gp.qprint_var(pels) 987 988 # We need to purge error logs between boots or they build up. 989 grk.run_key(delete_errlogs_cmd, ignore=1) 990 grk.run_key(delete_bmcdump_cmd, ignore=1) 991 if redfish_support_trans_state: 992 grk.run_key(delete_sysdump_cmd, ignore=1) 993 994 boot_results.print_report() 995 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 996 997 plug_in_setup() 998 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 999 call_point='stop_check', shell_rc=stop_test_rc(), 1000 stop_on_non_zero_rc=1) 1001 if shell_rc == stop_test_rc(): 1002 message = "Stopping as requested by user.\n" 1003 gp.qprint_time(message) 1004 BuiltIn().fail(message) 1005 1006 # This should help prevent ConnectionErrors. 1007 # Purge all redfish and REST connection sessions. 1008 if redfish_delete_sessions: 1009 grk.run_key_u("Close All Connections", ignore=1) 1010 grk.run_key_u("Delete All Redfish Sessions", ignore=1) 1011 1012 return True 1013 1014 1015def obmc_boot_test_teardown(): 1016 r""" 1017 Clean up after the main keyword. 1018 """ 1019 gp.qprint_executing() 1020 1021 if ga.psutil_imported: 1022 ga.terminate_descendants() 1023 1024 if cp_setup_called: 1025 plug_in_setup() 1026 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1027 call_point='cleanup', stop_on_plug_in_failure=0) 1028 1029 if 'boot_results_file_path' in globals(): 1030 # Save boot_results and boot_history objects to a file in case they are 1031 # needed again. 1032 gp.qprint_timen("Saving boot_results to the following path.") 1033 gp.qprint_var(boot_results_file_path) 1034 pickle.dump((boot_results, boot_history), 1035 open(boot_results_file_path, 'wb'), 1036 pickle.HIGHEST_PROTOCOL) 1037 1038 global save_stack 1039 # Restore any global values saved on the save_stack. 1040 for parm_name in main_func_parm_list: 1041 # Get the parm_value if it was saved on the stack. 1042 try: 1043 parm_value = save_stack.pop(parm_name) 1044 except BaseException: 1045 # If it was not saved, no further action is required. 1046 continue 1047 1048 # Restore the saved value. 1049 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1050 "}\", parm_value)" 1051 gp.dpissuing(cmd_buf) 1052 exec(cmd_buf) 1053 1054 gp.dprintn(save_stack.sprint_obj()) 1055 1056 1057def test_teardown(): 1058 r""" 1059 Clean up after this test case. 1060 """ 1061 1062 gp.qprintn() 1063 gp.qprint_executing() 1064 1065 if ga.psutil_imported: 1066 ga.terminate_descendants() 1067 1068 cmd_buf = ["Print Error", 1069 "A keyword timeout occurred ending this program.\n"] 1070 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1071 1072 if redfish_supported: 1073 redfish.logout() 1074 1075 gp.qprint_pgm_footer() 1076 1077 1078def post_stack(): 1079 r""" 1080 Process post_stack plug-in programs. 1081 """ 1082 1083 if not call_post_stack_plug: 1084 # The caller does not wish to have post_stack plug-in processing done. 1085 return 1086 1087 global boot_success 1088 1089 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1090 pre_boot_plug_in_setup() 1091 # For the purposes of the following plug-ins, mark the "boot" as a success. 1092 boot_success = 1 1093 plug_in_setup() 1094 rc, shell_rc, failed_plug_in_name, history =\ 1095 grpi.rprocess_plug_in_packages(call_point='post_stack', 1096 stop_on_plug_in_failure=0, 1097 return_history=True) 1098 for doing_msg in history: 1099 update_boot_history(boot_history, doing_msg, max_boot_history) 1100 if rc != 0: 1101 boot_success = 0 1102 1103 plug_in_setup() 1104 rc, shell_rc, failed_plug_in_name =\ 1105 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1106 shell_rc=dump_ffdc_rc(), 1107 stop_on_plug_in_failure=1, 1108 stop_on_non_zero_rc=1) 1109 if shell_rc == dump_ffdc_rc(): 1110 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1111 if status != 'PASS': 1112 gp.qprint_error("Call to my_ffdc failed.\n") 1113 # Leave a record for caller that "soft" errors occurred. 1114 soft_errors = 1 1115 gpu.save_plug_in_value(soft_errors, pgm_name) 1116 1117 plug_in_setup() 1118 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1119 call_point='stop_check', shell_rc=stop_test_rc(), 1120 stop_on_non_zero_rc=1) 1121 if shell_rc == stop_test_rc(): 1122 message = "Stopping as requested by user.\n" 1123 gp.qprint_time(message) 1124 BuiltIn().fail(message) 1125 1126 1127def obmc_boot_test_py(loc_boot_stack=None, 1128 loc_stack_mode=None, 1129 loc_quiet=None): 1130 r""" 1131 Do main program processing. 1132 """ 1133 1134 global save_stack 1135 1136 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']}) 1137 1138 gp.dprintn() 1139 # Process function parms. 1140 for parm_name in main_func_parm_list: 1141 # Get parm's value. 1142 parm_value = eval("loc_" + parm_name) 1143 gp.dpvars(parm_name, parm_value) 1144 1145 if parm_value is not None: 1146 # Save the global value on a stack. 1147 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1148 parm_name + "}\"), \"" + parm_name + "\")" 1149 gp.dpissuing(cmd_buf) 1150 exec(cmd_buf) 1151 1152 # Set the global value to the passed value. 1153 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1154 "}\", loc_" + parm_name + ")" 1155 gp.dpissuing(cmd_buf) 1156 exec(cmd_buf) 1157 1158 gp.dprintn(save_stack.sprint_obj()) 1159 1160 setup() 1161 1162 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1163 1164 if ffdc_only: 1165 gp.qprint_timen("Caller requested ffdc_only.") 1166 if do_pre_boot_plug_in_setup: 1167 pre_boot_plug_in_setup() 1168 grk.run_key_u("my_ffdc") 1169 return 1170 1171 if delete_errlogs: 1172 # print error logs before delete 1173 if redfish_support_trans_state: 1174 status, error_logs = grk.run_key_u("Get Redfish Event Logs") 1175 log.print_error_logs(error_logs, "AdditionalDataURI Message Severity") 1176 else: 1177 status, error_logs = grk.run_key_u("Get Error Logs") 1178 log.print_error_logs(error_logs, "AdditionalData Message Severity") 1179 pels = pel.peltool("-l", ignore_err=1) 1180 gp.qprint_var(pels) 1181 1182 # Delete errlogs prior to doing any boot tests. 1183 grk.run_key(delete_errlogs_cmd, ignore=1) 1184 grk.run_key(delete_bmcdump_cmd, ignore=1) 1185 if redfish_support_trans_state: 1186 grk.run_key(delete_sysdump_cmd, ignore=1) 1187 1188 # Process caller's boot_stack. 1189 while (len(boot_stack) > 0): 1190 test_loop_body() 1191 1192 gp.qprint_timen("Finished processing stack.") 1193 1194 post_stack() 1195 1196 # Process caller's boot_list. 1197 if len(boot_list) > 0: 1198 for ix in range(1, max_num_tests + 1): 1199 test_loop_body() 1200 1201 gp.qprint_timen("Completed all requested boot tests.") 1202 1203 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1204 new_fail = boot_fail - init_boot_fail 1205 if new_fail > boot_fail_threshold: 1206 error_message = "Boot failures exceed the boot failure" +\ 1207 " threshold:\n" +\ 1208 gp.sprint_var(new_fail) +\ 1209 gp.sprint_var(boot_fail_threshold) 1210 BuiltIn().fail(gp.sprint_error(error_message)) 1211