1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_arg as ga 27import gen_valid as gv 28import gen_misc as gm 29import gen_cmd as gc 30import gen_robot_keyword as grk 31import state as st 32import var_stack as vs 33import gen_plug_in_utils as gpu 34 35base_path = os.path.dirname(os.path.dirname( 36 imp.find_module("gen_robot_print")[1])) +\ 37 os.sep 38sys.path.append(base_path + "extended/") 39import run_keyword as rk 40 41# Setting master_pid correctly influences the behavior of plug-ins like 42# DB_Logging 43program_pid = os.getpid() 44master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 45pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 46 47# Set up boot data structures. 48os_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 49 50boot_lists = read_boot_lists() 51 52# The maximum number of entries that can be in the boot_history global variable. 53max_boot_history = 10 54boot_history = [] 55 56state = st.return_state_constant('default_state') 57cp_setup_called = 0 58next_boot = "" 59base_tool_dir_path = os.path.normpath(os.environ.get( 60 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 61 62ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 63boot_success = 0 64status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 65if status_dir_path != "": 66 status_dir_path = os.path.normpath(status_dir_path) + os.sep 67redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False) 68if redfish_supported: 69 redfish = BuiltIn().get_library_instance('redfish') 70 default_power_on = "Redfish Power On" 71 default_power_off = "Redfish Power Off" 72 delete_errlogs_cmd = "Delete Error Logs" 73 # TODO: delete_errlogs_cmd="Redfish Purge Event Log" 74 # TODO: default_set_power_policy = "Redfish Set Power Restore Policy AlwaysOff" 75 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 76else: 77 default_power_on = "REST Power On" 78 default_power_off = "REST Power Off" 79 delete_errlogs_cmd = "Delete Error Logs" 80 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 81boot_count = 0 82 83LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 84AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 85ffdc_prefix = AUTOBOOT_FFDC_PREFIX 86boot_start_time = "" 87boot_end_time = "" 88save_stack = vs.var_stack('save_stack') 89main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 90 91 92def dump_ffdc_rc(): 93 r""" 94 Return the constant dump ffdc test return code value. 95 96 When a plug-in call point program returns this value, it indicates that 97 this program should collect FFDC. 98 """ 99 100 return 0x00000200 101 102 103def stop_test_rc(): 104 r""" 105 Return the constant stop test return code value. 106 107 When a plug-in call point program returns this value, it indicates that 108 this program should stop running. 109 """ 110 111 return 0x00000200 112 113 114def process_host(host, 115 host_var_name=""): 116 r""" 117 Process a host by getting the associated host name and IP address and 118 setting them in global variables. 119 120 If the caller does not pass the host_var_name, this function will try to 121 figure out the name of the variable used by the caller for the host parm. 122 Callers are advised to explicitly specify the host_var_name when calling 123 with an exec command. In such cases, the get_arg_name cannot figure out 124 the host variable name. 125 126 This function will then create similar global variable names by 127 removing "_host" and appending "_host_name" or "_ip" to the host variable 128 name. 129 130 Example: 131 132 If a call is made like this: 133 process_host(openbmc_host) 134 135 Global variables openbmc_host_name and openbmc_ip will be set. 136 137 Description of argument(s): 138 host A host name or IP. The name of the variable used should 139 have a suffix of "_host". 140 host_var_name The name of the variable being used as the host parm. 141 """ 142 143 if host_var_name == "": 144 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 145 146 host_name_var_name = re.sub("host", "host_name", host_var_name) 147 ip_var_name = re.sub("host", "ip", host_var_name) 148 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 149 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 150 host + "')" 151 exec(cmd_buf) 152 153 154def process_pgm_parms(): 155 r""" 156 Process the program parameters by assigning them all to corresponding 157 globals. Also, set some global values that depend on program parameters. 158 """ 159 160 # Program parameter processing. 161 # Assign all program parms to python variables which are global to this 162 # module. 163 164 global parm_list 165 parm_list = BuiltIn().get_variable_value("${parm_list}") 166 # The following subset of parms should be processed as integers. 167 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 168 'boot_fail_threshold', 'delete_errlogs', 169 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 170 'test_mode', 'debug'] 171 for parm in parm_list: 172 if parm in int_list: 173 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 174 "}\", \"0\"))" 175 else: 176 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 177 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 178 gp.dpissuing(cmd_buf) 179 exec(cmd_buf) 180 if re.match(r".*_host$", parm): 181 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 182 exec(cmd_buf) 183 if re.match(r".*_password$", parm): 184 # Register the value of any parm whose name ends in _password. 185 # This will cause the print functions to replace passwords with 186 # asterisks in the output. 187 cmd_buf = "gp.register_passwords(" + parm + ")" 188 exec(cmd_buf) 189 190 global ffdc_dir_path_style 191 global boot_list 192 global boot_stack 193 global boot_results_file_path 194 global boot_results 195 global boot_history 196 global ffdc_list_file_path 197 global ffdc_report_list_path 198 global ffdc_summary_list_path 199 global boot_table 200 global valid_boot_types 201 202 if ffdc_dir_path_style == "": 203 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 204 205 # Convert these program parms to lists for easier processing.. 206 boot_list = list(filter(None, boot_list.split(":"))) 207 boot_stack = list(filter(None, boot_stack.split(":"))) 208 209 boot_table = create_boot_table(boot_table_path, os_host=os_host) 210 valid_boot_types = create_valid_boot_list(boot_table) 211 212 cleanup_boot_results_file() 213 boot_results_file_path = create_boot_results_file_path(pgm_name, 214 openbmc_nickname, 215 master_pid) 216 217 if os.path.isfile(boot_results_file_path): 218 # We've been called before in this run so we'll load the saved 219 # boot_results and boot_history objects. 220 boot_results, boot_history =\ 221 pickle.load(open(boot_results_file_path, 'rb')) 222 else: 223 boot_results = boot_results(boot_table, boot_pass, boot_fail) 224 225 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 226 "/FFDC_FILE_LIST" 227 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 228 "/FFDC_REPORT_FILE_LIST" 229 230 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 231 "/FFDC_SUMMARY_FILE_LIST" 232 233 234def initial_plug_in_setup(): 235 r""" 236 Initialize all plug-in environment variables which do not change for the 237 duration of the program. 238 239 """ 240 241 global LOG_LEVEL 242 BuiltIn().set_log_level("NONE") 243 244 BuiltIn().set_global_variable("${master_pid}", master_pid) 245 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 246 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 247 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 248 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 249 ffdc_list_file_path) 250 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 251 ffdc_report_list_path) 252 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 253 ffdc_summary_list_path) 254 255 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 256 ffdc_dir_path_style) 257 BuiltIn().set_global_variable("${FFDC_CHECK}", 258 ffdc_check) 259 260 # For each program parameter, set the corresponding AUTOBOOT_ environment 261 # variable value. Also, set an AUTOBOOT_ environment variable for every 262 # element in additional_values. 263 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 264 "status_dir_path", "base_tool_dir_path", 265 "ffdc_list_file_path", "ffdc_report_list_path", 266 "ffdc_summary_list_path", "execdir", "redfish_supported"] 267 268 plug_in_vars = parm_list + additional_values 269 270 for var_name in plug_in_vars: 271 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 272 var_name = var_name.upper() 273 if var_value is None: 274 var_value = "" 275 os.environ["AUTOBOOT_" + var_name] = str(var_value) 276 277 BuiltIn().set_log_level(LOG_LEVEL) 278 279 # Make sure the ffdc list directory exists. 280 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 281 if not os.path.exists(ffdc_list_dir_path): 282 os.makedirs(ffdc_list_dir_path) 283 284 285def plug_in_setup(): 286 r""" 287 Initialize all changing plug-in environment variables for use by the 288 plug-in programs. 289 """ 290 291 global LOG_LEVEL 292 global test_really_running 293 294 BuiltIn().set_log_level("NONE") 295 296 boot_pass, boot_fail = boot_results.return_total_pass_fail() 297 if boot_pass > 1: 298 test_really_running = 1 299 else: 300 test_really_running = 0 301 302 BuiltIn().set_global_variable("${test_really_running}", 303 test_really_running) 304 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 305 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 306 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 307 BuiltIn().set_global_variable("${boot_success}", boot_success) 308 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 309 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 310 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 311 312 # For each program parameter, set the corresponding AUTOBOOT_ environment 313 # variable value. Also, set an AUTOBOOT_ environment variable for every 314 # element in additional_values. 315 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 316 "boot_fail", "test_really_running", "ffdc_prefix", 317 "boot_start_time", "boot_end_time"] 318 319 plug_in_vars = additional_values 320 321 for var_name in plug_in_vars: 322 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 323 var_name = var_name.upper() 324 if var_value is None: 325 var_value = "" 326 os.environ["AUTOBOOT_" + var_name] = str(var_value) 327 328 if debug: 329 shell_rc, out_buf = \ 330 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 331 332 BuiltIn().set_log_level(LOG_LEVEL) 333 334 335def pre_boot_plug_in_setup(): 336 337 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 338 try: 339 os.remove(ffdc_list_file_path) 340 except OSError: 341 pass 342 343 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 344 try: 345 os.remove(ffdc_report_list_path) 346 except OSError: 347 pass 348 349 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 350 try: 351 os.remove(ffdc_summary_list_path) 352 except OSError: 353 pass 354 355 global ffdc_prefix 356 357 seconds = time.time() 358 loc_time = time.localtime(seconds) 359 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 360 361 ffdc_prefix = openbmc_nickname + "." + time_string 362 363 364def default_sigusr1(signal_number=0, 365 frame=None): 366 r""" 367 Handle SIGUSR1 by doing nothing. 368 369 This function assists in debugging SIGUSR1 processing by printing messages 370 to stdout and to the log.html file. 371 372 Description of argument(s): 373 signal_number The signal number (should always be 10 for SIGUSR1). 374 frame The frame data. 375 """ 376 377 gp.qprintn() 378 gp.qprint_executing() 379 gp.lprint_executing() 380 381 382def set_default_siguser1(): 383 r""" 384 Set the default_sigusr1 function to be the SIGUSR1 handler. 385 """ 386 387 gp.qprintn() 388 gp.qprint_executing() 389 gp.lprint_executing() 390 signal.signal(signal.SIGUSR1, default_sigusr1) 391 392 393def setup(): 394 r""" 395 Do general program setup tasks. 396 """ 397 398 global cp_setup_called 399 global transitional_boot_selected 400 401 gp.qprintn() 402 403 if redfish_supported: 404 redfish.login() 405 406 set_default_siguser1() 407 transitional_boot_selected = False 408 409 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 410 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 411 # If we can't find process_plug_in_packages.py, ssh_pw or 412 # validate_plug_ins.py, then we don't have our repo bin in PATH. 413 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 414 + " ssh_pw validate_plug_ins.py", quiet=1, 415 print_output=0, show_err=0) 416 if shell_rc != 0: 417 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 418 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 419 if robot_pgm_dir_path not in sys.path: 420 sys.path.append(robot_pgm_dir_path) 421 PYTHONPATH = os.environ.get("PYTHONPATH", "") 422 if PYTHONPATH == "": 423 os.environ['PYTHONPATH'] = robot_pgm_dir_path 424 else: 425 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 426 427 validate_parms() 428 429 gp.qprint_pgm_header() 430 431 grk.run_key_u(default_set_power_policy) 432 433 initial_plug_in_setup() 434 435 plug_in_setup() 436 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 437 call_point='setup') 438 if rc != 0: 439 error_message = "Plug-in setup failed.\n" 440 gp.print_error_report(error_message) 441 BuiltIn().fail(error_message) 442 # Setting cp_setup_called lets our Teardown know that it needs to call 443 # the cleanup plug-in call point. 444 cp_setup_called = 1 445 446 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 447 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 448 # FFDC_LOG_PATH is used by "FFDC" keyword. 449 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 450 451 # Also printed by FFDC. 452 global host_name 453 global host_ip 454 host = socket.gethostname() 455 host_name, host_ip = gm.get_host_name_ip(host) 456 457 gp.dprint_var(boot_table) 458 gp.dprint_var(boot_lists) 459 460 461def validate_parms(): 462 r""" 463 Validate all program parameters. 464 """ 465 466 process_pgm_parms() 467 468 gp.qprintn() 469 470 global openbmc_model 471 if openbmc_model == "": 472 status, ret_values =\ 473 grk.run_key_u("Get BMC System Model") 474 openbmc_model = ret_values 475 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 476 gv.set_exit_on_error(True) 477 gv.valid_value(openbmc_host) 478 gv.valid_value(openbmc_username) 479 gv.valid_value(openbmc_password) 480 gv.valid_value(rest_username) 481 gv.valid_value(rest_password) 482 gv.valid_value(ipmi_username) 483 gv.valid_value(ipmi_password) 484 if os_host != "": 485 gv.valid_value(os_username) 486 gv.valid_value(os_password) 487 if pdu_host != "": 488 gv.valid_value(pdu_username) 489 gv.valid_value(pdu_password) 490 gv.valid_integer(pdu_slot_no) 491 if openbmc_serial_host != "": 492 gv.valid_integer(openbmc_serial_port) 493 gv.valid_value(openbmc_model) 494 gv.valid_integer(max_num_tests) 495 gv.valid_integer(boot_pass) 496 gv.valid_integer(boot_fail) 497 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 498 BuiltIn().set_global_variable("${plug_in_packages_list}", 499 plug_in_packages_list) 500 gv.valid_value(stack_mode, valid_values=['normal', 'skip']) 501 gv.set_exit_on_error(False) 502 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 503 error_message = "You must provide either a value for either the" +\ 504 " boot_list or the boot_stack parm.\n" 505 BuiltIn().fail(gp.sprint_error(error_message)) 506 valid_boot_list(boot_list, valid_boot_types) 507 valid_boot_list(boot_stack, valid_boot_types) 508 selected_PDU_boots = list(set(boot_list + boot_stack) 509 & set(boot_lists['PDU_reboot'])) 510 if len(selected_PDU_boots) > 0 and pdu_host == "": 511 error_message = "You have selected the following boots which" +\ 512 " require a PDU host but no value for pdu_host:\n" 513 error_message += gp.sprint_var(selected_PDU_boots) 514 error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 515 BuiltIn().fail(gp.sprint_error(error_message)) 516 517 return 518 519 520def my_get_state(): 521 r""" 522 Get the system state plus a little bit of wrapping. 523 """ 524 525 global state 526 527 req_states = ['epoch_seconds'] + st.default_req_states 528 529 gp.qprint_timen("Getting system state.") 530 if test_mode: 531 state['epoch_seconds'] = int(time.time()) 532 else: 533 state = st.get_state(req_states=req_states, quiet=quiet) 534 gp.qprint_var(state) 535 536 537def valid_state(): 538 r""" 539 Verify that our state dictionary contains no blank values. If we don't get 540 valid state data, we cannot continue to work. 541 """ 542 543 if st.compare_states(state, st.invalid_state_match, 'or'): 544 error_message = "The state dictionary contains blank fields which" +\ 545 " is illegal.\n" + gp.sprint_var(state) 546 BuiltIn().fail(gp.sprint_error(error_message)) 547 548 549def select_boot(): 550 r""" 551 Select a boot test to be run based on our current state and return the 552 chosen boot type. 553 554 Description of arguments: 555 state The state of the machine. 556 """ 557 558 global transitional_boot_selected 559 global boot_stack 560 561 gp.qprint_timen("Selecting a boot test.") 562 563 if transitional_boot_selected and not boot_success: 564 prior_boot = next_boot 565 boot_candidate = boot_stack.pop() 566 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 567 + " transition to a valid state for '" + boot_candidate 568 + "' which was at the top of the boot_stack. Since" 569 + " the '" + next_boot + "' failed, the '" 570 + boot_candidate + "' has been removed from the stack" 571 + " to avoid and endless failure loop.") 572 if len(boot_stack) == 0: 573 return "" 574 575 my_get_state() 576 valid_state() 577 578 transitional_boot_selected = False 579 stack_popped = 0 580 if len(boot_stack) > 0: 581 stack_popped = 1 582 gp.qprint_dashes() 583 gp.qprint_var(boot_stack) 584 gp.qprint_dashes() 585 skip_boot_printed = 0 586 while len(boot_stack) > 0: 587 boot_candidate = boot_stack.pop() 588 if stack_mode == 'normal': 589 break 590 else: 591 if st.compare_states(state, boot_table[boot_candidate]['end']): 592 if not skip_boot_printed: 593 gp.qprint_var(stack_mode) 594 gp.qprintn() 595 gp.qprint_timen("Skipping the following boot tests" 596 + " which are unnecessary since their" 597 + " required end states match the" 598 + " current machine state:") 599 skip_boot_printed = 1 600 gp.qprint_var(boot_candidate) 601 boot_candidate = "" 602 if boot_candidate == "": 603 gp.qprint_dashes() 604 gp.qprint_var(boot_stack) 605 gp.qprint_dashes() 606 return boot_candidate 607 if st.compare_states(state, boot_table[boot_candidate]['start']): 608 gp.qprint_timen("The machine state is valid for a '" 609 + boot_candidate + "' boot test.") 610 gp.qprint_dashes() 611 gp.qprint_var(boot_stack) 612 gp.qprint_dashes() 613 return boot_candidate 614 else: 615 gp.qprint_timen("The machine state does not match the required" 616 + " starting state for a '" + boot_candidate 617 + "' boot test:") 618 gp.qprint_varx("boot_table_start_entry", 619 boot_table[boot_candidate]['start']) 620 boot_stack.append(boot_candidate) 621 transitional_boot_selected = True 622 popped_boot = boot_candidate 623 624 # Loop through your list selecting a boot_candidates 625 boot_candidates = [] 626 for boot_candidate in boot_list: 627 if st.compare_states(state, boot_table[boot_candidate]['start']): 628 if stack_popped: 629 if st.compare_states(boot_table[boot_candidate]['end'], 630 boot_table[popped_boot]['start']): 631 boot_candidates.append(boot_candidate) 632 else: 633 boot_candidates.append(boot_candidate) 634 635 if len(boot_candidates) == 0: 636 gp.qprint_timen("The user's boot list contained no boot tests" 637 + " which are valid for the current machine state.") 638 boot_candidate = default_power_on 639 if not st.compare_states(state, boot_table[default_power_on]['start']): 640 boot_candidate = default_power_off 641 boot_candidates.append(boot_candidate) 642 gp.qprint_timen("Using default '" + boot_candidate 643 + "' boot type to transition to valid state.") 644 645 gp.dprint_var(boot_candidates) 646 647 # Randomly select a boot from the candidate list. 648 boot = random.choice(boot_candidates) 649 650 return boot 651 652 653def print_defect_report(ffdc_file_list): 654 r""" 655 Print a defect report. 656 657 Description of argument(s): 658 ffdc_file_list A list of files which were collected by our ffdc functions. 659 """ 660 661 # Making deliberate choice to NOT run plug_in_setup(). We don't want 662 # ffdc_prefix updated. 663 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 664 call_point='ffdc_report', stop_on_plug_in_failure=0) 665 666 # Get additional header data which may have been created by ffdc plug-ins. 667 # Also, delete the individual header files to cleanup. 668 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 669 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 670 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 671 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 672 show_err=0) 673 674 # Get additional summary data which may have been created by ffdc plug-ins. 675 # Also, delete the individual header files to cleanup. 676 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 677 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 678 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 679 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 680 show_err=0) 681 682 # ffdc_list_file_path contains a list of any ffdc files created by plug- 683 # ins, etc. Read that data into a list. 684 try: 685 plug_in_ffdc_list = \ 686 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 687 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 688 except IOError: 689 plug_in_ffdc_list = [] 690 691 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 692 # in. Eliminate duplicates and sort the list. 693 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 694 695 if status_file_path != "": 696 ffdc_file_list.insert(0, status_file_path) 697 698 # Convert the list to a printable list. 699 printable_ffdc_file_list = "\n".join(ffdc_file_list) 700 701 # Open ffdc_file_list for writing. We will write a complete list of 702 # FFDC files to it for possible use by plug-ins like cp_stop_check. 703 ffdc_list_file = open(ffdc_list_file_path, 'w') 704 ffdc_list_file.write(printable_ffdc_file_list + "\n") 705 ffdc_list_file.close() 706 707 indent = 0 708 width = 90 709 linefeed = 1 710 char = "=" 711 712 gp.qprintn() 713 gp.qprint_dashes(indent, width, linefeed, char) 714 gp.qprintn("Copy this data to the defect:\n") 715 716 if len(more_header_info) > 0: 717 gp.qprintn(more_header_info) 718 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 719 openbmc_host_name, openbmc_ip, openbmc_username, 720 openbmc_password, rest_username, rest_password, ipmi_username, 721 ipmi_password, os_host, os_host_name, os_ip, os_username, 722 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 723 pdu_password, pdu_slot_no, openbmc_serial_host, 724 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 725 726 gp.qprintn() 727 print_boot_history(boot_history) 728 gp.qprintn() 729 gp.qprint_var(state) 730 gp.qprintn() 731 gp.qprintn("FFDC data files:") 732 gp.qprintn(printable_ffdc_file_list) 733 gp.qprintn() 734 735 if len(ffdc_summary_info) > 0: 736 gp.qprintn(ffdc_summary_info) 737 738 gp.qprint_dashes(indent, width, linefeed, char) 739 740 741def my_ffdc(): 742 r""" 743 Collect FFDC data. 744 """ 745 746 global state 747 748 plug_in_setup() 749 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 750 call_point='ffdc', stop_on_plug_in_failure=0) 751 752 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 753 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 754 + AUTOBOOT_FFDC_PREFIX 755 + " ffdc_function_list=" 756 + ffdc_function_list, ignore=1) 757 if status != 'PASS': 758 gp.qprint_error("Call to ffdc failed.\n") 759 if type(ffdc_file_list) is not list: 760 ffdc_file_list = [] 761 # Leave a record for caller that "soft" errors occurred. 762 soft_errors = 1 763 gpu.save_plug_in_value(soft_errors, pgm_name) 764 765 my_get_state() 766 767 print_defect_report(ffdc_file_list) 768 769 770def print_test_start_message(boot_keyword): 771 r""" 772 Print a message indicating what boot test is about to run. 773 774 Description of arguments: 775 boot_keyword The name of the boot which is to be run 776 (e.g. "BMC Power On"). 777 """ 778 779 global boot_history 780 global boot_start_time 781 782 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 783 784 # Set boot_start_time for use by plug-ins. 785 boot_start_time = doing_msg[1:33] 786 gp.qprint_var(boot_start_time) 787 788 gp.qprint(doing_msg) 789 790 update_boot_history(boot_history, doing_msg, max_boot_history) 791 792 793def stop_boot_test(signal_number=0, 794 frame=None): 795 r""" 796 Handle SIGUSR1 by aborting the boot test that is running. 797 798 Description of argument(s): 799 signal_number The signal number (should always be 10 for SIGUSR1). 800 frame The frame data. 801 """ 802 803 gp.qprintn() 804 gp.qprint_executing() 805 gp.lprint_executing() 806 807 # Restore original sigusr1 handler. 808 set_default_siguser1() 809 810 message = "The caller has asked that the boot test be stopped and marked" 811 message += " as a failure." 812 813 function_stack = gm.get_function_stack() 814 if "wait_state" in function_stack: 815 st.set_exit_wait_early_message(message) 816 else: 817 BuiltIn().fail(gp.sprint_error(message)) 818 819 820def run_boot(boot): 821 r""" 822 Run the specified boot. 823 824 Description of arguments: 825 boot The name of the boot test to be performed. 826 """ 827 828 global state 829 830 signal.signal(signal.SIGUSR1, stop_boot_test) 831 gp.qprint_timen("stop_boot_test is armed.") 832 833 print_test_start_message(boot) 834 835 plug_in_setup() 836 rc, shell_rc, failed_plug_in_name = \ 837 grpi.rprocess_plug_in_packages(call_point="pre_boot") 838 if rc != 0: 839 error_message = "Plug-in failed with non-zero return code.\n" +\ 840 gp.sprint_var(rc, fmt=gp.hexa()) 841 set_default_siguser1() 842 BuiltIn().fail(gp.sprint_error(error_message)) 843 844 if test_mode: 845 # In test mode, we'll pretend the boot worked by assigning its 846 # required end state to the default state value. 847 state = st.strip_anchor_state(boot_table[boot]['end']) 848 else: 849 # Assertion: We trust that the state data was made fresh by the 850 # caller. 851 852 gp.qprintn() 853 854 if boot_table[boot]['method_type'] == "keyword": 855 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 856 boot_table[boot]['method'], 857 quiet=quiet) 858 859 if boot_table[boot]['bmc_reboot']: 860 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 861 plug_in_setup() 862 rc, shell_rc, failed_plug_in_name = \ 863 grpi.rprocess_plug_in_packages(call_point="post_reboot") 864 if rc != 0: 865 error_message = "Plug-in failed with non-zero return code.\n" 866 error_message += gp.sprint_var(rc, fmt=gp.hexa()) 867 set_default_siguser1() 868 BuiltIn().fail(gp.sprint_error(error_message)) 869 else: 870 match_state = st.anchor_state(state) 871 del match_state['epoch_seconds'] 872 # Wait for the state to change in any way. 873 st.wait_state(match_state, wait_time=state_change_timeout, 874 interval="10 seconds", invert=1) 875 876 gp.qprintn() 877 if boot_table[boot]['end']['chassis'] == "Off": 878 boot_timeout = power_off_timeout 879 else: 880 boot_timeout = power_on_timeout 881 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 882 interval="10 seconds") 883 884 plug_in_setup() 885 rc, shell_rc, failed_plug_in_name = \ 886 grpi.rprocess_plug_in_packages(call_point="post_boot") 887 if rc != 0: 888 error_message = "Plug-in failed with non-zero return code.\n" +\ 889 gp.sprint_var(rc, fmt=gp.hexa()) 890 set_default_siguser1() 891 BuiltIn().fail(gp.sprint_error(error_message)) 892 893 # Restore original sigusr1 handler. 894 set_default_siguser1() 895 896 897def test_loop_body(): 898 r""" 899 The main loop body for the loop in main_py. 900 901 Description of arguments: 902 boot_count The iteration number (starts at 1). 903 """ 904 905 global boot_count 906 global state 907 global next_boot 908 global boot_success 909 global boot_end_time 910 911 gp.qprintn() 912 913 next_boot = select_boot() 914 if next_boot == "": 915 return True 916 917 boot_count += 1 918 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 919 920 pre_boot_plug_in_setup() 921 922 cmd_buf = ["run_boot", next_boot] 923 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 924 if boot_status == "FAIL": 925 gp.qprint(msg) 926 927 gp.qprintn() 928 if boot_status == "PASS": 929 boot_success = 1 930 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 931 + "\" succeeded.") 932 else: 933 boot_success = 0 934 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 935 + "\" failed.") 936 937 # Set boot_end_time for use by plug-ins. 938 boot_end_time = completion_msg[1:33] 939 gp.qprint_var(boot_end_time) 940 941 gp.qprint(completion_msg) 942 943 boot_results.update(next_boot, boot_status) 944 945 plug_in_setup() 946 # NOTE: A post_test_case call point failure is NOT counted as a boot 947 # failure. 948 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 949 call_point='post_test_case', stop_on_plug_in_failure=0) 950 951 plug_in_setup() 952 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 953 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 954 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 955 if ffdc_check == "All" or\ 956 shell_rc == dump_ffdc_rc(): 957 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 958 if status != 'PASS': 959 gp.qprint_error("Call to my_ffdc failed.\n") 960 # Leave a record for caller that "soft" errors occurred. 961 soft_errors = 1 962 gpu.save_plug_in_value(soft_errors, pgm_name) 963 964 if delete_errlogs: 965 # We need to purge error logs between boots or they build up. 966 grk.run_key(delete_errlogs_cmd, ignore=1) 967 968 boot_results.print_report() 969 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 970 971 plug_in_setup() 972 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 973 call_point='stop_check', shell_rc=stop_test_rc(), 974 stop_on_non_zero_rc=1) 975 if shell_rc == stop_test_rc(): 976 message = "Stopping as requested by user.\n" 977 gp.qprint_time(message) 978 BuiltIn().fail(message) 979 980 # This should help prevent ConnectionErrors. 981 grk.run_key_u("Close All Connections") 982 983 return True 984 985 986def obmc_boot_test_teardown(): 987 r""" 988 Clean up after the main keyword. 989 """ 990 gp.qprint_executing() 991 992 if ga.psutil_imported: 993 ga.terminate_descendants() 994 995 if cp_setup_called: 996 plug_in_setup() 997 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 998 call_point='cleanup', stop_on_plug_in_failure=0) 999 1000 if 'boot_results_file_path' in globals(): 1001 # Save boot_results and boot_history objects to a file in case they are 1002 # needed again. 1003 gp.qprint_timen("Saving boot_results to the following path.") 1004 gp.qprint_var(boot_results_file_path) 1005 pickle.dump((boot_results, boot_history), 1006 open(boot_results_file_path, 'wb'), 1007 pickle.HIGHEST_PROTOCOL) 1008 1009 global save_stack 1010 # Restore any global values saved on the save_stack. 1011 for parm_name in main_func_parm_list: 1012 # Get the parm_value if it was saved on the stack. 1013 try: 1014 parm_value = save_stack.pop(parm_name) 1015 except BaseException: 1016 # If it was not saved, no further action is required. 1017 continue 1018 1019 # Restore the saved value. 1020 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1021 "}\", parm_value)" 1022 gp.dpissuing(cmd_buf) 1023 exec(cmd_buf) 1024 1025 gp.dprintn(save_stack.sprint_obj()) 1026 1027 1028def test_teardown(): 1029 r""" 1030 Clean up after this test case. 1031 """ 1032 1033 gp.qprintn() 1034 gp.qprint_executing() 1035 1036 if ga.psutil_imported: 1037 ga.terminate_descendants() 1038 1039 cmd_buf = ["Print Error", 1040 "A keyword timeout occurred ending this program.\n"] 1041 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1042 1043 if redfish_supported: 1044 redfish.logout() 1045 1046 gp.qprint_pgm_footer() 1047 1048 1049def post_stack(): 1050 r""" 1051 Process post_stack plug-in programs. 1052 """ 1053 1054 if not call_post_stack_plug: 1055 # The caller does not wish to have post_stack plug-in processing done. 1056 return 1057 1058 global boot_success 1059 1060 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1061 pre_boot_plug_in_setup() 1062 # For the purposes of the following plug-ins, mark the "boot" as a success. 1063 boot_success = 1 1064 plug_in_setup() 1065 rc, shell_rc, failed_plug_in_name, history =\ 1066 grpi.rprocess_plug_in_packages(call_point='post_stack', 1067 stop_on_plug_in_failure=0, 1068 return_history=True) 1069 for doing_msg in history: 1070 update_boot_history(boot_history, doing_msg, max_boot_history) 1071 if rc != 0: 1072 boot_success = 0 1073 1074 plug_in_setup() 1075 rc, shell_rc, failed_plug_in_name =\ 1076 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1077 shell_rc=dump_ffdc_rc(), 1078 stop_on_plug_in_failure=1, 1079 stop_on_non_zero_rc=1) 1080 if shell_rc == dump_ffdc_rc(): 1081 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1082 if status != 'PASS': 1083 gp.qprint_error("Call to my_ffdc failed.\n") 1084 # Leave a record for caller that "soft" errors occurred. 1085 soft_errors = 1 1086 gpu.save_plug_in_value(soft_errors, pgm_name) 1087 1088 plug_in_setup() 1089 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1090 call_point='stop_check', shell_rc=stop_test_rc(), 1091 stop_on_non_zero_rc=1) 1092 if shell_rc == stop_test_rc(): 1093 message = "Stopping as requested by user.\n" 1094 gp.qprint_time(message) 1095 BuiltIn().fail(message) 1096 1097 1098def obmc_boot_test_py(loc_boot_stack=None, 1099 loc_stack_mode=None, 1100 loc_quiet=None): 1101 r""" 1102 Do main program processing. 1103 """ 1104 1105 global save_stack 1106 1107 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']}) 1108 1109 gp.dprintn() 1110 # Process function parms. 1111 for parm_name in main_func_parm_list: 1112 # Get parm's value. 1113 parm_value = eval("loc_" + parm_name) 1114 gp.dpvars(parm_name, parm_value) 1115 1116 if parm_value is not None: 1117 # Save the global value on a stack. 1118 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1119 parm_name + "}\"), \"" + parm_name + "\")" 1120 gp.dpissuing(cmd_buf) 1121 exec(cmd_buf) 1122 1123 # Set the global value to the passed value. 1124 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1125 "}\", loc_" + parm_name + ")" 1126 gp.dpissuing(cmd_buf) 1127 exec(cmd_buf) 1128 1129 gp.dprintn(save_stack.sprint_obj()) 1130 1131 setup() 1132 1133 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1134 1135 if ffdc_only: 1136 gp.qprint_timen("Caller requested ffdc_only.") 1137 if do_pre_boot_plug_in_setup: 1138 pre_boot_plug_in_setup() 1139 grk.run_key_u("my_ffdc") 1140 return 1141 1142 if delete_errlogs: 1143 # Delete errlogs prior to doing any boot tests. 1144 grk.run_key(delete_errlogs_cmd, ignore=1) 1145 1146 # Process caller's boot_stack. 1147 while (len(boot_stack) > 0): 1148 test_loop_body() 1149 1150 gp.qprint_timen("Finished processing stack.") 1151 1152 post_stack() 1153 1154 # Process caller's boot_list. 1155 if len(boot_list) > 0: 1156 for ix in range(1, max_num_tests + 1): 1157 test_loop_body() 1158 1159 gp.qprint_timen("Completed all requested boot tests.") 1160 1161 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1162 new_fail = boot_fail - init_boot_fail 1163 if new_fail > boot_fail_threshold: 1164 error_message = "Boot failures exceed the boot failure" +\ 1165 " threshold:\n" +\ 1166 gp.sprint_var(new_fail) +\ 1167 gp.sprint_var(boot_fail_threshold) 1168 BuiltIn().fail(gp.sprint_error(error_message)) 1169