1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_arg as ga 27import gen_valid as gv 28import gen_misc as gm 29import gen_cmd as gc 30import gen_robot_keyword as grk 31import state as st 32import var_stack as vs 33import gen_plug_in_utils as gpu 34 35base_path = os.path.dirname(os.path.dirname( 36 imp.find_module("gen_robot_print")[1])) +\ 37 os.sep 38sys.path.append(base_path + "extended/") 39import run_keyword as rk 40 41redfish = BuiltIn().get_library_instance('redfish') 42 43# Setting master_pid correctly influences the behavior of plug-ins like 44# DB_Logging 45program_pid = os.getpid() 46master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 47pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 48 49# Set up boot data structures. 50os_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 51 52boot_lists = read_boot_lists() 53 54# The maximum number of entries that can be in the boot_history global variable. 55max_boot_history = 10 56boot_history = [] 57 58state = st.return_state_constant('default_state') 59cp_setup_called = 0 60next_boot = "" 61base_tool_dir_path = os.path.normpath(os.environ.get( 62 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 63 64ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 65boot_success = 0 66status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 67if status_dir_path != "": 68 status_dir_path = os.path.normpath(status_dir_path) + os.sep 69redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False) 70if redfish_supported: 71 default_power_on = "Redfish Power On" 72 default_power_off = "Redfish Power Off" 73 delete_errlogs_cmd = "Delete Error Logs" 74 # TODO: delete_errlogs_cmd="Redfish Purge Event Log" 75 # TODO: default_set_power_policy = "Redfish Set Power Restore Policy AlwaysOff" 76 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 77else: 78 default_power_on = "REST Power On" 79 default_power_off = "REST Power Off" 80 delete_errlogs_cmd = "Delete Error Logs" 81 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 82boot_count = 0 83 84LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 85AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 86ffdc_prefix = AUTOBOOT_FFDC_PREFIX 87boot_start_time = "" 88boot_end_time = "" 89save_stack = vs.var_stack('save_stack') 90main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 91 92 93def dump_ffdc_rc(): 94 r""" 95 Return the constant dump ffdc test return code value. 96 97 When a plug-in call point program returns this value, it indicates that 98 this program should collect FFDC. 99 """ 100 101 return 0x00000200 102 103 104def stop_test_rc(): 105 r""" 106 Return the constant stop test return code value. 107 108 When a plug-in call point program returns this value, it indicates that 109 this program should stop running. 110 """ 111 112 return 0x00000200 113 114 115def process_host(host, 116 host_var_name=""): 117 r""" 118 Process a host by getting the associated host name and IP address and 119 setting them in global variables. 120 121 If the caller does not pass the host_var_name, this function will try to 122 figure out the name of the variable used by the caller for the host parm. 123 Callers are advised to explicitly specify the host_var_name when calling 124 with an exec command. In such cases, the get_arg_name cannot figure out 125 the host variable name. 126 127 This function will then create similar global variable names by 128 removing "_host" and appending "_host_name" or "_ip" to the host variable 129 name. 130 131 Example: 132 133 If a call is made like this: 134 process_host(openbmc_host) 135 136 Global variables openbmc_host_name and openbmc_ip will be set. 137 138 Description of argument(s): 139 host A host name or IP. The name of the variable used should 140 have a suffix of "_host". 141 host_var_name The name of the variable being used as the host parm. 142 """ 143 144 if host_var_name == "": 145 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 146 147 host_name_var_name = re.sub("host", "host_name", host_var_name) 148 ip_var_name = re.sub("host", "ip", host_var_name) 149 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 150 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 151 host + "')" 152 exec(cmd_buf) 153 154 155def process_pgm_parms(): 156 r""" 157 Process the program parameters by assigning them all to corresponding 158 globals. Also, set some global values that depend on program parameters. 159 """ 160 161 # Program parameter processing. 162 # Assign all program parms to python variables which are global to this 163 # module. 164 165 global parm_list 166 parm_list = BuiltIn().get_variable_value("${parm_list}") 167 # The following subset of parms should be processed as integers. 168 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 169 'boot_fail_threshold', 'delete_errlogs', 170 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 171 'test_mode', 'debug'] 172 for parm in parm_list: 173 if parm in int_list: 174 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 175 "}\", \"0\"))" 176 else: 177 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 178 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 179 gp.dpissuing(cmd_buf) 180 exec(cmd_buf) 181 if re.match(r".*_host$", parm): 182 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 183 exec(cmd_buf) 184 if re.match(r".*_password$", parm): 185 # Register the value of any parm whose name ends in _password. 186 # This will cause the print functions to replace passwords with 187 # asterisks in the output. 188 cmd_buf = "gp.register_passwords(" + parm + ")" 189 exec(cmd_buf) 190 191 global ffdc_dir_path_style 192 global boot_list 193 global boot_stack 194 global boot_results_file_path 195 global boot_results 196 global boot_history 197 global ffdc_list_file_path 198 global ffdc_report_list_path 199 global ffdc_summary_list_path 200 global boot_table 201 global valid_boot_types 202 203 if ffdc_dir_path_style == "": 204 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 205 206 # Convert these program parms to lists for easier processing.. 207 boot_list = list(filter(None, boot_list.split(":"))) 208 boot_stack = list(filter(None, boot_stack.split(":"))) 209 210 boot_table = create_boot_table(boot_table_path, os_host=os_host) 211 valid_boot_types = create_valid_boot_list(boot_table) 212 213 cleanup_boot_results_file() 214 boot_results_file_path = create_boot_results_file_path(pgm_name, 215 openbmc_nickname, 216 master_pid) 217 218 if os.path.isfile(boot_results_file_path): 219 # We've been called before in this run so we'll load the saved 220 # boot_results and boot_history objects. 221 boot_results, boot_history =\ 222 pickle.load(open(boot_results_file_path, 'rb')) 223 else: 224 boot_results = boot_results(boot_table, boot_pass, boot_fail) 225 226 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 227 "/FFDC_FILE_LIST" 228 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 229 "/FFDC_REPORT_FILE_LIST" 230 231 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 232 "/FFDC_SUMMARY_FILE_LIST" 233 234 235def initial_plug_in_setup(): 236 r""" 237 Initialize all plug-in environment variables which do not change for the 238 duration of the program. 239 240 """ 241 242 global LOG_LEVEL 243 BuiltIn().set_log_level("NONE") 244 245 BuiltIn().set_global_variable("${master_pid}", master_pid) 246 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 247 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 248 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 249 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 250 ffdc_list_file_path) 251 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 252 ffdc_report_list_path) 253 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 254 ffdc_summary_list_path) 255 256 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 257 ffdc_dir_path_style) 258 BuiltIn().set_global_variable("${FFDC_CHECK}", 259 ffdc_check) 260 261 # For each program parameter, set the corresponding AUTOBOOT_ environment 262 # variable value. Also, set an AUTOBOOT_ environment variable for every 263 # element in additional_values. 264 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 265 "status_dir_path", "base_tool_dir_path", 266 "ffdc_list_file_path", "ffdc_report_list_path", 267 "ffdc_summary_list_path", "execdir", "redfish_supported"] 268 269 plug_in_vars = parm_list + additional_values 270 271 for var_name in plug_in_vars: 272 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 273 var_name = var_name.upper() 274 if var_value is None: 275 var_value = "" 276 os.environ["AUTOBOOT_" + var_name] = str(var_value) 277 278 BuiltIn().set_log_level(LOG_LEVEL) 279 280 # Make sure the ffdc list directory exists. 281 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 282 if not os.path.exists(ffdc_list_dir_path): 283 os.makedirs(ffdc_list_dir_path) 284 285 286def plug_in_setup(): 287 r""" 288 Initialize all changing plug-in environment variables for use by the 289 plug-in programs. 290 """ 291 292 global LOG_LEVEL 293 global test_really_running 294 295 BuiltIn().set_log_level("NONE") 296 297 boot_pass, boot_fail = boot_results.return_total_pass_fail() 298 if boot_pass > 1: 299 test_really_running = 1 300 else: 301 test_really_running = 0 302 303 BuiltIn().set_global_variable("${test_really_running}", 304 test_really_running) 305 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 306 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 307 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 308 BuiltIn().set_global_variable("${boot_success}", boot_success) 309 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 310 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 311 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 312 313 # For each program parameter, set the corresponding AUTOBOOT_ environment 314 # variable value. Also, set an AUTOBOOT_ environment variable for every 315 # element in additional_values. 316 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 317 "boot_fail", "test_really_running", "ffdc_prefix", 318 "boot_start_time", "boot_end_time"] 319 320 plug_in_vars = additional_values 321 322 for var_name in plug_in_vars: 323 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 324 var_name = var_name.upper() 325 if var_value is None: 326 var_value = "" 327 os.environ["AUTOBOOT_" + var_name] = str(var_value) 328 329 if debug: 330 shell_rc, out_buf = \ 331 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 332 333 BuiltIn().set_log_level(LOG_LEVEL) 334 335 336def pre_boot_plug_in_setup(): 337 338 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 339 try: 340 os.remove(ffdc_list_file_path) 341 except OSError: 342 pass 343 344 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 345 try: 346 os.remove(ffdc_report_list_path) 347 except OSError: 348 pass 349 350 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 351 try: 352 os.remove(ffdc_summary_list_path) 353 except OSError: 354 pass 355 356 global ffdc_prefix 357 358 seconds = time.time() 359 loc_time = time.localtime(seconds) 360 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 361 362 ffdc_prefix = openbmc_nickname + "." + time_string 363 364 365def default_sigusr1(signal_number=0, 366 frame=None): 367 r""" 368 Handle SIGUSR1 by doing nothing. 369 370 This function assists in debugging SIGUSR1 processing by printing messages 371 to stdout and to the log.html file. 372 373 Description of argument(s): 374 signal_number The signal number (should always be 10 for SIGUSR1). 375 frame The frame data. 376 """ 377 378 gp.qprintn() 379 gp.qprint_executing() 380 gp.lprint_executing() 381 382 383def set_default_siguser1(): 384 r""" 385 Set the default_sigusr1 function to be the SIGUSR1 handler. 386 """ 387 388 gp.qprintn() 389 gp.qprint_executing() 390 gp.lprint_executing() 391 signal.signal(signal.SIGUSR1, default_sigusr1) 392 393 394def setup(): 395 r""" 396 Do general program setup tasks. 397 """ 398 399 global cp_setup_called 400 global transitional_boot_selected 401 402 gp.qprintn() 403 404 if redfish_supported: 405 redfish.login() 406 407 set_default_siguser1() 408 transitional_boot_selected = False 409 410 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 411 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 412 # If we can't find process_plug_in_packages.py, ssh_pw or 413 # validate_plug_ins.py, then we don't have our repo bin in PATH. 414 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 415 + " ssh_pw validate_plug_ins.py", quiet=1, 416 print_output=0, show_err=0) 417 if shell_rc != 0: 418 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 419 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 420 if robot_pgm_dir_path not in sys.path: 421 sys.path.append(robot_pgm_dir_path) 422 PYTHONPATH = os.environ.get("PYTHONPATH", "") 423 if PYTHONPATH == "": 424 os.environ['PYTHONPATH'] = robot_pgm_dir_path 425 else: 426 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 427 428 validate_parms() 429 430 gp.qprint_pgm_header() 431 432 grk.run_key_u(default_set_power_policy) 433 434 initial_plug_in_setup() 435 436 plug_in_setup() 437 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 438 call_point='setup') 439 if rc != 0: 440 error_message = "Plug-in setup failed.\n" 441 gp.print_error_report(error_message) 442 BuiltIn().fail(error_message) 443 # Setting cp_setup_called lets our Teardown know that it needs to call 444 # the cleanup plug-in call point. 445 cp_setup_called = 1 446 447 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 448 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 449 # FFDC_LOG_PATH is used by "FFDC" keyword. 450 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 451 452 # Also printed by FFDC. 453 global host_name 454 global host_ip 455 host = socket.gethostname() 456 host_name, host_ip = gm.get_host_name_ip(host) 457 458 gp.dprint_var(boot_table) 459 gp.dprint_var(boot_lists) 460 461 462def validate_parms(): 463 r""" 464 Validate all program parameters. 465 """ 466 467 process_pgm_parms() 468 469 gp.qprintn() 470 471 global openbmc_model 472 if openbmc_model == "": 473 status, ret_values =\ 474 grk.run_key_u("Get BMC System Model") 475 openbmc_model = ret_values 476 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 477 gv.set_exit_on_error(True) 478 gv.valid_value(openbmc_host) 479 gv.valid_value(openbmc_username) 480 gv.valid_value(openbmc_password) 481 gv.valid_value(rest_username) 482 gv.valid_value(rest_password) 483 gv.valid_value(ipmi_username) 484 gv.valid_value(ipmi_password) 485 if os_host != "": 486 gv.valid_value(os_username) 487 gv.valid_value(os_password) 488 if pdu_host != "": 489 gv.valid_value(pdu_username) 490 gv.valid_value(pdu_password) 491 gv.valid_integer(pdu_slot_no) 492 if openbmc_serial_host != "": 493 gv.valid_integer(openbmc_serial_port) 494 gv.valid_value(openbmc_model) 495 gv.valid_integer(max_num_tests) 496 gv.valid_integer(boot_pass) 497 gv.valid_integer(boot_fail) 498 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 499 BuiltIn().set_global_variable("${plug_in_packages_list}", 500 plug_in_packages_list) 501 gv.valid_value(stack_mode, valid_values=['normal', 'skip']) 502 gv.set_exit_on_error(False) 503 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 504 error_message = "You must provide either a value for either the" +\ 505 " boot_list or the boot_stack parm.\n" 506 BuiltIn().fail(gp.sprint_error(error_message)) 507 valid_boot_list(boot_list, valid_boot_types) 508 valid_boot_list(boot_stack, valid_boot_types) 509 selected_PDU_boots = list(set(boot_list + boot_stack) 510 & set(boot_lists['PDU_reboot'])) 511 if len(selected_PDU_boots) > 0 and pdu_host == "": 512 error_message = "You have selected the following boots which" +\ 513 " require a PDU host but no value for pdu_host:\n" 514 error_message += gp.sprint_var(selected_PDU_boots) 515 error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 516 BuiltIn().fail(gp.sprint_error(error_message)) 517 518 return 519 520 521def my_get_state(): 522 r""" 523 Get the system state plus a little bit of wrapping. 524 """ 525 526 global state 527 528 req_states = ['epoch_seconds'] + st.default_req_states 529 530 gp.qprint_timen("Getting system state.") 531 if test_mode: 532 state['epoch_seconds'] = int(time.time()) 533 else: 534 state = st.get_state(req_states=req_states, quiet=quiet) 535 gp.qprint_var(state) 536 537 538def valid_state(): 539 r""" 540 Verify that our state dictionary contains no blank values. If we don't get 541 valid state data, we cannot continue to work. 542 """ 543 544 if st.compare_states(state, st.invalid_state_match, 'or'): 545 error_message = "The state dictionary contains blank fields which" +\ 546 " is illegal.\n" + gp.sprint_var(state) 547 BuiltIn().fail(gp.sprint_error(error_message)) 548 549 550def select_boot(): 551 r""" 552 Select a boot test to be run based on our current state and return the 553 chosen boot type. 554 555 Description of arguments: 556 state The state of the machine. 557 """ 558 559 global transitional_boot_selected 560 global boot_stack 561 562 gp.qprint_timen("Selecting a boot test.") 563 564 if transitional_boot_selected and not boot_success: 565 prior_boot = next_boot 566 boot_candidate = boot_stack.pop() 567 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 568 + " transition to a valid state for '" + boot_candidate 569 + "' which was at the top of the boot_stack. Since" 570 + " the '" + next_boot + "' failed, the '" 571 + boot_candidate + "' has been removed from the stack" 572 + " to avoid and endless failure loop.") 573 if len(boot_stack) == 0: 574 return "" 575 576 my_get_state() 577 valid_state() 578 579 transitional_boot_selected = False 580 stack_popped = 0 581 if len(boot_stack) > 0: 582 stack_popped = 1 583 gp.qprint_dashes() 584 gp.qprint_var(boot_stack) 585 gp.qprint_dashes() 586 skip_boot_printed = 0 587 while len(boot_stack) > 0: 588 boot_candidate = boot_stack.pop() 589 if stack_mode == 'normal': 590 break 591 else: 592 if st.compare_states(state, boot_table[boot_candidate]['end']): 593 if not skip_boot_printed: 594 gp.qprint_var(stack_mode) 595 gp.qprintn() 596 gp.qprint_timen("Skipping the following boot tests" 597 + " which are unnecessary since their" 598 + " required end states match the" 599 + " current machine state:") 600 skip_boot_printed = 1 601 gp.qprint_var(boot_candidate) 602 boot_candidate = "" 603 if boot_candidate == "": 604 gp.qprint_dashes() 605 gp.qprint_var(boot_stack) 606 gp.qprint_dashes() 607 return boot_candidate 608 if st.compare_states(state, boot_table[boot_candidate]['start']): 609 gp.qprint_timen("The machine state is valid for a '" 610 + boot_candidate + "' boot test.") 611 gp.qprint_dashes() 612 gp.qprint_var(boot_stack) 613 gp.qprint_dashes() 614 return boot_candidate 615 else: 616 gp.qprint_timen("The machine state does not match the required" 617 + " starting state for a '" + boot_candidate 618 + "' boot test:") 619 gp.qprint_varx("boot_table_start_entry", 620 boot_table[boot_candidate]['start']) 621 boot_stack.append(boot_candidate) 622 transitional_boot_selected = True 623 popped_boot = boot_candidate 624 625 # Loop through your list selecting a boot_candidates 626 boot_candidates = [] 627 for boot_candidate in boot_list: 628 if st.compare_states(state, boot_table[boot_candidate]['start']): 629 if stack_popped: 630 if st.compare_states(boot_table[boot_candidate]['end'], 631 boot_table[popped_boot]['start']): 632 boot_candidates.append(boot_candidate) 633 else: 634 boot_candidates.append(boot_candidate) 635 636 if len(boot_candidates) == 0: 637 gp.qprint_timen("The user's boot list contained no boot tests" 638 + " which are valid for the current machine state.") 639 boot_candidate = default_power_on 640 if not st.compare_states(state, boot_table[default_power_on]['start']): 641 boot_candidate = default_power_off 642 boot_candidates.append(boot_candidate) 643 gp.qprint_timen("Using default '" + boot_candidate 644 + "' boot type to transition to valid state.") 645 646 gp.dprint_var(boot_candidates) 647 648 # Randomly select a boot from the candidate list. 649 boot = random.choice(boot_candidates) 650 651 return boot 652 653 654def print_defect_report(ffdc_file_list): 655 r""" 656 Print a defect report. 657 658 Description of argument(s): 659 ffdc_file_list A list of files which were collected by our ffdc functions. 660 """ 661 662 # Making deliberate choice to NOT run plug_in_setup(). We don't want 663 # ffdc_prefix updated. 664 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 665 call_point='ffdc_report', stop_on_plug_in_failure=0) 666 667 # Get additional header data which may have been created by ffdc plug-ins. 668 # Also, delete the individual header files to cleanup. 669 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 670 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 671 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 672 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 673 show_err=0) 674 675 # Get additional summary data which may have been created by ffdc plug-ins. 676 # Also, delete the individual header files to cleanup. 677 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 678 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 679 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 680 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 681 show_err=0) 682 683 # ffdc_list_file_path contains a list of any ffdc files created by plug- 684 # ins, etc. Read that data into a list. 685 try: 686 plug_in_ffdc_list = \ 687 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 688 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 689 except IOError: 690 plug_in_ffdc_list = [] 691 692 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 693 # in. Eliminate duplicates and sort the list. 694 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 695 696 if status_file_path != "": 697 ffdc_file_list.insert(0, status_file_path) 698 699 # Convert the list to a printable list. 700 printable_ffdc_file_list = "\n".join(ffdc_file_list) 701 702 # Open ffdc_file_list for writing. We will write a complete list of 703 # FFDC files to it for possible use by plug-ins like cp_stop_check. 704 ffdc_list_file = open(ffdc_list_file_path, 'w') 705 ffdc_list_file.write(printable_ffdc_file_list + "\n") 706 ffdc_list_file.close() 707 708 indent = 0 709 width = 90 710 linefeed = 1 711 char = "=" 712 713 gp.qprintn() 714 gp.qprint_dashes(indent, width, linefeed, char) 715 gp.qprintn("Copy this data to the defect:\n") 716 717 if len(more_header_info) > 0: 718 gp.qprintn(more_header_info) 719 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 720 openbmc_host_name, openbmc_ip, openbmc_username, 721 openbmc_password, rest_username, rest_password, ipmi_username, 722 ipmi_password, os_host, os_host_name, os_ip, os_username, 723 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 724 pdu_password, pdu_slot_no, openbmc_serial_host, 725 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 726 727 gp.qprintn() 728 print_boot_history(boot_history) 729 gp.qprintn() 730 gp.qprint_var(state) 731 gp.qprintn() 732 gp.qprintn("FFDC data files:") 733 gp.qprintn(printable_ffdc_file_list) 734 gp.qprintn() 735 736 if len(ffdc_summary_info) > 0: 737 gp.qprintn(ffdc_summary_info) 738 739 gp.qprint_dashes(indent, width, linefeed, char) 740 741 742def my_ffdc(): 743 r""" 744 Collect FFDC data. 745 """ 746 747 global state 748 749 plug_in_setup() 750 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 751 call_point='ffdc', stop_on_plug_in_failure=0) 752 753 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 754 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 755 + AUTOBOOT_FFDC_PREFIX 756 + " ffdc_function_list=" 757 + ffdc_function_list, ignore=1) 758 if status != 'PASS': 759 gp.qprint_error("Call to ffdc failed.\n") 760 if type(ffdc_file_list) is not list: 761 ffdc_file_list = [] 762 # Leave a record for caller that "soft" errors occurred. 763 soft_errors = 1 764 gpu.save_plug_in_value(soft_errors, pgm_name) 765 766 my_get_state() 767 768 print_defect_report(ffdc_file_list) 769 770 771def print_test_start_message(boot_keyword): 772 r""" 773 Print a message indicating what boot test is about to run. 774 775 Description of arguments: 776 boot_keyword The name of the boot which is to be run 777 (e.g. "BMC Power On"). 778 """ 779 780 global boot_history 781 global boot_start_time 782 783 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 784 785 # Set boot_start_time for use by plug-ins. 786 boot_start_time = doing_msg[1:33] 787 gp.qprint_var(boot_start_time) 788 789 gp.qprint(doing_msg) 790 791 update_boot_history(boot_history, doing_msg, max_boot_history) 792 793 794def stop_boot_test(signal_number=0, 795 frame=None): 796 r""" 797 Handle SIGUSR1 by aborting the boot test that is running. 798 799 Description of argument(s): 800 signal_number The signal number (should always be 10 for SIGUSR1). 801 frame The frame data. 802 """ 803 804 gp.qprintn() 805 gp.qprint_executing() 806 gp.lprint_executing() 807 808 # Restore original sigusr1 handler. 809 set_default_siguser1() 810 811 message = "The caller has asked that the boot test be stopped and marked" 812 message += " as a failure." 813 814 function_stack = gm.get_function_stack() 815 if "wait_state" in function_stack: 816 st.set_exit_wait_early_message(message) 817 else: 818 BuiltIn().fail(gp.sprint_error(message)) 819 820 821def run_boot(boot): 822 r""" 823 Run the specified boot. 824 825 Description of arguments: 826 boot The name of the boot test to be performed. 827 """ 828 829 global state 830 831 signal.signal(signal.SIGUSR1, stop_boot_test) 832 gp.qprint_timen("stop_boot_test is armed.") 833 834 print_test_start_message(boot) 835 836 plug_in_setup() 837 rc, shell_rc, failed_plug_in_name = \ 838 grpi.rprocess_plug_in_packages(call_point="pre_boot") 839 if rc != 0: 840 error_message = "Plug-in failed with non-zero return code.\n" +\ 841 gp.sprint_var(rc, fmt=gp.hexa()) 842 set_default_siguser1() 843 BuiltIn().fail(gp.sprint_error(error_message)) 844 845 if test_mode: 846 # In test mode, we'll pretend the boot worked by assigning its 847 # required end state to the default state value. 848 state = st.strip_anchor_state(boot_table[boot]['end']) 849 else: 850 # Assertion: We trust that the state data was made fresh by the 851 # caller. 852 853 gp.qprintn() 854 855 if boot_table[boot]['method_type'] == "keyword": 856 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 857 boot_table[boot]['method'], 858 quiet=quiet) 859 860 if boot_table[boot]['bmc_reboot']: 861 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 862 plug_in_setup() 863 rc, shell_rc, failed_plug_in_name = \ 864 grpi.rprocess_plug_in_packages(call_point="post_reboot") 865 if rc != 0: 866 error_message = "Plug-in failed with non-zero return code.\n" 867 error_message += gp.sprint_var(rc, fmt=gp.hexa()) 868 set_default_siguser1() 869 BuiltIn().fail(gp.sprint_error(error_message)) 870 else: 871 match_state = st.anchor_state(state) 872 del match_state['epoch_seconds'] 873 # Wait for the state to change in any way. 874 st.wait_state(match_state, wait_time=state_change_timeout, 875 interval="10 seconds", invert=1) 876 877 gp.qprintn() 878 if boot_table[boot]['end']['chassis'] == "Off": 879 boot_timeout = power_off_timeout 880 else: 881 boot_timeout = power_on_timeout 882 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 883 interval="10 seconds") 884 885 plug_in_setup() 886 rc, shell_rc, failed_plug_in_name = \ 887 grpi.rprocess_plug_in_packages(call_point="post_boot") 888 if rc != 0: 889 error_message = "Plug-in failed with non-zero return code.\n" +\ 890 gp.sprint_var(rc, fmt=gp.hexa()) 891 set_default_siguser1() 892 BuiltIn().fail(gp.sprint_error(error_message)) 893 894 # Restore original sigusr1 handler. 895 set_default_siguser1() 896 897 898def test_loop_body(): 899 r""" 900 The main loop body for the loop in main_py. 901 902 Description of arguments: 903 boot_count The iteration number (starts at 1). 904 """ 905 906 global boot_count 907 global state 908 global next_boot 909 global boot_success 910 global boot_end_time 911 912 gp.qprintn() 913 914 next_boot = select_boot() 915 if next_boot == "": 916 return True 917 918 boot_count += 1 919 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 920 921 pre_boot_plug_in_setup() 922 923 cmd_buf = ["run_boot", next_boot] 924 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 925 if boot_status == "FAIL": 926 gp.qprint(msg) 927 928 gp.qprintn() 929 if boot_status == "PASS": 930 boot_success = 1 931 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 932 + "\" succeeded.") 933 else: 934 boot_success = 0 935 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 936 + "\" failed.") 937 938 # Set boot_end_time for use by plug-ins. 939 boot_end_time = completion_msg[1:33] 940 gp.qprint_var(boot_end_time) 941 942 gp.qprint(completion_msg) 943 944 boot_results.update(next_boot, boot_status) 945 946 plug_in_setup() 947 # NOTE: A post_test_case call point failure is NOT counted as a boot 948 # failure. 949 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 950 call_point='post_test_case', stop_on_plug_in_failure=0) 951 952 plug_in_setup() 953 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 954 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 955 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 956 if ffdc_check == "All" or\ 957 shell_rc == dump_ffdc_rc(): 958 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 959 if status != 'PASS': 960 gp.qprint_error("Call to my_ffdc failed.\n") 961 # Leave a record for caller that "soft" errors occurred. 962 soft_errors = 1 963 gpu.save_plug_in_value(soft_errors, pgm_name) 964 965 if delete_errlogs: 966 # We need to purge error logs between boots or they build up. 967 grk.run_key(delete_errlogs_cmd, ignore=1) 968 969 boot_results.print_report() 970 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 971 972 plug_in_setup() 973 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 974 call_point='stop_check', shell_rc=stop_test_rc(), 975 stop_on_non_zero_rc=1) 976 if shell_rc == stop_test_rc(): 977 message = "Stopping as requested by user.\n" 978 gp.qprint_time(message) 979 BuiltIn().fail(message) 980 981 # This should help prevent ConnectionErrors. 982 grk.run_key_u("Close All Connections") 983 984 return True 985 986 987def obmc_boot_test_teardown(): 988 r""" 989 Clean up after the main keyword. 990 """ 991 gp.qprint_executing() 992 993 if ga.psutil_imported: 994 ga.terminate_descendants() 995 996 if cp_setup_called: 997 plug_in_setup() 998 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 999 call_point='cleanup', stop_on_plug_in_failure=0) 1000 1001 if 'boot_results_file_path' in globals(): 1002 # Save boot_results and boot_history objects to a file in case they are 1003 # needed again. 1004 gp.qprint_timen("Saving boot_results to the following path.") 1005 gp.qprint_var(boot_results_file_path) 1006 pickle.dump((boot_results, boot_history), 1007 open(boot_results_file_path, 'wb'), 1008 pickle.HIGHEST_PROTOCOL) 1009 1010 global save_stack 1011 # Restore any global values saved on the save_stack. 1012 for parm_name in main_func_parm_list: 1013 # Get the parm_value if it was saved on the stack. 1014 try: 1015 parm_value = save_stack.pop(parm_name) 1016 except BaseException: 1017 # If it was not saved, no further action is required. 1018 continue 1019 1020 # Restore the saved value. 1021 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1022 "}\", parm_value)" 1023 gp.dpissuing(cmd_buf) 1024 exec(cmd_buf) 1025 1026 gp.dprintn(save_stack.sprint_obj()) 1027 1028 1029def test_teardown(): 1030 r""" 1031 Clean up after this test case. 1032 """ 1033 1034 gp.qprintn() 1035 gp.qprint_executing() 1036 1037 if ga.psutil_imported: 1038 ga.terminate_descendants() 1039 1040 cmd_buf = ["Print Error", 1041 "A keyword timeout occurred ending this program.\n"] 1042 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1043 1044 if redfish_supported: 1045 redfish.logout() 1046 1047 gp.qprint_pgm_footer() 1048 1049 1050def post_stack(): 1051 r""" 1052 Process post_stack plug-in programs. 1053 """ 1054 1055 if not call_post_stack_plug: 1056 # The caller does not wish to have post_stack plug-in processing done. 1057 return 1058 1059 global boot_success 1060 1061 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1062 pre_boot_plug_in_setup() 1063 # For the purposes of the following plug-ins, mark the "boot" as a success. 1064 boot_success = 1 1065 plug_in_setup() 1066 rc, shell_rc, failed_plug_in_name, history =\ 1067 grpi.rprocess_plug_in_packages(call_point='post_stack', 1068 stop_on_plug_in_failure=0, 1069 return_history=True) 1070 for doing_msg in history: 1071 update_boot_history(boot_history, doing_msg, max_boot_history) 1072 if rc != 0: 1073 boot_success = 0 1074 1075 plug_in_setup() 1076 rc, shell_rc, failed_plug_in_name =\ 1077 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1078 shell_rc=dump_ffdc_rc(), 1079 stop_on_plug_in_failure=1, 1080 stop_on_non_zero_rc=1) 1081 if shell_rc == dump_ffdc_rc(): 1082 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1083 if status != 'PASS': 1084 gp.qprint_error("Call to my_ffdc failed.\n") 1085 # Leave a record for caller that "soft" errors occurred. 1086 soft_errors = 1 1087 gpu.save_plug_in_value(soft_errors, pgm_name) 1088 1089 plug_in_setup() 1090 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1091 call_point='stop_check', shell_rc=stop_test_rc(), 1092 stop_on_non_zero_rc=1) 1093 if shell_rc == stop_test_rc(): 1094 message = "Stopping as requested by user.\n" 1095 gp.qprint_time(message) 1096 BuiltIn().fail(message) 1097 1098 1099def obmc_boot_test_py(loc_boot_stack=None, 1100 loc_stack_mode=None, 1101 loc_quiet=None): 1102 r""" 1103 Do main program processing. 1104 """ 1105 1106 global save_stack 1107 1108 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']}) 1109 1110 gp.dprintn() 1111 # Process function parms. 1112 for parm_name in main_func_parm_list: 1113 # Get parm's value. 1114 parm_value = eval("loc_" + parm_name) 1115 gp.dpvars(parm_name, parm_value) 1116 1117 if parm_value is not None: 1118 # Save the global value on a stack. 1119 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1120 parm_name + "}\"), \"" + parm_name + "\")" 1121 gp.dpissuing(cmd_buf) 1122 exec(cmd_buf) 1123 1124 # Set the global value to the passed value. 1125 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1126 "}\", loc_" + parm_name + ")" 1127 gp.dpissuing(cmd_buf) 1128 exec(cmd_buf) 1129 1130 gp.dprintn(save_stack.sprint_obj()) 1131 1132 setup() 1133 1134 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1135 1136 if ffdc_only: 1137 gp.qprint_timen("Caller requested ffdc_only.") 1138 if do_pre_boot_plug_in_setup: 1139 pre_boot_plug_in_setup() 1140 grk.run_key_u("my_ffdc") 1141 return 1142 1143 if delete_errlogs: 1144 # Delete errlogs prior to doing any boot tests. 1145 grk.run_key(delete_errlogs_cmd, ignore=1) 1146 1147 # Process caller's boot_stack. 1148 while (len(boot_stack) > 0): 1149 test_loop_body() 1150 1151 gp.qprint_timen("Finished processing stack.") 1152 1153 post_stack() 1154 1155 # Process caller's boot_list. 1156 if len(boot_list) > 0: 1157 for ix in range(1, max_num_tests + 1): 1158 test_loop_body() 1159 1160 gp.qprint_timen("Completed all requested boot tests.") 1161 1162 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1163 new_fail = boot_fail - init_boot_fail 1164 if new_fail > boot_fail_threshold: 1165 error_message = "Boot failures exceed the boot failure" +\ 1166 " threshold:\n" +\ 1167 gp.sprint_var(new_fail) +\ 1168 gp.sprint_var(boot_fail_threshold) 1169 BuiltIn().fail(gp.sprint_error(error_message)) 1170