1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_arg as ga 27import gen_valid as gv 28import gen_misc as gm 29import gen_cmd as gc 30import gen_robot_keyword as grk 31import state as st 32import var_stack as vs 33import gen_plug_in_utils as gpu 34import pel_utils as pel 35import logging_utils as log 36 37base_path = os.path.dirname(os.path.dirname( 38 imp.find_module("gen_robot_print")[1])) +\ 39 os.sep 40sys.path.append(base_path + "extended/") 41import run_keyword as rk 42 43# Setting master_pid correctly influences the behavior of plug-ins like 44# DB_Logging 45program_pid = os.getpid() 46master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 47pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 48 49# Set up boot data structures. 50os_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 51 52boot_lists = read_boot_lists() 53 54# The maximum number of entries that can be in the boot_history global variable. 55max_boot_history = 10 56boot_history = [] 57 58state = st.return_state_constant('default_state') 59cp_setup_called = 0 60next_boot = "" 61base_tool_dir_path = os.path.normpath(os.environ.get( 62 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 63 64ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 65boot_success = 0 66status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 67if status_dir_path != "": 68 status_dir_path = os.path.normpath(status_dir_path) + os.sep 69redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False) 70redfish_rest_supported = BuiltIn().get_variable_value("${REDFISH_REST_SUPPORTED}", default=False) 71if redfish_supported: 72 redfish = BuiltIn().get_library_instance('redfish') 73 default_power_on = "Redfish Power On" 74 default_power_off = "Redfish Power Off" 75 if redfish_rest_supported: 76 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 77 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 78 else: 79 delete_errlogs_cmd = "Redfish Purge Event Log" 80 default_set_power_policy = "Redfish Set Power Restore Policy AlwaysOff" 81else: 82 default_power_on = "REST Power On" 83 default_power_off = "REST Power Off" 84 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 85 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 86boot_count = 0 87 88LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 89AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 90ffdc_prefix = AUTOBOOT_FFDC_PREFIX 91boot_start_time = "" 92boot_end_time = "" 93save_stack = vs.var_stack('save_stack') 94main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 95 96 97def dump_ffdc_rc(): 98 r""" 99 Return the constant dump ffdc test return code value. 100 101 When a plug-in call point program returns this value, it indicates that 102 this program should collect FFDC. 103 """ 104 105 return 0x00000200 106 107 108def stop_test_rc(): 109 r""" 110 Return the constant stop test return code value. 111 112 When a plug-in call point program returns this value, it indicates that 113 this program should stop running. 114 """ 115 116 return 0x00000200 117 118 119def process_host(host, 120 host_var_name=""): 121 r""" 122 Process a host by getting the associated host name and IP address and 123 setting them in global variables. 124 125 If the caller does not pass the host_var_name, this function will try to 126 figure out the name of the variable used by the caller for the host parm. 127 Callers are advised to explicitly specify the host_var_name when calling 128 with an exec command. In such cases, the get_arg_name cannot figure out 129 the host variable name. 130 131 This function will then create similar global variable names by 132 removing "_host" and appending "_host_name" or "_ip" to the host variable 133 name. 134 135 Example: 136 137 If a call is made like this: 138 process_host(openbmc_host) 139 140 Global variables openbmc_host_name and openbmc_ip will be set. 141 142 Description of argument(s): 143 host A host name or IP. The name of the variable used should 144 have a suffix of "_host". 145 host_var_name The name of the variable being used as the host parm. 146 """ 147 148 if host_var_name == "": 149 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 150 151 host_name_var_name = re.sub("host", "host_name", host_var_name) 152 ip_var_name = re.sub("host", "ip", host_var_name) 153 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 154 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 155 host + "')" 156 exec(cmd_buf) 157 158 159def process_pgm_parms(): 160 r""" 161 Process the program parameters by assigning them all to corresponding 162 globals. Also, set some global values that depend on program parameters. 163 """ 164 165 # Program parameter processing. 166 # Assign all program parms to python variables which are global to this 167 # module. 168 169 global parm_list 170 parm_list = BuiltIn().get_variable_value("${parm_list}") 171 # The following subset of parms should be processed as integers. 172 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 173 'boot_fail_threshold', 'delete_errlogs', 174 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 175 'test_mode', 'debug'] 176 for parm in parm_list: 177 if parm in int_list: 178 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 179 "}\", \"0\"))" 180 else: 181 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 182 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 183 gp.dpissuing(cmd_buf) 184 exec(cmd_buf) 185 if re.match(r".*_host$", parm): 186 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 187 exec(cmd_buf) 188 if re.match(r".*_password$", parm): 189 # Register the value of any parm whose name ends in _password. 190 # This will cause the print functions to replace passwords with 191 # asterisks in the output. 192 cmd_buf = "gp.register_passwords(" + parm + ")" 193 exec(cmd_buf) 194 195 global ffdc_dir_path_style 196 global boot_list 197 global boot_stack 198 global boot_results_file_path 199 global boot_results 200 global boot_history 201 global ffdc_list_file_path 202 global ffdc_report_list_path 203 global ffdc_summary_list_path 204 global boot_table 205 global valid_boot_types 206 207 if ffdc_dir_path_style == "": 208 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 209 210 # Convert these program parms to lists for easier processing.. 211 boot_list = list(filter(None, boot_list.split(":"))) 212 boot_stack = list(filter(None, boot_stack.split(":"))) 213 214 boot_table = create_boot_table(boot_table_path, os_host=os_host) 215 valid_boot_types = create_valid_boot_list(boot_table) 216 217 cleanup_boot_results_file() 218 boot_results_file_path = create_boot_results_file_path(pgm_name, 219 openbmc_nickname, 220 master_pid) 221 222 if os.path.isfile(boot_results_file_path): 223 # We've been called before in this run so we'll load the saved 224 # boot_results and boot_history objects. 225 boot_results, boot_history =\ 226 pickle.load(open(boot_results_file_path, 'rb')) 227 else: 228 boot_results = boot_results(boot_table, boot_pass, boot_fail) 229 230 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 231 "/FFDC_FILE_LIST" 232 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 233 "/FFDC_REPORT_FILE_LIST" 234 235 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 236 "/FFDC_SUMMARY_FILE_LIST" 237 238 239def initial_plug_in_setup(): 240 r""" 241 Initialize all plug-in environment variables which do not change for the 242 duration of the program. 243 244 """ 245 246 global LOG_LEVEL 247 BuiltIn().set_log_level("NONE") 248 249 BuiltIn().set_global_variable("${master_pid}", master_pid) 250 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 251 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 252 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 253 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 254 ffdc_list_file_path) 255 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 256 ffdc_report_list_path) 257 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 258 ffdc_summary_list_path) 259 260 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 261 ffdc_dir_path_style) 262 BuiltIn().set_global_variable("${FFDC_CHECK}", 263 ffdc_check) 264 265 # For each program parameter, set the corresponding AUTOBOOT_ environment 266 # variable value. Also, set an AUTOBOOT_ environment variable for every 267 # element in additional_values. 268 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 269 "status_dir_path", "base_tool_dir_path", 270 "ffdc_list_file_path", "ffdc_report_list_path", 271 "ffdc_summary_list_path", "execdir", "redfish_supported", 272 "redfish_rest_supported"] 273 274 plug_in_vars = parm_list + additional_values 275 276 for var_name in plug_in_vars: 277 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 278 var_name = var_name.upper() 279 if var_value is None: 280 var_value = "" 281 os.environ["AUTOBOOT_" + var_name] = str(var_value) 282 283 BuiltIn().set_log_level(LOG_LEVEL) 284 285 # Make sure the ffdc list directory exists. 286 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 287 if not os.path.exists(ffdc_list_dir_path): 288 os.makedirs(ffdc_list_dir_path) 289 290 291def plug_in_setup(): 292 r""" 293 Initialize all changing plug-in environment variables for use by the 294 plug-in programs. 295 """ 296 297 global LOG_LEVEL 298 global test_really_running 299 300 BuiltIn().set_log_level("NONE") 301 302 boot_pass, boot_fail = boot_results.return_total_pass_fail() 303 if boot_pass > 1: 304 test_really_running = 1 305 else: 306 test_really_running = 0 307 308 BuiltIn().set_global_variable("${test_really_running}", 309 test_really_running) 310 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 311 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 312 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 313 BuiltIn().set_global_variable("${boot_success}", boot_success) 314 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 315 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 316 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 317 318 # For each program parameter, set the corresponding AUTOBOOT_ environment 319 # variable value. Also, set an AUTOBOOT_ environment variable for every 320 # element in additional_values. 321 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 322 "boot_fail", "test_really_running", "ffdc_prefix", 323 "boot_start_time", "boot_end_time"] 324 325 plug_in_vars = additional_values 326 327 for var_name in plug_in_vars: 328 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 329 var_name = var_name.upper() 330 if var_value is None: 331 var_value = "" 332 os.environ["AUTOBOOT_" + var_name] = str(var_value) 333 334 if debug: 335 shell_rc, out_buf = \ 336 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 337 338 BuiltIn().set_log_level(LOG_LEVEL) 339 340 341def pre_boot_plug_in_setup(): 342 343 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 344 try: 345 os.remove(ffdc_list_file_path) 346 except OSError: 347 pass 348 349 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 350 try: 351 os.remove(ffdc_report_list_path) 352 except OSError: 353 pass 354 355 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 356 try: 357 os.remove(ffdc_summary_list_path) 358 except OSError: 359 pass 360 361 global ffdc_prefix 362 363 seconds = time.time() 364 loc_time = time.localtime(seconds) 365 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 366 367 ffdc_prefix = openbmc_nickname + "." + time_string 368 369 370def default_sigusr1(signal_number=0, 371 frame=None): 372 r""" 373 Handle SIGUSR1 by doing nothing. 374 375 This function assists in debugging SIGUSR1 processing by printing messages 376 to stdout and to the log.html file. 377 378 Description of argument(s): 379 signal_number The signal number (should always be 10 for SIGUSR1). 380 frame The frame data. 381 """ 382 383 gp.qprintn() 384 gp.qprint_executing() 385 gp.lprint_executing() 386 387 388def set_default_siguser1(): 389 r""" 390 Set the default_sigusr1 function to be the SIGUSR1 handler. 391 """ 392 393 gp.qprintn() 394 gp.qprint_executing() 395 gp.lprint_executing() 396 signal.signal(signal.SIGUSR1, default_sigusr1) 397 398 399def setup(): 400 r""" 401 Do general program setup tasks. 402 """ 403 404 global cp_setup_called 405 global transitional_boot_selected 406 407 gp.qprintn() 408 409 if redfish_supported: 410 redfish.login() 411 412 set_default_siguser1() 413 transitional_boot_selected = False 414 415 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 416 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 417 # If we can't find process_plug_in_packages.py, ssh_pw or 418 # validate_plug_ins.py, then we don't have our repo bin in PATH. 419 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 420 + " ssh_pw validate_plug_ins.py", quiet=1, 421 print_output=0, show_err=0) 422 if shell_rc != 0: 423 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 424 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 425 if robot_pgm_dir_path not in sys.path: 426 sys.path.append(robot_pgm_dir_path) 427 PYTHONPATH = os.environ.get("PYTHONPATH", "") 428 if PYTHONPATH == "": 429 os.environ['PYTHONPATH'] = robot_pgm_dir_path 430 else: 431 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 432 433 validate_parms() 434 435 gp.qprint_pgm_header() 436 437 grk.run_key_u(default_set_power_policy) 438 439 initial_plug_in_setup() 440 441 plug_in_setup() 442 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 443 call_point='setup') 444 if rc != 0: 445 error_message = "Plug-in setup failed.\n" 446 gp.print_error_report(error_message) 447 BuiltIn().fail(error_message) 448 # Setting cp_setup_called lets our Teardown know that it needs to call 449 # the cleanup plug-in call point. 450 cp_setup_called = 1 451 452 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 453 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 454 # FFDC_LOG_PATH is used by "FFDC" keyword. 455 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 456 457 # Also printed by FFDC. 458 global host_name 459 global host_ip 460 host = socket.gethostname() 461 host_name, host_ip = gm.get_host_name_ip(host) 462 463 gp.dprint_var(boot_table) 464 gp.dprint_var(boot_lists) 465 466 467def validate_parms(): 468 r""" 469 Validate all program parameters. 470 """ 471 472 process_pgm_parms() 473 474 gp.qprintn() 475 476 global openbmc_model 477 if openbmc_model == "": 478 status, ret_values =\ 479 grk.run_key_u("Get BMC System Model") 480 openbmc_model = ret_values 481 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 482 gv.set_exit_on_error(True) 483 gv.valid_value(openbmc_host) 484 gv.valid_value(openbmc_username) 485 gv.valid_value(openbmc_password) 486 gv.valid_value(rest_username) 487 gv.valid_value(rest_password) 488 gv.valid_value(ipmi_username) 489 gv.valid_value(ipmi_password) 490 if os_host != "": 491 gv.valid_value(os_username) 492 gv.valid_value(os_password) 493 if pdu_host != "": 494 gv.valid_value(pdu_username) 495 gv.valid_value(pdu_password) 496 gv.valid_integer(pdu_slot_no) 497 if openbmc_serial_host != "": 498 gv.valid_integer(openbmc_serial_port) 499 gv.valid_value(openbmc_model) 500 gv.valid_integer(max_num_tests) 501 gv.valid_integer(boot_pass) 502 gv.valid_integer(boot_fail) 503 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 504 BuiltIn().set_global_variable("${plug_in_packages_list}", 505 plug_in_packages_list) 506 gv.valid_value(stack_mode, valid_values=['normal', 'skip']) 507 gv.set_exit_on_error(False) 508 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 509 error_message = "You must provide either a value for either the" +\ 510 " boot_list or the boot_stack parm.\n" 511 BuiltIn().fail(gp.sprint_error(error_message)) 512 valid_boot_list(boot_list, valid_boot_types) 513 valid_boot_list(boot_stack, valid_boot_types) 514 selected_PDU_boots = list(set(boot_list + boot_stack) 515 & set(boot_lists['PDU_reboot'])) 516 if len(selected_PDU_boots) > 0 and pdu_host == "": 517 error_message = "You have selected the following boots which" +\ 518 " require a PDU host but no value for pdu_host:\n" 519 error_message += gp.sprint_var(selected_PDU_boots) 520 error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 521 BuiltIn().fail(gp.sprint_error(error_message)) 522 523 return 524 525 526def my_get_state(): 527 r""" 528 Get the system state plus a little bit of wrapping. 529 """ 530 531 global state 532 533 req_states = ['epoch_seconds'] + st.default_req_states 534 535 gp.qprint_timen("Getting system state.") 536 if test_mode: 537 state['epoch_seconds'] = int(time.time()) 538 else: 539 state = st.get_state(req_states=req_states, quiet=quiet) 540 gp.qprint_var(state) 541 542 543def valid_state(): 544 r""" 545 Verify that our state dictionary contains no blank values. If we don't get 546 valid state data, we cannot continue to work. 547 """ 548 549 if st.compare_states(state, st.invalid_state_match, 'or'): 550 error_message = "The state dictionary contains blank fields which" +\ 551 " is illegal.\n" + gp.sprint_var(state) 552 BuiltIn().fail(gp.sprint_error(error_message)) 553 554 555def select_boot(): 556 r""" 557 Select a boot test to be run based on our current state and return the 558 chosen boot type. 559 560 Description of arguments: 561 state The state of the machine. 562 """ 563 564 global transitional_boot_selected 565 global boot_stack 566 567 gp.qprint_timen("Selecting a boot test.") 568 569 if transitional_boot_selected and not boot_success: 570 prior_boot = next_boot 571 boot_candidate = boot_stack.pop() 572 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 573 + " transition to a valid state for '" + boot_candidate 574 + "' which was at the top of the boot_stack. Since" 575 + " the '" + next_boot + "' failed, the '" 576 + boot_candidate + "' has been removed from the stack" 577 + " to avoid and endless failure loop.") 578 if len(boot_stack) == 0: 579 return "" 580 581 my_get_state() 582 valid_state() 583 584 transitional_boot_selected = False 585 stack_popped = 0 586 if len(boot_stack) > 0: 587 stack_popped = 1 588 gp.qprint_dashes() 589 gp.qprint_var(boot_stack) 590 gp.qprint_dashes() 591 skip_boot_printed = 0 592 while len(boot_stack) > 0: 593 boot_candidate = boot_stack.pop() 594 if stack_mode == 'normal': 595 break 596 else: 597 if st.compare_states(state, boot_table[boot_candidate]['end']): 598 if not skip_boot_printed: 599 gp.qprint_var(stack_mode) 600 gp.qprintn() 601 gp.qprint_timen("Skipping the following boot tests" 602 + " which are unnecessary since their" 603 + " required end states match the" 604 + " current machine state:") 605 skip_boot_printed = 1 606 gp.qprint_var(boot_candidate) 607 boot_candidate = "" 608 if boot_candidate == "": 609 gp.qprint_dashes() 610 gp.qprint_var(boot_stack) 611 gp.qprint_dashes() 612 return boot_candidate 613 if st.compare_states(state, boot_table[boot_candidate]['start']): 614 gp.qprint_timen("The machine state is valid for a '" 615 + boot_candidate + "' boot test.") 616 gp.qprint_dashes() 617 gp.qprint_var(boot_stack) 618 gp.qprint_dashes() 619 return boot_candidate 620 else: 621 gp.qprint_timen("The machine state does not match the required" 622 + " starting state for a '" + boot_candidate 623 + "' boot test:") 624 gp.qprint_varx("boot_table_start_entry", 625 boot_table[boot_candidate]['start']) 626 boot_stack.append(boot_candidate) 627 transitional_boot_selected = True 628 popped_boot = boot_candidate 629 630 # Loop through your list selecting a boot_candidates 631 boot_candidates = [] 632 for boot_candidate in boot_list: 633 if st.compare_states(state, boot_table[boot_candidate]['start']): 634 if stack_popped: 635 if st.compare_states(boot_table[boot_candidate]['end'], 636 boot_table[popped_boot]['start']): 637 boot_candidates.append(boot_candidate) 638 else: 639 boot_candidates.append(boot_candidate) 640 641 if len(boot_candidates) == 0: 642 gp.qprint_timen("The user's boot list contained no boot tests" 643 + " which are valid for the current machine state.") 644 boot_candidate = default_power_on 645 if not st.compare_states(state, boot_table[default_power_on]['start']): 646 boot_candidate = default_power_off 647 boot_candidates.append(boot_candidate) 648 gp.qprint_timen("Using default '" + boot_candidate 649 + "' boot type to transition to valid state.") 650 651 gp.dprint_var(boot_candidates) 652 653 # Randomly select a boot from the candidate list. 654 boot = random.choice(boot_candidates) 655 656 return boot 657 658 659def print_defect_report(ffdc_file_list): 660 r""" 661 Print a defect report. 662 663 Description of argument(s): 664 ffdc_file_list A list of files which were collected by our ffdc functions. 665 """ 666 667 # Making deliberate choice to NOT run plug_in_setup(). We don't want 668 # ffdc_prefix updated. 669 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 670 call_point='ffdc_report', stop_on_plug_in_failure=0) 671 672 # Get additional header data which may have been created by ffdc plug-ins. 673 # Also, delete the individual header files to cleanup. 674 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 675 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 676 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 677 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 678 show_err=0) 679 680 # Get additional summary data which may have been created by ffdc plug-ins. 681 # Also, delete the individual header files to cleanup. 682 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 683 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 684 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 685 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 686 show_err=0) 687 688 # ffdc_list_file_path contains a list of any ffdc files created by plug- 689 # ins, etc. Read that data into a list. 690 try: 691 plug_in_ffdc_list = \ 692 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 693 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 694 except IOError: 695 plug_in_ffdc_list = [] 696 697 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 698 # in. Eliminate duplicates and sort the list. 699 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 700 701 if status_file_path != "": 702 ffdc_file_list.insert(0, status_file_path) 703 704 # Convert the list to a printable list. 705 printable_ffdc_file_list = "\n".join(ffdc_file_list) 706 707 # Open ffdc_file_list for writing. We will write a complete list of 708 # FFDC files to it for possible use by plug-ins like cp_stop_check. 709 ffdc_list_file = open(ffdc_list_file_path, 'w') 710 ffdc_list_file.write(printable_ffdc_file_list + "\n") 711 ffdc_list_file.close() 712 713 indent = 0 714 width = 90 715 linefeed = 1 716 char = "=" 717 718 gp.qprintn() 719 gp.qprint_dashes(indent, width, linefeed, char) 720 gp.qprintn("Copy this data to the defect:\n") 721 722 if len(more_header_info) > 0: 723 gp.qprintn(more_header_info) 724 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 725 openbmc_host_name, openbmc_ip, openbmc_username, 726 openbmc_password, rest_username, rest_password, ipmi_username, 727 ipmi_password, os_host, os_host_name, os_ip, os_username, 728 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 729 pdu_password, pdu_slot_no, openbmc_serial_host, 730 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 731 732 gp.qprintn() 733 print_boot_history(boot_history) 734 gp.qprintn() 735 gp.qprint_var(state) 736 gp.qprintn() 737 gp.qprintn("FFDC data files:") 738 gp.qprintn(printable_ffdc_file_list) 739 gp.qprintn() 740 741 if len(ffdc_summary_info) > 0: 742 gp.qprintn(ffdc_summary_info) 743 744 gp.qprint_dashes(indent, width, linefeed, char) 745 746 747def my_ffdc(): 748 r""" 749 Collect FFDC data. 750 """ 751 752 global state 753 754 plug_in_setup() 755 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 756 call_point='ffdc', stop_on_plug_in_failure=0) 757 758 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 759 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 760 + AUTOBOOT_FFDC_PREFIX 761 + " ffdc_function_list=" 762 + ffdc_function_list, ignore=1) 763 if status != 'PASS': 764 gp.qprint_error("Call to ffdc failed.\n") 765 if type(ffdc_file_list) is not list: 766 ffdc_file_list = [] 767 # Leave a record for caller that "soft" errors occurred. 768 soft_errors = 1 769 gpu.save_plug_in_value(soft_errors, pgm_name) 770 771 my_get_state() 772 773 print_defect_report(ffdc_file_list) 774 775 776def print_test_start_message(boot_keyword): 777 r""" 778 Print a message indicating what boot test is about to run. 779 780 Description of arguments: 781 boot_keyword The name of the boot which is to be run 782 (e.g. "BMC Power On"). 783 """ 784 785 global boot_history 786 global boot_start_time 787 788 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 789 790 # Set boot_start_time for use by plug-ins. 791 boot_start_time = doing_msg[1:33] 792 gp.qprint_var(boot_start_time) 793 794 gp.qprint(doing_msg) 795 796 update_boot_history(boot_history, doing_msg, max_boot_history) 797 798 799def stop_boot_test(signal_number=0, 800 frame=None): 801 r""" 802 Handle SIGUSR1 by aborting the boot test that is running. 803 804 Description of argument(s): 805 signal_number The signal number (should always be 10 for SIGUSR1). 806 frame The frame data. 807 """ 808 809 gp.qprintn() 810 gp.qprint_executing() 811 gp.lprint_executing() 812 813 # Restore original sigusr1 handler. 814 set_default_siguser1() 815 816 message = "The caller has asked that the boot test be stopped and marked" 817 message += " as a failure." 818 819 function_stack = gm.get_function_stack() 820 if "wait_state" in function_stack: 821 st.set_exit_wait_early_message(message) 822 else: 823 BuiltIn().fail(gp.sprint_error(message)) 824 825 826def run_boot(boot): 827 r""" 828 Run the specified boot. 829 830 Description of arguments: 831 boot The name of the boot test to be performed. 832 """ 833 834 global state 835 836 signal.signal(signal.SIGUSR1, stop_boot_test) 837 gp.qprint_timen("stop_boot_test is armed.") 838 839 print_test_start_message(boot) 840 841 plug_in_setup() 842 rc, shell_rc, failed_plug_in_name = \ 843 grpi.rprocess_plug_in_packages(call_point="pre_boot") 844 if rc != 0: 845 error_message = "Plug-in failed with non-zero return code.\n" +\ 846 gp.sprint_var(rc, fmt=gp.hexa()) 847 set_default_siguser1() 848 BuiltIn().fail(gp.sprint_error(error_message)) 849 850 if test_mode: 851 # In test mode, we'll pretend the boot worked by assigning its 852 # required end state to the default state value. 853 state = st.strip_anchor_state(boot_table[boot]['end']) 854 else: 855 # Assertion: We trust that the state data was made fresh by the 856 # caller. 857 858 gp.qprintn() 859 860 if boot_table[boot]['method_type'] == "keyword": 861 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 862 boot_table[boot]['method'], 863 quiet=quiet) 864 865 if boot_table[boot]['bmc_reboot']: 866 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 867 plug_in_setup() 868 rc, shell_rc, failed_plug_in_name = \ 869 grpi.rprocess_plug_in_packages(call_point="post_reboot") 870 if rc != 0: 871 error_message = "Plug-in failed with non-zero return code.\n" 872 error_message += gp.sprint_var(rc, fmt=gp.hexa()) 873 set_default_siguser1() 874 BuiltIn().fail(gp.sprint_error(error_message)) 875 else: 876 match_state = st.anchor_state(state) 877 del match_state['epoch_seconds'] 878 # Wait for the state to change in any way. 879 st.wait_state(match_state, wait_time=state_change_timeout, 880 interval="10 seconds", invert=1) 881 882 gp.qprintn() 883 if boot_table[boot]['end']['chassis'] == "Off": 884 boot_timeout = power_off_timeout 885 else: 886 boot_timeout = power_on_timeout 887 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 888 interval="10 seconds") 889 890 plug_in_setup() 891 rc, shell_rc, failed_plug_in_name = \ 892 grpi.rprocess_plug_in_packages(call_point="post_boot") 893 if rc != 0: 894 error_message = "Plug-in failed with non-zero return code.\n" +\ 895 gp.sprint_var(rc, fmt=gp.hexa()) 896 set_default_siguser1() 897 BuiltIn().fail(gp.sprint_error(error_message)) 898 899 # Restore original sigusr1 handler. 900 set_default_siguser1() 901 902 903def test_loop_body(): 904 r""" 905 The main loop body for the loop in main_py. 906 907 Description of arguments: 908 boot_count The iteration number (starts at 1). 909 """ 910 911 global boot_count 912 global state 913 global next_boot 914 global boot_success 915 global boot_end_time 916 917 gp.qprintn() 918 919 next_boot = select_boot() 920 if next_boot == "": 921 return True 922 923 boot_count += 1 924 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 925 926 pre_boot_plug_in_setup() 927 928 cmd_buf = ["run_boot", next_boot] 929 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 930 if boot_status == "FAIL": 931 gp.qprint(msg) 932 933 gp.qprintn() 934 if boot_status == "PASS": 935 boot_success = 1 936 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 937 + "\" succeeded.") 938 else: 939 boot_success = 0 940 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 941 + "\" failed.") 942 943 # Set boot_end_time for use by plug-ins. 944 boot_end_time = completion_msg[1:33] 945 gp.qprint_var(boot_end_time) 946 947 gp.qprint(completion_msg) 948 949 boot_results.update(next_boot, boot_status) 950 951 plug_in_setup() 952 # NOTE: A post_test_case call point failure is NOT counted as a boot 953 # failure. 954 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 955 call_point='post_test_case', stop_on_plug_in_failure=0) 956 957 plug_in_setup() 958 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 959 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 960 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 961 if ffdc_check == "All" or\ 962 shell_rc == dump_ffdc_rc(): 963 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 964 if status != 'PASS': 965 gp.qprint_error("Call to my_ffdc failed.\n") 966 # Leave a record for caller that "soft" errors occurred. 967 soft_errors = 1 968 gpu.save_plug_in_value(soft_errors, pgm_name) 969 970 if delete_errlogs: 971 # print error logs before delete 972 status, error_logs = grk.run_key_u("Get Error Logs") 973 pels = pel.peltool("-l", ignore_err=1) 974 log.print_error_logs(error_logs, "AdditionalData Message Severity") 975 gp.qprint_var(pels) 976 977 # We need to purge error logs between boots or they build up. 978 grk.run_key(delete_errlogs_cmd, ignore=1) 979 980 boot_results.print_report() 981 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 982 983 plug_in_setup() 984 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 985 call_point='stop_check', shell_rc=stop_test_rc(), 986 stop_on_non_zero_rc=1) 987 if shell_rc == stop_test_rc(): 988 message = "Stopping as requested by user.\n" 989 gp.qprint_time(message) 990 BuiltIn().fail(message) 991 992 # This should help prevent ConnectionErrors. 993 grk.run_key_u("Close All Connections") 994 995 return True 996 997 998def obmc_boot_test_teardown(): 999 r""" 1000 Clean up after the main keyword. 1001 """ 1002 gp.qprint_executing() 1003 1004 if ga.psutil_imported: 1005 ga.terminate_descendants() 1006 1007 if cp_setup_called: 1008 plug_in_setup() 1009 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1010 call_point='cleanup', stop_on_plug_in_failure=0) 1011 1012 if 'boot_results_file_path' in globals(): 1013 # Save boot_results and boot_history objects to a file in case they are 1014 # needed again. 1015 gp.qprint_timen("Saving boot_results to the following path.") 1016 gp.qprint_var(boot_results_file_path) 1017 pickle.dump((boot_results, boot_history), 1018 open(boot_results_file_path, 'wb'), 1019 pickle.HIGHEST_PROTOCOL) 1020 1021 global save_stack 1022 # Restore any global values saved on the save_stack. 1023 for parm_name in main_func_parm_list: 1024 # Get the parm_value if it was saved on the stack. 1025 try: 1026 parm_value = save_stack.pop(parm_name) 1027 except BaseException: 1028 # If it was not saved, no further action is required. 1029 continue 1030 1031 # Restore the saved value. 1032 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1033 "}\", parm_value)" 1034 gp.dpissuing(cmd_buf) 1035 exec(cmd_buf) 1036 1037 gp.dprintn(save_stack.sprint_obj()) 1038 1039 1040def test_teardown(): 1041 r""" 1042 Clean up after this test case. 1043 """ 1044 1045 gp.qprintn() 1046 gp.qprint_executing() 1047 1048 if ga.psutil_imported: 1049 ga.terminate_descendants() 1050 1051 cmd_buf = ["Print Error", 1052 "A keyword timeout occurred ending this program.\n"] 1053 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1054 1055 if redfish_supported: 1056 redfish.logout() 1057 1058 gp.qprint_pgm_footer() 1059 1060 1061def post_stack(): 1062 r""" 1063 Process post_stack plug-in programs. 1064 """ 1065 1066 if not call_post_stack_plug: 1067 # The caller does not wish to have post_stack plug-in processing done. 1068 return 1069 1070 global boot_success 1071 1072 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1073 pre_boot_plug_in_setup() 1074 # For the purposes of the following plug-ins, mark the "boot" as a success. 1075 boot_success = 1 1076 plug_in_setup() 1077 rc, shell_rc, failed_plug_in_name, history =\ 1078 grpi.rprocess_plug_in_packages(call_point='post_stack', 1079 stop_on_plug_in_failure=0, 1080 return_history=True) 1081 for doing_msg in history: 1082 update_boot_history(boot_history, doing_msg, max_boot_history) 1083 if rc != 0: 1084 boot_success = 0 1085 1086 plug_in_setup() 1087 rc, shell_rc, failed_plug_in_name =\ 1088 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1089 shell_rc=dump_ffdc_rc(), 1090 stop_on_plug_in_failure=1, 1091 stop_on_non_zero_rc=1) 1092 if shell_rc == dump_ffdc_rc(): 1093 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1094 if status != 'PASS': 1095 gp.qprint_error("Call to my_ffdc failed.\n") 1096 # Leave a record for caller that "soft" errors occurred. 1097 soft_errors = 1 1098 gpu.save_plug_in_value(soft_errors, pgm_name) 1099 1100 plug_in_setup() 1101 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1102 call_point='stop_check', shell_rc=stop_test_rc(), 1103 stop_on_non_zero_rc=1) 1104 if shell_rc == stop_test_rc(): 1105 message = "Stopping as requested by user.\n" 1106 gp.qprint_time(message) 1107 BuiltIn().fail(message) 1108 1109 1110def obmc_boot_test_py(loc_boot_stack=None, 1111 loc_stack_mode=None, 1112 loc_quiet=None): 1113 r""" 1114 Do main program processing. 1115 """ 1116 1117 global save_stack 1118 1119 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']}) 1120 1121 gp.dprintn() 1122 # Process function parms. 1123 for parm_name in main_func_parm_list: 1124 # Get parm's value. 1125 parm_value = eval("loc_" + parm_name) 1126 gp.dpvars(parm_name, parm_value) 1127 1128 if parm_value is not None: 1129 # Save the global value on a stack. 1130 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1131 parm_name + "}\"), \"" + parm_name + "\")" 1132 gp.dpissuing(cmd_buf) 1133 exec(cmd_buf) 1134 1135 # Set the global value to the passed value. 1136 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1137 "}\", loc_" + parm_name + ")" 1138 gp.dpissuing(cmd_buf) 1139 exec(cmd_buf) 1140 1141 gp.dprintn(save_stack.sprint_obj()) 1142 1143 setup() 1144 1145 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1146 1147 if ffdc_only: 1148 gp.qprint_timen("Caller requested ffdc_only.") 1149 if do_pre_boot_plug_in_setup: 1150 pre_boot_plug_in_setup() 1151 grk.run_key_u("my_ffdc") 1152 return 1153 1154 if delete_errlogs: 1155 # print error logs before delete 1156 status, error_logs = grk.run_key_u("Get Error Logs") 1157 pels = pel.peltool("-l", ignore_err=1) 1158 log.print_error_logs(error_logs, "AdditionalData Message Severity") 1159 gp.qprint_var(pels) 1160 1161 # Delete errlogs prior to doing any boot tests. 1162 grk.run_key(delete_errlogs_cmd, ignore=1) 1163 1164 # Process caller's boot_stack. 1165 while (len(boot_stack) > 0): 1166 test_loop_body() 1167 1168 gp.qprint_timen("Finished processing stack.") 1169 1170 post_stack() 1171 1172 # Process caller's boot_list. 1173 if len(boot_list) > 0: 1174 for ix in range(1, max_num_tests + 1): 1175 test_loop_body() 1176 1177 gp.qprint_timen("Completed all requested boot tests.") 1178 1179 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1180 new_fail = boot_fail - init_boot_fail 1181 if new_fail > boot_fail_threshold: 1182 error_message = "Boot failures exceed the boot failure" +\ 1183 " threshold:\n" +\ 1184 gp.sprint_var(new_fail) +\ 1185 gp.sprint_var(boot_fail_threshold) 1186 BuiltIn().fail(gp.sprint_error(error_message)) 1187