1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_arg as ga 27import gen_valid as gv 28import gen_misc as gm 29import gen_cmd as gc 30import gen_robot_keyword as grk 31import state as st 32import var_stack as vs 33import gen_plug_in_utils as gpu 34 35base_path = os.path.dirname(os.path.dirname( 36 imp.find_module("gen_robot_print")[1])) +\ 37 os.sep 38sys.path.append(base_path + "extended/") 39import run_keyword as rk 40 41# Setting master_pid correctly influences the behavior of plug-ins like 42# DB_Logging 43program_pid = os.getpid() 44master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 45pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 46 47# Set up boot data structures. 48os_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 49 50boot_lists = read_boot_lists() 51 52# The maximum number of entries that can be in the boot_history global variable. 53max_boot_history = 10 54boot_history = [] 55 56state = st.return_state_constant('default_state') 57cp_setup_called = 0 58next_boot = "" 59base_tool_dir_path = os.path.normpath(os.environ.get( 60 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 61 62ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 63boot_success = 0 64status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 65if status_dir_path != "": 66 status_dir_path = os.path.normpath(status_dir_path) + os.sep 67redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False) 68if redfish_supported: 69 default_power_on = "Redfish Power On" 70 default_power_off = "Redfish Power Off" 71 delete_errlogs_cmd = "Delete Error Logs" 72 # TODO: delete_errlogs_cmd="Redfish Purge Event Log" 73else: 74 default_power_on = "REST Power On" 75 default_power_off = "REST Power Off" 76 delete_errlogs_cmd = "Delete Error Logs" 77boot_count = 0 78 79LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 80AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 81ffdc_prefix = AUTOBOOT_FFDC_PREFIX 82boot_start_time = "" 83boot_end_time = "" 84save_stack = vs.var_stack('save_stack') 85main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 86 87 88def dump_ffdc_rc(): 89 r""" 90 Return the constant dump ffdc test return code value. 91 92 When a plug-in call point program returns this value, it indicates that 93 this program should collect FFDC. 94 """ 95 96 return 0x00000200 97 98 99def stop_test_rc(): 100 r""" 101 Return the constant stop test return code value. 102 103 When a plug-in call point program returns this value, it indicates that 104 this program should stop running. 105 """ 106 107 return 0x00000200 108 109 110def process_host(host, 111 host_var_name=""): 112 r""" 113 Process a host by getting the associated host name and IP address and 114 setting them in global variables. 115 116 If the caller does not pass the host_var_name, this function will try to 117 figure out the name of the variable used by the caller for the host parm. 118 Callers are advised to explicitly specify the host_var_name when calling 119 with an exec command. In such cases, the get_arg_name cannot figure out 120 the host variable name. 121 122 This function will then create similar global variable names by 123 removing "_host" and appending "_host_name" or "_ip" to the host variable 124 name. 125 126 Example: 127 128 If a call is made like this: 129 process_host(openbmc_host) 130 131 Global variables openbmc_host_name and openbmc_ip will be set. 132 133 Description of argument(s): 134 host A host name or IP. The name of the variable used should 135 have a suffix of "_host". 136 host_var_name The name of the variable being used as the host parm. 137 """ 138 139 if host_var_name == "": 140 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 141 142 host_name_var_name = re.sub("host", "host_name", host_var_name) 143 ip_var_name = re.sub("host", "ip", host_var_name) 144 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 145 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 146 host + "')" 147 exec(cmd_buf) 148 149 150def process_pgm_parms(): 151 r""" 152 Process the program parameters by assigning them all to corresponding 153 globals. Also, set some global values that depend on program parameters. 154 """ 155 156 # Program parameter processing. 157 # Assign all program parms to python variables which are global to this 158 # module. 159 160 global parm_list 161 parm_list = BuiltIn().get_variable_value("${parm_list}") 162 # The following subset of parms should be processed as integers. 163 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 164 'boot_fail_threshold', 'delete_errlogs', 165 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 166 'test_mode', 'debug'] 167 for parm in parm_list: 168 if parm in int_list: 169 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 170 "}\", \"0\"))" 171 else: 172 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 173 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 174 gp.dpissuing(cmd_buf) 175 exec(cmd_buf) 176 if re.match(r".*_host$", parm): 177 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 178 exec(cmd_buf) 179 if re.match(r".*_password$", parm): 180 # Register the value of any parm whose name ends in _password. 181 # This will cause the print functions to replace passwords with 182 # asterisks in the output. 183 cmd_buf = "gp.register_passwords(" + parm + ")" 184 exec(cmd_buf) 185 186 global ffdc_dir_path_style 187 global boot_list 188 global boot_stack 189 global boot_results_file_path 190 global boot_results 191 global boot_history 192 global ffdc_list_file_path 193 global ffdc_report_list_path 194 global ffdc_summary_list_path 195 global boot_table 196 global valid_boot_types 197 198 if ffdc_dir_path_style == "": 199 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 200 201 # Convert these program parms to lists for easier processing.. 202 boot_list = list(filter(None, boot_list.split(":"))) 203 boot_stack = list(filter(None, boot_stack.split(":"))) 204 205 boot_table = create_boot_table(boot_table_path, os_host=os_host) 206 valid_boot_types = create_valid_boot_list(boot_table) 207 208 cleanup_boot_results_file() 209 boot_results_file_path = create_boot_results_file_path(pgm_name, 210 openbmc_nickname, 211 master_pid) 212 213 if os.path.isfile(boot_results_file_path): 214 # We've been called before in this run so we'll load the saved 215 # boot_results and boot_history objects. 216 boot_results, boot_history =\ 217 pickle.load(open(boot_results_file_path, 'rb')) 218 else: 219 boot_results = boot_results(boot_table, boot_pass, boot_fail) 220 221 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 222 "/FFDC_FILE_LIST" 223 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 224 "/FFDC_REPORT_FILE_LIST" 225 226 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 227 "/FFDC_SUMMARY_FILE_LIST" 228 229 230def initial_plug_in_setup(): 231 r""" 232 Initialize all plug-in environment variables which do not change for the 233 duration of the program. 234 235 """ 236 237 global LOG_LEVEL 238 BuiltIn().set_log_level("NONE") 239 240 BuiltIn().set_global_variable("${master_pid}", master_pid) 241 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 242 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 243 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 244 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 245 ffdc_list_file_path) 246 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 247 ffdc_report_list_path) 248 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 249 ffdc_summary_list_path) 250 251 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 252 ffdc_dir_path_style) 253 BuiltIn().set_global_variable("${FFDC_CHECK}", 254 ffdc_check) 255 256 # For each program parameter, set the corresponding AUTOBOOT_ environment 257 # variable value. Also, set an AUTOBOOT_ environment variable for every 258 # element in additional_values. 259 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 260 "status_dir_path", "base_tool_dir_path", 261 "ffdc_list_file_path", "ffdc_report_list_path", 262 "ffdc_summary_list_path", "execdir"] 263 264 plug_in_vars = parm_list + additional_values 265 266 for var_name in plug_in_vars: 267 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 268 var_name = var_name.upper() 269 if var_value is None: 270 var_value = "" 271 os.environ["AUTOBOOT_" + var_name] = str(var_value) 272 273 BuiltIn().set_log_level(LOG_LEVEL) 274 275 # Make sure the ffdc list directory exists. 276 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 277 if not os.path.exists(ffdc_list_dir_path): 278 os.makedirs(ffdc_list_dir_path) 279 280 281def plug_in_setup(): 282 r""" 283 Initialize all changing plug-in environment variables for use by the 284 plug-in programs. 285 """ 286 287 global LOG_LEVEL 288 global test_really_running 289 290 BuiltIn().set_log_level("NONE") 291 292 boot_pass, boot_fail = boot_results.return_total_pass_fail() 293 if boot_pass > 1: 294 test_really_running = 1 295 else: 296 test_really_running = 0 297 298 BuiltIn().set_global_variable("${test_really_running}", 299 test_really_running) 300 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 301 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 302 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 303 BuiltIn().set_global_variable("${boot_success}", boot_success) 304 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 305 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 306 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 307 308 # For each program parameter, set the corresponding AUTOBOOT_ environment 309 # variable value. Also, set an AUTOBOOT_ environment variable for every 310 # element in additional_values. 311 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 312 "boot_fail", "test_really_running", "ffdc_prefix", 313 "boot_start_time", "boot_end_time"] 314 315 plug_in_vars = additional_values 316 317 for var_name in plug_in_vars: 318 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 319 var_name = var_name.upper() 320 if var_value is None: 321 var_value = "" 322 os.environ["AUTOBOOT_" + var_name] = str(var_value) 323 324 if debug: 325 shell_rc, out_buf = \ 326 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 327 328 BuiltIn().set_log_level(LOG_LEVEL) 329 330 331def pre_boot_plug_in_setup(): 332 333 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 334 try: 335 os.remove(ffdc_list_file_path) 336 except OSError: 337 pass 338 339 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 340 try: 341 os.remove(ffdc_report_list_path) 342 except OSError: 343 pass 344 345 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 346 try: 347 os.remove(ffdc_summary_list_path) 348 except OSError: 349 pass 350 351 global ffdc_prefix 352 353 seconds = time.time() 354 loc_time = time.localtime(seconds) 355 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 356 357 ffdc_prefix = openbmc_nickname + "." + time_string 358 359 360def default_sigusr1(signal_number=0, 361 frame=None): 362 r""" 363 Handle SIGUSR1 by doing nothing. 364 365 This function assists in debugging SIGUSR1 processing by printing messages 366 to stdout and to the log.html file. 367 368 Description of argument(s): 369 signal_number The signal number (should always be 10 for SIGUSR1). 370 frame The frame data. 371 """ 372 373 gp.qprintn() 374 gp.qprint_executing() 375 gp.lprint_executing() 376 377 378def set_default_siguser1(): 379 r""" 380 Set the default_sigusr1 function to be the SIGUSR1 handler. 381 """ 382 383 gp.qprintn() 384 gp.qprint_executing() 385 gp.lprint_executing() 386 signal.signal(signal.SIGUSR1, default_sigusr1) 387 388 389def setup(): 390 r""" 391 Do general program setup tasks. 392 """ 393 394 global cp_setup_called 395 global transitional_boot_selected 396 397 gp.qprintn() 398 399 set_default_siguser1() 400 transitional_boot_selected = False 401 402 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 403 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 404 # If we can't find process_plug_in_packages.py, ssh_pw or 405 # validate_plug_ins.py, then we don't have our repo bin in PATH. 406 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 407 + " ssh_pw validate_plug_ins.py", quiet=1, 408 print_output=0, show_err=0) 409 if shell_rc != 0: 410 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 411 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 412 if robot_pgm_dir_path not in sys.path: 413 sys.path.append(robot_pgm_dir_path) 414 PYTHONPATH = os.environ.get("PYTHONPATH", "") 415 if PYTHONPATH == "": 416 os.environ['PYTHONPATH'] = robot_pgm_dir_path 417 else: 418 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 419 420 validate_parms() 421 422 gp.qprint_pgm_header() 423 424 grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF") 425 426 initial_plug_in_setup() 427 428 plug_in_setup() 429 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 430 call_point='setup') 431 if rc != 0: 432 error_message = "Plug-in setup failed.\n" 433 gp.print_error_report(error_message) 434 BuiltIn().fail(error_message) 435 # Setting cp_setup_called lets our Teardown know that it needs to call 436 # the cleanup plug-in call point. 437 cp_setup_called = 1 438 439 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 440 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 441 # FFDC_LOG_PATH is used by "FFDC" keyword. 442 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 443 444 # Also printed by FFDC. 445 global host_name 446 global host_ip 447 host = socket.gethostname() 448 host_name, host_ip = gm.get_host_name_ip(host) 449 450 gp.dprint_var(boot_table) 451 gp.dprint_var(boot_lists) 452 453 454def validate_parms(): 455 r""" 456 Validate all program parameters. 457 """ 458 459 process_pgm_parms() 460 461 gp.qprintn() 462 463 global openbmc_model 464 gv.valid_value(openbmc_host) 465 gv.valid_value(openbmc_username) 466 gv.valid_value(openbmc_password) 467 gv.valid_value(rest_username) 468 gv.valid_value(rest_password) 469 gv.valid_value(ipmi_username) 470 gv.valid_value(ipmi_password) 471 if os_host != "": 472 gv.valid_value(os_username) 473 gv.valid_value(os_password) 474 475 if pdu_host != "": 476 gv.valid_value(pdu_username) 477 gv.valid_value(pdu_password) 478 gv.valid_integer(pdu_slot_no) 479 if openbmc_serial_host != "": 480 gv.valid_integer(openbmc_serial_port) 481 if openbmc_model == "": 482 status, ret_values =\ 483 grk.run_key_u("Get BMC System Model") 484 openbmc_model = ret_values 485 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 486 gv.valid_value(openbmc_model) 487 gv.valid_integer(max_num_tests) 488 gv.valid_integer(boot_pass) 489 gv.valid_integer(boot_fail) 490 491 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 492 BuiltIn().set_global_variable("${plug_in_packages_list}", 493 plug_in_packages_list) 494 495 gv.valid_value(stack_mode, valid_values=['normal', 'skip']) 496 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 497 error_message = "You must provide either a value for either the" +\ 498 " boot_list or the boot_stack parm.\n" 499 BuiltIn().fail(gp.sprint_error(error_message)) 500 501 valid_boot_list(boot_list, valid_boot_types) 502 valid_boot_list(boot_stack, valid_boot_types) 503 504 selected_PDU_boots = list(set(boot_list + boot_stack) 505 & set(boot_lists['PDU_reboot'])) 506 507 if len(selected_PDU_boots) > 0 and pdu_host == "": 508 error_message = "You have selected the following boots which" +\ 509 " require a PDU host but no value for pdu_host:\n" 510 error_message += gp.sprint_var(selected_PDU_boots) 511 error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 512 BuiltIn().fail(gp.sprint_error(error_message)) 513 514 return 515 516 517def my_get_state(): 518 r""" 519 Get the system state plus a little bit of wrapping. 520 """ 521 522 global state 523 524 req_states = ['epoch_seconds'] + st.default_req_states 525 526 gp.qprint_timen("Getting system state.") 527 if test_mode: 528 state['epoch_seconds'] = int(time.time()) 529 else: 530 state = st.get_state(req_states=req_states, quiet=quiet) 531 gp.qprint_var(state) 532 533 534def valid_state(): 535 r""" 536 Verify that our state dictionary contains no blank values. If we don't get 537 valid state data, we cannot continue to work. 538 """ 539 540 if st.compare_states(state, st.invalid_state_match, 'or'): 541 error_message = "The state dictionary contains blank fields which" +\ 542 " is illegal.\n" + gp.sprint_var(state) 543 BuiltIn().fail(gp.sprint_error(error_message)) 544 545 546def select_boot(): 547 r""" 548 Select a boot test to be run based on our current state and return the 549 chosen boot type. 550 551 Description of arguments: 552 state The state of the machine. 553 """ 554 555 global transitional_boot_selected 556 global boot_stack 557 558 gp.qprint_timen("Selecting a boot test.") 559 560 if transitional_boot_selected and not boot_success: 561 prior_boot = next_boot 562 boot_candidate = boot_stack.pop() 563 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 564 + " transition to a valid state for '" + boot_candidate 565 + "' which was at the top of the boot_stack. Since" 566 + " the '" + next_boot + "' failed, the '" 567 + boot_candidate + "' has been removed from the stack" 568 + " to avoid and endless failure loop.") 569 if len(boot_stack) == 0: 570 return "" 571 572 my_get_state() 573 valid_state() 574 575 transitional_boot_selected = False 576 stack_popped = 0 577 if len(boot_stack) > 0: 578 stack_popped = 1 579 gp.qprint_dashes() 580 gp.qprint_var(boot_stack) 581 gp.qprint_dashes() 582 skip_boot_printed = 0 583 while len(boot_stack) > 0: 584 boot_candidate = boot_stack.pop() 585 if stack_mode == 'normal': 586 break 587 else: 588 if st.compare_states(state, boot_table[boot_candidate]['end']): 589 if not skip_boot_printed: 590 gp.qprint_var(stack_mode) 591 gp.qprintn() 592 gp.qprint_timen("Skipping the following boot tests" 593 + " which are unnecessary since their" 594 + " required end states match the" 595 + " current machine state:") 596 skip_boot_printed = 1 597 gp.qprint_var(boot_candidate) 598 boot_candidate = "" 599 if boot_candidate == "": 600 gp.qprint_dashes() 601 gp.qprint_var(boot_stack) 602 gp.qprint_dashes() 603 return boot_candidate 604 if st.compare_states(state, boot_table[boot_candidate]['start']): 605 gp.qprint_timen("The machine state is valid for a '" 606 + boot_candidate + "' boot test.") 607 gp.qprint_dashes() 608 gp.qprint_var(boot_stack) 609 gp.qprint_dashes() 610 return boot_candidate 611 else: 612 gp.qprint_timen("The machine state does not match the required" 613 + " starting state for a '" + boot_candidate 614 + "' boot test:") 615 gp.qprint_varx("boot_table_start_entry", 616 boot_table[boot_candidate]['start']) 617 boot_stack.append(boot_candidate) 618 transitional_boot_selected = True 619 popped_boot = boot_candidate 620 621 # Loop through your list selecting a boot_candidates 622 boot_candidates = [] 623 for boot_candidate in boot_list: 624 if st.compare_states(state, boot_table[boot_candidate]['start']): 625 if stack_popped: 626 if st.compare_states(boot_table[boot_candidate]['end'], 627 boot_table[popped_boot]['start']): 628 boot_candidates.append(boot_candidate) 629 else: 630 boot_candidates.append(boot_candidate) 631 632 if len(boot_candidates) == 0: 633 gp.qprint_timen("The user's boot list contained no boot tests" 634 + " which are valid for the current machine state.") 635 boot_candidate = default_power_on 636 if not st.compare_states(state, boot_table[default_power_on]['start']): 637 boot_candidate = default_power_off 638 boot_candidates.append(boot_candidate) 639 gp.qprint_timen("Using default '" + boot_candidate 640 + "' boot type to transition to valid state.") 641 642 gp.dprint_var(boot_candidates) 643 644 # Randomly select a boot from the candidate list. 645 boot = random.choice(boot_candidates) 646 647 return boot 648 649 650def print_defect_report(ffdc_file_list): 651 r""" 652 Print a defect report. 653 654 Description of argument(s): 655 ffdc_file_list A list of files which were collected by our ffdc functions. 656 """ 657 658 # Making deliberate choice to NOT run plug_in_setup(). We don't want 659 # ffdc_prefix updated. 660 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 661 call_point='ffdc_report', stop_on_plug_in_failure=0) 662 663 # Get additional header data which may have been created by ffdc plug-ins. 664 # Also, delete the individual header files to cleanup. 665 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 666 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 667 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 668 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 669 show_err=0) 670 671 # Get additional summary data which may have been created by ffdc plug-ins. 672 # Also, delete the individual header files to cleanup. 673 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 674 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 675 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 676 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 677 show_err=0) 678 679 # ffdc_list_file_path contains a list of any ffdc files created by plug- 680 # ins, etc. Read that data into a list. 681 try: 682 plug_in_ffdc_list = \ 683 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 684 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 685 except IOError: 686 plug_in_ffdc_list = [] 687 688 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 689 # in. Eliminate duplicates and sort the list. 690 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 691 692 if status_file_path != "": 693 ffdc_file_list.insert(0, status_file_path) 694 695 # Convert the list to a printable list. 696 printable_ffdc_file_list = "\n".join(ffdc_file_list) 697 698 # Open ffdc_file_list for writing. We will write a complete list of 699 # FFDC files to it for possible use by plug-ins like cp_stop_check. 700 ffdc_list_file = open(ffdc_list_file_path, 'w') 701 ffdc_list_file.write(printable_ffdc_file_list + "\n") 702 ffdc_list_file.close() 703 704 indent = 0 705 width = 90 706 linefeed = 1 707 char = "=" 708 709 gp.qprintn() 710 gp.qprint_dashes(indent, width, linefeed, char) 711 gp.qprintn("Copy this data to the defect:\n") 712 713 if len(more_header_info) > 0: 714 gp.qprintn(more_header_info) 715 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 716 openbmc_host_name, openbmc_ip, openbmc_username, 717 openbmc_password, rest_username, rest_password, ipmi_username, 718 ipmi_password, os_host, os_host_name, os_ip, os_username, 719 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 720 pdu_password, pdu_slot_no, openbmc_serial_host, 721 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 722 723 gp.qprintn() 724 print_boot_history(boot_history) 725 gp.qprintn() 726 gp.qprint_var(state) 727 gp.qprintn() 728 gp.qprintn("FFDC data files:") 729 gp.qprintn(printable_ffdc_file_list) 730 gp.qprintn() 731 732 if len(ffdc_summary_info) > 0: 733 gp.qprintn(ffdc_summary_info) 734 735 gp.qprint_dashes(indent, width, linefeed, char) 736 737 738def my_ffdc(): 739 r""" 740 Collect FFDC data. 741 """ 742 743 global state 744 745 plug_in_setup() 746 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 747 call_point='ffdc', stop_on_plug_in_failure=0) 748 749 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 750 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 751 + AUTOBOOT_FFDC_PREFIX 752 + " ffdc_function_list=" 753 + ffdc_function_list, ignore=1) 754 if status != 'PASS': 755 gp.qprint_error("Call to ffdc failed.\n") 756 if type(ffdc_file_list) is not list: 757 ffdc_file_list = [] 758 # Leave a record for caller that "soft" errors occurred. 759 soft_errors = 1 760 gpu.save_plug_in_value(soft_errors, pgm_name) 761 762 my_get_state() 763 764 print_defect_report(ffdc_file_list) 765 766 767def print_test_start_message(boot_keyword): 768 r""" 769 Print a message indicating what boot test is about to run. 770 771 Description of arguments: 772 boot_keyword The name of the boot which is to be run 773 (e.g. "BMC Power On"). 774 """ 775 776 global boot_history 777 global boot_start_time 778 779 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 780 781 # Set boot_start_time for use by plug-ins. 782 boot_start_time = doing_msg[1:33] 783 gp.qprint_var(boot_start_time) 784 785 gp.qprint(doing_msg) 786 787 update_boot_history(boot_history, doing_msg, max_boot_history) 788 789 790def stop_boot_test(signal_number=0, 791 frame=None): 792 r""" 793 Handle SIGUSR1 by aborting the boot test that is running. 794 795 Description of argument(s): 796 signal_number The signal number (should always be 10 for SIGUSR1). 797 frame The frame data. 798 """ 799 800 gp.qprintn() 801 gp.qprint_executing() 802 gp.lprint_executing() 803 804 # Restore original sigusr1 handler. 805 set_default_siguser1() 806 807 message = "The caller has asked that the boot test be stopped and marked" 808 message += " as a failure." 809 810 function_stack = gm.get_function_stack() 811 if "wait_state" in function_stack: 812 st.set_exit_wait_early_message(message) 813 else: 814 BuiltIn().fail(gp.sprint_error(message)) 815 816 817def run_boot(boot): 818 r""" 819 Run the specified boot. 820 821 Description of arguments: 822 boot The name of the boot test to be performed. 823 """ 824 825 global state 826 827 signal.signal(signal.SIGUSR1, stop_boot_test) 828 gp.qprint_timen("stop_boot_test is armed.") 829 830 print_test_start_message(boot) 831 832 plug_in_setup() 833 rc, shell_rc, failed_plug_in_name = \ 834 grpi.rprocess_plug_in_packages(call_point="pre_boot") 835 if rc != 0: 836 error_message = "Plug-in failed with non-zero return code.\n" +\ 837 gp.sprint_var(rc, fmt=gp.hexa()) 838 set_default_siguser1() 839 BuiltIn().fail(gp.sprint_error(error_message)) 840 841 if test_mode: 842 # In test mode, we'll pretend the boot worked by assigning its 843 # required end state to the default state value. 844 state = st.strip_anchor_state(boot_table[boot]['end']) 845 else: 846 # Assertion: We trust that the state data was made fresh by the 847 # caller. 848 849 gp.qprintn() 850 851 if boot_table[boot]['method_type'] == "keyword": 852 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 853 boot_table[boot]['method'], 854 quiet=quiet) 855 856 if boot_table[boot]['bmc_reboot']: 857 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 858 plug_in_setup() 859 rc, shell_rc, failed_plug_in_name = \ 860 grpi.rprocess_plug_in_packages(call_point="post_reboot") 861 if rc != 0: 862 error_message = "Plug-in failed with non-zero return code.\n" 863 error_message += gp.sprint_var(rc, fmt=gp.hexa()) 864 set_default_siguser1() 865 BuiltIn().fail(gp.sprint_error(error_message)) 866 else: 867 match_state = st.anchor_state(state) 868 del match_state['epoch_seconds'] 869 # Wait for the state to change in any way. 870 st.wait_state(match_state, wait_time=state_change_timeout, 871 interval="10 seconds", invert=1) 872 873 gp.qprintn() 874 if boot_table[boot]['end']['chassis'] == "Off": 875 boot_timeout = power_off_timeout 876 else: 877 boot_timeout = power_on_timeout 878 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 879 interval="10 seconds") 880 881 plug_in_setup() 882 rc, shell_rc, failed_plug_in_name = \ 883 grpi.rprocess_plug_in_packages(call_point="post_boot") 884 if rc != 0: 885 error_message = "Plug-in failed with non-zero return code.\n" +\ 886 gp.sprint_var(rc, fmt=gp.hexa()) 887 set_default_siguser1() 888 BuiltIn().fail(gp.sprint_error(error_message)) 889 890 # Restore original sigusr1 handler. 891 set_default_siguser1() 892 893 894def test_loop_body(): 895 r""" 896 The main loop body for the loop in main_py. 897 898 Description of arguments: 899 boot_count The iteration number (starts at 1). 900 """ 901 902 global boot_count 903 global state 904 global next_boot 905 global boot_success 906 global boot_end_time 907 908 gp.qprintn() 909 910 next_boot = select_boot() 911 if next_boot == "": 912 return True 913 914 boot_count += 1 915 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 916 917 pre_boot_plug_in_setup() 918 919 cmd_buf = ["run_boot", next_boot] 920 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 921 if boot_status == "FAIL": 922 gp.qprint(msg) 923 924 gp.qprintn() 925 if boot_status == "PASS": 926 boot_success = 1 927 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 928 + "\" succeeded.") 929 else: 930 boot_success = 0 931 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 932 + "\" failed.") 933 934 # Set boot_end_time for use by plug-ins. 935 boot_end_time = completion_msg[1:33] 936 gp.qprint_var(boot_end_time) 937 938 gp.qprint(completion_msg) 939 940 boot_results.update(next_boot, boot_status) 941 942 plug_in_setup() 943 # NOTE: A post_test_case call point failure is NOT counted as a boot 944 # failure. 945 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 946 call_point='post_test_case', stop_on_plug_in_failure=0) 947 948 plug_in_setup() 949 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 950 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 951 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 952 if ffdc_check == "All" or\ 953 shell_rc == dump_ffdc_rc(): 954 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 955 if status != 'PASS': 956 gp.qprint_error("Call to my_ffdc failed.\n") 957 # Leave a record for caller that "soft" errors occurred. 958 soft_errors = 1 959 gpu.save_plug_in_value(soft_errors, pgm_name) 960 961 if delete_errlogs: 962 # We need to purge error logs between boots or they build up. 963 grk.run_key(delete_errlogs_cmd, ignore=1) 964 965 boot_results.print_report() 966 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 967 968 plug_in_setup() 969 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 970 call_point='stop_check', shell_rc=stop_test_rc(), 971 stop_on_non_zero_rc=1) 972 if shell_rc == stop_test_rc(): 973 message = "Stopping as requested by user.\n" 974 gp.qprint_time(message) 975 BuiltIn().fail(message) 976 977 # This should help prevent ConnectionErrors. 978 grk.run_key_u("Close All Connections") 979 980 return True 981 982 983def obmc_boot_test_teardown(): 984 r""" 985 Clean up after the main keyword. 986 """ 987 gp.qprint_executing() 988 989 if ga.psutil_imported: 990 ga.terminate_descendants() 991 992 if cp_setup_called: 993 plug_in_setup() 994 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 995 call_point='cleanup', stop_on_plug_in_failure=0) 996 997 if 'boot_results_file_path' in globals(): 998 # Save boot_results and boot_history objects to a file in case they are 999 # needed again. 1000 gp.qprint_timen("Saving boot_results to the following path.") 1001 gp.qprint_var(boot_results_file_path) 1002 pickle.dump((boot_results, boot_history), 1003 open(boot_results_file_path, 'wb'), 1004 pickle.HIGHEST_PROTOCOL) 1005 1006 global save_stack 1007 # Restore any global values saved on the save_stack. 1008 for parm_name in main_func_parm_list: 1009 # Get the parm_value if it was saved on the stack. 1010 try: 1011 parm_value = save_stack.pop(parm_name) 1012 except BaseException: 1013 # If it was not saved, no further action is required. 1014 continue 1015 1016 # Restore the saved value. 1017 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1018 "}\", parm_value)" 1019 gp.dpissuing(cmd_buf) 1020 exec(cmd_buf) 1021 1022 gp.dprintn(save_stack.sprint_obj()) 1023 1024 1025def test_teardown(): 1026 r""" 1027 Clean up after this test case. 1028 """ 1029 1030 gp.qprintn() 1031 gp.qprint_executing() 1032 1033 if ga.psutil_imported: 1034 ga.terminate_descendants() 1035 1036 cmd_buf = ["Print Error", 1037 "A keyword timeout occurred ending this program.\n"] 1038 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1039 1040 gp.qprint_pgm_footer() 1041 1042 1043def post_stack(): 1044 r""" 1045 Process post_stack plug-in programs. 1046 """ 1047 1048 if not call_post_stack_plug: 1049 # The caller does not wish to have post_stack plug-in processing done. 1050 return 1051 1052 global boot_success 1053 1054 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1055 pre_boot_plug_in_setup() 1056 # For the purposes of the following plug-ins, mark the "boot" as a success. 1057 boot_success = 1 1058 plug_in_setup() 1059 rc, shell_rc, failed_plug_in_name, history =\ 1060 grpi.rprocess_plug_in_packages(call_point='post_stack', 1061 stop_on_plug_in_failure=0, 1062 return_history=True) 1063 for doing_msg in history: 1064 update_boot_history(boot_history, doing_msg, max_boot_history) 1065 if rc != 0: 1066 boot_success = 0 1067 1068 plug_in_setup() 1069 rc, shell_rc, failed_plug_in_name =\ 1070 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1071 shell_rc=dump_ffdc_rc(), 1072 stop_on_plug_in_failure=1, 1073 stop_on_non_zero_rc=1) 1074 if shell_rc == dump_ffdc_rc(): 1075 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1076 if status != 'PASS': 1077 gp.qprint_error("Call to my_ffdc failed.\n") 1078 # Leave a record for caller that "soft" errors occurred. 1079 soft_errors = 1 1080 gpu.save_plug_in_value(soft_errors, pgm_name) 1081 1082 plug_in_setup() 1083 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1084 call_point='stop_check', shell_rc=stop_test_rc(), 1085 stop_on_non_zero_rc=1) 1086 if shell_rc == stop_test_rc(): 1087 message = "Stopping as requested by user.\n" 1088 gp.qprint_time(message) 1089 BuiltIn().fail(message) 1090 1091 1092def obmc_boot_test_py(loc_boot_stack=None, 1093 loc_stack_mode=None, 1094 loc_quiet=None): 1095 r""" 1096 Do main program processing. 1097 """ 1098 1099 global save_stack 1100 1101 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']}) 1102 1103 gp.dprintn() 1104 # Process function parms. 1105 for parm_name in main_func_parm_list: 1106 # Get parm's value. 1107 parm_value = eval("loc_" + parm_name) 1108 gp.dpvars(parm_name, parm_value) 1109 1110 if parm_value is not None: 1111 # Save the global value on a stack. 1112 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1113 parm_name + "}\"), \"" + parm_name + "\")" 1114 gp.dpissuing(cmd_buf) 1115 exec(cmd_buf) 1116 1117 # Set the global value to the passed value. 1118 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1119 "}\", loc_" + parm_name + ")" 1120 gp.dpissuing(cmd_buf) 1121 exec(cmd_buf) 1122 1123 gp.dprintn(save_stack.sprint_obj()) 1124 1125 setup() 1126 1127 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1128 1129 if ffdc_only: 1130 gp.qprint_timen("Caller requested ffdc_only.") 1131 if do_pre_boot_plug_in_setup: 1132 pre_boot_plug_in_setup() 1133 grk.run_key_u("my_ffdc") 1134 return 1135 1136 if delete_errlogs: 1137 # Delete errlogs prior to doing any boot tests. 1138 grk.run_key(delete_errlogs_cmd, ignore=1) 1139 1140 # Process caller's boot_stack. 1141 while (len(boot_stack) > 0): 1142 test_loop_body() 1143 1144 gp.qprint_timen("Finished processing stack.") 1145 1146 post_stack() 1147 1148 # Process caller's boot_list. 1149 if len(boot_list) > 0: 1150 for ix in range(1, max_num_tests + 1): 1151 test_loop_body() 1152 1153 gp.qprint_timen("Completed all requested boot tests.") 1154 1155 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1156 new_fail = boot_fail - init_boot_fail 1157 if new_fail > boot_fail_threshold: 1158 error_message = "Boot failures exceed the boot failure" +\ 1159 " threshold:\n" +\ 1160 gp.sprint_var(new_fail) +\ 1161 gp.sprint_var(boot_fail_threshold) 1162 BuiltIn().fail(gp.sprint_error(error_message)) 1163