1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_valid as gv 27import gen_misc as gm 28import gen_cmd as gc 29import gen_robot_keyword as grk 30import state as st 31import var_stack as vs 32import gen_plug_in_utils as gpu 33 34base_path = os.path.dirname(os.path.dirname( 35 imp.find_module("gen_robot_print")[1])) +\ 36 os.sep 37sys.path.append(base_path + "extended/") 38import run_keyword as rk 39 40# Setting master_pid correctly influences the behavior of plug-ins like 41# DB_Logging 42program_pid = os.getpid() 43master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 44pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 45 46# Set up boot data structures. 47os_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 48boot_table = create_boot_table(os_host=os_host) 49valid_boot_types = create_valid_boot_list(boot_table) 50 51boot_lists = read_boot_lists() 52 53# The maximum number of entries that can be in the boot_history global variable. 54max_boot_history = 10 55boot_history = [] 56 57state = st.return_state_constant('default_state') 58cp_setup_called = 0 59next_boot = "" 60base_tool_dir_path = os.path.normpath(os.environ.get( 61 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 62 63ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 64boot_success = 0 65status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 66if status_dir_path != "": 67 status_dir_path = os.path.normpath(status_dir_path) + os.sep 68redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False) 69if redfish_supported: 70 default_power_on = "Redfish Power On" 71 default_power_off = "Redfish Power Off" 72else: 73 default_power_on = "REST Power On" 74 default_power_off = "REST Power Off" 75boot_count = 0 76 77LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 78AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 79ffdc_prefix = AUTOBOOT_FFDC_PREFIX 80boot_start_time = "" 81boot_end_time = "" 82save_stack = vs.var_stack('save_stack') 83main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 84 85 86def dump_ffdc_rc(): 87 r""" 88 Return the constant dump ffdc test return code value. 89 90 When a plug-in call point program returns this value, it indicates that 91 this program should collect FFDC. 92 """ 93 94 return 0x00000200 95 96 97def stop_test_rc(): 98 r""" 99 Return the constant stop test return code value. 100 101 When a plug-in call point program returns this value, it indicates that 102 this program should stop running. 103 """ 104 105 return 0x00000200 106 107 108def process_host(host, 109 host_var_name=""): 110 r""" 111 Process a host by getting the associated host name and IP address and 112 setting them in global variables. 113 114 If the caller does not pass the host_var_name, this function will try to 115 figure out the name of the variable used by the caller for the host parm. 116 Callers are advised to explicitly specify the host_var_name when calling 117 with an exec command. In such cases, the get_arg_name cannot figure out 118 the host variable name. 119 120 This function will then create similar global variable names by 121 removing "_host" and appending "_host_name" or "_ip" to the host variable 122 name. 123 124 Example: 125 126 If a call is made like this: 127 process_host(openbmc_host) 128 129 Global variables openbmc_host_name and openbmc_ip will be set. 130 131 Description of argument(s): 132 host A host name or IP. The name of the variable used should 133 have a suffix of "_host". 134 host_var_name The name of the variable being used as the host parm. 135 """ 136 137 if host_var_name == "": 138 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 139 140 host_name_var_name = re.sub("host", "host_name", host_var_name) 141 ip_var_name = re.sub("host", "ip", host_var_name) 142 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 143 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 144 host + "')" 145 exec(cmd_buf) 146 147 148def process_pgm_parms(): 149 r""" 150 Process the program parameters by assigning them all to corresponding 151 globals. Also, set some global values that depend on program parameters. 152 """ 153 154 # Program parameter processing. 155 # Assign all program parms to python variables which are global to this 156 # module. 157 158 global parm_list 159 parm_list = BuiltIn().get_variable_value("${parm_list}") 160 # The following subset of parms should be processed as integers. 161 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 162 'boot_fail_threshold', 'delete_errlogs', 163 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 164 'test_mode', 'debug'] 165 for parm in parm_list: 166 if parm in int_list: 167 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 168 "}\", \"0\"))" 169 else: 170 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 171 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 172 gp.dpissuing(cmd_buf) 173 exec(cmd_buf) 174 if re.match(r".*_host$", parm): 175 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 176 exec(cmd_buf) 177 if re.match(r".*_password$", parm): 178 # Register the value of any parm whose name ends in _password. 179 # This will cause the print functions to replace passwords with 180 # asterisks in the output. 181 cmd_buf = "gp.register_passwords(" + parm + ")" 182 exec(cmd_buf) 183 184 global ffdc_dir_path_style 185 global boot_list 186 global boot_stack 187 global boot_results_file_path 188 global boot_results 189 global boot_history 190 global ffdc_list_file_path 191 global ffdc_report_list_path 192 global ffdc_summary_list_path 193 194 if ffdc_dir_path_style == "": 195 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 196 197 # Convert these program parms to lists for easier processing.. 198 boot_list = list(filter(None, boot_list.split(":"))) 199 boot_stack = list(filter(None, boot_stack.split(":"))) 200 201 cleanup_boot_results_file() 202 boot_results_file_path = create_boot_results_file_path(pgm_name, 203 openbmc_nickname, 204 master_pid) 205 206 if os.path.isfile(boot_results_file_path): 207 # We've been called before in this run so we'll load the saved 208 # boot_results and boot_history objects. 209 boot_results, boot_history =\ 210 pickle.load(open(boot_results_file_path, 'rb')) 211 else: 212 boot_results = boot_results(boot_table, boot_pass, boot_fail) 213 214 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 215 "/FFDC_FILE_LIST" 216 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 217 "/FFDC_REPORT_FILE_LIST" 218 219 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 220 "/FFDC_SUMMARY_FILE_LIST" 221 222 223def initial_plug_in_setup(): 224 r""" 225 Initialize all plug-in environment variables which do not change for the 226 duration of the program. 227 228 """ 229 230 global LOG_LEVEL 231 BuiltIn().set_log_level("NONE") 232 233 BuiltIn().set_global_variable("${master_pid}", master_pid) 234 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 235 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 236 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 237 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 238 ffdc_list_file_path) 239 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 240 ffdc_report_list_path) 241 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 242 ffdc_summary_list_path) 243 244 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 245 ffdc_dir_path_style) 246 BuiltIn().set_global_variable("${FFDC_CHECK}", 247 ffdc_check) 248 249 # For each program parameter, set the corresponding AUTOBOOT_ environment 250 # variable value. Also, set an AUTOBOOT_ environment variable for every 251 # element in additional_values. 252 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 253 "status_dir_path", "base_tool_dir_path", 254 "ffdc_list_file_path", "ffdc_report_list_path", 255 "ffdc_summary_list_path", "execdir"] 256 257 plug_in_vars = parm_list + additional_values 258 259 for var_name in plug_in_vars: 260 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 261 var_name = var_name.upper() 262 if var_value is None: 263 var_value = "" 264 os.environ["AUTOBOOT_" + var_name] = str(var_value) 265 266 BuiltIn().set_log_level(LOG_LEVEL) 267 268 # Make sure the ffdc list directory exists. 269 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 270 if not os.path.exists(ffdc_list_dir_path): 271 os.makedirs(ffdc_list_dir_path) 272 273 274def plug_in_setup(): 275 r""" 276 Initialize all changing plug-in environment variables for use by the 277 plug-in programs. 278 """ 279 280 global LOG_LEVEL 281 global test_really_running 282 283 BuiltIn().set_log_level("NONE") 284 285 boot_pass, boot_fail = boot_results.return_total_pass_fail() 286 if boot_pass > 1: 287 test_really_running = 1 288 else: 289 test_really_running = 0 290 291 BuiltIn().set_global_variable("${test_really_running}", 292 test_really_running) 293 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 294 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 295 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 296 BuiltIn().set_global_variable("${boot_success}", boot_success) 297 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 298 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 299 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 300 301 # For each program parameter, set the corresponding AUTOBOOT_ environment 302 # variable value. Also, set an AUTOBOOT_ environment variable for every 303 # element in additional_values. 304 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 305 "boot_fail", "test_really_running", "ffdc_prefix", 306 "boot_start_time", "boot_end_time"] 307 308 plug_in_vars = additional_values 309 310 for var_name in plug_in_vars: 311 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 312 var_name = var_name.upper() 313 if var_value is None: 314 var_value = "" 315 os.environ["AUTOBOOT_" + var_name] = str(var_value) 316 317 if debug: 318 shell_rc, out_buf = \ 319 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 320 321 BuiltIn().set_log_level(LOG_LEVEL) 322 323 324def pre_boot_plug_in_setup(): 325 326 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 327 try: 328 os.remove(ffdc_list_file_path) 329 except OSError: 330 pass 331 332 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 333 try: 334 os.remove(ffdc_report_list_path) 335 except OSError: 336 pass 337 338 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 339 try: 340 os.remove(ffdc_summary_list_path) 341 except OSError: 342 pass 343 344 global ffdc_prefix 345 346 seconds = time.time() 347 loc_time = time.localtime(seconds) 348 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 349 350 ffdc_prefix = openbmc_nickname + "." + time_string 351 352 353def default_sigusr1(signal_number=0, 354 frame=None): 355 r""" 356 Handle SIGUSR1 by doing nothing. 357 358 This function assists in debugging SIGUSR1 processing by printing messages 359 to stdout and to the log.html file. 360 361 Description of argument(s): 362 signal_number The signal number (should always be 10 for SIGUSR1). 363 frame The frame data. 364 """ 365 366 gp.qprintn() 367 gp.qprint_executing() 368 gp.lprint_executing() 369 370 371def set_default_siguser1(): 372 r""" 373 Set the default_sigusr1 function to be the SIGUSR1 handler. 374 """ 375 376 gp.qprintn() 377 gp.qprint_executing() 378 gp.lprint_executing() 379 signal.signal(signal.SIGUSR1, default_sigusr1) 380 381 382def setup(): 383 r""" 384 Do general program setup tasks. 385 """ 386 387 global cp_setup_called 388 global transitional_boot_selected 389 390 gp.qprintn() 391 392 set_default_siguser1() 393 transitional_boot_selected = False 394 395 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 396 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 397 # If we can't find process_plug_in_packages.py, ssh_pw or 398 # validate_plug_ins.py, then we don't have our repo bin in PATH. 399 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 400 + " ssh_pw validate_plug_ins.py", quiet=1, 401 print_output=0, show_err=0) 402 if shell_rc != 0: 403 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 404 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 405 if robot_pgm_dir_path not in sys.path: 406 sys.path.append(robot_pgm_dir_path) 407 PYTHONPATH = os.environ.get("PYTHONPATH", "") 408 if PYTHONPATH == "": 409 os.environ['PYTHONPATH'] = robot_pgm_dir_path 410 else: 411 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 412 413 validate_parms() 414 415 gp.qprint_pgm_header() 416 417 grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF") 418 419 initial_plug_in_setup() 420 421 plug_in_setup() 422 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 423 call_point='setup') 424 if rc != 0: 425 error_message = "Plug-in setup failed.\n" 426 gp.print_error_report(error_message) 427 BuiltIn().fail(error_message) 428 # Setting cp_setup_called lets our Teardown know that it needs to call 429 # the cleanup plug-in call point. 430 cp_setup_called = 1 431 432 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 433 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 434 # FFDC_LOG_PATH is used by "FFDC" keyword. 435 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 436 437 # Also printed by FFDC. 438 global host_name 439 global host_ip 440 host = socket.gethostname() 441 host_name, host_ip = gm.get_host_name_ip(host) 442 443 gp.dprint_var(boot_table) 444 gp.dprint_var(boot_lists) 445 446 447def validate_parms(): 448 r""" 449 Validate all program parameters. 450 """ 451 452 process_pgm_parms() 453 454 gp.qprintn() 455 456 global openbmc_model 457 gv.valid_value(openbmc_host) 458 gv.valid_value(openbmc_username) 459 gv.valid_value(openbmc_password) 460 gv.valid_value(rest_username) 461 gv.valid_value(rest_password) 462 gv.valid_value(ipmi_username) 463 gv.valid_value(ipmi_password) 464 if os_host != "": 465 gv.valid_value(os_username) 466 gv.valid_value(os_password) 467 468 if pdu_host != "": 469 gv.valid_value(pdu_username) 470 gv.valid_value(pdu_password) 471 gv.valid_integer(pdu_slot_no) 472 if openbmc_serial_host != "": 473 gv.valid_integer(openbmc_serial_port) 474 if openbmc_model == "": 475 status, ret_values =\ 476 grk.run_key_u("Get BMC System Model") 477 openbmc_model = ret_values 478 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 479 gv.valid_value(openbmc_model) 480 gv.valid_integer(max_num_tests) 481 gv.valid_integer(boot_pass) 482 gv.valid_integer(boot_fail) 483 484 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 485 BuiltIn().set_global_variable("${plug_in_packages_list}", 486 plug_in_packages_list) 487 488 gv.valid_value(stack_mode, valid_values=['normal', 'skip']) 489 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 490 error_message = "You must provide either a value for either the" +\ 491 " boot_list or the boot_stack parm.\n" 492 BuiltIn().fail(gp.sprint_error(error_message)) 493 494 valid_boot_list(boot_list, valid_boot_types) 495 valid_boot_list(boot_stack, valid_boot_types) 496 497 selected_PDU_boots = list(set(boot_list + boot_stack) 498 & set(boot_lists['PDU_reboot'])) 499 500 if len(selected_PDU_boots) > 0 and pdu_host == "": 501 error_message = "You have selected the following boots which" +\ 502 " require a PDU host but no value for pdu_host:\n" 503 error_message += gp.sprint_var(selected_PDU_boots) 504 error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 505 BuiltIn().fail(gp.sprint_error(error_message)) 506 507 return 508 509 510def my_get_state(): 511 r""" 512 Get the system state plus a little bit of wrapping. 513 """ 514 515 global state 516 517 req_states = ['epoch_seconds'] + st.default_req_states 518 519 gp.qprint_timen("Getting system state.") 520 if test_mode: 521 state['epoch_seconds'] = int(time.time()) 522 else: 523 state = st.get_state(req_states=req_states, quiet=quiet) 524 gp.qprint_var(state) 525 526 527def valid_state(): 528 r""" 529 Verify that our state dictionary contains no blank values. If we don't get 530 valid state data, we cannot continue to work. 531 """ 532 533 if st.compare_states(state, st.invalid_state_match, 'or'): 534 error_message = "The state dictionary contains blank fields which" +\ 535 " is illegal.\n" + gp.sprint_var(state) 536 BuiltIn().fail(gp.sprint_error(error_message)) 537 538 539def select_boot(): 540 r""" 541 Select a boot test to be run based on our current state and return the 542 chosen boot type. 543 544 Description of arguments: 545 state The state of the machine. 546 """ 547 548 global transitional_boot_selected 549 global boot_stack 550 551 gp.qprint_timen("Selecting a boot test.") 552 553 if transitional_boot_selected and not boot_success: 554 prior_boot = next_boot 555 boot_candidate = boot_stack.pop() 556 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 557 + " transition to a valid state for '" + boot_candidate 558 + "' which was at the top of the boot_stack. Since" 559 + " the '" + next_boot + "' failed, the '" 560 + boot_candidate + "' has been removed from the stack" 561 + " to avoid and endless failure loop.") 562 if len(boot_stack) == 0: 563 return "" 564 565 my_get_state() 566 valid_state() 567 568 transitional_boot_selected = False 569 stack_popped = 0 570 if len(boot_stack) > 0: 571 stack_popped = 1 572 gp.qprint_dashes() 573 gp.qprint_var(boot_stack) 574 gp.qprint_dashes() 575 skip_boot_printed = 0 576 while len(boot_stack) > 0: 577 boot_candidate = boot_stack.pop() 578 if stack_mode == 'normal': 579 break 580 else: 581 if st.compare_states(state, boot_table[boot_candidate]['end']): 582 if not skip_boot_printed: 583 gp.qprint_var(stack_mode) 584 gp.qprintn() 585 gp.qprint_timen("Skipping the following boot tests" 586 + " which are unnecessary since their" 587 + " required end states match the" 588 + " current machine state:") 589 skip_boot_printed = 1 590 gp.qprint_var(boot_candidate) 591 boot_candidate = "" 592 if boot_candidate == "": 593 gp.qprint_dashes() 594 gp.qprint_var(boot_stack) 595 gp.qprint_dashes() 596 return boot_candidate 597 if st.compare_states(state, boot_table[boot_candidate]['start']): 598 gp.qprint_timen("The machine state is valid for a '" 599 + boot_candidate + "' boot test.") 600 gp.qprint_dashes() 601 gp.qprint_var(boot_stack) 602 gp.qprint_dashes() 603 return boot_candidate 604 else: 605 gp.qprint_timen("The machine state does not match the required" 606 + " starting state for a '" + boot_candidate 607 + "' boot test:") 608 gp.qprint_varx("boot_table_start_entry", 609 boot_table[boot_candidate]['start']) 610 boot_stack.append(boot_candidate) 611 transitional_boot_selected = True 612 popped_boot = boot_candidate 613 614 # Loop through your list selecting a boot_candidates 615 boot_candidates = [] 616 for boot_candidate in boot_list: 617 if st.compare_states(state, boot_table[boot_candidate]['start']): 618 if stack_popped: 619 if st.compare_states(boot_table[boot_candidate]['end'], 620 boot_table[popped_boot]['start']): 621 boot_candidates.append(boot_candidate) 622 else: 623 boot_candidates.append(boot_candidate) 624 625 if len(boot_candidates) == 0: 626 gp.qprint_timen("The user's boot list contained no boot tests" 627 + " which are valid for the current machine state.") 628 boot_candidate = default_power_on 629 if not st.compare_states(state, boot_table[default_power_on]['start']): 630 boot_candidate = default_power_off 631 boot_candidates.append(boot_candidate) 632 gp.qprint_timen("Using default '" + boot_candidate 633 + "' boot type to transition to valid state.") 634 635 gp.dprint_var(boot_candidates) 636 637 # Randomly select a boot from the candidate list. 638 boot = random.choice(boot_candidates) 639 640 return boot 641 642 643def print_defect_report(ffdc_file_list): 644 r""" 645 Print a defect report. 646 647 Description of argument(s): 648 ffdc_file_list A list of files which were collected by our ffdc functions. 649 """ 650 651 # Making deliberate choice to NOT run plug_in_setup(). We don't want 652 # ffdc_prefix updated. 653 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 654 call_point='ffdc_report', stop_on_plug_in_failure=0) 655 656 # Get additional header data which may have been created by ffdc plug-ins. 657 # Also, delete the individual header files to cleanup. 658 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 659 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 660 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 661 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 662 show_err=0) 663 664 # Get additional summary data which may have been created by ffdc plug-ins. 665 # Also, delete the individual header files to cleanup. 666 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 667 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 668 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 669 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 670 show_err=0) 671 672 # ffdc_list_file_path contains a list of any ffdc files created by plug- 673 # ins, etc. Read that data into a list. 674 try: 675 plug_in_ffdc_list = \ 676 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 677 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 678 except IOError: 679 plug_in_ffdc_list = [] 680 681 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 682 # in. Eliminate duplicates and sort the list. 683 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 684 685 if status_file_path != "": 686 ffdc_file_list.insert(0, status_file_path) 687 688 # Convert the list to a printable list. 689 printable_ffdc_file_list = "\n".join(ffdc_file_list) 690 691 # Open ffdc_file_list for writing. We will write a complete list of 692 # FFDC files to it for possible use by plug-ins like cp_stop_check. 693 ffdc_list_file = open(ffdc_list_file_path, 'w') 694 ffdc_list_file.write(printable_ffdc_file_list + "\n") 695 ffdc_list_file.close() 696 697 indent = 0 698 width = 90 699 linefeed = 1 700 char = "=" 701 702 gp.qprintn() 703 gp.qprint_dashes(indent, width, linefeed, char) 704 gp.qprintn("Copy this data to the defect:\n") 705 706 if len(more_header_info) > 0: 707 gp.qprintn(more_header_info) 708 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 709 openbmc_host_name, openbmc_ip, openbmc_username, 710 openbmc_password, rest_username, rest_password, ipmi_username, 711 ipmi_password, os_host, os_host_name, os_ip, os_username, 712 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 713 pdu_password, pdu_slot_no, openbmc_serial_host, 714 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 715 716 gp.qprintn() 717 print_boot_history(boot_history) 718 gp.qprintn() 719 gp.qprint_var(state) 720 gp.qprintn() 721 gp.qprintn("FFDC data files:") 722 gp.qprintn(printable_ffdc_file_list) 723 gp.qprintn() 724 725 if len(ffdc_summary_info) > 0: 726 gp.qprintn(ffdc_summary_info) 727 728 gp.qprint_dashes(indent, width, linefeed, char) 729 730 731def my_ffdc(): 732 r""" 733 Collect FFDC data. 734 """ 735 736 global state 737 738 plug_in_setup() 739 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 740 call_point='ffdc', stop_on_plug_in_failure=0) 741 742 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 743 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 744 + AUTOBOOT_FFDC_PREFIX 745 + " ffdc_function_list=" 746 + ffdc_function_list, ignore=1) 747 if status != 'PASS': 748 gp.qprint_error("Call to ffdc failed.\n") 749 if type(ffdc_file_list) is not list: 750 ffdc_file_list = [] 751 # Leave a record for caller that "soft" errors occurred. 752 soft_errors = 1 753 gpu.save_plug_in_value(soft_errors, pgm_name) 754 755 my_get_state() 756 757 print_defect_report(ffdc_file_list) 758 759 760def print_test_start_message(boot_keyword): 761 r""" 762 Print a message indicating what boot test is about to run. 763 764 Description of arguments: 765 boot_keyword The name of the boot which is to be run 766 (e.g. "BMC Power On"). 767 """ 768 769 global boot_history 770 global boot_start_time 771 772 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 773 774 # Set boot_start_time for use by plug-ins. 775 boot_start_time = doing_msg[1:33] 776 gp.qprint_var(boot_start_time) 777 778 gp.qprint(doing_msg) 779 780 update_boot_history(boot_history, doing_msg, max_boot_history) 781 782 783def stop_boot_test(signal_number=0, 784 frame=None): 785 r""" 786 Handle SIGUSR1 by aborting the boot test that is running. 787 788 Description of argument(s): 789 signal_number The signal number (should always be 10 for SIGUSR1). 790 frame The frame data. 791 """ 792 793 gp.qprintn() 794 gp.qprint_executing() 795 gp.lprint_executing() 796 797 # Restore original sigusr1 handler. 798 set_default_siguser1() 799 800 message = "The caller has asked that the boot test be stopped and marked" 801 message += " as a failure." 802 803 function_stack = gm.get_function_stack() 804 if "wait_state" in function_stack: 805 st.set_exit_wait_early_message(message) 806 else: 807 BuiltIn().fail(gp.sprint_error(message)) 808 809 810def run_boot(boot): 811 r""" 812 Run the specified boot. 813 814 Description of arguments: 815 boot The name of the boot test to be performed. 816 """ 817 818 global state 819 820 signal.signal(signal.SIGUSR1, stop_boot_test) 821 gp.qprint_timen("stop_boot_test is armed.") 822 823 print_test_start_message(boot) 824 825 plug_in_setup() 826 rc, shell_rc, failed_plug_in_name = \ 827 grpi.rprocess_plug_in_packages(call_point="pre_boot") 828 if rc != 0: 829 error_message = "Plug-in failed with non-zero return code.\n" +\ 830 gp.sprint_var(rc, fmt=gp.hexa()) 831 set_default_siguser1() 832 BuiltIn().fail(gp.sprint_error(error_message)) 833 834 if test_mode: 835 # In test mode, we'll pretend the boot worked by assigning its 836 # required end state to the default state value. 837 state = st.strip_anchor_state(boot_table[boot]['end']) 838 else: 839 # Assertion: We trust that the state data was made fresh by the 840 # caller. 841 842 gp.qprintn() 843 844 if boot_table[boot]['method_type'] == "keyword": 845 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 846 boot_table[boot]['method'], 847 quiet=quiet) 848 849 if boot_table[boot]['bmc_reboot']: 850 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 851 plug_in_setup() 852 rc, shell_rc, failed_plug_in_name = \ 853 grpi.rprocess_plug_in_packages(call_point="post_reboot") 854 if rc != 0: 855 error_message = "Plug-in failed with non-zero return code.\n" 856 error_message += gp.sprint_var(rc, fmt=gp.hexa()) 857 set_default_siguser1() 858 BuiltIn().fail(gp.sprint_error(error_message)) 859 else: 860 match_state = st.anchor_state(state) 861 del match_state['epoch_seconds'] 862 # Wait for the state to change in any way. 863 st.wait_state(match_state, wait_time=state_change_timeout, 864 interval="10 seconds", invert=1) 865 866 gp.qprintn() 867 if boot_table[boot]['end']['chassis'] == "Off": 868 boot_timeout = power_off_timeout 869 else: 870 boot_timeout = power_on_timeout 871 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 872 interval="10 seconds") 873 874 plug_in_setup() 875 rc, shell_rc, failed_plug_in_name = \ 876 grpi.rprocess_plug_in_packages(call_point="post_boot") 877 if rc != 0: 878 error_message = "Plug-in failed with non-zero return code.\n" +\ 879 gp.sprint_var(rc, fmt=gp.hexa()) 880 set_default_siguser1() 881 BuiltIn().fail(gp.sprint_error(error_message)) 882 883 # Restore original sigusr1 handler. 884 set_default_siguser1() 885 886 887def test_loop_body(): 888 r""" 889 The main loop body for the loop in main_py. 890 891 Description of arguments: 892 boot_count The iteration number (starts at 1). 893 """ 894 895 global boot_count 896 global state 897 global next_boot 898 global boot_success 899 global boot_end_time 900 901 gp.qprintn() 902 903 next_boot = select_boot() 904 if next_boot == "": 905 return True 906 907 boot_count += 1 908 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 909 910 pre_boot_plug_in_setup() 911 912 cmd_buf = ["run_boot", next_boot] 913 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 914 if boot_status == "FAIL": 915 gp.qprint(msg) 916 917 gp.qprintn() 918 if boot_status == "PASS": 919 boot_success = 1 920 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 921 + "\" succeeded.") 922 else: 923 boot_success = 0 924 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 925 + "\" failed.") 926 927 # Set boot_end_time for use by plug-ins. 928 boot_end_time = completion_msg[1:33] 929 gp.qprint_var(boot_end_time) 930 931 gp.qprint(completion_msg) 932 933 boot_results.update(next_boot, boot_status) 934 935 plug_in_setup() 936 # NOTE: A post_test_case call point failure is NOT counted as a boot 937 # failure. 938 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 939 call_point='post_test_case', stop_on_plug_in_failure=0) 940 941 plug_in_setup() 942 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 943 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 944 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 945 if ffdc_check == "All" or\ 946 shell_rc == dump_ffdc_rc(): 947 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 948 if status != 'PASS': 949 gp.qprint_error("Call to my_ffdc failed.\n") 950 # Leave a record for caller that "soft" errors occurred. 951 soft_errors = 1 952 gpu.save_plug_in_value(soft_errors, pgm_name) 953 954 if delete_errlogs: 955 # We need to purge error logs between boots or they build up. 956 grk.run_key("Delete Error logs", ignore=1) 957 958 boot_results.print_report() 959 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 960 961 plug_in_setup() 962 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 963 call_point='stop_check', shell_rc=stop_test_rc(), 964 stop_on_non_zero_rc=1) 965 if shell_rc == stop_test_rc(): 966 message = "Stopping as requested by user.\n" 967 gp.qprint_time(message) 968 BuiltIn().fail(message) 969 970 # This should help prevent ConnectionErrors. 971 grk.run_key_u("Close All Connections") 972 973 return True 974 975 976def obmc_boot_test_teardown(): 977 r""" 978 Clean up after the Main keyword. 979 """ 980 981 if cp_setup_called: 982 plug_in_setup() 983 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 984 call_point='cleanup', stop_on_plug_in_failure=0) 985 986 if 'boot_results_file_path' in globals(): 987 # Save boot_results and boot_history objects to a file in case they are 988 # needed again. 989 gp.qprint_timen("Saving boot_results to the following path.") 990 gp.qprint_var(boot_results_file_path) 991 pickle.dump((boot_results, boot_history), 992 open(boot_results_file_path, 'wb'), 993 pickle.HIGHEST_PROTOCOL) 994 995 global save_stack 996 # Restore any global values saved on the save_stack. 997 for parm_name in main_func_parm_list: 998 # Get the parm_value if it was saved on the stack. 999 try: 1000 parm_value = save_stack.pop(parm_name) 1001 except BaseException: 1002 # If it was not saved, no further action is required. 1003 continue 1004 1005 # Restore the saved value. 1006 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1007 "}\", parm_value)" 1008 gp.dpissuing(cmd_buf) 1009 exec(cmd_buf) 1010 1011 gp.dprintn(save_stack.sprint_obj()) 1012 1013 1014def test_teardown(): 1015 r""" 1016 Clean up after this test case. 1017 """ 1018 1019 gp.qprintn() 1020 cmd_buf = ["Print Error", 1021 "A keyword timeout occurred ending this program.\n"] 1022 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1023 1024 gp.qprint_pgm_footer() 1025 1026 1027def post_stack(): 1028 r""" 1029 Process post_stack plug-in programs. 1030 """ 1031 1032 if not call_post_stack_plug: 1033 # The caller does not wish to have post_stack plug-in processing done. 1034 return 1035 1036 global boot_success 1037 1038 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1039 pre_boot_plug_in_setup() 1040 # For the purposes of the following plug-ins, mark the "boot" as a success. 1041 boot_success = 1 1042 plug_in_setup() 1043 rc, shell_rc, failed_plug_in_name, history =\ 1044 grpi.rprocess_plug_in_packages(call_point='post_stack', 1045 stop_on_plug_in_failure=0, 1046 return_history=True) 1047 for doing_msg in history: 1048 update_boot_history(boot_history, doing_msg, max_boot_history) 1049 if rc != 0: 1050 boot_success = 0 1051 1052 plug_in_setup() 1053 rc, shell_rc, failed_plug_in_name =\ 1054 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1055 shell_rc=dump_ffdc_rc(), 1056 stop_on_plug_in_failure=1, 1057 stop_on_non_zero_rc=1) 1058 if shell_rc == dump_ffdc_rc(): 1059 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1060 if status != 'PASS': 1061 gp.qprint_error("Call to my_ffdc failed.\n") 1062 # Leave a record for caller that "soft" errors occurred. 1063 soft_errors = 1 1064 gpu.save_plug_in_value(soft_errors, pgm_name) 1065 1066 plug_in_setup() 1067 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1068 call_point='stop_check', shell_rc=stop_test_rc(), 1069 stop_on_non_zero_rc=1) 1070 if shell_rc == stop_test_rc(): 1071 message = "Stopping as requested by user.\n" 1072 gp.qprint_time(message) 1073 BuiltIn().fail(message) 1074 1075 1076def obmc_boot_test_py(loc_boot_stack=None, 1077 loc_stack_mode=None, 1078 loc_quiet=None): 1079 r""" 1080 Do main program processing. 1081 """ 1082 1083 global save_stack 1084 1085 gp.dprintn() 1086 # Process function parms. 1087 for parm_name in main_func_parm_list: 1088 # Get parm's value. 1089 parm_value = eval("loc_" + parm_name) 1090 gp.dpvars(parm_name, parm_value) 1091 1092 if parm_value is not None: 1093 # Save the global value on a stack. 1094 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1095 parm_name + "}\"), \"" + parm_name + "\")" 1096 gp.dpissuing(cmd_buf) 1097 exec(cmd_buf) 1098 1099 # Set the global value to the passed value. 1100 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1101 "}\", loc_" + parm_name + ")" 1102 gp.dpissuing(cmd_buf) 1103 exec(cmd_buf) 1104 1105 gp.dprintn(save_stack.sprint_obj()) 1106 1107 setup() 1108 1109 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1110 1111 if ffdc_only: 1112 gp.qprint_timen("Caller requested ffdc_only.") 1113 if do_pre_boot_plug_in_setup: 1114 pre_boot_plug_in_setup() 1115 grk.run_key_u("my_ffdc") 1116 return 1117 1118 # Process caller's boot_stack. 1119 while (len(boot_stack) > 0): 1120 test_loop_body() 1121 1122 gp.qprint_timen("Finished processing stack.") 1123 1124 post_stack() 1125 1126 # Process caller's boot_list. 1127 if len(boot_list) > 0: 1128 for ix in range(1, max_num_tests + 1): 1129 test_loop_body() 1130 1131 gp.qprint_timen("Completed all requested boot tests.") 1132 1133 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1134 new_fail = boot_fail - init_boot_fail 1135 if new_fail > boot_fail_threshold: 1136 error_message = "Boot failures exceed the boot failure" +\ 1137 " threshold:\n" +\ 1138 gp.sprint_var(new_fail) +\ 1139 gp.sprint_var(boot_fail_threshold) 1140 BuiltIn().fail(gp.sprint_error(error_message)) 1141