1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_robot_valid as grv 27import gen_misc as gm 28import gen_cmd as gc 29import gen_robot_keyword as grk 30import state as st 31import var_stack as vs 32import gen_plug_in_utils as gpu 33 34base_path = os.path.dirname(os.path.dirname( 35 imp.find_module("gen_robot_print")[1])) +\ 36 os.sep 37sys.path.append(base_path + "extended/") 38import run_keyword as rk 39 40# Setting master_pid correctly influences the behavior of plug-ins like 41# DB_Logging 42program_pid = os.getpid() 43master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 44pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 45 46# Set up boot data structures. 47os_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 48boot_table = create_boot_table(os_host=os_host) 49valid_boot_types = create_valid_boot_list(boot_table) 50 51boot_lists = read_boot_lists() 52 53# The maximum number of entries that can be in the boot_history global variable. 54max_boot_history = 10 55boot_history = [] 56 57state = st.return_state_constant('default_state') 58cp_setup_called = 0 59next_boot = "" 60base_tool_dir_path = os.path.normpath(os.environ.get( 61 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 62 63ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 64boot_success = 0 65status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 66if status_dir_path != "": 67 status_dir_path = os.path.normpath(status_dir_path) + os.sep 68default_power_on = "REST Power On" 69default_power_off = "REST Power Off" 70boot_count = 0 71 72LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 73AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 74ffdc_prefix = AUTOBOOT_FFDC_PREFIX 75boot_start_time = "" 76boot_end_time = "" 77save_stack = vs.var_stack('save_stack') 78main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 79 80 81def dump_ffdc_rc(): 82 r""" 83 Return the constant dump ffdc test return code value. 84 85 When a plug-in call point program returns this value, it indicates that 86 this program should collect FFDC. 87 """ 88 89 return 0x00000200 90 91 92def stop_test_rc(): 93 r""" 94 Return the constant stop test return code value. 95 96 When a plug-in call point program returns this value, it indicates that 97 this program should stop running. 98 """ 99 100 return 0x00000200 101 102 103def process_host(host, 104 host_var_name=""): 105 r""" 106 Process a host by getting the associated host name and IP address and 107 setting them in global variables. 108 109 If the caller does not pass the host_var_name, this function will try to 110 figure out the name of the variable used by the caller for the host parm. 111 Callers are advised to explicitly specify the host_var_name when calling 112 with an exec command. In such cases, the get_arg_name cannot figure out 113 the host variable name. 114 115 This function will then create similar global variable names by 116 removing "_host" and appending "_host_name" or "_ip" to the host variable 117 name. 118 119 Example: 120 121 If a call is made like this: 122 process_host(openbmc_host) 123 124 Global variables openbmc_host_name and openbmc_ip will be set. 125 126 Description of argument(s): 127 host A host name or IP. The name of the variable used should 128 have a suffix of "_host". 129 host_var_name The name of the variable being used as the host parm. 130 """ 131 132 if host_var_name == "": 133 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 134 135 host_name_var_name = re.sub("host", "host_name", host_var_name) 136 ip_var_name = re.sub("host", "ip", host_var_name) 137 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 138 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 139 host + "')" 140 exec(cmd_buf) 141 142 143def process_pgm_parms(): 144 r""" 145 Process the program parameters by assigning them all to corresponding 146 globals. Also, set some global values that depend on program parameters. 147 """ 148 149 # Program parameter processing. 150 # Assign all program parms to python variables which are global to this 151 # module. 152 153 global parm_list 154 parm_list = BuiltIn().get_variable_value("${parm_list}") 155 # The following subset of parms should be processed as integers. 156 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 157 'boot_fail_threshold', 'delete_errlogs', 158 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 159 'test_mode', 'debug'] 160 for parm in parm_list: 161 if parm in int_list: 162 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 163 "}\", \"0\"))" 164 else: 165 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 166 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 167 gp.dpissuing(cmd_buf) 168 exec(cmd_buf) 169 if re.match(r".*_host$", parm): 170 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 171 exec(cmd_buf) 172 if re.match(r".*_password$", parm): 173 # Register the value of any parm whose name ends in _password. 174 # This will cause the print functions to replace passwords with 175 # asterisks in the output. 176 cmd_buf = "gp.register_passwords(" + parm + ")" 177 exec(cmd_buf) 178 179 global ffdc_dir_path_style 180 global boot_list 181 global boot_stack 182 global boot_results_file_path 183 global boot_results 184 global boot_history 185 global ffdc_list_file_path 186 global ffdc_report_list_path 187 global ffdc_summary_list_path 188 189 if ffdc_dir_path_style == "": 190 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 191 192 # Convert these program parms to lists for easier processing.. 193 boot_list = list(filter(None, boot_list.split(":"))) 194 boot_stack = list(filter(None, boot_stack.split(":"))) 195 196 cleanup_boot_results_file() 197 boot_results_file_path = create_boot_results_file_path(pgm_name, 198 openbmc_nickname, 199 master_pid) 200 201 if os.path.isfile(boot_results_file_path): 202 # We've been called before in this run so we'll load the saved 203 # boot_results and boot_history objects. 204 boot_results, boot_history =\ 205 pickle.load(open(boot_results_file_path, 'rb')) 206 else: 207 boot_results = boot_results(boot_table, boot_pass, boot_fail) 208 209 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 210 "/FFDC_FILE_LIST" 211 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 212 "/FFDC_REPORT_FILE_LIST" 213 214 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 215 "/FFDC_SUMMARY_FILE_LIST" 216 217 218def initial_plug_in_setup(): 219 r""" 220 Initialize all plug-in environment variables which do not change for the 221 duration of the program. 222 223 """ 224 225 global LOG_LEVEL 226 BuiltIn().set_log_level("NONE") 227 228 BuiltIn().set_global_variable("${master_pid}", master_pid) 229 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 230 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 231 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 232 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 233 ffdc_list_file_path) 234 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 235 ffdc_report_list_path) 236 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 237 ffdc_summary_list_path) 238 239 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 240 ffdc_dir_path_style) 241 BuiltIn().set_global_variable("${FFDC_CHECK}", 242 ffdc_check) 243 244 # For each program parameter, set the corresponding AUTOBOOT_ environment 245 # variable value. Also, set an AUTOBOOT_ environment variable for every 246 # element in additional_values. 247 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 248 "status_dir_path", "base_tool_dir_path", 249 "ffdc_list_file_path", "ffdc_report_list_path", 250 "ffdc_summary_list_path", "execdir"] 251 252 plug_in_vars = parm_list + additional_values 253 254 for var_name in plug_in_vars: 255 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 256 var_name = var_name.upper() 257 if var_value is None: 258 var_value = "" 259 os.environ["AUTOBOOT_" + var_name] = str(var_value) 260 261 BuiltIn().set_log_level(LOG_LEVEL) 262 263 # Make sure the ffdc list directory exists. 264 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 265 if not os.path.exists(ffdc_list_dir_path): 266 os.makedirs(ffdc_list_dir_path) 267 268 269def plug_in_setup(): 270 r""" 271 Initialize all changing plug-in environment variables for use by the 272 plug-in programs. 273 """ 274 275 global LOG_LEVEL 276 global test_really_running 277 278 BuiltIn().set_log_level("NONE") 279 280 boot_pass, boot_fail = boot_results.return_total_pass_fail() 281 if boot_pass > 1: 282 test_really_running = 1 283 else: 284 test_really_running = 0 285 286 BuiltIn().set_global_variable("${test_really_running}", 287 test_really_running) 288 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 289 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 290 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 291 BuiltIn().set_global_variable("${boot_success}", boot_success) 292 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 293 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 294 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 295 296 # For each program parameter, set the corresponding AUTOBOOT_ environment 297 # variable value. Also, set an AUTOBOOT_ environment variable for every 298 # element in additional_values. 299 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 300 "boot_fail", "test_really_running", "ffdc_prefix", 301 "boot_start_time", "boot_end_time"] 302 303 plug_in_vars = additional_values 304 305 for var_name in plug_in_vars: 306 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 307 var_name = var_name.upper() 308 if var_value is None: 309 var_value = "" 310 os.environ["AUTOBOOT_" + var_name] = str(var_value) 311 312 if debug: 313 shell_rc, out_buf = \ 314 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 315 316 BuiltIn().set_log_level(LOG_LEVEL) 317 318 319def pre_boot_plug_in_setup(): 320 321 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 322 try: 323 os.remove(ffdc_list_file_path) 324 except OSError: 325 pass 326 327 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 328 try: 329 os.remove(ffdc_report_list_path) 330 except OSError: 331 pass 332 333 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 334 try: 335 os.remove(ffdc_summary_list_path) 336 except OSError: 337 pass 338 339 global ffdc_prefix 340 341 seconds = time.time() 342 loc_time = time.localtime(seconds) 343 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 344 345 ffdc_prefix = openbmc_nickname + "." + time_string 346 347 348def default_sigusr1(signal_number=0, 349 frame=None): 350 r""" 351 Handle SIGUSR1 by doing nothing. 352 353 This function assists in debugging SIGUSR1 processing by printing messages 354 to stdout and to the log.html file. 355 356 Description of argument(s): 357 signal_number The signal number (should always be 10 for SIGUSR1). 358 frame The frame data. 359 """ 360 361 gp.printn() 362 gp.print_executing() 363 gp.lprint_executing() 364 365 366def set_default_siguser1(): 367 r""" 368 Set the default_sigusr1 function to be the SIGUSR1 handler. 369 """ 370 371 gp.printn() 372 gp.print_executing() 373 gp.lprint_executing() 374 signal.signal(signal.SIGUSR1, default_sigusr1) 375 376 377def setup(): 378 r""" 379 Do general program setup tasks. 380 """ 381 382 global cp_setup_called 383 global transitional_boot_selected 384 385 gp.qprintn() 386 387 set_default_siguser1() 388 transitional_boot_selected = False 389 390 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 391 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 392 # If we can't find process_plug_in_packages.py, ssh_pw or 393 # validate_plug_ins.py, then we don't have our repo bin in PATH. 394 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 395 + " ssh_pw validate_plug_ins.py", quiet=1, 396 print_output=0, show_err=0) 397 if shell_rc != 0: 398 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 399 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 400 if robot_pgm_dir_path not in sys.path: 401 sys.path.append(robot_pgm_dir_path) 402 PYTHONPATH = os.environ.get("PYTHONPATH", "") 403 if PYTHONPATH == "": 404 os.environ['PYTHONPATH'] = robot_pgm_dir_path 405 else: 406 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 407 408 validate_parms() 409 410 gp.qprint_pgm_header() 411 412 grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF") 413 414 initial_plug_in_setup() 415 416 plug_in_setup() 417 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 418 call_point='setup') 419 if rc != 0: 420 error_message = "Plug-in setup failed.\n" 421 gp.print_error_report(error_message) 422 BuiltIn().fail(error_message) 423 # Setting cp_setup_called lets our Teardown know that it needs to call 424 # the cleanup plug-in call point. 425 cp_setup_called = 1 426 427 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 428 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 429 # FFDC_LOG_PATH is used by "FFDC" keyword. 430 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 431 432 # Also printed by FFDC. 433 global host_name 434 global host_ip 435 host = socket.gethostname() 436 host_name, host_ip = gm.get_host_name_ip(host) 437 438 gp.dprint_var(boot_table) 439 gp.dprint_var(boot_lists) 440 441 442def validate_parms(): 443 r""" 444 Validate all program parameters. 445 """ 446 447 process_pgm_parms() 448 449 gp.qprintn() 450 451 global openbmc_model 452 grv.rvalid_value("openbmc_host") 453 grv.rvalid_value("openbmc_username") 454 grv.rvalid_value("openbmc_password") 455 grv.rvalid_value("rest_username") 456 grv.rvalid_value("rest_password") 457 grv.rvalid_value("ipmi_username") 458 grv.rvalid_value("ipmi_password") 459 if os_host != "": 460 grv.rvalid_value("os_username") 461 grv.rvalid_value("os_password") 462 463 if pdu_host != "": 464 grv.rvalid_value("pdu_username") 465 grv.rvalid_value("pdu_password") 466 grv.rvalid_integer("pdu_slot_no") 467 if openbmc_serial_host != "": 468 grv.rvalid_integer("openbmc_serial_port") 469 if openbmc_model == "": 470 status, ret_values =\ 471 grk.run_key_u("Get BMC System Model") 472 openbmc_model = ret_values 473 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 474 grv.rvalid_value("openbmc_model") 475 grv.rvalid_integer("max_num_tests") 476 grv.rvalid_integer("boot_pass") 477 grv.rvalid_integer("boot_fail") 478 479 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 480 BuiltIn().set_global_variable("${plug_in_packages_list}", 481 plug_in_packages_list) 482 483 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) 484 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 485 error_message = "You must provide either a value for either the" +\ 486 " boot_list or the boot_stack parm.\n" 487 BuiltIn().fail(gp.sprint_error(error_message)) 488 489 valid_boot_list(boot_list, valid_boot_types) 490 valid_boot_list(boot_stack, valid_boot_types) 491 492 selected_PDU_boots = list(set(boot_list + boot_stack) 493 & set(boot_lists['PDU_reboot'])) 494 495 if len(selected_PDU_boots) > 0 and pdu_host == "": 496 error_message = "You have selected the following boots which" +\ 497 " require a PDU host but no value for pdu_host:\n" 498 error_message += gp.sprint_var(selected_PDU_boots) 499 error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 500 BuiltIn().fail(gp.sprint_error(error_message)) 501 502 return 503 504 505def my_get_state(): 506 r""" 507 Get the system state plus a little bit of wrapping. 508 """ 509 510 global state 511 512 req_states = ['epoch_seconds'] + st.default_req_states 513 514 gp.qprint_timen("Getting system state.") 515 if test_mode: 516 state['epoch_seconds'] = int(time.time()) 517 else: 518 state = st.get_state(req_states=req_states, quiet=quiet) 519 gp.qprint_var(state) 520 521 522def valid_state(): 523 r""" 524 Verify that our state dictionary contains no blank values. If we don't get 525 valid state data, we cannot continue to work. 526 """ 527 528 if st.compare_states(state, st.invalid_state_match, 'or'): 529 error_message = "The state dictionary contains blank fields which" +\ 530 " is illegal.\n" + gp.sprint_var(state) 531 BuiltIn().fail(gp.sprint_error(error_message)) 532 533 534def select_boot(): 535 r""" 536 Select a boot test to be run based on our current state and return the 537 chosen boot type. 538 539 Description of arguments: 540 state The state of the machine. 541 """ 542 543 global transitional_boot_selected 544 global boot_stack 545 546 gp.qprint_timen("Selecting a boot test.") 547 548 if transitional_boot_selected and not boot_success: 549 prior_boot = next_boot 550 boot_candidate = boot_stack.pop() 551 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 552 + " transition to a valid state for '" + boot_candidate 553 + "' which was at the top of the boot_stack. Since" 554 + " the '" + next_boot + "' failed, the '" 555 + boot_candidate + "' has been removed from the stack" 556 + " to avoid and endless failure loop.") 557 if len(boot_stack) == 0: 558 return "" 559 560 my_get_state() 561 valid_state() 562 563 transitional_boot_selected = False 564 stack_popped = 0 565 if len(boot_stack) > 0: 566 stack_popped = 1 567 gp.qprint_dashes() 568 gp.qprint_var(boot_stack) 569 gp.qprint_dashes() 570 skip_boot_printed = 0 571 while len(boot_stack) > 0: 572 boot_candidate = boot_stack.pop() 573 if stack_mode == 'normal': 574 break 575 else: 576 if st.compare_states(state, boot_table[boot_candidate]['end']): 577 if not skip_boot_printed: 578 gp.qprint_var(stack_mode) 579 gp.qprintn() 580 gp.qprint_timen("Skipping the following boot tests" 581 + " which are unnecessary since their" 582 + " required end states match the" 583 + " current machine state:") 584 skip_boot_printed = 1 585 gp.qprint_var(boot_candidate) 586 boot_candidate = "" 587 if boot_candidate == "": 588 gp.qprint_dashes() 589 gp.qprint_var(boot_stack) 590 gp.qprint_dashes() 591 return boot_candidate 592 if st.compare_states(state, boot_table[boot_candidate]['start']): 593 gp.qprint_timen("The machine state is valid for a '" 594 + boot_candidate + "' boot test.") 595 gp.qprint_dashes() 596 gp.qprint_var(boot_stack) 597 gp.qprint_dashes() 598 return boot_candidate 599 else: 600 gp.qprint_timen("The machine state does not match the required" 601 + " starting state for a '" + boot_candidate 602 + "' boot test:") 603 gp.qprint_varx("boot_table_start_entry", 604 boot_table[boot_candidate]['start']) 605 boot_stack.append(boot_candidate) 606 transitional_boot_selected = True 607 popped_boot = boot_candidate 608 609 # Loop through your list selecting a boot_candidates 610 boot_candidates = [] 611 for boot_candidate in boot_list: 612 if st.compare_states(state, boot_table[boot_candidate]['start']): 613 if stack_popped: 614 if st.compare_states(boot_table[boot_candidate]['end'], 615 boot_table[popped_boot]['start']): 616 boot_candidates.append(boot_candidate) 617 else: 618 boot_candidates.append(boot_candidate) 619 620 if len(boot_candidates) == 0: 621 gp.qprint_timen("The user's boot list contained no boot tests" 622 + " which are valid for the current machine state.") 623 boot_candidate = default_power_on 624 if not st.compare_states(state, boot_table[default_power_on]['start']): 625 boot_candidate = default_power_off 626 boot_candidates.append(boot_candidate) 627 gp.qprint_timen("Using default '" + boot_candidate 628 + "' boot type to transition to valid state.") 629 630 gp.dprint_var(boot_candidates) 631 632 # Randomly select a boot from the candidate list. 633 boot = random.choice(boot_candidates) 634 635 return boot 636 637 638def print_defect_report(ffdc_file_list): 639 r""" 640 Print a defect report. 641 642 Description of argument(s): 643 ffdc_file_list A list of files which were collected by our ffdc functions. 644 """ 645 646 # Making deliberate choice to NOT run plug_in_setup(). We don't want 647 # ffdc_prefix updated. 648 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 649 call_point='ffdc_report', stop_on_plug_in_failure=0) 650 651 # Get additional header data which may have been created by ffdc plug-ins. 652 # Also, delete the individual header files to cleanup. 653 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 654 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 655 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 656 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 657 show_err=0) 658 659 # Get additional summary data which may have been created by ffdc plug-ins. 660 # Also, delete the individual header files to cleanup. 661 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 662 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 663 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 664 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 665 show_err=0) 666 667 # ffdc_list_file_path contains a list of any ffdc files created by plug- 668 # ins, etc. Read that data into a list. 669 try: 670 plug_in_ffdc_list = \ 671 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 672 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 673 except IOError: 674 plug_in_ffdc_list = [] 675 676 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 677 # in. Eliminate duplicates and sort the list. 678 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 679 680 if status_file_path != "": 681 ffdc_file_list.insert(0, status_file_path) 682 683 # Convert the list to a printable list. 684 printable_ffdc_file_list = "\n".join(ffdc_file_list) 685 686 # Open ffdc_file_list for writing. We will write a complete list of 687 # FFDC files to it for possible use by plug-ins like cp_stop_check. 688 ffdc_list_file = open(ffdc_list_file_path, 'w') 689 ffdc_list_file.write(printable_ffdc_file_list + "\n") 690 ffdc_list_file.close() 691 692 indent = 0 693 width = 90 694 linefeed = 1 695 char = "=" 696 697 gp.qprintn() 698 gp.qprint_dashes(indent, width, linefeed, char) 699 gp.qprintn("Copy this data to the defect:\n") 700 701 if len(more_header_info) > 0: 702 gp.qprintn(more_header_info) 703 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 704 openbmc_host_name, openbmc_ip, openbmc_username, 705 openbmc_password, rest_username, rest_password, ipmi_username, 706 ipmi_password, os_host, os_host_name, os_ip, os_username, 707 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 708 pdu_password, pdu_slot_no, openbmc_serial_host, 709 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 710 711 gp.qprintn() 712 print_boot_history(boot_history) 713 gp.qprintn() 714 gp.qprint_var(state) 715 gp.qprintn() 716 gp.qprintn("FFDC data files:") 717 gp.qprintn(printable_ffdc_file_list) 718 gp.qprintn() 719 720 if len(ffdc_summary_info) > 0: 721 gp.qprintn(ffdc_summary_info) 722 723 gp.qprint_dashes(indent, width, linefeed, char) 724 725 726def my_ffdc(): 727 r""" 728 Collect FFDC data. 729 """ 730 731 global state 732 733 plug_in_setup() 734 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 735 call_point='ffdc', stop_on_plug_in_failure=0) 736 737 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 738 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 739 + AUTOBOOT_FFDC_PREFIX 740 + " ffdc_function_list=" 741 + ffdc_function_list, ignore=1) 742 if status != 'PASS': 743 gp.qprint_error("Call to ffdc failed.\n") 744 if type(ffdc_file_list) is not list: 745 ffdc_file_list = [] 746 # Leave a record for caller that "soft" errors occurred. 747 soft_errors = 1 748 gpu.save_plug_in_value(soft_errors, pgm_name) 749 750 my_get_state() 751 752 print_defect_report(ffdc_file_list) 753 754 755def print_test_start_message(boot_keyword): 756 r""" 757 Print a message indicating what boot test is about to run. 758 759 Description of arguments: 760 boot_keyword The name of the boot which is to be run 761 (e.g. "BMC Power On"). 762 """ 763 764 global boot_history 765 global boot_start_time 766 767 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 768 769 # Set boot_start_time for use by plug-ins. 770 boot_start_time = doing_msg[1:33] 771 gp.qprint_var(boot_start_time) 772 773 gp.qprint(doing_msg) 774 775 update_boot_history(boot_history, doing_msg, max_boot_history) 776 777 778def stop_boot_test(signal_number=0, 779 frame=None): 780 r""" 781 Handle SIGUSR1 by aborting the boot test that is running. 782 783 Description of argument(s): 784 signal_number The signal number (should always be 10 for SIGUSR1). 785 frame The frame data. 786 """ 787 788 gp.printn() 789 gp.print_executing() 790 gp.lprint_executing() 791 792 # Restore original sigusr1 handler. 793 set_default_siguser1() 794 795 message = "The caller has asked that the boot test be stopped and marked" 796 message += " as a failure." 797 798 function_stack = gm.get_function_stack() 799 if "wait_state" in function_stack: 800 st.set_exit_wait_early_message(message) 801 else: 802 BuiltIn().fail(gp.sprint_error(message)) 803 804 805def run_boot(boot): 806 r""" 807 Run the specified boot. 808 809 Description of arguments: 810 boot The name of the boot test to be performed. 811 """ 812 813 global state 814 815 signal.signal(signal.SIGUSR1, stop_boot_test) 816 gp.qprint_timen("stop_boot_test is armed.") 817 818 print_test_start_message(boot) 819 820 plug_in_setup() 821 rc, shell_rc, failed_plug_in_name = \ 822 grpi.rprocess_plug_in_packages(call_point="pre_boot") 823 if rc != 0: 824 error_message = "Plug-in failed with non-zero return code.\n" +\ 825 gp.sprint_var(rc, fmt=gp.hexa()) 826 set_default_siguser1() 827 BuiltIn().fail(gp.sprint_error(error_message)) 828 829 if test_mode: 830 # In test mode, we'll pretend the boot worked by assigning its 831 # required end state to the default state value. 832 state = st.strip_anchor_state(boot_table[boot]['end']) 833 else: 834 # Assertion: We trust that the state data was made fresh by the 835 # caller. 836 837 gp.qprintn() 838 839 if boot_table[boot]['method_type'] == "keyword": 840 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 841 boot_table[boot]['method'], 842 quiet=quiet) 843 844 if boot_table[boot]['bmc_reboot']: 845 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 846 plug_in_setup() 847 rc, shell_rc, failed_plug_in_name = \ 848 grpi.rprocess_plug_in_packages(call_point="post_reboot") 849 if rc != 0: 850 error_message = "Plug-in failed with non-zero return code.\n" 851 error_message += gp.sprint_var(rc, fmt=gp.hexa()) 852 set_default_siguser1() 853 BuiltIn().fail(gp.sprint_error(error_message)) 854 else: 855 match_state = st.anchor_state(state) 856 del match_state['epoch_seconds'] 857 # Wait for the state to change in any way. 858 st.wait_state(match_state, wait_time=state_change_timeout, 859 interval="10 seconds", invert=1) 860 861 gp.qprintn() 862 if boot_table[boot]['end']['chassis'] == "Off": 863 boot_timeout = power_off_timeout 864 else: 865 boot_timeout = power_on_timeout 866 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 867 interval="10 seconds") 868 869 plug_in_setup() 870 rc, shell_rc, failed_plug_in_name = \ 871 grpi.rprocess_plug_in_packages(call_point="post_boot") 872 if rc != 0: 873 error_message = "Plug-in failed with non-zero return code.\n" +\ 874 gp.sprint_var(rc, fmt=gp.hexa()) 875 set_default_siguser1() 876 BuiltIn().fail(gp.sprint_error(error_message)) 877 878 # Restore original sigusr1 handler. 879 set_default_siguser1() 880 881 882def test_loop_body(): 883 r""" 884 The main loop body for the loop in main_py. 885 886 Description of arguments: 887 boot_count The iteration number (starts at 1). 888 """ 889 890 global boot_count 891 global state 892 global next_boot 893 global boot_success 894 global boot_end_time 895 896 gp.qprintn() 897 898 next_boot = select_boot() 899 if next_boot == "": 900 return True 901 902 boot_count += 1 903 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 904 905 pre_boot_plug_in_setup() 906 907 cmd_buf = ["run_boot", next_boot] 908 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 909 if boot_status == "FAIL": 910 gp.qprint(msg) 911 912 gp.qprintn() 913 if boot_status == "PASS": 914 boot_success = 1 915 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 916 + "\" succeeded.") 917 else: 918 boot_success = 0 919 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 920 + "\" failed.") 921 922 # Set boot_end_time for use by plug-ins. 923 boot_end_time = completion_msg[1:33] 924 gp.qprint_var(boot_end_time) 925 926 gp.qprint(completion_msg) 927 928 boot_results.update(next_boot, boot_status) 929 930 plug_in_setup() 931 # NOTE: A post_test_case call point failure is NOT counted as a boot 932 # failure. 933 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 934 call_point='post_test_case', stop_on_plug_in_failure=0) 935 936 plug_in_setup() 937 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 938 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 939 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 940 if ffdc_check == "All" or\ 941 shell_rc == dump_ffdc_rc(): 942 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 943 if status != 'PASS': 944 gp.qprint_error("Call to my_ffdc failed.\n") 945 # Leave a record for caller that "soft" errors occurred. 946 soft_errors = 1 947 gpu.save_plug_in_value(soft_errors, pgm_name) 948 949 if delete_errlogs: 950 # We need to purge error logs between boots or they build up. 951 grk.run_key("Delete Error logs", ignore=1) 952 953 boot_results.print_report() 954 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 955 956 plug_in_setup() 957 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 958 call_point='stop_check', shell_rc=stop_test_rc(), 959 stop_on_non_zero_rc=1) 960 if shell_rc == stop_test_rc(): 961 message = "Stopping as requested by user.\n" 962 gp.print_time(message) 963 BuiltIn().fail(message) 964 965 # This should help prevent ConnectionErrors. 966 grk.run_key_u("Close All Connections") 967 968 return True 969 970 971def obmc_boot_test_teardown(): 972 r""" 973 Clean up after the Main keyword. 974 """ 975 976 if cp_setup_called: 977 plug_in_setup() 978 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 979 call_point='cleanup', stop_on_plug_in_failure=0) 980 981 if 'boot_results_file_path' in globals(): 982 # Save boot_results and boot_history objects to a file in case they are 983 # needed again. 984 gp.qprint_timen("Saving boot_results to the following path.") 985 gp.qprint_var(boot_results_file_path) 986 pickle.dump((boot_results, boot_history), 987 open(boot_results_file_path, 'wb'), 988 pickle.HIGHEST_PROTOCOL) 989 990 global save_stack 991 # Restore any global values saved on the save_stack. 992 for parm_name in main_func_parm_list: 993 # Get the parm_value if it was saved on the stack. 994 try: 995 parm_value = save_stack.pop(parm_name) 996 except BaseException: 997 # If it was not saved, no further action is required. 998 continue 999 1000 # Restore the saved value. 1001 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1002 "}\", parm_value)" 1003 gp.dpissuing(cmd_buf) 1004 exec(cmd_buf) 1005 1006 gp.dprintn(save_stack.sprint_obj()) 1007 1008 1009def test_teardown(): 1010 r""" 1011 Clean up after this test case. 1012 """ 1013 1014 gp.qprintn() 1015 cmd_buf = ["Print Error", 1016 "A keyword timeout occurred ending this program.\n"] 1017 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1018 1019 gp.qprint_pgm_footer() 1020 1021 1022def post_stack(): 1023 r""" 1024 Process post_stack plug-in programs. 1025 """ 1026 1027 if not call_post_stack_plug: 1028 # The caller does not wish to have post_stack plug-in processing done. 1029 return 1030 1031 global boot_success 1032 1033 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1034 pre_boot_plug_in_setup() 1035 # For the purposes of the following plug-ins, mark the "boot" as a success. 1036 boot_success = 1 1037 plug_in_setup() 1038 rc, shell_rc, failed_plug_in_name, history =\ 1039 grpi.rprocess_plug_in_packages(call_point='post_stack', 1040 stop_on_plug_in_failure=0, 1041 return_history=True) 1042 for doing_msg in history: 1043 update_boot_history(boot_history, doing_msg, max_boot_history) 1044 if rc != 0: 1045 boot_success = 0 1046 1047 plug_in_setup() 1048 rc, shell_rc, failed_plug_in_name =\ 1049 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1050 shell_rc=dump_ffdc_rc(), 1051 stop_on_plug_in_failure=1, 1052 stop_on_non_zero_rc=1) 1053 if shell_rc == dump_ffdc_rc(): 1054 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1055 if status != 'PASS': 1056 gp.qprint_error("Call to my_ffdc failed.\n") 1057 # Leave a record for caller that "soft" errors occurred. 1058 soft_errors = 1 1059 gpu.save_plug_in_value(soft_errors, pgm_name) 1060 1061 plug_in_setup() 1062 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1063 call_point='stop_check', shell_rc=stop_test_rc(), 1064 stop_on_non_zero_rc=1) 1065 if shell_rc == stop_test_rc(): 1066 message = "Stopping as requested by user.\n" 1067 gp.print_time(message) 1068 BuiltIn().fail(message) 1069 1070 1071def obmc_boot_test_py(loc_boot_stack=None, 1072 loc_stack_mode=None, 1073 loc_quiet=None): 1074 r""" 1075 Do main program processing. 1076 """ 1077 1078 global save_stack 1079 1080 gp.dprintn() 1081 # Process function parms. 1082 for parm_name in main_func_parm_list: 1083 # Get parm's value. 1084 parm_value = eval("loc_" + parm_name) 1085 gp.dpvars(parm_name, parm_value) 1086 1087 if parm_value is not None: 1088 # Save the global value on a stack. 1089 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1090 parm_name + "}\"), \"" + parm_name + "\")" 1091 gp.dpissuing(cmd_buf) 1092 exec(cmd_buf) 1093 1094 # Set the global value to the passed value. 1095 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1096 "}\", loc_" + parm_name + ")" 1097 gp.dpissuing(cmd_buf) 1098 exec(cmd_buf) 1099 1100 gp.dprintn(save_stack.sprint_obj()) 1101 1102 setup() 1103 1104 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1105 1106 if ffdc_only: 1107 gp.qprint_timen("Caller requested ffdc_only.") 1108 if do_pre_boot_plug_in_setup: 1109 pre_boot_plug_in_setup() 1110 grk.run_key_u("my_ffdc") 1111 return 1112 1113 # Process caller's boot_stack. 1114 while (len(boot_stack) > 0): 1115 test_loop_body() 1116 1117 gp.qprint_timen("Finished processing stack.") 1118 1119 post_stack() 1120 1121 # Process caller's boot_list. 1122 if len(boot_list) > 0: 1123 for ix in range(1, max_num_tests + 1): 1124 test_loop_body() 1125 1126 gp.qprint_timen("Completed all requested boot tests.") 1127 1128 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1129 new_fail = boot_fail - init_boot_fail 1130 if new_fail > boot_fail_threshold: 1131 error_message = "Boot failures exceed the boot failure" +\ 1132 " threshold:\n" +\ 1133 gp.sprint_var(new_fail) +\ 1134 gp.sprint_var(boot_fail_threshold) 1135 BuiltIn().fail(gp.sprint_error(error_message)) 1136