1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_robot_valid as grv 27import gen_misc as gm 28import gen_cmd as gc 29import gen_robot_keyword as grk 30import state as st 31import var_stack as vs 32import gen_plug_in_utils as gpu 33 34base_path = os.path.dirname(os.path.dirname( 35 imp.find_module("gen_robot_print")[1])) +\ 36 os.sep 37sys.path.append(base_path + "extended/") 38import run_keyword as rk 39 40# Setting master_pid correctly influences the behavior of plug-ins like 41# DB_Logging 42program_pid = os.getpid() 43master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 44pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 45 46# Set up boot data structures. 47boot_table = create_boot_table() 48valid_boot_types = create_valid_boot_list(boot_table) 49 50boot_lists = read_boot_lists() 51# The maximum number of entries that can be in the last_ten global variable. 52max_boot_history = 10 53last_ten = [] 54 55state = st.return_state_constant('default_state') 56cp_setup_called = 0 57next_boot = "" 58base_tool_dir_path = os.path.normpath(os.environ.get( 59 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 60 61ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 62boot_success = 0 63status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 64if status_dir_path != "": 65 status_dir_path = os.path.normpath(status_dir_path) + os.sep 66default_power_on = "REST Power On" 67default_power_off = "REST Power Off" 68boot_count = 0 69 70LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 71ffdc_prefix = "" 72boot_start_time = "" 73boot_end_time = "" 74save_stack = vs.var_stack('save_stack') 75main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 76 77 78def dump_ffdc_rc(): 79 r""" 80 Return the constant dump ffdc test return code value. 81 82 When a plug-in call point program returns this value, it indicates that 83 this program should collect FFDC. 84 """ 85 86 return 0x00000200 87 88 89def stop_test_rc(): 90 r""" 91 Return the constant stop test return code value. 92 93 When a plug-in call point program returns this value, it indicates that 94 this program should stop running. 95 """ 96 97 return 0x00000200 98 99 100def process_host(host, 101 host_var_name=""): 102 r""" 103 Process a host by getting the associated host name and IP address and 104 setting them in global variables. 105 106 If the caller does not pass the host_var_name, this function will try to 107 figure out the name of the variable used by the caller for the host parm. 108 Callers are advised to explicitly specify the host_var_name when calling 109 with an exec command. In such cases, the get_arg_name cannot figure out 110 the host variable name. 111 112 This function will then create similar global variable names by 113 removing "_host" and appending "_host_name" or "_ip" to the host variable 114 name. 115 116 Example: 117 118 If a call is made like this: 119 process_host(openbmc_host) 120 121 Global variables openbmc_host_name and openbmc_ip will be set. 122 123 Description of argument(s): 124 host A host name or IP. The name of the variable used should 125 have a suffix of "_host". 126 host_var_name The name of the variable being used as the host parm. 127 """ 128 129 if host_var_name == "": 130 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 131 132 host_name_var_name = re.sub("host", "host_name", host_var_name) 133 ip_var_name = re.sub("host", "ip", host_var_name) 134 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 135 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 136 host + "')" 137 exec(cmd_buf) 138 139 140def process_pgm_parms(): 141 r""" 142 Process the program parameters by assigning them all to corresponding 143 globals. Also, set some global values that depend on program parameters. 144 """ 145 146 # Program parameter processing. 147 # Assign all program parms to python variables which are global to this 148 # module. 149 150 global parm_list 151 parm_list = BuiltIn().get_variable_value("${parm_list}") 152 # The following subset of parms should be processed as integers. 153 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 154 'boot_fail_threshold', 'delete_errlogs', 155 'call_post_stack_plug', 'quiet', 'test_mode', 'debug'] 156 for parm in parm_list: 157 if parm in int_list: 158 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 159 "}\", \"0\"))" 160 else: 161 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 162 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 163 gp.dpissuing(cmd_buf) 164 exec(cmd_buf) 165 if re.match(r".*_host$", parm): 166 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 167 exec(cmd_buf) 168 if re.match(r".*_password$", parm): 169 # Register the value of any parm whose name ends in _password. 170 # This will cause the print functions to replace passwords with 171 # asterisks in the output. 172 cmd_buf = "gp.register_passwords(" + parm + ")" 173 exec(cmd_buf) 174 175 global ffdc_dir_path_style 176 global boot_list 177 global boot_stack 178 global boot_results_file_path 179 global boot_results 180 global last_ten 181 global ffdc_list_file_path 182 global ffdc_report_list_path 183 global ffdc_summary_list_path 184 185 if ffdc_dir_path_style == "": 186 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 187 188 # Convert these program parms to lists for easier processing.. 189 boot_list = list(filter(None, boot_list.split(":"))) 190 boot_stack = list(filter(None, boot_stack.split(":"))) 191 192 cleanup_boot_results_file() 193 boot_results_file_path = create_boot_results_file_path(pgm_name, 194 openbmc_nickname, 195 master_pid) 196 197 if os.path.isfile(boot_results_file_path): 198 # We've been called before in this run so we'll load the saved 199 # boot_results and last_ten objects. 200 boot_results, last_ten =\ 201 pickle.load(open(boot_results_file_path, 'rb')) 202 else: 203 boot_results = boot_results(boot_table, boot_pass, boot_fail) 204 205 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 206 "/FFDC_FILE_LIST" 207 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 208 "/FFDC_REPORT_FILE_LIST" 209 210 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 211 "/FFDC_SUMMARY_FILE_LIST" 212 213 214def initial_plug_in_setup(): 215 r""" 216 Initialize all plug-in environment variables which do not change for the 217 duration of the program. 218 219 """ 220 221 global LOG_LEVEL 222 BuiltIn().set_log_level("NONE") 223 224 BuiltIn().set_global_variable("${master_pid}", master_pid) 225 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 226 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 227 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 228 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 229 ffdc_list_file_path) 230 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 231 ffdc_report_list_path) 232 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 233 ffdc_summary_list_path) 234 235 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 236 ffdc_dir_path_style) 237 BuiltIn().set_global_variable("${FFDC_CHECK}", 238 ffdc_check) 239 240 # For each program parameter, set the corresponding AUTOBOOT_ environment 241 # variable value. Also, set an AUTOBOOT_ environment variable for every 242 # element in additional_values. 243 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 244 "status_dir_path", "base_tool_dir_path", 245 "ffdc_list_file_path", "ffdc_report_list_path", 246 "ffdc_summary_list_path", "execdir"] 247 248 plug_in_vars = parm_list + additional_values 249 250 for var_name in plug_in_vars: 251 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 252 var_name = var_name.upper() 253 if var_value is None: 254 var_value = "" 255 os.environ["AUTOBOOT_" + var_name] = str(var_value) 256 257 BuiltIn().set_log_level(LOG_LEVEL) 258 259 # Make sure the ffdc list directory exists. 260 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 261 if not os.path.exists(ffdc_list_dir_path): 262 os.makedirs(ffdc_list_dir_path) 263 264 265def plug_in_setup(): 266 r""" 267 Initialize all changing plug-in environment variables for use by the 268 plug-in programs. 269 """ 270 271 global LOG_LEVEL 272 global test_really_running 273 274 BuiltIn().set_log_level("NONE") 275 276 boot_pass, boot_fail = boot_results.return_total_pass_fail() 277 if boot_pass > 1: 278 test_really_running = 1 279 else: 280 test_really_running = 0 281 282 BuiltIn().set_global_variable("${test_really_running}", 283 test_really_running) 284 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 285 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 286 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 287 BuiltIn().set_global_variable("${boot_success}", boot_success) 288 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 289 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 290 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 291 292 # For each program parameter, set the corresponding AUTOBOOT_ environment 293 # variable value. Also, set an AUTOBOOT_ environment variable for every 294 # element in additional_values. 295 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 296 "boot_fail", "test_really_running", "ffdc_prefix", 297 "boot_start_time", "boot_end_time"] 298 299 plug_in_vars = additional_values 300 301 for var_name in plug_in_vars: 302 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 303 var_name = var_name.upper() 304 if var_value is None: 305 var_value = "" 306 os.environ["AUTOBOOT_" + var_name] = str(var_value) 307 308 if debug: 309 shell_rc, out_buf = \ 310 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 311 312 BuiltIn().set_log_level(LOG_LEVEL) 313 314 315def pre_boot_plug_in_setup(): 316 317 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 318 try: 319 os.remove(ffdc_list_file_path) 320 except OSError: 321 pass 322 323 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 324 try: 325 os.remove(ffdc_report_list_path) 326 except OSError: 327 pass 328 329 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 330 try: 331 os.remove(ffdc_summary_list_path) 332 except OSError: 333 pass 334 335 global ffdc_prefix 336 337 seconds = time.time() 338 loc_time = time.localtime(seconds) 339 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 340 341 ffdc_prefix = openbmc_nickname + "." + time_string 342 343 344def default_sigusr1(signal_number=0, 345 frame=None): 346 r""" 347 Handle SIGUSR1 by doing nothing. 348 349 This function assists in debugging SIGUSR1 processing by printing messages 350 to stdout and to the log.html file. 351 352 Description of argument(s): 353 signal_number The signal number (should always be 10 for SIGUSR1). 354 frame The frame data. 355 """ 356 357 gp.printn() 358 gp.print_executing() 359 gp.lprint_executing() 360 361 362def set_default_siguser1(): 363 r""" 364 Set the default_sigusr1 function to be the SIGUSR1 handler. 365 """ 366 367 gp.printn() 368 gp.print_executing() 369 gp.lprint_executing() 370 signal.signal(signal.SIGUSR1, default_sigusr1) 371 372 373def setup(): 374 r""" 375 Do general program setup tasks. 376 """ 377 378 global cp_setup_called 379 global transitional_boot_selected 380 381 gp.qprintn() 382 383 set_default_siguser1() 384 transitional_boot_selected = False 385 386 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 387 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 388 # If we can't find process_plug_in_packages.py, ssh_pw or 389 # validate_plug_ins.py, then we don't have our repo bin in PATH. 390 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 391 + " ssh_pw validate_plug_ins.py", quiet=1, 392 print_output=0, show_err=0) 393 if shell_rc != 0: 394 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 395 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 396 if robot_pgm_dir_path not in sys.path: 397 sys.path.append(robot_pgm_dir_path) 398 PYTHONPATH = os.environ.get("PYTHONPATH", "") 399 if PYTHONPATH == "": 400 os.environ['PYTHONPATH'] = robot_pgm_dir_path 401 else: 402 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 403 404 validate_parms() 405 406 gp.qprint_pgm_header() 407 408 grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF") 409 410 initial_plug_in_setup() 411 412 plug_in_setup() 413 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 414 call_point='setup') 415 if rc != 0: 416 error_message = "Plug-in setup failed.\n" 417 gp.print_error_report(error_message) 418 BuiltIn().fail(error_message) 419 # Setting cp_setup_called lets our Teardown know that it needs to call 420 # the cleanup plug-in call point. 421 cp_setup_called = 1 422 423 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 424 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 425 # FFDC_LOG_PATH is used by "FFDC" keyword. 426 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 427 428 # Also printed by FFDC. 429 global host_name 430 global host_ip 431 host = socket.gethostname() 432 host_name, host_ip = gm.get_host_name_ip(host) 433 434 gp.dprint_var(boot_table, 1) 435 gp.dprint_var(boot_lists) 436 437 438def validate_parms(): 439 r""" 440 Validate all program parameters. 441 """ 442 443 process_pgm_parms() 444 445 gp.qprintn() 446 447 global openbmc_model 448 grv.rvalid_value("openbmc_host") 449 grv.rvalid_value("openbmc_username") 450 grv.rvalid_value("openbmc_password") 451 grv.rvalid_value("rest_username") 452 grv.rvalid_value("rest_password") 453 grv.rvalid_value("ipmi_username") 454 grv.rvalid_value("ipmi_password") 455 if os_host != "": 456 grv.rvalid_value("os_username") 457 grv.rvalid_value("os_password") 458 459 if pdu_host != "": 460 grv.rvalid_value("pdu_username") 461 grv.rvalid_value("pdu_password") 462 grv.rvalid_integer("pdu_slot_no") 463 if openbmc_serial_host != "": 464 grv.rvalid_integer("openbmc_serial_port") 465 if openbmc_model == "": 466 status, ret_values =\ 467 grk.run_key_u("Get BMC System Model") 468 openbmc_model = ret_values 469 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 470 grv.rvalid_value("openbmc_model") 471 grv.rvalid_integer("max_num_tests") 472 grv.rvalid_integer("boot_pass") 473 grv.rvalid_integer("boot_fail") 474 475 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 476 BuiltIn().set_global_variable("${plug_in_packages_list}", 477 plug_in_packages_list) 478 479 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) 480 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 481 error_message = "You must provide either a value for either the" +\ 482 " boot_list or the boot_stack parm.\n" 483 BuiltIn().fail(gp.sprint_error(error_message)) 484 485 valid_boot_list(boot_list, valid_boot_types) 486 valid_boot_list(boot_stack, valid_boot_types) 487 488 selected_PDU_boots = list(set(boot_list + boot_stack) 489 & set(boot_lists['PDU_reboot'])) 490 491 if len(selected_PDU_boots) > 0 and pdu_host == "": 492 error_message = "You have selected the following boots which" +\ 493 " require a PDU host but no value for pdu_host:\n" 494 error_message += gp.sprint_var(selected_PDU_boots) 495 error_message += gp.sprint_var(pdu_host, 2) 496 BuiltIn().fail(gp.sprint_error(error_message)) 497 498 return 499 500 501def my_get_state(): 502 r""" 503 Get the system state plus a little bit of wrapping. 504 """ 505 506 global state 507 508 req_states = ['epoch_seconds'] + st.default_req_states 509 510 gp.qprint_timen("Getting system state.") 511 if test_mode: 512 state['epoch_seconds'] = int(time.time()) 513 else: 514 state = st.get_state(req_states=req_states, quiet=quiet) 515 gp.qprint_var(state) 516 517 518def valid_state(): 519 r""" 520 Verify that our state dictionary contains no blank values. If we don't get 521 valid state data, we cannot continue to work. 522 """ 523 524 if st.compare_states(state, st.invalid_state_match, 'or'): 525 error_message = "The state dictionary contains blank fields which" +\ 526 " is illegal.\n" + gp.sprint_var(state) 527 BuiltIn().fail(gp.sprint_error(error_message)) 528 529 530def select_boot(): 531 r""" 532 Select a boot test to be run based on our current state and return the 533 chosen boot type. 534 535 Description of arguments: 536 state The state of the machine. 537 """ 538 539 global transitional_boot_selected 540 global boot_stack 541 542 gp.qprint_timen("Selecting a boot test.") 543 544 if transitional_boot_selected and not boot_success: 545 prior_boot = next_boot 546 boot_candidate = boot_stack.pop() 547 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 548 + " transition to a valid state for '" + boot_candidate 549 + "' which was at the top of the boot_stack. Since" 550 + " the '" + next_boot + "' failed, the '" 551 + boot_candidate + "' has been removed from the stack" 552 + " to avoid and endless failure loop.") 553 if len(boot_stack) == 0: 554 return "" 555 556 my_get_state() 557 valid_state() 558 559 transitional_boot_selected = False 560 stack_popped = 0 561 if len(boot_stack) > 0: 562 stack_popped = 1 563 gp.qprint_dashes() 564 gp.qprint_var(boot_stack) 565 gp.qprint_dashes() 566 skip_boot_printed = 0 567 while len(boot_stack) > 0: 568 boot_candidate = boot_stack.pop() 569 if stack_mode == 'normal': 570 break 571 else: 572 if st.compare_states(state, boot_table[boot_candidate]['end']): 573 if not skip_boot_printed: 574 gp.qprint_var(stack_mode) 575 gp.qprintn() 576 gp.qprint_timen("Skipping the following boot tests" 577 + " which are unnecessary since their" 578 + " required end states match the" 579 + " current machine state:") 580 skip_boot_printed = 1 581 gp.qprint_var(boot_candidate) 582 boot_candidate = "" 583 if boot_candidate == "": 584 gp.qprint_dashes() 585 gp.qprint_var(boot_stack) 586 gp.qprint_dashes() 587 return boot_candidate 588 if st.compare_states(state, boot_table[boot_candidate]['start']): 589 gp.qprint_timen("The machine state is valid for a '" 590 + boot_candidate + "' boot test.") 591 gp.qprint_dashes() 592 gp.qprint_var(boot_stack) 593 gp.qprint_dashes() 594 return boot_candidate 595 else: 596 gp.qprint_timen("The machine state does not match the required" 597 + " starting state for a '" + boot_candidate 598 + "' boot test:") 599 gp.qprint_varx("boot_table[" + boot_candidate + "][start]", 600 boot_table[boot_candidate]['start'], 1) 601 boot_stack.append(boot_candidate) 602 transitional_boot_selected = True 603 popped_boot = boot_candidate 604 605 # Loop through your list selecting a boot_candidates 606 boot_candidates = [] 607 for boot_candidate in boot_list: 608 if st.compare_states(state, boot_table[boot_candidate]['start']): 609 if stack_popped: 610 if st.compare_states(boot_table[boot_candidate]['end'], 611 boot_table[popped_boot]['start']): 612 boot_candidates.append(boot_candidate) 613 else: 614 boot_candidates.append(boot_candidate) 615 616 if len(boot_candidates) == 0: 617 gp.qprint_timen("The user's boot list contained no boot tests" 618 + " which are valid for the current machine state.") 619 boot_candidate = default_power_on 620 if not st.compare_states(state, boot_table[default_power_on]['start']): 621 boot_candidate = default_power_off 622 boot_candidates.append(boot_candidate) 623 gp.qprint_timen("Using default '" + boot_candidate 624 + "' boot type to transition to valid state.") 625 626 gp.dprint_var(boot_candidates) 627 628 # Randomly select a boot from the candidate list. 629 boot = random.choice(boot_candidates) 630 631 return boot 632 633 634def print_last_boots(): 635 r""" 636 Print the last ten boots done with their time stamps. 637 """ 638 639 # indent 0, 90 chars wide, linefeed, char is "=" 640 gp.qprint_dashes(0, 90) 641 gp.qprintn("Last 10 boots:\n") 642 643 for boot_entry in last_ten: 644 gp.qprint(boot_entry) 645 gp.qprint_dashes(0, 90) 646 647 648def print_defect_report(ffdc_file_list): 649 r""" 650 Print a defect report. 651 652 Description of argument(s): 653 ffdc_file_list A list of files which were collected by our ffdc functions. 654 """ 655 656 # Making deliberate choice to NOT run plug_in_setup(). We don't want 657 # ffdc_prefix updated. 658 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 659 call_point='ffdc_report', stop_on_plug_in_failure=0) 660 661 # Get additional header data which may have been created by ffdc plug-ins. 662 # Also, delete the individual header files to cleanup. 663 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 664 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 665 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 666 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 667 show_err=0) 668 669 # Get additional summary data which may have been created by ffdc plug-ins. 670 # Also, delete the individual header files to cleanup. 671 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 672 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 673 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 674 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 675 show_err=0) 676 677 # ffdc_list_file_path contains a list of any ffdc files created by plug- 678 # ins, etc. Read that data into a list. 679 try: 680 plug_in_ffdc_list = \ 681 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 682 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 683 except IOError: 684 plug_in_ffdc_list = [] 685 686 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 687 # in. Eliminate duplicates and sort the list. 688 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 689 690 if status_file_path != "": 691 ffdc_file_list.insert(0, status_file_path) 692 693 # Convert the list to a printable list. 694 printable_ffdc_file_list = "\n".join(ffdc_file_list) 695 696 # Open ffdc_file_list for writing. We will write a complete list of 697 # FFDC files to it for possible use by plug-ins like cp_stop_check. 698 ffdc_list_file = open(ffdc_list_file_path, 'w') 699 ffdc_list_file.write(printable_ffdc_file_list + "\n") 700 ffdc_list_file.close() 701 702 indent = 0 703 width = 90 704 linefeed = 1 705 char = "=" 706 707 gp.qprintn() 708 gp.qprint_dashes(indent, width, linefeed, char) 709 gp.qprintn("Copy this data to the defect:\n") 710 711 if len(more_header_info) > 0: 712 gp.qprintn(more_header_info) 713 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 714 openbmc_host_name, openbmc_ip, openbmc_username, 715 openbmc_password, rest_username, rest_password, ipmi_username, 716 ipmi_password, os_host, os_host_name, os_ip, os_username, 717 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 718 pdu_password, pdu_slot_no, openbmc_serial_host, 719 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 720 721 gp.qprintn() 722 print_last_boots() 723 gp.qprintn() 724 gp.qprint_var(state) 725 gp.qprintn() 726 gp.qprintn("FFDC data files:") 727 gp.qprintn(printable_ffdc_file_list) 728 gp.qprintn() 729 730 if len(ffdc_summary_info) > 0: 731 gp.qprintn(ffdc_summary_info) 732 733 gp.qprint_dashes(indent, width, linefeed, char) 734 735 736def my_ffdc(): 737 r""" 738 Collect FFDC data. 739 """ 740 741 global state 742 743 plug_in_setup() 744 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 745 call_point='ffdc', stop_on_plug_in_failure=0) 746 747 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 748 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 749 + AUTOBOOT_FFDC_PREFIX 750 + " ffdc_function_list=" 751 + ffdc_function_list, ignore=1) 752 if status != 'PASS': 753 gp.qprint_error("Call to ffdc failed.\n") 754 if type(ffdc_file_list) is not list: 755 ffdc_file_list = [] 756 # Leave a record for caller that "soft" errors occurred. 757 soft_errors = 1 758 gpu.save_plug_in_value(soft_errors, pgm_name) 759 760 my_get_state() 761 762 print_defect_report(ffdc_file_list) 763 764 765def print_test_start_message(boot_keyword): 766 r""" 767 Print a message indicating what boot test is about to run. 768 769 Description of arguments: 770 boot_keyword The name of the boot which is to be run 771 (e.g. "BMC Power On"). 772 """ 773 774 global last_ten 775 global boot_start_time 776 777 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 778 779 # Set boot_start_time for use by plug-ins. 780 boot_start_time = doing_msg[1:33] 781 gp.qprint_var(boot_start_time) 782 783 gp.qprint(doing_msg) 784 785 last_ten.append(doing_msg) 786 787 # Trim list to max number of entries. 788 del last_ten[:max(0, len(last_ten) - max_boot_history)] 789 790 791def stop_boot_test(signal_number=0, 792 frame=None): 793 r""" 794 Handle SIGUSR1 by aborting the boot test that is running. 795 796 Description of argument(s): 797 signal_number The signal number (should always be 10 for SIGUSR1). 798 frame The frame data. 799 """ 800 801 gp.printn() 802 gp.print_executing() 803 gp.lprint_executing() 804 805 # Restore original sigusr1 handler. 806 set_default_siguser1() 807 808 message = "The caller has asked that the boot test be stopped and marked" 809 message += " as a failure." 810 811 function_stack = gm.get_function_stack() 812 if "wait_state" in function_stack: 813 st.set_exit_wait_early_message(message) 814 else: 815 BuiltIn().fail(gp.sprint_error(message)) 816 817 818def run_boot(boot): 819 r""" 820 Run the specified boot. 821 822 Description of arguments: 823 boot The name of the boot test to be performed. 824 """ 825 826 global state 827 828 signal.signal(signal.SIGUSR1, stop_boot_test) 829 gp.qprint_timen("stop_boot_test is armed.") 830 831 print_test_start_message(boot) 832 833 plug_in_setup() 834 rc, shell_rc, failed_plug_in_name = \ 835 grpi.rprocess_plug_in_packages(call_point="pre_boot") 836 if rc != 0: 837 error_message = "Plug-in failed with non-zero return code.\n" +\ 838 gp.sprint_var(rc, 1) 839 set_default_siguser1() 840 BuiltIn().fail(gp.sprint_error(error_message)) 841 842 if test_mode: 843 # In test mode, we'll pretend the boot worked by assigning its 844 # required end state to the default state value. 845 state = st.strip_anchor_state(boot_table[boot]['end']) 846 else: 847 # Assertion: We trust that the state data was made fresh by the 848 # caller. 849 850 gp.qprintn() 851 852 if boot_table[boot]['method_type'] == "keyword": 853 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 854 boot_table[boot]['method'], 855 quiet=quiet) 856 857 if boot_table[boot]['bmc_reboot']: 858 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 859 plug_in_setup() 860 rc, shell_rc, failed_plug_in_name = \ 861 grpi.rprocess_plug_in_packages(call_point="post_reboot") 862 if rc != 0: 863 error_message = "Plug-in failed with non-zero return code.\n" 864 error_message += gp.sprint_var(rc, 1) 865 set_default_siguser1() 866 BuiltIn().fail(gp.sprint_error(error_message)) 867 else: 868 match_state = st.anchor_state(state) 869 del match_state['epoch_seconds'] 870 # Wait for the state to change in any way. 871 st.wait_state(match_state, wait_time=state_change_timeout, 872 interval="10 seconds", invert=1) 873 874 gp.qprintn() 875 if boot_table[boot]['end']['chassis'] == "Off": 876 boot_timeout = power_off_timeout 877 else: 878 boot_timeout = power_on_timeout 879 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 880 interval="10 seconds") 881 882 plug_in_setup() 883 rc, shell_rc, failed_plug_in_name = \ 884 grpi.rprocess_plug_in_packages(call_point="post_boot") 885 if rc != 0: 886 error_message = "Plug-in failed with non-zero return code.\n" +\ 887 gp.sprint_var(rc, 1) 888 set_default_siguser1() 889 BuiltIn().fail(gp.sprint_error(error_message)) 890 891 # Restore original sigusr1 handler. 892 set_default_siguser1() 893 894 895def test_loop_body(): 896 r""" 897 The main loop body for the loop in main_py. 898 899 Description of arguments: 900 boot_count The iteration number (starts at 1). 901 """ 902 903 global boot_count 904 global state 905 global next_boot 906 global boot_success 907 global boot_end_time 908 909 gp.qprintn() 910 911 next_boot = select_boot() 912 if next_boot == "": 913 return True 914 915 boot_count += 1 916 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 917 918 pre_boot_plug_in_setup() 919 920 cmd_buf = ["run_boot", next_boot] 921 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 922 if boot_status == "FAIL": 923 gp.qprint(msg) 924 925 gp.qprintn() 926 if boot_status == "PASS": 927 boot_success = 1 928 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 929 + "\" succeeded.") 930 else: 931 boot_success = 0 932 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 933 + "\" failed.") 934 935 # Set boot_end_time for use by plug-ins. 936 boot_end_time = completion_msg[1:33] 937 gp.qprint_var(boot_end_time) 938 939 gp.qprint(completion_msg) 940 941 boot_results.update(next_boot, boot_status) 942 943 plug_in_setup() 944 # NOTE: A post_test_case call point failure is NOT counted as a boot 945 # failure. 946 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 947 call_point='post_test_case', stop_on_plug_in_failure=0) 948 949 plug_in_setup() 950 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 951 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 952 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 953 if ffdc_check == "All" or\ 954 shell_rc == dump_ffdc_rc(): 955 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 956 if status != 'PASS': 957 gp.qprint_error("Call to my_ffdc failed.\n") 958 # Leave a record for caller that "soft" errors occurred. 959 soft_errors = 1 960 gpu.save_plug_in_value(soft_errors, pgm_name) 961 962 if delete_errlogs: 963 # We need to purge error logs between boots or they build up. 964 grk.run_key("Delete Error logs", ignore=1) 965 966 boot_results.print_report() 967 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 968 969 plug_in_setup() 970 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 971 call_point='stop_check', shell_rc=stop_test_rc(), 972 stop_on_non_zero_rc=1) 973 if shell_rc == stop_test_rc(): 974 message = "Stopping as requested by user.\n" 975 gp.print_time(message) 976 BuiltIn().fail(message) 977 978 # This should help prevent ConnectionErrors. 979 grk.run_key_u("Close All Connections") 980 981 return True 982 983 984def obmc_boot_test_teardown(): 985 r""" 986 Clean up after the Main keyword. 987 """ 988 989 if cp_setup_called: 990 plug_in_setup() 991 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 992 call_point='cleanup', stop_on_plug_in_failure=0) 993 994 if 'boot_results_file_path' in globals(): 995 # Save boot_results and last_ten objects to a file in case they are 996 # needed again. 997 gp.qprint_timen("Saving boot_results to the following path.") 998 gp.qprint_var(boot_results_file_path) 999 pickle.dump((boot_results, last_ten), 1000 open(boot_results_file_path, 'wb'), 1001 pickle.HIGHEST_PROTOCOL) 1002 1003 global save_stack 1004 # Restore any global values saved on the save_stack. 1005 for parm_name in main_func_parm_list: 1006 # Get the parm_value if it was saved on the stack. 1007 try: 1008 parm_value = save_stack.pop(parm_name) 1009 except BaseException: 1010 # If it was not saved, no further action is required. 1011 continue 1012 1013 # Restore the saved value. 1014 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1015 "}\", parm_value)" 1016 gp.dpissuing(cmd_buf) 1017 exec(cmd_buf) 1018 1019 gp.dprintn(save_stack.sprint_obj()) 1020 1021 1022def test_teardown(): 1023 r""" 1024 Clean up after this test case. 1025 """ 1026 1027 gp.qprintn() 1028 cmd_buf = ["Print Error", 1029 "A keyword timeout occurred ending this program.\n"] 1030 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1031 1032 gp.qprint_pgm_footer() 1033 1034 1035def post_stack(): 1036 r""" 1037 Process post_stack plug-in programs. 1038 """ 1039 1040 if not call_post_stack_plug: 1041 # The caller does not wish to have post_stack plug-in processing done. 1042 return 1043 1044 global boot_success 1045 1046 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1047 pre_boot_plug_in_setup() 1048 # For the purposes of the following plug-ins, mark the "boot" as a success. 1049 boot_success = 1 1050 plug_in_setup() 1051 rc, shell_rc, failed_plug_in_name, history =\ 1052 grpi.rprocess_plug_in_packages(call_point='post_stack', 1053 stop_on_plug_in_failure=0, 1054 return_history=True) 1055 last_ten.extend(history) 1056 # Trim list to max number of entries. 1057 del last_ten[:max(0, len(last_ten) - max_boot_history)] 1058 if rc != 0: 1059 boot_success = 0 1060 1061 plug_in_setup() 1062 rc, shell_rc, failed_plug_in_name =\ 1063 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1064 shell_rc=dump_ffdc_rc(), 1065 stop_on_plug_in_failure=1, 1066 stop_on_non_zero_rc=1) 1067 if shell_rc == dump_ffdc_rc(): 1068 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1069 if status != 'PASS': 1070 gp.qprint_error("Call to my_ffdc failed.\n") 1071 # Leave a record for caller that "soft" errors occurred. 1072 soft_errors = 1 1073 gpu.save_plug_in_value(soft_errors, pgm_name) 1074 1075 plug_in_setup() 1076 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1077 call_point='stop_check', shell_rc=stop_test_rc(), 1078 stop_on_non_zero_rc=1) 1079 if shell_rc == stop_test_rc(): 1080 message = "Stopping as requested by user.\n" 1081 gp.print_time(message) 1082 BuiltIn().fail(message) 1083 1084 1085def obmc_boot_test_py(loc_boot_stack=None, 1086 loc_stack_mode=None, 1087 loc_quiet=None): 1088 r""" 1089 Do main program processing. 1090 """ 1091 1092 global save_stack 1093 1094 gp.dprintn() 1095 # Process function parms. 1096 for parm_name in main_func_parm_list: 1097 # Get parm's value. 1098 parm_value = eval("loc_" + parm_name) 1099 gp.dpvars(parm_name, parm_value) 1100 1101 if parm_value is not None: 1102 # Save the global value on a stack. 1103 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1104 parm_name + "}\"), \"" + parm_name + "\")" 1105 gp.dpissuing(cmd_buf) 1106 exec(cmd_buf) 1107 1108 # Set the global value to the passed value. 1109 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1110 "}\", loc_" + parm_name + ")" 1111 gp.dpissuing(cmd_buf) 1112 exec(cmd_buf) 1113 1114 gp.dprintn(save_stack.sprint_obj()) 1115 1116 setup() 1117 1118 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1119 1120 if ffdc_only: 1121 gp.qprint_timen("Caller requested ffdc_only.") 1122 pre_boot_plug_in_setup() 1123 grk.run_key_u("my_ffdc") 1124 return 1125 1126 # Process caller's boot_stack. 1127 while (len(boot_stack) > 0): 1128 test_loop_body() 1129 1130 gp.qprint_timen("Finished processing stack.") 1131 1132 post_stack() 1133 1134 # Process caller's boot_list. 1135 if len(boot_list) > 0: 1136 for ix in range(1, max_num_tests + 1): 1137 test_loop_body() 1138 1139 gp.qprint_timen("Completed all requested boot tests.") 1140 1141 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1142 new_fail = boot_fail - init_boot_fail 1143 if new_fail > boot_fail_threshold: 1144 error_message = "Boot failures exceed the boot failure" +\ 1145 " threshold:\n" +\ 1146 gp.sprint_var(new_fail) +\ 1147 gp.sprint_var(boot_fail_threshold) 1148 BuiltIn().fail(gp.sprint_error(error_message)) 1149