1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_print as grp 26import gen_robot_plug_in as grpi 27import gen_robot_valid as grv 28import gen_misc as gm 29import gen_cmd as gc 30import gen_robot_keyword as grk 31import state as st 32import var_stack as vs 33 34base_path = os.path.dirname(os.path.dirname( 35 imp.find_module("gen_robot_print")[1])) +\ 36 os.sep 37sys.path.append(base_path + "extended/") 38import run_keyword as rk 39 40# Setting master_pid correctly influences the behavior of plug-ins like 41# DB_Logging 42program_pid = os.getpid() 43master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 44pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 45 46# Set up boot data structures. 47boot_table = create_boot_table() 48valid_boot_types = create_valid_boot_list(boot_table) 49 50boot_lists = read_boot_lists() 51# The maximum number of entries that can be in the last_ten global variable. 52max_boot_history = 10 53last_ten = [] 54 55state = st.return_state_constant('default_state') 56cp_setup_called = 0 57next_boot = "" 58base_tool_dir_path = os.path.normpath(os.environ.get( 59 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 60 61ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 62boot_success = 0 63status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 64if status_dir_path != "": 65 status_dir_path = os.path.normpath(status_dir_path) + os.sep 66default_power_on = "REST Power On" 67default_power_off = "REST Power Off" 68boot_count = 0 69 70LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 71ffdc_prefix = "" 72boot_start_time = "" 73boot_end_time = "" 74save_stack = vs.var_stack('save_stack') 75main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 76 77 78def dump_ffdc_rc(): 79 r""" 80 Return the constant dump ffdc test return code value. 81 82 When a plug-in call point program returns this value, it indicates that 83 this program should collect FFDC. 84 """ 85 86 return 0x00000200 87 88 89def stop_test_rc(): 90 r""" 91 Return the constant stop test return code value. 92 93 When a plug-in call point program returns this value, it indicates that 94 this program should stop running. 95 """ 96 97 return 0x00000200 98 99 100def process_host(host, 101 host_var_name=""): 102 r""" 103 Process a host by getting the associated host name and IP address and 104 setting them in global variables. 105 106 If the caller does not pass the host_var_name, this function will try to 107 figure out the name of the variable used by the caller for the host parm. 108 Callers are advised to explicitly specify the host_var_name when calling 109 with an exec command. In such cases, the get_arg_name cannot figure out 110 the host variable name. 111 112 This function will then create similar global variable names by 113 removing "_host" and appending "_host_name" or "_ip" to the host variable 114 name. 115 116 Example: 117 118 If a call is made like this: 119 process_host(openbmc_host) 120 121 Global variables openbmc_host_name and openbmc_ip will be set. 122 123 Description of argument(s): 124 host A host name or IP. The name of the variable used should 125 have a suffix of "_host". 126 host_var_name The name of the variable being used as the host parm. 127 """ 128 129 if host_var_name == "": 130 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 131 132 host_name_var_name = re.sub("host", "host_name", host_var_name) 133 ip_var_name = re.sub("host", "ip", host_var_name) 134 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 135 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 136 host + "')" 137 exec(cmd_buf) 138 139 140def process_pgm_parms(): 141 r""" 142 Process the program parameters by assigning them all to corresponding 143 globals. Also, set some global values that depend on program parameters. 144 """ 145 146 # Program parameter processing. 147 # Assign all program parms to python variables which are global to this 148 # module. 149 150 global parm_list 151 parm_list = BuiltIn().get_variable_value("${parm_list}") 152 # The following subset of parms should be processed as integers. 153 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 154 'boot_fail_threshold', 'delete_errlogs', 155 'call_post_stack_plug', 'quiet', 'test_mode', 'debug'] 156 for parm in parm_list: 157 if parm in int_list: 158 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 159 "}\", \"0\"))" 160 else: 161 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 162 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 163 gp.dpissuing(cmd_buf) 164 exec(cmd_buf) 165 if re.match(r".*_host$", parm): 166 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 167 exec(cmd_buf) 168 if re.match(r".*_password$", parm): 169 # Register the value of any parm whose name ends in _password. 170 # This will cause the print functions to replace passwords with 171 # asterisks in the output. 172 cmd_buf = "gp.register_passwords(" + parm + ")" 173 exec(cmd_buf) 174 175 global ffdc_dir_path_style 176 global boot_list 177 global boot_stack 178 global boot_results_file_path 179 global boot_results 180 global last_ten 181 global ffdc_list_file_path 182 global ffdc_report_list_path 183 global ffdc_summary_list_path 184 185 if ffdc_dir_path_style == "": 186 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 187 188 # Convert these program parms to lists for easier processing.. 189 boot_list = list(filter(None, boot_list.split(":"))) 190 boot_stack = list(filter(None, boot_stack.split(":"))) 191 192 cleanup_boot_results_file() 193 boot_results_file_path = create_boot_results_file_path(pgm_name, 194 openbmc_nickname, 195 master_pid) 196 197 if os.path.isfile(boot_results_file_path): 198 # We've been called before in this run so we'll load the saved 199 # boot_results and last_ten objects. 200 boot_results, last_ten =\ 201 pickle.load(open(boot_results_file_path, 'rb')) 202 else: 203 boot_results = boot_results(boot_table, boot_pass, boot_fail) 204 205 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 206 "/FFDC_FILE_LIST" 207 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 208 "/FFDC_REPORT_FILE_LIST" 209 210 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 211 "/FFDC_SUMMARY_FILE_LIST" 212 213 214def initial_plug_in_setup(): 215 r""" 216 Initialize all plug-in environment variables which do not change for the 217 duration of the program. 218 219 """ 220 221 global LOG_LEVEL 222 BuiltIn().set_log_level("NONE") 223 224 BuiltIn().set_global_variable("${master_pid}", master_pid) 225 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 226 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 227 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 228 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 229 ffdc_list_file_path) 230 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 231 ffdc_report_list_path) 232 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 233 ffdc_summary_list_path) 234 235 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 236 ffdc_dir_path_style) 237 BuiltIn().set_global_variable("${FFDC_CHECK}", 238 ffdc_check) 239 240 # For each program parameter, set the corresponding AUTOBOOT_ environment 241 # variable value. Also, set an AUTOBOOT_ environment variable for every 242 # element in additional_values. 243 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 244 "status_dir_path", "base_tool_dir_path", 245 "ffdc_list_file_path", "ffdc_report_list_path", 246 "ffdc_summary_list_path", "execdir"] 247 248 plug_in_vars = parm_list + additional_values 249 250 for var_name in plug_in_vars: 251 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 252 var_name = var_name.upper() 253 if var_value is None: 254 var_value = "" 255 os.environ["AUTOBOOT_" + var_name] = str(var_value) 256 257 BuiltIn().set_log_level(LOG_LEVEL) 258 259 # Make sure the ffdc list directory exists. 260 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 261 if not os.path.exists(ffdc_list_dir_path): 262 os.makedirs(ffdc_list_dir_path) 263 264 265def plug_in_setup(): 266 r""" 267 Initialize all changing plug-in environment variables for use by the 268 plug-in programs. 269 """ 270 271 global LOG_LEVEL 272 global test_really_running 273 274 BuiltIn().set_log_level("NONE") 275 276 boot_pass, boot_fail = boot_results.return_total_pass_fail() 277 if boot_pass > 1: 278 test_really_running = 1 279 else: 280 test_really_running = 0 281 282 BuiltIn().set_global_variable("${test_really_running}", 283 test_really_running) 284 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 285 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 286 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 287 BuiltIn().set_global_variable("${boot_success}", boot_success) 288 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 289 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 290 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 291 292 # For each program parameter, set the corresponding AUTOBOOT_ environment 293 # variable value. Also, set an AUTOBOOT_ environment variable for every 294 # element in additional_values. 295 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 296 "boot_fail", "test_really_running", "ffdc_prefix", 297 "boot_start_time", "boot_end_time"] 298 299 plug_in_vars = additional_values 300 301 for var_name in plug_in_vars: 302 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 303 var_name = var_name.upper() 304 if var_value is None: 305 var_value = "" 306 os.environ["AUTOBOOT_" + var_name] = str(var_value) 307 308 if debug: 309 shell_rc, out_buf = \ 310 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 311 312 BuiltIn().set_log_level(LOG_LEVEL) 313 314 315def pre_boot_plug_in_setup(): 316 317 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 318 try: 319 os.remove(ffdc_list_file_path) 320 except OSError: 321 pass 322 323 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 324 try: 325 os.remove(ffdc_report_list_path) 326 except OSError: 327 pass 328 329 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 330 try: 331 os.remove(ffdc_summary_list_path) 332 except OSError: 333 pass 334 335 global ffdc_prefix 336 337 seconds = time.time() 338 loc_time = time.localtime(seconds) 339 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 340 341 ffdc_prefix = openbmc_nickname + "." + time_string 342 343 344def default_sigusr1(signal_number=0, 345 frame=None): 346 r""" 347 Handle SIGUSR1 by doing nothing. 348 349 This function assists in debugging SIGUSR1 processing by printing messages 350 to stdout and to the log.html file. 351 352 Description of argument(s): 353 signal_number The signal number (should always be 10 for SIGUSR1). 354 frame The frame data. 355 """ 356 357 gp.printn() 358 gp.print_executing() 359 gp.lprint_executing() 360 361 362def set_default_siguser1(): 363 r""" 364 Set the default_sigusr1 function to be the SIGUSR1 handler. 365 """ 366 367 gp.printn() 368 gp.print_executing() 369 gp.lprint_executing() 370 signal.signal(signal.SIGUSR1, default_sigusr1) 371 372 373def setup(): 374 r""" 375 Do general program setup tasks. 376 """ 377 378 global cp_setup_called 379 global transitional_boot_selected 380 381 gp.qprintn() 382 383 set_default_siguser1() 384 transitional_boot_selected = False 385 386 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 387 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 388 # If we can't find process_plug_in_packages.py, ssh_pw or 389 # validate_plug_ins.py, then we don't have our repo bin in PATH. 390 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 391 + " ssh_pw validate_plug_ins.py", quiet=1, 392 print_output=0, show_err=0) 393 if shell_rc != 0: 394 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 395 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 396 if robot_pgm_dir_path not in sys.path: 397 sys.path.append(robot_pgm_dir_path) 398 PYTHONPATH = os.environ.get("PYTHONPATH", "") 399 if PYTHONPATH == "": 400 os.environ['PYTHONPATH'] = robot_pgm_dir_path 401 else: 402 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 403 404 validate_parms() 405 406 grp.rqprint_pgm_header() 407 408 grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF") 409 410 initial_plug_in_setup() 411 412 plug_in_setup() 413 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 414 call_point='setup') 415 if rc != 0: 416 error_message = "Plug-in setup failed.\n" 417 grp.rprint_error_report(error_message) 418 BuiltIn().fail(error_message) 419 # Setting cp_setup_called lets our Teardown know that it needs to call 420 # the cleanup plug-in call point. 421 cp_setup_called = 1 422 423 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 424 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 425 # FFDC_LOG_PATH is used by "FFDC" keyword. 426 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 427 428 # Also printed by FFDC. 429 global host_name 430 global host_ip 431 host = socket.gethostname() 432 host_name, host_ip = gm.get_host_name_ip(host) 433 434 gp.dprint_var(boot_table, 1) 435 gp.dprint_var(boot_lists) 436 437 438def validate_parms(): 439 r""" 440 Validate all program parameters. 441 """ 442 443 process_pgm_parms() 444 445 gp.qprintn() 446 447 global openbmc_model 448 grv.rvalid_value("openbmc_host") 449 grv.rvalid_value("openbmc_username") 450 grv.rvalid_value("openbmc_password") 451 grv.rvalid_value("rest_username") 452 grv.rvalid_value("rest_password") 453 grv.rvalid_value("ipmi_username") 454 grv.rvalid_value("ipmi_password") 455 if os_host != "": 456 grv.rvalid_value("os_username") 457 grv.rvalid_value("os_password") 458 459 if pdu_host != "": 460 grv.rvalid_value("pdu_username") 461 grv.rvalid_value("pdu_password") 462 grv.rvalid_integer("pdu_slot_no") 463 if openbmc_serial_host != "": 464 grv.rvalid_integer("openbmc_serial_port") 465 if openbmc_model == "": 466 status, ret_values =\ 467 grk.run_key_u("Get BMC System Model") 468 openbmc_model = ret_values 469 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 470 grv.rvalid_value("openbmc_model") 471 grv.rvalid_integer("max_num_tests") 472 grv.rvalid_integer("boot_pass") 473 grv.rvalid_integer("boot_fail") 474 475 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 476 BuiltIn().set_global_variable("${plug_in_packages_list}", 477 plug_in_packages_list) 478 479 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) 480 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 481 error_message = "You must provide either a value for either the" +\ 482 " boot_list or the boot_stack parm.\n" 483 BuiltIn().fail(gp.sprint_error(error_message)) 484 485 valid_boot_list(boot_list, valid_boot_types) 486 valid_boot_list(boot_stack, valid_boot_types) 487 488 selected_PDU_boots = list(set(boot_list + boot_stack) 489 & set(boot_lists['PDU_reboot'])) 490 491 if len(selected_PDU_boots) > 0 and pdu_host == "": 492 error_message = "You have selected the following boots which" +\ 493 " require a PDU host but no value for pdu_host:\n" 494 error_message += gp.sprint_var(selected_PDU_boots) 495 error_message += gp.sprint_var(pdu_host, 2) 496 BuiltIn().fail(gp.sprint_error(error_message)) 497 498 return 499 500 501def my_get_state(): 502 r""" 503 Get the system state plus a little bit of wrapping. 504 """ 505 506 global state 507 508 req_states = ['epoch_seconds'] + st.default_req_states 509 510 gp.qprint_timen("Getting system state.") 511 if test_mode: 512 state['epoch_seconds'] = int(time.time()) 513 else: 514 state = st.get_state(req_states=req_states, quiet=quiet) 515 gp.qprint_var(state) 516 517 518def valid_state(): 519 r""" 520 Verify that our state dictionary contains no blank values. If we don't get 521 valid state data, we cannot continue to work. 522 """ 523 524 if st.compare_states(state, st.invalid_state_match, 'or'): 525 error_message = "The state dictionary contains blank fields which" +\ 526 " is illegal.\n" + gp.sprint_var(state) 527 BuiltIn().fail(gp.sprint_error(error_message)) 528 529 530def select_boot(): 531 r""" 532 Select a boot test to be run based on our current state and return the 533 chosen boot type. 534 535 Description of arguments: 536 state The state of the machine. 537 """ 538 539 global transitional_boot_selected 540 global boot_stack 541 542 gp.qprint_timen("Selecting a boot test.") 543 544 if transitional_boot_selected and not boot_success: 545 prior_boot = next_boot 546 boot_candidate = boot_stack.pop() 547 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 548 + " transition to a valid state for '" + boot_candidate 549 + "' which was at the top of the boot_stack. Since" 550 + " the '" + next_boot + "' failed, the '" 551 + boot_candidate + "' has been removed from the stack" 552 + " to avoid and endless failure loop.") 553 if len(boot_stack) == 0: 554 return "" 555 556 my_get_state() 557 valid_state() 558 559 transitional_boot_selected = False 560 stack_popped = 0 561 if len(boot_stack) > 0: 562 stack_popped = 1 563 gp.qprint_dashes() 564 gp.qprint_var(boot_stack) 565 gp.qprint_dashes() 566 skip_boot_printed = 0 567 while len(boot_stack) > 0: 568 boot_candidate = boot_stack.pop() 569 if stack_mode == 'normal': 570 break 571 else: 572 if st.compare_states(state, boot_table[boot_candidate]['end']): 573 if not skip_boot_printed: 574 gp.qprint_var(stack_mode) 575 gp.qprintn() 576 gp.qprint_timen("Skipping the following boot tests" 577 + " which are unnecessary since their" 578 + " required end states match the" 579 + " current machine state:") 580 skip_boot_printed = 1 581 gp.qprint_var(boot_candidate) 582 boot_candidate = "" 583 if boot_candidate == "": 584 gp.qprint_dashes() 585 gp.qprint_var(boot_stack) 586 gp.qprint_dashes() 587 return boot_candidate 588 if st.compare_states(state, boot_table[boot_candidate]['start']): 589 gp.qprint_timen("The machine state is valid for a '" 590 + boot_candidate + "' boot test.") 591 gp.qprint_dashes() 592 gp.qprint_var(boot_stack) 593 gp.qprint_dashes() 594 return boot_candidate 595 else: 596 gp.qprint_timen("The machine state does not match the required" 597 + " starting state for a '" + boot_candidate 598 + "' boot test:") 599 gp.qprint_varx("boot_table[" + boot_candidate + "][start]", 600 boot_table[boot_candidate]['start'], 1) 601 boot_stack.append(boot_candidate) 602 transitional_boot_selected = True 603 popped_boot = boot_candidate 604 605 # Loop through your list selecting a boot_candidates 606 boot_candidates = [] 607 for boot_candidate in boot_list: 608 if st.compare_states(state, boot_table[boot_candidate]['start']): 609 if stack_popped: 610 if st.compare_states(boot_table[boot_candidate]['end'], 611 boot_table[popped_boot]['start']): 612 boot_candidates.append(boot_candidate) 613 else: 614 boot_candidates.append(boot_candidate) 615 616 if len(boot_candidates) == 0: 617 gp.qprint_timen("The user's boot list contained no boot tests" 618 + " which are valid for the current machine state.") 619 boot_candidate = default_power_on 620 if not st.compare_states(state, boot_table[default_power_on]['start']): 621 boot_candidate = default_power_off 622 boot_candidates.append(boot_candidate) 623 gp.qprint_timen("Using default '" + boot_candidate 624 + "' boot type to transition to valid state.") 625 626 gp.dprint_var(boot_candidates) 627 628 # Randomly select a boot from the candidate list. 629 boot = random.choice(boot_candidates) 630 631 return boot 632 633 634def print_last_boots(): 635 r""" 636 Print the last ten boots done with their time stamps. 637 """ 638 639 # indent 0, 90 chars wide, linefeed, char is "=" 640 gp.qprint_dashes(0, 90) 641 gp.qprintn("Last 10 boots:\n") 642 643 for boot_entry in last_ten: 644 grp.rqprint(boot_entry) 645 gp.qprint_dashes(0, 90) 646 647 648def print_defect_report(ffdc_file_list): 649 r""" 650 Print a defect report. 651 652 Description of argument(s): 653 ffdc_file_list A list of files which were collected by our ffdc functions. 654 """ 655 656 # Making deliberate choice to NOT run plug_in_setup(). We don't want 657 # ffdc_prefix updated. 658 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 659 call_point='ffdc_report', stop_on_plug_in_failure=0) 660 661 # Get additional header data which may have been created by ffdc plug-ins. 662 # Also, delete the individual header files to cleanup. 663 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 664 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 665 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 666 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 667 show_err=0) 668 669 # Get additional summary data which may have been created by ffdc plug-ins. 670 # Also, delete the individual header files to cleanup. 671 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 672 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 673 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 674 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 675 show_err=0) 676 677 # ffdc_list_file_path contains a list of any ffdc files created by plug- 678 # ins, etc. Read that data into a list. 679 try: 680 plug_in_ffdc_list = \ 681 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 682 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 683 except IOError: 684 plug_in_ffdc_list = [] 685 686 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 687 # in. Eliminate duplicates and sort the list. 688 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 689 690 if status_file_path != "": 691 ffdc_file_list.insert(0, status_file_path) 692 693 # Convert the list to a printable list. 694 printable_ffdc_file_list = "\n".join(ffdc_file_list) 695 696 # Open ffdc_file_list for writing. We will write a complete list of 697 # FFDC files to it for possible use by plug-ins like cp_stop_check. 698 ffdc_list_file = open(ffdc_list_file_path, 'w') 699 ffdc_list_file.write(printable_ffdc_file_list + "\n") 700 ffdc_list_file.close() 701 702 indent = 0 703 width = 90 704 linefeed = 1 705 char = "=" 706 707 gp.qprintn() 708 gp.qprint_dashes(indent, width, linefeed, char) 709 gp.qprintn("Copy this data to the defect:\n") 710 711 if len(more_header_info) > 0: 712 gp.qprintn(more_header_info) 713 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 714 openbmc_host_name, openbmc_ip, openbmc_username, 715 openbmc_password, rest_username, rest_password, ipmi_username, 716 ipmi_password, os_host, os_host_name, os_ip, os_username, 717 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 718 pdu_password, pdu_slot_no, openbmc_serial_host, 719 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 720 721 gp.qprintn() 722 print_last_boots() 723 gp.qprintn() 724 gp.qprint_var(state) 725 gp.qprintn() 726 gp.qprintn("FFDC data files:") 727 gp.qprintn(printable_ffdc_file_list) 728 gp.qprintn() 729 730 if len(ffdc_summary_info) > 0: 731 gp.qprintn(ffdc_summary_info) 732 733 gp.qprint_dashes(indent, width, linefeed, char) 734 735 736def my_ffdc(): 737 r""" 738 Collect FFDC data. 739 """ 740 741 global state 742 743 plug_in_setup() 744 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 745 call_point='ffdc', stop_on_plug_in_failure=0) 746 747 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 748 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 749 + AUTOBOOT_FFDC_PREFIX 750 + " ffdc_function_list=" 751 + ffdc_function_list, ignore=1) 752 if status != 'PASS': 753 gp.qprint_error("Call to ffdc failed.\n") 754 755 my_get_state() 756 757 print_defect_report(ffdc_file_list) 758 759 760def print_test_start_message(boot_keyword): 761 r""" 762 Print a message indicating what boot test is about to run. 763 764 Description of arguments: 765 boot_keyword The name of the boot which is to be run 766 (e.g. "BMC Power On"). 767 """ 768 769 global last_ten 770 global boot_start_time 771 772 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 773 774 # Set boot_start_time for use by plug-ins. 775 boot_start_time = doing_msg[1:33] 776 gp.qprint_var(boot_start_time) 777 778 gp.qprint(doing_msg) 779 780 last_ten.append(doing_msg) 781 782 # Trim list to max number of entries. 783 del last_ten[:max(0, len(last_ten) - max_boot_history)] 784 785 786def stop_boot_test(signal_number=0, 787 frame=None): 788 r""" 789 Handle SIGUSR1 by aborting the boot test that is running. 790 791 Description of argument(s): 792 signal_number The signal number (should always be 10 for SIGUSR1). 793 frame The frame data. 794 """ 795 796 gp.printn() 797 gp.print_executing() 798 gp.lprint_executing() 799 800 # Restore original sigusr1 handler. 801 set_default_siguser1() 802 803 message = "The caller has asked that the boot test be stopped and marked" 804 message += " as a failure." 805 806 function_stack = gm.get_function_stack() 807 if "wait_state" in function_stack: 808 st.set_wait_early_exit_message(message) 809 else: 810 BuiltIn().fail(gp.sprint_error(message)) 811 812 813def run_boot(boot): 814 r""" 815 Run the specified boot. 816 817 Description of arguments: 818 boot The name of the boot test to be performed. 819 """ 820 821 global state 822 823 signal.signal(signal.SIGUSR1, stop_boot_test) 824 gp.qprint_timen("stop_boot_test is armed.") 825 826 print_test_start_message(boot) 827 828 plug_in_setup() 829 rc, shell_rc, failed_plug_in_name = \ 830 grpi.rprocess_plug_in_packages(call_point="pre_boot") 831 if rc != 0: 832 error_message = "Plug-in failed with non-zero return code.\n" +\ 833 gp.sprint_var(rc, 1) 834 set_default_siguser1() 835 BuiltIn().fail(gp.sprint_error(error_message)) 836 837 if test_mode: 838 # In test mode, we'll pretend the boot worked by assigning its 839 # required end state to the default state value. 840 state = st.strip_anchor_state(boot_table[boot]['end']) 841 else: 842 # Assertion: We trust that the state data was made fresh by the 843 # caller. 844 845 gp.qprintn() 846 847 if boot_table[boot]['method_type'] == "keyword": 848 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 849 boot_table[boot]['method'], 850 quiet=quiet) 851 852 if boot_table[boot]['bmc_reboot']: 853 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 854 plug_in_setup() 855 rc, shell_rc, failed_plug_in_name = \ 856 grpi.rprocess_plug_in_packages(call_point="post_reboot") 857 if rc != 0: 858 error_message = "Plug-in failed with non-zero return code.\n" 859 error_message += gp.sprint_var(rc, 1) 860 set_default_siguser1() 861 BuiltIn().fail(gp.sprint_error(error_message)) 862 else: 863 match_state = st.anchor_state(state) 864 del match_state['epoch_seconds'] 865 # Wait for the state to change in any way. 866 st.wait_state(match_state, wait_time=state_change_timeout, 867 interval="10 seconds", invert=1) 868 869 gp.qprintn() 870 if boot_table[boot]['end']['chassis'] == "Off": 871 boot_timeout = power_off_timeout 872 else: 873 boot_timeout = power_on_timeout 874 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 875 interval="10 seconds") 876 877 plug_in_setup() 878 rc, shell_rc, failed_plug_in_name = \ 879 grpi.rprocess_plug_in_packages(call_point="post_boot") 880 if rc != 0: 881 error_message = "Plug-in failed with non-zero return code.\n" +\ 882 gp.sprint_var(rc, 1) 883 set_default_siguser1() 884 BuiltIn().fail(gp.sprint_error(error_message)) 885 886 # Restore original sigusr1 handler. 887 set_default_siguser1() 888 889 890def test_loop_body(): 891 r""" 892 The main loop body for the loop in main_py. 893 894 Description of arguments: 895 boot_count The iteration number (starts at 1). 896 """ 897 898 global boot_count 899 global state 900 global next_boot 901 global boot_success 902 global boot_end_time 903 904 gp.qprintn() 905 906 next_boot = select_boot() 907 if next_boot == "": 908 return True 909 910 boot_count += 1 911 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 912 913 pre_boot_plug_in_setup() 914 915 cmd_buf = ["run_boot", next_boot] 916 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 917 if boot_status == "FAIL": 918 gp.qprint(msg) 919 920 gp.qprintn() 921 if boot_status == "PASS": 922 boot_success = 1 923 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 924 + "\" succeeded.") 925 else: 926 boot_success = 0 927 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 928 + "\" failed.") 929 930 # Set boot_end_time for use by plug-ins. 931 boot_end_time = completion_msg[1:33] 932 gp.qprint_var(boot_end_time) 933 934 gp.qprint(completion_msg) 935 936 boot_results.update(next_boot, boot_status) 937 938 plug_in_setup() 939 # NOTE: A post_test_case call point failure is NOT counted as a boot 940 # failure. 941 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 942 call_point='post_test_case', stop_on_plug_in_failure=0) 943 944 plug_in_setup() 945 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 946 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 947 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 948 if ffdc_check == "All" or\ 949 shell_rc == dump_ffdc_rc(): 950 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 951 if status != 'PASS': 952 gp.qprint_error("Call to my_ffdc failed.\n") 953 954 if delete_errlogs: 955 # We need to purge error logs between boots or they build up. 956 grk.run_key("Delete Error logs", ignore=1) 957 958 boot_results.print_report() 959 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 960 961 plug_in_setup() 962 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 963 call_point='stop_check', shell_rc=stop_test_rc(), 964 stop_on_non_zero_rc=1) 965 if shell_rc == stop_test_rc(): 966 message = "Stopping as requested by user.\n" 967 gp.print_time(message) 968 BuiltIn().fail(message) 969 970 # This should help prevent ConnectionErrors. 971 grk.run_key_u("Close All Connections") 972 973 return True 974 975 976def obmc_boot_test_teardown(): 977 r""" 978 Clean up after the Main keyword. 979 """ 980 981 if cp_setup_called: 982 plug_in_setup() 983 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 984 call_point='cleanup', stop_on_plug_in_failure=0) 985 986 if 'boot_results_file_path' in globals(): 987 # Save boot_results and last_ten objects to a file in case they are 988 # needed again. 989 gp.qprint_timen("Saving boot_results to the following path.") 990 gp.qprint_var(boot_results_file_path) 991 pickle.dump((boot_results, last_ten), 992 open(boot_results_file_path, 'wb'), 993 pickle.HIGHEST_PROTOCOL) 994 995 global save_stack 996 # Restore any global values saved on the save_stack. 997 for parm_name in main_func_parm_list: 998 # Get the parm_value if it was saved on the stack. 999 try: 1000 parm_value = save_stack.pop(parm_name) 1001 except BaseException: 1002 # If it was not saved, no further action is required. 1003 continue 1004 1005 # Restore the saved value. 1006 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1007 "}\", parm_value)" 1008 gp.dpissuing(cmd_buf) 1009 exec(cmd_buf) 1010 1011 gp.dprintn(save_stack.sprint_obj()) 1012 1013 1014def test_teardown(): 1015 r""" 1016 Clean up after this test case. 1017 """ 1018 1019 gp.qprintn() 1020 cmd_buf = ["Print Error", 1021 "A keyword timeout occurred ending this program.\n"] 1022 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1023 1024 grp.rqprint_pgm_footer() 1025 1026 1027def post_stack(): 1028 r""" 1029 Process post_stack plug-in programs. 1030 """ 1031 1032 if not call_post_stack_plug: 1033 # The caller does not wish to have post_stack plug-in processing done. 1034 return 1035 1036 global boot_success 1037 1038 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1039 pre_boot_plug_in_setup() 1040 # For the purposes of the following plug-ins, mark the "boot" as a success. 1041 boot_success = 1 1042 plug_in_setup() 1043 rc, shell_rc, failed_plug_in_name, history =\ 1044 grpi.rprocess_plug_in_packages(call_point='post_stack', 1045 stop_on_plug_in_failure=0, 1046 return_history=True) 1047 last_ten.extend(history) 1048 # Trim list to max number of entries. 1049 del last_ten[:max(0, len(last_ten) - max_boot_history)] 1050 if rc != 0: 1051 boot_success = 0 1052 1053 plug_in_setup() 1054 rc, shell_rc, failed_plug_in_name =\ 1055 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1056 shell_rc=dump_ffdc_rc(), 1057 stop_on_plug_in_failure=1, 1058 stop_on_non_zero_rc=1) 1059 if shell_rc == dump_ffdc_rc(): 1060 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1061 if status != 'PASS': 1062 gp.qprint_error("Call to my_ffdc failed.\n") 1063 1064 plug_in_setup() 1065 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1066 call_point='stop_check', shell_rc=stop_test_rc(), 1067 stop_on_non_zero_rc=1) 1068 if shell_rc == stop_test_rc(): 1069 message = "Stopping as requested by user.\n" 1070 gp.print_time(message) 1071 BuiltIn().fail(message) 1072 1073 1074def obmc_boot_test_py(loc_boot_stack=None, 1075 loc_stack_mode=None, 1076 loc_quiet=None): 1077 r""" 1078 Do main program processing. 1079 """ 1080 1081 global save_stack 1082 1083 gp.dprintn() 1084 # Process function parms. 1085 for parm_name in main_func_parm_list: 1086 # Get parm's value. 1087 parm_value = eval("loc_" + parm_name) 1088 gp.dpvars(parm_name, parm_value) 1089 1090 if parm_value is not None: 1091 # Save the global value on a stack. 1092 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1093 parm_name + "}\"), \"" + parm_name + "\")" 1094 gp.dpissuing(cmd_buf) 1095 exec(cmd_buf) 1096 1097 # Set the global value to the passed value. 1098 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1099 "}\", loc_" + parm_name + ")" 1100 gp.dpissuing(cmd_buf) 1101 exec(cmd_buf) 1102 1103 gp.dprintn(save_stack.sprint_obj()) 1104 1105 setup() 1106 1107 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1108 1109 if ffdc_only: 1110 gp.qprint_timen("Caller requested ffdc_only.") 1111 pre_boot_plug_in_setup() 1112 grk.run_key_u("my_ffdc") 1113 return 1114 1115 # Process caller's boot_stack. 1116 while (len(boot_stack) > 0): 1117 test_loop_body() 1118 1119 gp.qprint_timen("Finished processing stack.") 1120 1121 post_stack() 1122 1123 # Process caller's boot_list. 1124 if len(boot_list) > 0: 1125 for ix in range(1, max_num_tests + 1): 1126 test_loop_body() 1127 1128 gp.qprint_timen("Completed all requested boot tests.") 1129 1130 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1131 new_fail = boot_fail - init_boot_fail 1132 if new_fail > boot_fail_threshold: 1133 error_message = "Boot failures exceed the boot failure" +\ 1134 " threshold:\n" +\ 1135 gp.sprint_var(new_fail) +\ 1136 gp.sprint_var(boot_fail_threshold) 1137 BuiltIn().fail(gp.sprint_error(error_message)) 1138