1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_robot_valid as grv 27import gen_misc as gm 28import gen_cmd as gc 29import gen_robot_keyword as grk 30import state as st 31import var_stack as vs 32 33base_path = os.path.dirname(os.path.dirname( 34 imp.find_module("gen_robot_print")[1])) +\ 35 os.sep 36sys.path.append(base_path + "extended/") 37import run_keyword as rk 38 39# Setting master_pid correctly influences the behavior of plug-ins like 40# DB_Logging 41program_pid = os.getpid() 42master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 43pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 44 45# Set up boot data structures. 46boot_table = create_boot_table() 47valid_boot_types = create_valid_boot_list(boot_table) 48 49boot_lists = read_boot_lists() 50# The maximum number of entries that can be in the last_ten global variable. 51max_boot_history = 10 52last_ten = [] 53 54state = st.return_state_constant('default_state') 55cp_setup_called = 0 56next_boot = "" 57base_tool_dir_path = os.path.normpath(os.environ.get( 58 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 59 60ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 61boot_success = 0 62status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 63if status_dir_path != "": 64 status_dir_path = os.path.normpath(status_dir_path) + os.sep 65default_power_on = "REST Power On" 66default_power_off = "REST Power Off" 67boot_count = 0 68 69LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 70ffdc_prefix = "" 71boot_start_time = "" 72boot_end_time = "" 73save_stack = vs.var_stack('save_stack') 74main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 75 76 77def dump_ffdc_rc(): 78 r""" 79 Return the constant dump ffdc test return code value. 80 81 When a plug-in call point program returns this value, it indicates that 82 this program should collect FFDC. 83 """ 84 85 return 0x00000200 86 87 88def stop_test_rc(): 89 r""" 90 Return the constant stop test return code value. 91 92 When a plug-in call point program returns this value, it indicates that 93 this program should stop running. 94 """ 95 96 return 0x00000200 97 98 99def process_host(host, 100 host_var_name=""): 101 r""" 102 Process a host by getting the associated host name and IP address and 103 setting them in global variables. 104 105 If the caller does not pass the host_var_name, this function will try to 106 figure out the name of the variable used by the caller for the host parm. 107 Callers are advised to explicitly specify the host_var_name when calling 108 with an exec command. In such cases, the get_arg_name cannot figure out 109 the host variable name. 110 111 This function will then create similar global variable names by 112 removing "_host" and appending "_host_name" or "_ip" to the host variable 113 name. 114 115 Example: 116 117 If a call is made like this: 118 process_host(openbmc_host) 119 120 Global variables openbmc_host_name and openbmc_ip will be set. 121 122 Description of argument(s): 123 host A host name or IP. The name of the variable used should 124 have a suffix of "_host". 125 host_var_name The name of the variable being used as the host parm. 126 """ 127 128 if host_var_name == "": 129 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 130 131 host_name_var_name = re.sub("host", "host_name", host_var_name) 132 ip_var_name = re.sub("host", "ip", host_var_name) 133 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 134 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 135 host + "')" 136 exec(cmd_buf) 137 138 139def process_pgm_parms(): 140 r""" 141 Process the program parameters by assigning them all to corresponding 142 globals. Also, set some global values that depend on program parameters. 143 """ 144 145 # Program parameter processing. 146 # Assign all program parms to python variables which are global to this 147 # module. 148 149 global parm_list 150 parm_list = BuiltIn().get_variable_value("${parm_list}") 151 # The following subset of parms should be processed as integers. 152 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 153 'boot_fail_threshold', 'delete_errlogs', 154 'call_post_stack_plug', 'quiet', 'test_mode', 'debug'] 155 for parm in parm_list: 156 if parm in int_list: 157 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 158 "}\", \"0\"))" 159 else: 160 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 161 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 162 gp.dpissuing(cmd_buf) 163 exec(cmd_buf) 164 if re.match(r".*_host$", parm): 165 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 166 exec(cmd_buf) 167 if re.match(r".*_password$", parm): 168 # Register the value of any parm whose name ends in _password. 169 # This will cause the print functions to replace passwords with 170 # asterisks in the output. 171 cmd_buf = "gp.register_passwords(" + parm + ")" 172 exec(cmd_buf) 173 174 global ffdc_dir_path_style 175 global boot_list 176 global boot_stack 177 global boot_results_file_path 178 global boot_results 179 global last_ten 180 global ffdc_list_file_path 181 global ffdc_report_list_path 182 global ffdc_summary_list_path 183 184 if ffdc_dir_path_style == "": 185 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 186 187 # Convert these program parms to lists for easier processing.. 188 boot_list = list(filter(None, boot_list.split(":"))) 189 boot_stack = list(filter(None, boot_stack.split(":"))) 190 191 cleanup_boot_results_file() 192 boot_results_file_path = create_boot_results_file_path(pgm_name, 193 openbmc_nickname, 194 master_pid) 195 196 if os.path.isfile(boot_results_file_path): 197 # We've been called before in this run so we'll load the saved 198 # boot_results and last_ten objects. 199 boot_results, last_ten =\ 200 pickle.load(open(boot_results_file_path, 'rb')) 201 else: 202 boot_results = boot_results(boot_table, boot_pass, boot_fail) 203 204 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 205 "/FFDC_FILE_LIST" 206 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 207 "/FFDC_REPORT_FILE_LIST" 208 209 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 210 "/FFDC_SUMMARY_FILE_LIST" 211 212 213def initial_plug_in_setup(): 214 r""" 215 Initialize all plug-in environment variables which do not change for the 216 duration of the program. 217 218 """ 219 220 global LOG_LEVEL 221 BuiltIn().set_log_level("NONE") 222 223 BuiltIn().set_global_variable("${master_pid}", master_pid) 224 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 225 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 226 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 227 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 228 ffdc_list_file_path) 229 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 230 ffdc_report_list_path) 231 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 232 ffdc_summary_list_path) 233 234 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 235 ffdc_dir_path_style) 236 BuiltIn().set_global_variable("${FFDC_CHECK}", 237 ffdc_check) 238 239 # For each program parameter, set the corresponding AUTOBOOT_ environment 240 # variable value. Also, set an AUTOBOOT_ environment variable for every 241 # element in additional_values. 242 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 243 "status_dir_path", "base_tool_dir_path", 244 "ffdc_list_file_path", "ffdc_report_list_path", 245 "ffdc_summary_list_path", "execdir"] 246 247 plug_in_vars = parm_list + additional_values 248 249 for var_name in plug_in_vars: 250 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 251 var_name = var_name.upper() 252 if var_value is None: 253 var_value = "" 254 os.environ["AUTOBOOT_" + var_name] = str(var_value) 255 256 BuiltIn().set_log_level(LOG_LEVEL) 257 258 # Make sure the ffdc list directory exists. 259 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 260 if not os.path.exists(ffdc_list_dir_path): 261 os.makedirs(ffdc_list_dir_path) 262 263 264def plug_in_setup(): 265 r""" 266 Initialize all changing plug-in environment variables for use by the 267 plug-in programs. 268 """ 269 270 global LOG_LEVEL 271 global test_really_running 272 273 BuiltIn().set_log_level("NONE") 274 275 boot_pass, boot_fail = boot_results.return_total_pass_fail() 276 if boot_pass > 1: 277 test_really_running = 1 278 else: 279 test_really_running = 0 280 281 BuiltIn().set_global_variable("${test_really_running}", 282 test_really_running) 283 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 284 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 285 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 286 BuiltIn().set_global_variable("${boot_success}", boot_success) 287 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 288 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 289 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 290 291 # For each program parameter, set the corresponding AUTOBOOT_ environment 292 # variable value. Also, set an AUTOBOOT_ environment variable for every 293 # element in additional_values. 294 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 295 "boot_fail", "test_really_running", "ffdc_prefix", 296 "boot_start_time", "boot_end_time"] 297 298 plug_in_vars = additional_values 299 300 for var_name in plug_in_vars: 301 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 302 var_name = var_name.upper() 303 if var_value is None: 304 var_value = "" 305 os.environ["AUTOBOOT_" + var_name] = str(var_value) 306 307 if debug: 308 shell_rc, out_buf = \ 309 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 310 311 BuiltIn().set_log_level(LOG_LEVEL) 312 313 314def pre_boot_plug_in_setup(): 315 316 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 317 try: 318 os.remove(ffdc_list_file_path) 319 except OSError: 320 pass 321 322 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 323 try: 324 os.remove(ffdc_report_list_path) 325 except OSError: 326 pass 327 328 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 329 try: 330 os.remove(ffdc_summary_list_path) 331 except OSError: 332 pass 333 334 global ffdc_prefix 335 336 seconds = time.time() 337 loc_time = time.localtime(seconds) 338 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 339 340 ffdc_prefix = openbmc_nickname + "." + time_string 341 342 343def default_sigusr1(signal_number=0, 344 frame=None): 345 r""" 346 Handle SIGUSR1 by doing nothing. 347 348 This function assists in debugging SIGUSR1 processing by printing messages 349 to stdout and to the log.html file. 350 351 Description of argument(s): 352 signal_number The signal number (should always be 10 for SIGUSR1). 353 frame The frame data. 354 """ 355 356 gp.printn() 357 gp.print_executing() 358 gp.lprint_executing() 359 360 361def set_default_siguser1(): 362 r""" 363 Set the default_sigusr1 function to be the SIGUSR1 handler. 364 """ 365 366 gp.printn() 367 gp.print_executing() 368 gp.lprint_executing() 369 signal.signal(signal.SIGUSR1, default_sigusr1) 370 371 372def setup(): 373 r""" 374 Do general program setup tasks. 375 """ 376 377 global cp_setup_called 378 global transitional_boot_selected 379 380 gp.qprintn() 381 382 set_default_siguser1() 383 transitional_boot_selected = False 384 385 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 386 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 387 # If we can't find process_plug_in_packages.py, ssh_pw or 388 # validate_plug_ins.py, then we don't have our repo bin in PATH. 389 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 390 + " ssh_pw validate_plug_ins.py", quiet=1, 391 print_output=0, show_err=0) 392 if shell_rc != 0: 393 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 394 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 395 if robot_pgm_dir_path not in sys.path: 396 sys.path.append(robot_pgm_dir_path) 397 PYTHONPATH = os.environ.get("PYTHONPATH", "") 398 if PYTHONPATH == "": 399 os.environ['PYTHONPATH'] = robot_pgm_dir_path 400 else: 401 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 402 403 validate_parms() 404 405 gp.qprint_pgm_header() 406 407 grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF") 408 409 initial_plug_in_setup() 410 411 plug_in_setup() 412 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 413 call_point='setup') 414 if rc != 0: 415 error_message = "Plug-in setup failed.\n" 416 gp.print_error_report(error_message) 417 BuiltIn().fail(error_message) 418 # Setting cp_setup_called lets our Teardown know that it needs to call 419 # the cleanup plug-in call point. 420 cp_setup_called = 1 421 422 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 423 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 424 # FFDC_LOG_PATH is used by "FFDC" keyword. 425 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 426 427 # Also printed by FFDC. 428 global host_name 429 global host_ip 430 host = socket.gethostname() 431 host_name, host_ip = gm.get_host_name_ip(host) 432 433 gp.dprint_var(boot_table, 1) 434 gp.dprint_var(boot_lists) 435 436 437def validate_parms(): 438 r""" 439 Validate all program parameters. 440 """ 441 442 process_pgm_parms() 443 444 gp.qprintn() 445 446 global openbmc_model 447 grv.rvalid_value("openbmc_host") 448 grv.rvalid_value("openbmc_username") 449 grv.rvalid_value("openbmc_password") 450 grv.rvalid_value("rest_username") 451 grv.rvalid_value("rest_password") 452 grv.rvalid_value("ipmi_username") 453 grv.rvalid_value("ipmi_password") 454 if os_host != "": 455 grv.rvalid_value("os_username") 456 grv.rvalid_value("os_password") 457 458 if pdu_host != "": 459 grv.rvalid_value("pdu_username") 460 grv.rvalid_value("pdu_password") 461 grv.rvalid_integer("pdu_slot_no") 462 if openbmc_serial_host != "": 463 grv.rvalid_integer("openbmc_serial_port") 464 if openbmc_model == "": 465 status, ret_values =\ 466 grk.run_key_u("Get BMC System Model") 467 openbmc_model = ret_values 468 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 469 grv.rvalid_value("openbmc_model") 470 grv.rvalid_integer("max_num_tests") 471 grv.rvalid_integer("boot_pass") 472 grv.rvalid_integer("boot_fail") 473 474 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 475 BuiltIn().set_global_variable("${plug_in_packages_list}", 476 plug_in_packages_list) 477 478 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) 479 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 480 error_message = "You must provide either a value for either the" +\ 481 " boot_list or the boot_stack parm.\n" 482 BuiltIn().fail(gp.sprint_error(error_message)) 483 484 valid_boot_list(boot_list, valid_boot_types) 485 valid_boot_list(boot_stack, valid_boot_types) 486 487 selected_PDU_boots = list(set(boot_list + boot_stack) 488 & set(boot_lists['PDU_reboot'])) 489 490 if len(selected_PDU_boots) > 0 and pdu_host == "": 491 error_message = "You have selected the following boots which" +\ 492 " require a PDU host but no value for pdu_host:\n" 493 error_message += gp.sprint_var(selected_PDU_boots) 494 error_message += gp.sprint_var(pdu_host, 2) 495 BuiltIn().fail(gp.sprint_error(error_message)) 496 497 return 498 499 500def my_get_state(): 501 r""" 502 Get the system state plus a little bit of wrapping. 503 """ 504 505 global state 506 507 req_states = ['epoch_seconds'] + st.default_req_states 508 509 gp.qprint_timen("Getting system state.") 510 if test_mode: 511 state['epoch_seconds'] = int(time.time()) 512 else: 513 state = st.get_state(req_states=req_states, quiet=quiet) 514 gp.qprint_var(state) 515 516 517def valid_state(): 518 r""" 519 Verify that our state dictionary contains no blank values. If we don't get 520 valid state data, we cannot continue to work. 521 """ 522 523 if st.compare_states(state, st.invalid_state_match, 'or'): 524 error_message = "The state dictionary contains blank fields which" +\ 525 " is illegal.\n" + gp.sprint_var(state) 526 BuiltIn().fail(gp.sprint_error(error_message)) 527 528 529def select_boot(): 530 r""" 531 Select a boot test to be run based on our current state and return the 532 chosen boot type. 533 534 Description of arguments: 535 state The state of the machine. 536 """ 537 538 global transitional_boot_selected 539 global boot_stack 540 541 gp.qprint_timen("Selecting a boot test.") 542 543 if transitional_boot_selected and not boot_success: 544 prior_boot = next_boot 545 boot_candidate = boot_stack.pop() 546 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 547 + " transition to a valid state for '" + boot_candidate 548 + "' which was at the top of the boot_stack. Since" 549 + " the '" + next_boot + "' failed, the '" 550 + boot_candidate + "' has been removed from the stack" 551 + " to avoid and endless failure loop.") 552 if len(boot_stack) == 0: 553 return "" 554 555 my_get_state() 556 valid_state() 557 558 transitional_boot_selected = False 559 stack_popped = 0 560 if len(boot_stack) > 0: 561 stack_popped = 1 562 gp.qprint_dashes() 563 gp.qprint_var(boot_stack) 564 gp.qprint_dashes() 565 skip_boot_printed = 0 566 while len(boot_stack) > 0: 567 boot_candidate = boot_stack.pop() 568 if stack_mode == 'normal': 569 break 570 else: 571 if st.compare_states(state, boot_table[boot_candidate]['end']): 572 if not skip_boot_printed: 573 gp.qprint_var(stack_mode) 574 gp.qprintn() 575 gp.qprint_timen("Skipping the following boot tests" 576 + " which are unnecessary since their" 577 + " required end states match the" 578 + " current machine state:") 579 skip_boot_printed = 1 580 gp.qprint_var(boot_candidate) 581 boot_candidate = "" 582 if boot_candidate == "": 583 gp.qprint_dashes() 584 gp.qprint_var(boot_stack) 585 gp.qprint_dashes() 586 return boot_candidate 587 if st.compare_states(state, boot_table[boot_candidate]['start']): 588 gp.qprint_timen("The machine state is valid for a '" 589 + boot_candidate + "' boot test.") 590 gp.qprint_dashes() 591 gp.qprint_var(boot_stack) 592 gp.qprint_dashes() 593 return boot_candidate 594 else: 595 gp.qprint_timen("The machine state does not match the required" 596 + " starting state for a '" + boot_candidate 597 + "' boot test:") 598 gp.qprint_varx("boot_table[" + boot_candidate + "][start]", 599 boot_table[boot_candidate]['start'], 1) 600 boot_stack.append(boot_candidate) 601 transitional_boot_selected = True 602 popped_boot = boot_candidate 603 604 # Loop through your list selecting a boot_candidates 605 boot_candidates = [] 606 for boot_candidate in boot_list: 607 if st.compare_states(state, boot_table[boot_candidate]['start']): 608 if stack_popped: 609 if st.compare_states(boot_table[boot_candidate]['end'], 610 boot_table[popped_boot]['start']): 611 boot_candidates.append(boot_candidate) 612 else: 613 boot_candidates.append(boot_candidate) 614 615 if len(boot_candidates) == 0: 616 gp.qprint_timen("The user's boot list contained no boot tests" 617 + " which are valid for the current machine state.") 618 boot_candidate = default_power_on 619 if not st.compare_states(state, boot_table[default_power_on]['start']): 620 boot_candidate = default_power_off 621 boot_candidates.append(boot_candidate) 622 gp.qprint_timen("Using default '" + boot_candidate 623 + "' boot type to transition to valid state.") 624 625 gp.dprint_var(boot_candidates) 626 627 # Randomly select a boot from the candidate list. 628 boot = random.choice(boot_candidates) 629 630 return boot 631 632 633def print_last_boots(): 634 r""" 635 Print the last ten boots done with their time stamps. 636 """ 637 638 # indent 0, 90 chars wide, linefeed, char is "=" 639 gp.qprint_dashes(0, 90) 640 gp.qprintn("Last 10 boots:\n") 641 642 for boot_entry in last_ten: 643 gp.qprint(boot_entry) 644 gp.qprint_dashes(0, 90) 645 646 647def print_defect_report(ffdc_file_list): 648 r""" 649 Print a defect report. 650 651 Description of argument(s): 652 ffdc_file_list A list of files which were collected by our ffdc functions. 653 """ 654 655 # Making deliberate choice to NOT run plug_in_setup(). We don't want 656 # ffdc_prefix updated. 657 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 658 call_point='ffdc_report', stop_on_plug_in_failure=0) 659 660 # Get additional header data which may have been created by ffdc plug-ins. 661 # Also, delete the individual header files to cleanup. 662 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 663 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 664 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 665 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 666 show_err=0) 667 668 # Get additional summary data which may have been created by ffdc plug-ins. 669 # Also, delete the individual header files to cleanup. 670 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 671 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 672 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 673 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 674 show_err=0) 675 676 # ffdc_list_file_path contains a list of any ffdc files created by plug- 677 # ins, etc. Read that data into a list. 678 try: 679 plug_in_ffdc_list = \ 680 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 681 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 682 except IOError: 683 plug_in_ffdc_list = [] 684 685 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 686 # in. Eliminate duplicates and sort the list. 687 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 688 689 if status_file_path != "": 690 ffdc_file_list.insert(0, status_file_path) 691 692 # Convert the list to a printable list. 693 printable_ffdc_file_list = "\n".join(ffdc_file_list) 694 695 # Open ffdc_file_list for writing. We will write a complete list of 696 # FFDC files to it for possible use by plug-ins like cp_stop_check. 697 ffdc_list_file = open(ffdc_list_file_path, 'w') 698 ffdc_list_file.write(printable_ffdc_file_list + "\n") 699 ffdc_list_file.close() 700 701 indent = 0 702 width = 90 703 linefeed = 1 704 char = "=" 705 706 gp.qprintn() 707 gp.qprint_dashes(indent, width, linefeed, char) 708 gp.qprintn("Copy this data to the defect:\n") 709 710 if len(more_header_info) > 0: 711 gp.qprintn(more_header_info) 712 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 713 openbmc_host_name, openbmc_ip, openbmc_username, 714 openbmc_password, rest_username, rest_password, ipmi_username, 715 ipmi_password, os_host, os_host_name, os_ip, os_username, 716 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 717 pdu_password, pdu_slot_no, openbmc_serial_host, 718 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 719 720 gp.qprintn() 721 print_last_boots() 722 gp.qprintn() 723 gp.qprint_var(state) 724 gp.qprintn() 725 gp.qprintn("FFDC data files:") 726 gp.qprintn(printable_ffdc_file_list) 727 gp.qprintn() 728 729 if len(ffdc_summary_info) > 0: 730 gp.qprintn(ffdc_summary_info) 731 732 gp.qprint_dashes(indent, width, linefeed, char) 733 734 735def my_ffdc(): 736 r""" 737 Collect FFDC data. 738 """ 739 740 global state 741 742 plug_in_setup() 743 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 744 call_point='ffdc', stop_on_plug_in_failure=0) 745 746 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 747 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 748 + AUTOBOOT_FFDC_PREFIX 749 + " ffdc_function_list=" 750 + ffdc_function_list, ignore=1) 751 if status != 'PASS': 752 gp.qprint_error("Call to ffdc failed.\n") 753 754 my_get_state() 755 756 print_defect_report(ffdc_file_list) 757 758 759def print_test_start_message(boot_keyword): 760 r""" 761 Print a message indicating what boot test is about to run. 762 763 Description of arguments: 764 boot_keyword The name of the boot which is to be run 765 (e.g. "BMC Power On"). 766 """ 767 768 global last_ten 769 global boot_start_time 770 771 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 772 773 # Set boot_start_time for use by plug-ins. 774 boot_start_time = doing_msg[1:33] 775 gp.qprint_var(boot_start_time) 776 777 gp.qprint(doing_msg) 778 779 last_ten.append(doing_msg) 780 781 # Trim list to max number of entries. 782 del last_ten[:max(0, len(last_ten) - max_boot_history)] 783 784 785def stop_boot_test(signal_number=0, 786 frame=None): 787 r""" 788 Handle SIGUSR1 by aborting the boot test that is running. 789 790 Description of argument(s): 791 signal_number The signal number (should always be 10 for SIGUSR1). 792 frame The frame data. 793 """ 794 795 gp.printn() 796 gp.print_executing() 797 gp.lprint_executing() 798 799 # Restore original sigusr1 handler. 800 set_default_siguser1() 801 802 message = "The caller has asked that the boot test be stopped and marked" 803 message += " as a failure." 804 805 function_stack = gm.get_function_stack() 806 if "wait_state" in function_stack: 807 st.set_wait_early_exit_message(message) 808 else: 809 BuiltIn().fail(gp.sprint_error(message)) 810 811 812def run_boot(boot): 813 r""" 814 Run the specified boot. 815 816 Description of arguments: 817 boot The name of the boot test to be performed. 818 """ 819 820 global state 821 822 signal.signal(signal.SIGUSR1, stop_boot_test) 823 gp.qprint_timen("stop_boot_test is armed.") 824 825 print_test_start_message(boot) 826 827 plug_in_setup() 828 rc, shell_rc, failed_plug_in_name = \ 829 grpi.rprocess_plug_in_packages(call_point="pre_boot") 830 if rc != 0: 831 error_message = "Plug-in failed with non-zero return code.\n" +\ 832 gp.sprint_var(rc, 1) 833 set_default_siguser1() 834 BuiltIn().fail(gp.sprint_error(error_message)) 835 836 if test_mode: 837 # In test mode, we'll pretend the boot worked by assigning its 838 # required end state to the default state value. 839 state = st.strip_anchor_state(boot_table[boot]['end']) 840 else: 841 # Assertion: We trust that the state data was made fresh by the 842 # caller. 843 844 gp.qprintn() 845 846 if boot_table[boot]['method_type'] == "keyword": 847 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 848 boot_table[boot]['method'], 849 quiet=quiet) 850 851 if boot_table[boot]['bmc_reboot']: 852 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 853 plug_in_setup() 854 rc, shell_rc, failed_plug_in_name = \ 855 grpi.rprocess_plug_in_packages(call_point="post_reboot") 856 if rc != 0: 857 error_message = "Plug-in failed with non-zero return code.\n" 858 error_message += gp.sprint_var(rc, 1) 859 set_default_siguser1() 860 BuiltIn().fail(gp.sprint_error(error_message)) 861 else: 862 match_state = st.anchor_state(state) 863 del match_state['epoch_seconds'] 864 # Wait for the state to change in any way. 865 st.wait_state(match_state, wait_time=state_change_timeout, 866 interval="10 seconds", invert=1) 867 868 gp.qprintn() 869 if boot_table[boot]['end']['chassis'] == "Off": 870 boot_timeout = power_off_timeout 871 else: 872 boot_timeout = power_on_timeout 873 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 874 interval="10 seconds") 875 876 plug_in_setup() 877 rc, shell_rc, failed_plug_in_name = \ 878 grpi.rprocess_plug_in_packages(call_point="post_boot") 879 if rc != 0: 880 error_message = "Plug-in failed with non-zero return code.\n" +\ 881 gp.sprint_var(rc, 1) 882 set_default_siguser1() 883 BuiltIn().fail(gp.sprint_error(error_message)) 884 885 # Restore original sigusr1 handler. 886 set_default_siguser1() 887 888 889def test_loop_body(): 890 r""" 891 The main loop body for the loop in main_py. 892 893 Description of arguments: 894 boot_count The iteration number (starts at 1). 895 """ 896 897 global boot_count 898 global state 899 global next_boot 900 global boot_success 901 global boot_end_time 902 903 gp.qprintn() 904 905 next_boot = select_boot() 906 if next_boot == "": 907 return True 908 909 boot_count += 1 910 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 911 912 pre_boot_plug_in_setup() 913 914 cmd_buf = ["run_boot", next_boot] 915 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 916 if boot_status == "FAIL": 917 gp.qprint(msg) 918 919 gp.qprintn() 920 if boot_status == "PASS": 921 boot_success = 1 922 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 923 + "\" succeeded.") 924 else: 925 boot_success = 0 926 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 927 + "\" failed.") 928 929 # Set boot_end_time for use by plug-ins. 930 boot_end_time = completion_msg[1:33] 931 gp.qprint_var(boot_end_time) 932 933 gp.qprint(completion_msg) 934 935 boot_results.update(next_boot, boot_status) 936 937 plug_in_setup() 938 # NOTE: A post_test_case call point failure is NOT counted as a boot 939 # failure. 940 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 941 call_point='post_test_case', stop_on_plug_in_failure=0) 942 943 plug_in_setup() 944 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 945 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 946 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 947 if ffdc_check == "All" or\ 948 shell_rc == dump_ffdc_rc(): 949 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 950 if status != 'PASS': 951 gp.qprint_error("Call to my_ffdc failed.\n") 952 953 if delete_errlogs: 954 # We need to purge error logs between boots or they build up. 955 grk.run_key("Delete Error logs", ignore=1) 956 957 boot_results.print_report() 958 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 959 960 plug_in_setup() 961 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 962 call_point='stop_check', shell_rc=stop_test_rc(), 963 stop_on_non_zero_rc=1) 964 if shell_rc == stop_test_rc(): 965 message = "Stopping as requested by user.\n" 966 gp.print_time(message) 967 BuiltIn().fail(message) 968 969 # This should help prevent ConnectionErrors. 970 grk.run_key_u("Close All Connections") 971 972 return True 973 974 975def obmc_boot_test_teardown(): 976 r""" 977 Clean up after the Main keyword. 978 """ 979 980 if cp_setup_called: 981 plug_in_setup() 982 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 983 call_point='cleanup', stop_on_plug_in_failure=0) 984 985 if 'boot_results_file_path' in globals(): 986 # Save boot_results and last_ten objects to a file in case they are 987 # needed again. 988 gp.qprint_timen("Saving boot_results to the following path.") 989 gp.qprint_var(boot_results_file_path) 990 pickle.dump((boot_results, last_ten), 991 open(boot_results_file_path, 'wb'), 992 pickle.HIGHEST_PROTOCOL) 993 994 global save_stack 995 # Restore any global values saved on the save_stack. 996 for parm_name in main_func_parm_list: 997 # Get the parm_value if it was saved on the stack. 998 try: 999 parm_value = save_stack.pop(parm_name) 1000 except BaseException: 1001 # If it was not saved, no further action is required. 1002 continue 1003 1004 # Restore the saved value. 1005 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1006 "}\", parm_value)" 1007 gp.dpissuing(cmd_buf) 1008 exec(cmd_buf) 1009 1010 gp.dprintn(save_stack.sprint_obj()) 1011 1012 1013def test_teardown(): 1014 r""" 1015 Clean up after this test case. 1016 """ 1017 1018 gp.qprintn() 1019 cmd_buf = ["Print Error", 1020 "A keyword timeout occurred ending this program.\n"] 1021 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1022 1023 gp.qprint_pgm_footer() 1024 1025 1026def post_stack(): 1027 r""" 1028 Process post_stack plug-in programs. 1029 """ 1030 1031 if not call_post_stack_plug: 1032 # The caller does not wish to have post_stack plug-in processing done. 1033 return 1034 1035 global boot_success 1036 1037 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1038 pre_boot_plug_in_setup() 1039 # For the purposes of the following plug-ins, mark the "boot" as a success. 1040 boot_success = 1 1041 plug_in_setup() 1042 rc, shell_rc, failed_plug_in_name, history =\ 1043 grpi.rprocess_plug_in_packages(call_point='post_stack', 1044 stop_on_plug_in_failure=0, 1045 return_history=True) 1046 last_ten.extend(history) 1047 # Trim list to max number of entries. 1048 del last_ten[:max(0, len(last_ten) - max_boot_history)] 1049 if rc != 0: 1050 boot_success = 0 1051 1052 plug_in_setup() 1053 rc, shell_rc, failed_plug_in_name =\ 1054 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1055 shell_rc=dump_ffdc_rc(), 1056 stop_on_plug_in_failure=1, 1057 stop_on_non_zero_rc=1) 1058 if shell_rc == dump_ffdc_rc(): 1059 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1060 if status != 'PASS': 1061 gp.qprint_error("Call to my_ffdc failed.\n") 1062 1063 plug_in_setup() 1064 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1065 call_point='stop_check', shell_rc=stop_test_rc(), 1066 stop_on_non_zero_rc=1) 1067 if shell_rc == stop_test_rc(): 1068 message = "Stopping as requested by user.\n" 1069 gp.print_time(message) 1070 BuiltIn().fail(message) 1071 1072 1073def obmc_boot_test_py(loc_boot_stack=None, 1074 loc_stack_mode=None, 1075 loc_quiet=None): 1076 r""" 1077 Do main program processing. 1078 """ 1079 1080 global save_stack 1081 1082 gp.dprintn() 1083 # Process function parms. 1084 for parm_name in main_func_parm_list: 1085 # Get parm's value. 1086 parm_value = eval("loc_" + parm_name) 1087 gp.dpvars(parm_name, parm_value) 1088 1089 if parm_value is not None: 1090 # Save the global value on a stack. 1091 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1092 parm_name + "}\"), \"" + parm_name + "\")" 1093 gp.dpissuing(cmd_buf) 1094 exec(cmd_buf) 1095 1096 # Set the global value to the passed value. 1097 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1098 "}\", loc_" + parm_name + ")" 1099 gp.dpissuing(cmd_buf) 1100 exec(cmd_buf) 1101 1102 gp.dprintn(save_stack.sprint_obj()) 1103 1104 setup() 1105 1106 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1107 1108 if ffdc_only: 1109 gp.qprint_timen("Caller requested ffdc_only.") 1110 pre_boot_plug_in_setup() 1111 grk.run_key_u("my_ffdc") 1112 return 1113 1114 # Process caller's boot_stack. 1115 while (len(boot_stack) > 0): 1116 test_loop_body() 1117 1118 gp.qprint_timen("Finished processing stack.") 1119 1120 post_stack() 1121 1122 # Process caller's boot_list. 1123 if len(boot_list) > 0: 1124 for ix in range(1, max_num_tests + 1): 1125 test_loop_body() 1126 1127 gp.qprint_timen("Completed all requested boot tests.") 1128 1129 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1130 new_fail = boot_fail - init_boot_fail 1131 if new_fail > boot_fail_threshold: 1132 error_message = "Boot failures exceed the boot failure" +\ 1133 " threshold:\n" +\ 1134 gp.sprint_var(new_fail) +\ 1135 gp.sprint_var(boot_fail_threshold) 1136 BuiltIn().fail(gp.sprint_error(error_message)) 1137