1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13try: 14 import cPickle as pickle 15except ImportError: 16 import pickle 17import socket 18 19from robot.utils import DotDict 20from robot.libraries.BuiltIn import BuiltIn 21 22from boot_data import * 23import gen_print as gp 24import gen_robot_print as grp 25import gen_robot_plug_in as grpi 26import gen_robot_valid as grv 27import gen_misc as gm 28import gen_cmd as gc 29import gen_robot_keyword as grk 30import state as st 31import var_stack as vs 32 33base_path = os.path.dirname(os.path.dirname( 34 imp.find_module("gen_robot_print")[1])) +\ 35 os.sep 36sys.path.append(base_path + "extended/") 37import run_keyword as rk 38 39# Setting master_pid correctly influences the behavior of plug-ins like 40# DB_Logging 41program_pid = os.getpid() 42master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 43pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 44 45# Set up boot data structures. 46boot_table = create_boot_table() 47valid_boot_types = create_valid_boot_list(boot_table) 48 49boot_lists = read_boot_lists() 50# The maximum number of entries that can be in the last_ten global variable. 51max_boot_history = 10 52last_ten = [] 53 54state = st.return_state_constant('default_state') 55cp_setup_called = 0 56next_boot = "" 57base_tool_dir_path = os.path.normpath(os.environ.get( 58 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 59 60ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 61boot_success = 0 62status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 63if status_dir_path != "": 64 status_dir_path = os.path.normpath(status_dir_path) + os.sep 65default_power_on = "REST Power On" 66default_power_off = "REST Power Off" 67boot_count = 0 68 69LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 70ffdc_prefix = "" 71boot_start_time = "" 72boot_end_time = "" 73save_stack = vs.var_stack('save_stack') 74main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 75 76 77def dump_ffdc_rc(): 78 r""" 79 Return the constant dump ffdc test return code value. 80 81 When a plug-in call point program returns this value, it indicates that 82 this program should collect FFDC. 83 """ 84 85 return 0x00000200 86 87 88def stop_test_rc(): 89 r""" 90 Return the constant stop test return code value. 91 92 When a plug-in call point program returns this value, it indicates that 93 this program should stop running. 94 """ 95 96 return 0x00000200 97 98 99def process_host(host, 100 host_var_name=""): 101 r""" 102 Process a host by getting the associated host name and IP address and 103 setting them in global variables. 104 105 If the caller does not pass the host_var_name, this function will try to 106 figure out the name of the variable used by the caller for the host parm. 107 Callers are advised to explicitly specify the host_var_name when calling 108 with an exec command. In such cases, the get_arg_name cannot figure out 109 the host variable name. 110 111 This function will then create similar global variable names by 112 removing "_host" and appending "_host_name" or "_ip" to the host variable 113 name. 114 115 Example: 116 117 If a call is made like this: 118 process_host(openbmc_host) 119 120 Global variables openbmc_host_name and openbmc_ip will be set. 121 122 Description of argument(s): 123 host A host name or IP. The name of the variable used should 124 have a suffix of "_host". 125 host_var_name The name of the variable being used as the host parm. 126 """ 127 128 if host_var_name == "": 129 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 130 131 host_name_var_name = re.sub("host", "host_name", host_var_name) 132 ip_var_name = re.sub("host", "ip", host_var_name) 133 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 134 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 135 host + "')" 136 exec(cmd_buf) 137 138 139def process_pgm_parms(): 140 r""" 141 Process the program parameters by assigning them all to corresponding 142 globals. Also, set some global values that depend on program parameters. 143 """ 144 145 # Program parameter processing. 146 # Assign all program parms to python variables which are global to this 147 # module. 148 149 global parm_list 150 parm_list = BuiltIn().get_variable_value("${parm_list}") 151 # The following subset of parms should be processed as integers. 152 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 153 'boot_fail_threshold', 'delete_errlogs', 154 'call_post_stack_plug', 'quiet', 'test_mode', 'debug'] 155 for parm in parm_list: 156 if parm in int_list: 157 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 158 "}\", \"0\"))" 159 else: 160 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 161 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 162 gp.dpissuing(cmd_buf) 163 exec(cmd_buf) 164 if re.match(r".*_host$", parm): 165 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 166 exec(cmd_buf) 167 if re.match(r".*_password$", parm): 168 # Register the value of any parm whose name ends in _password. 169 # This will cause the print functions to replace passwords with 170 # asterisks in the output. 171 cmd_buf = "gp.register_passwords(" + parm + ")" 172 exec(cmd_buf) 173 174 global ffdc_dir_path_style 175 global boot_list 176 global boot_stack 177 global boot_results_file_path 178 global boot_results 179 global last_ten 180 global ffdc_list_file_path 181 global ffdc_report_list_path 182 global ffdc_summary_list_path 183 184 if ffdc_dir_path_style == "": 185 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 186 187 # Convert these program parms to lists for easier processing.. 188 boot_list = list(filter(None, boot_list.split(":"))) 189 boot_stack = list(filter(None, boot_stack.split(":"))) 190 191 cleanup_boot_results_file() 192 boot_results_file_path = create_boot_results_file_path(pgm_name, 193 openbmc_nickname, 194 master_pid) 195 196 if os.path.isfile(boot_results_file_path): 197 # We've been called before in this run so we'll load the saved 198 # boot_results and last_ten objects. 199 boot_results, last_ten =\ 200 pickle.load(open(boot_results_file_path, 'rb')) 201 else: 202 boot_results = boot_results(boot_table, boot_pass, boot_fail) 203 204 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 205 "/FFDC_FILE_LIST" 206 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 207 "/FFDC_REPORT_FILE_LIST" 208 209 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 210 "/FFDC_SUMMARY_FILE_LIST" 211 212 213def initial_plug_in_setup(): 214 r""" 215 Initialize all plug-in environment variables which do not change for the 216 duration of the program. 217 218 """ 219 220 global LOG_LEVEL 221 BuiltIn().set_log_level("NONE") 222 223 BuiltIn().set_global_variable("${master_pid}", master_pid) 224 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 225 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 226 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 227 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 228 ffdc_list_file_path) 229 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 230 ffdc_report_list_path) 231 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 232 ffdc_summary_list_path) 233 234 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 235 ffdc_dir_path_style) 236 BuiltIn().set_global_variable("${FFDC_CHECK}", 237 ffdc_check) 238 239 # For each program parameter, set the corresponding AUTOBOOT_ environment 240 # variable value. Also, set an AUTOBOOT_ environment variable for every 241 # element in additional_values. 242 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 243 "status_dir_path", "base_tool_dir_path", 244 "ffdc_list_file_path", "ffdc_report_list_path", 245 "ffdc_summary_list_path"] 246 247 plug_in_vars = parm_list + additional_values 248 249 for var_name in plug_in_vars: 250 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 251 var_name = var_name.upper() 252 if var_value is None: 253 var_value = "" 254 os.environ["AUTOBOOT_" + var_name] = str(var_value) 255 256 BuiltIn().set_log_level(LOG_LEVEL) 257 258 # Make sure the ffdc list directory exists. 259 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 260 if not os.path.exists(ffdc_list_dir_path): 261 os.makedirs(ffdc_list_dir_path) 262 263 264def plug_in_setup(): 265 r""" 266 Initialize all changing plug-in environment variables for use by the 267 plug-in programs. 268 """ 269 270 global LOG_LEVEL 271 global test_really_running 272 273 BuiltIn().set_log_level("NONE") 274 275 boot_pass, boot_fail = boot_results.return_total_pass_fail() 276 if boot_pass > 1: 277 test_really_running = 1 278 else: 279 test_really_running = 0 280 281 BuiltIn().set_global_variable("${test_really_running}", 282 test_really_running) 283 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 284 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 285 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 286 BuiltIn().set_global_variable("${boot_success}", boot_success) 287 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 288 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 289 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 290 291 # For each program parameter, set the corresponding AUTOBOOT_ environment 292 # variable value. Also, set an AUTOBOOT_ environment variable for every 293 # element in additional_values. 294 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 295 "boot_fail", "test_really_running", "ffdc_prefix", 296 "boot_start_time", "boot_end_time"] 297 298 plug_in_vars = additional_values 299 300 for var_name in plug_in_vars: 301 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 302 var_name = var_name.upper() 303 if var_value is None: 304 var_value = "" 305 os.environ["AUTOBOOT_" + var_name] = str(var_value) 306 307 if debug: 308 shell_rc, out_buf = \ 309 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 310 311 BuiltIn().set_log_level(LOG_LEVEL) 312 313 314def pre_boot_plug_in_setup(): 315 316 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 317 try: 318 os.remove(ffdc_list_file_path) 319 except OSError: 320 pass 321 322 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 323 try: 324 os.remove(ffdc_report_list_path) 325 except OSError: 326 pass 327 328 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 329 try: 330 os.remove(ffdc_summary_list_path) 331 except OSError: 332 pass 333 334 global ffdc_prefix 335 336 seconds = time.time() 337 loc_time = time.localtime(seconds) 338 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 339 340 ffdc_prefix = openbmc_nickname + "." + time_string 341 342 343def setup(): 344 r""" 345 Do general program setup tasks. 346 """ 347 348 global cp_setup_called 349 global transitional_boot_selected 350 351 gp.qprintn() 352 353 transitional_boot_selected = False 354 355 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 356 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 357 # If we can't find process_plug_in_packages.py, ssh_pw or 358 # validate_plug_ins.py, then we don't have our repo bin in PATH. 359 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 360 + " ssh_pw validate_plug_ins.py", quiet=1, 361 print_output=0, show_err=0) 362 if shell_rc != 0: 363 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 364 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 365 if robot_pgm_dir_path not in sys.path: 366 sys.path.append(robot_pgm_dir_path) 367 PYTHONPATH = os.environ.get("PYTHONPATH", "") 368 if PYTHONPATH == "": 369 os.environ['PYTHONPATH'] = robot_pgm_dir_path 370 else: 371 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 372 373 validate_parms() 374 375 grp.rqprint_pgm_header() 376 377 grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF") 378 379 initial_plug_in_setup() 380 381 plug_in_setup() 382 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 383 call_point='setup') 384 if rc != 0: 385 error_message = "Plug-in setup failed.\n" 386 grp.rprint_error_report(error_message) 387 BuiltIn().fail(error_message) 388 # Setting cp_setup_called lets our Teardown know that it needs to call 389 # the cleanup plug-in call point. 390 cp_setup_called = 1 391 392 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 393 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 394 # FFDC_LOG_PATH is used by "FFDC" keyword. 395 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 396 397 # Also printed by FFDC. 398 global host_name 399 global host_ip 400 host = socket.gethostname() 401 host_name, host_ip = gm.get_host_name_ip(host) 402 403 gp.dprint_var(boot_table, 1) 404 gp.dprint_var(boot_lists) 405 406 407def validate_parms(): 408 r""" 409 Validate all program parameters. 410 """ 411 412 process_pgm_parms() 413 414 gp.qprintn() 415 416 global openbmc_model 417 grv.rvalid_value("openbmc_host") 418 grv.rvalid_value("openbmc_username") 419 grv.rvalid_value("openbmc_password") 420 if os_host != "": 421 grv.rvalid_value("os_username") 422 grv.rvalid_value("os_password") 423 424 if pdu_host != "": 425 grv.rvalid_value("pdu_username") 426 grv.rvalid_value("pdu_password") 427 grv.rvalid_integer("pdu_slot_no") 428 if openbmc_serial_host != "": 429 grv.rvalid_integer("openbmc_serial_port") 430 if openbmc_model == "": 431 status, ret_values =\ 432 grk.run_key_u("Get BMC System Model") 433 openbmc_model = ret_values 434 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 435 grv.rvalid_value("openbmc_model") 436 grv.rvalid_integer("max_num_tests") 437 grv.rvalid_integer("boot_pass") 438 grv.rvalid_integer("boot_fail") 439 440 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 441 BuiltIn().set_global_variable("${plug_in_packages_list}", 442 plug_in_packages_list) 443 444 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) 445 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 446 error_message = "You must provide either a value for either the" +\ 447 " boot_list or the boot_stack parm.\n" 448 BuiltIn().fail(gp.sprint_error(error_message)) 449 450 valid_boot_list(boot_list, valid_boot_types) 451 valid_boot_list(boot_stack, valid_boot_types) 452 453 selected_PDU_boots = list(set(boot_list + boot_stack) 454 & set(boot_lists['PDU_reboot'])) 455 456 if len(selected_PDU_boots) > 0 and pdu_host == "": 457 error_message = "You have selected the following boots which" +\ 458 " require a PDU host but no value for pdu_host:\n" 459 error_message += gp.sprint_var(selected_PDU_boots) 460 error_message += gp.sprint_var(pdu_host, 2) 461 BuiltIn().fail(gp.sprint_error(error_message)) 462 463 return 464 465 466def my_get_state(): 467 r""" 468 Get the system state plus a little bit of wrapping. 469 """ 470 471 global state 472 473 req_states = ['epoch_seconds'] + st.default_req_states 474 475 gp.qprint_timen("Getting system state.") 476 if test_mode: 477 state['epoch_seconds'] = int(time.time()) 478 else: 479 state = st.get_state(req_states=req_states, quiet=quiet) 480 gp.qprint_var(state) 481 482 483def valid_state(): 484 r""" 485 Verify that our state dictionary contains no blank values. If we don't get 486 valid state data, we cannot continue to work. 487 """ 488 489 if st.compare_states(state, st.invalid_state_match, 'or'): 490 error_message = "The state dictionary contains blank fields which" +\ 491 " is illegal.\n" + gp.sprint_var(state) 492 BuiltIn().fail(gp.sprint_error(error_message)) 493 494 495def select_boot(): 496 r""" 497 Select a boot test to be run based on our current state and return the 498 chosen boot type. 499 500 Description of arguments: 501 state The state of the machine. 502 """ 503 504 global transitional_boot_selected 505 global boot_stack 506 507 gp.qprint_timen("Selecting a boot test.") 508 509 if transitional_boot_selected and not boot_success: 510 prior_boot = next_boot 511 boot_candidate = boot_stack.pop() 512 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 513 + " transition to a valid state for '" + boot_candidate 514 + "' which was at the top of the boot_stack. Since" 515 + " the '" + next_boot + "' failed, the '" 516 + boot_candidate + "' has been removed from the stack" 517 + " to avoid and endless failure loop.") 518 if len(boot_stack) == 0: 519 return "" 520 521 my_get_state() 522 valid_state() 523 524 transitional_boot_selected = False 525 stack_popped = 0 526 if len(boot_stack) > 0: 527 stack_popped = 1 528 gp.qprint_dashes() 529 gp.qprint_var(boot_stack) 530 gp.qprint_dashes() 531 skip_boot_printed = 0 532 while len(boot_stack) > 0: 533 boot_candidate = boot_stack.pop() 534 if stack_mode == 'normal': 535 break 536 else: 537 if st.compare_states(state, boot_table[boot_candidate]['end']): 538 if not skip_boot_printed: 539 gp.qprint_var(stack_mode) 540 gp.qprintn() 541 gp.qprint_timen("Skipping the following boot tests" 542 + " which are unnecessary since their" 543 + " required end states match the" 544 + " current machine state:") 545 skip_boot_printed = 1 546 gp.qprint_var(boot_candidate) 547 boot_candidate = "" 548 if boot_candidate == "": 549 gp.qprint_dashes() 550 gp.qprint_var(boot_stack) 551 gp.qprint_dashes() 552 return boot_candidate 553 if st.compare_states(state, boot_table[boot_candidate]['start']): 554 gp.qprint_timen("The machine state is valid for a '" 555 + boot_candidate + "' boot test.") 556 gp.qprint_dashes() 557 gp.qprint_var(boot_stack) 558 gp.qprint_dashes() 559 return boot_candidate 560 else: 561 gp.qprint_timen("The machine state does not match the required" 562 + " starting state for a '" + boot_candidate 563 + "' boot test:") 564 gp.qprint_varx("boot_table[" + boot_candidate + "][start]", 565 boot_table[boot_candidate]['start'], 1) 566 boot_stack.append(boot_candidate) 567 transitional_boot_selected = True 568 popped_boot = boot_candidate 569 570 # Loop through your list selecting a boot_candidates 571 boot_candidates = [] 572 for boot_candidate in boot_list: 573 if st.compare_states(state, boot_table[boot_candidate]['start']): 574 if stack_popped: 575 if st.compare_states(boot_table[boot_candidate]['end'], 576 boot_table[popped_boot]['start']): 577 boot_candidates.append(boot_candidate) 578 else: 579 boot_candidates.append(boot_candidate) 580 581 if len(boot_candidates) == 0: 582 gp.qprint_timen("The user's boot list contained no boot tests" 583 + " which are valid for the current machine state.") 584 boot_candidate = default_power_on 585 if not st.compare_states(state, boot_table[default_power_on]['start']): 586 boot_candidate = default_power_off 587 boot_candidates.append(boot_candidate) 588 gp.qprint_timen("Using default '" + boot_candidate 589 + "' boot type to transition to valid state.") 590 591 gp.dprint_var(boot_candidates) 592 593 # Randomly select a boot from the candidate list. 594 boot = random.choice(boot_candidates) 595 596 return boot 597 598 599def print_last_boots(): 600 r""" 601 Print the last ten boots done with their time stamps. 602 """ 603 604 # indent 0, 90 chars wide, linefeed, char is "=" 605 gp.qprint_dashes(0, 90) 606 gp.qprintn("Last 10 boots:\n") 607 608 for boot_entry in last_ten: 609 grp.rqprint(boot_entry) 610 gp.qprint_dashes(0, 90) 611 612 613def print_defect_report(ffdc_file_list): 614 r""" 615 Print a defect report. 616 617 Description of argument(s): 618 ffdc_file_list A list of files which were collected by our ffdc functions. 619 """ 620 621 # Making deliberate choice to NOT run plug_in_setup(). We don't want 622 # ffdc_prefix updated. 623 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 624 call_point='ffdc_report', stop_on_plug_in_failure=0) 625 626 # Get additional header data which may have been created by ffdc plug-ins. 627 # Also, delete the individual header files to cleanup. 628 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 629 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 630 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 631 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 632 show_err=0) 633 634 # Get additional summary data which may have been created by ffdc plug-ins. 635 # Also, delete the individual header files to cleanup. 636 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 637 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 638 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 639 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 640 show_err=0) 641 642 # ffdc_list_file_path contains a list of any ffdc files created by plug- 643 # ins, etc. Read that data into a list. 644 try: 645 plug_in_ffdc_list = \ 646 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 647 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 648 except IOError: 649 plug_in_ffdc_list = [] 650 651 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 652 # in. Eliminate duplicates and sort the list. 653 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 654 655 if status_file_path != "": 656 ffdc_file_list.insert(0, status_file_path) 657 658 # Convert the list to a printable list. 659 printable_ffdc_file_list = "\n".join(ffdc_file_list) 660 661 # Open ffdc_file_list for writing. We will write a complete list of 662 # FFDC files to it for possible use by plug-ins like cp_stop_check. 663 ffdc_list_file = open(ffdc_list_file_path, 'w') 664 ffdc_list_file.write(printable_ffdc_file_list + "\n") 665 ffdc_list_file.close() 666 667 indent = 0 668 width = 90 669 linefeed = 1 670 char = "=" 671 672 gp.qprintn() 673 gp.qprint_dashes(indent, width, linefeed, char) 674 gp.qprintn("Copy this data to the defect:\n") 675 676 if len(more_header_info) > 0: 677 gp.qprintn(more_header_info) 678 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 679 openbmc_host_name, openbmc_ip, openbmc_username, 680 openbmc_password, os_host, os_host_name, os_ip, os_username, 681 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 682 pdu_password, pdu_slot_no, openbmc_serial_host, 683 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 684 685 gp.qprintn() 686 print_last_boots() 687 gp.qprintn() 688 gp.qprint_var(state) 689 gp.qprintn() 690 gp.qprintn("FFDC data files:") 691 gp.qprintn(printable_ffdc_file_list) 692 gp.qprintn() 693 694 if len(ffdc_summary_info) > 0: 695 gp.qprintn(ffdc_summary_info) 696 697 gp.qprint_dashes(indent, width, linefeed, char) 698 699 700def my_ffdc(): 701 r""" 702 Collect FFDC data. 703 """ 704 705 global state 706 707 plug_in_setup() 708 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 709 call_point='ffdc', stop_on_plug_in_failure=0) 710 711 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 712 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 713 + AUTOBOOT_FFDC_PREFIX 714 + " ffdc_function_list=" 715 + ffdc_function_list, ignore=1) 716 if status != 'PASS': 717 gp.qprint_error("Call to ffdc failed.\n") 718 719 my_get_state() 720 721 print_defect_report(ffdc_file_list) 722 723 724def print_test_start_message(boot_keyword): 725 r""" 726 Print a message indicating what boot test is about to run. 727 728 Description of arguments: 729 boot_keyword The name of the boot which is to be run 730 (e.g. "BMC Power On"). 731 """ 732 733 global last_ten 734 global boot_start_time 735 736 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 737 738 # Set boot_start_time for use by plug-ins. 739 boot_start_time = doing_msg[1:33] 740 gp.qprint_var(boot_start_time) 741 742 gp.qprint(doing_msg) 743 744 last_ten.append(doing_msg) 745 746 # Trim list to max number of entries. 747 del last_ten[:max(0, len(last_ten) - max_boot_history)] 748 749 750def run_boot(boot): 751 r""" 752 Run the specified boot. 753 754 Description of arguments: 755 boot The name of the boot test to be performed. 756 """ 757 758 global state 759 760 print_test_start_message(boot) 761 762 plug_in_setup() 763 rc, shell_rc, failed_plug_in_name = \ 764 grpi.rprocess_plug_in_packages(call_point="pre_boot") 765 if rc != 0: 766 error_message = "Plug-in failed with non-zero return code.\n" +\ 767 gp.sprint_var(rc, 1) 768 BuiltIn().fail(gp.sprint_error(error_message)) 769 770 if test_mode: 771 # In test mode, we'll pretend the boot worked by assigning its 772 # required end state to the default state value. 773 state = st.strip_anchor_state(boot_table[boot]['end']) 774 else: 775 # Assertion: We trust that the state data was made fresh by the 776 # caller. 777 778 gp.qprintn() 779 780 if boot_table[boot]['method_type'] == "keyword": 781 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 782 boot_table[boot]['method'], 783 quiet=quiet) 784 785 if boot_table[boot]['bmc_reboot']: 786 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 787 plug_in_setup() 788 rc, shell_rc, failed_plug_in_name = \ 789 grpi.rprocess_plug_in_packages(call_point="post_reboot") 790 if rc != 0: 791 error_message = "Plug-in failed with non-zero return code.\n" 792 error_message += gp.sprint_var(rc, 1) 793 BuiltIn().fail(gp.sprint_error(error_message)) 794 else: 795 match_state = st.anchor_state(state) 796 del match_state['epoch_seconds'] 797 # Wait for the state to change in any way. 798 st.wait_state(match_state, wait_time=state_change_timeout, 799 interval="10 seconds", invert=1) 800 801 gp.qprintn() 802 if boot_table[boot]['end']['chassis'] == "Off": 803 boot_timeout = power_off_timeout 804 else: 805 boot_timeout = power_on_timeout 806 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 807 interval="10 seconds") 808 809 plug_in_setup() 810 rc, shell_rc, failed_plug_in_name = \ 811 grpi.rprocess_plug_in_packages(call_point="post_boot") 812 if rc != 0: 813 error_message = "Plug-in failed with non-zero return code.\n" +\ 814 gp.sprint_var(rc, 1) 815 BuiltIn().fail(gp.sprint_error(error_message)) 816 817 818def test_loop_body(): 819 r""" 820 The main loop body for the loop in main_py. 821 822 Description of arguments: 823 boot_count The iteration number (starts at 1). 824 """ 825 826 global boot_count 827 global state 828 global next_boot 829 global boot_success 830 global boot_end_time 831 832 gp.qprintn() 833 834 next_boot = select_boot() 835 if next_boot == "": 836 return True 837 838 boot_count += 1 839 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 840 841 pre_boot_plug_in_setup() 842 843 cmd_buf = ["run_boot", next_boot] 844 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 845 if boot_status == "FAIL": 846 gp.qprint(msg) 847 848 gp.qprintn() 849 if boot_status == "PASS": 850 boot_success = 1 851 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 852 + "\" succeeded.") 853 else: 854 boot_success = 0 855 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 856 + "\" failed.") 857 858 # Set boot_end_time for use by plug-ins. 859 boot_end_time = completion_msg[1:33] 860 gp.qprint_var(boot_end_time) 861 862 gp.qprint(completion_msg) 863 864 boot_results.update(next_boot, boot_status) 865 866 plug_in_setup() 867 # NOTE: A post_test_case call point failure is NOT counted as a boot 868 # failure. 869 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 870 call_point='post_test_case', stop_on_plug_in_failure=0) 871 872 plug_in_setup() 873 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 874 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 875 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 876 if boot_status != "PASS" or ffdc_check == "All" or\ 877 shell_rc == dump_ffdc_rc(): 878 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 879 if status != 'PASS': 880 gp.qprint_error("Call to my_ffdc failed.\n") 881 882 if delete_errlogs: 883 # We need to purge error logs between boots or they build up. 884 grk.run_key("Delete Error logs", ignore=1) 885 886 boot_results.print_report() 887 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 888 889 plug_in_setup() 890 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 891 call_point='stop_check', shell_rc=stop_test_rc(), 892 stop_on_non_zero_rc=1) 893 if shell_rc == stop_test_rc(): 894 message = "Stopping as requested by user.\n" 895 gp.print_time(message) 896 BuiltIn().fail(message) 897 898 # This should help prevent ConnectionErrors. 899 grk.run_key_u("Close All Connections") 900 901 return True 902 903 904def obmc_boot_test_teardown(): 905 r""" 906 Clean up after the Main keyword. 907 """ 908 909 if cp_setup_called: 910 plug_in_setup() 911 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 912 call_point='cleanup', stop_on_plug_in_failure=0) 913 914 if 'boot_results_file_path' in globals(): 915 # Save boot_results and last_ten objects to a file in case they are 916 # needed again. 917 gp.qprint_timen("Saving boot_results to the following path.") 918 gp.qprint_var(boot_results_file_path) 919 pickle.dump((boot_results, last_ten), 920 open(boot_results_file_path, 'wb'), 921 pickle.HIGHEST_PROTOCOL) 922 923 global save_stack 924 # Restore any global values saved on the save_stack. 925 for parm_name in main_func_parm_list: 926 # Get the parm_value if it was saved on the stack. 927 try: 928 parm_value = save_stack.pop(parm_name) 929 except BaseException: 930 # If it was not saved, no further action is required. 931 continue 932 933 # Restore the saved value. 934 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 935 "}\", parm_value)" 936 gp.dpissuing(cmd_buf) 937 exec(cmd_buf) 938 939 gp.dprintn(save_stack.sprint_obj()) 940 941 942def test_teardown(): 943 r""" 944 Clean up after this test case. 945 """ 946 947 gp.qprintn() 948 cmd_buf = ["Print Error", 949 "A keyword timeout occurred ending this program.\n"] 950 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 951 952 grp.rqprint_pgm_footer() 953 954 955def post_stack(): 956 r""" 957 Process post_stack plug-in programs. 958 """ 959 960 if not call_post_stack_plug: 961 # The caller does not wish to have post_stack plug-in processing done. 962 return 963 964 global boot_success 965 966 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 967 pre_boot_plug_in_setup() 968 # For the purposes of the following plug-ins, mark the "boot" as a success. 969 boot_success = 1 970 plug_in_setup() 971 rc, shell_rc, failed_plug_in_name, history =\ 972 grpi.rprocess_plug_in_packages(call_point='post_stack', 973 stop_on_plug_in_failure=0, 974 return_history=True) 975 last_ten.extend(history) 976 # Trim list to max number of entries. 977 del last_ten[:max(0, len(last_ten) - max_boot_history)] 978 if rc != 0: 979 boot_success = 0 980 981 plug_in_setup() 982 rc, shell_rc, failed_plug_in_name =\ 983 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 984 shell_rc=dump_ffdc_rc(), 985 stop_on_plug_in_failure=1, 986 stop_on_non_zero_rc=1) 987 if shell_rc == dump_ffdc_rc(): 988 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 989 if status != 'PASS': 990 gp.qprint_error("Call to my_ffdc failed.\n") 991 992 plug_in_setup() 993 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 994 call_point='stop_check', shell_rc=stop_test_rc(), 995 stop_on_non_zero_rc=1) 996 if shell_rc == stop_test_rc(): 997 message = "Stopping as requested by user.\n" 998 gp.print_time(message) 999 BuiltIn().fail(message) 1000 1001 1002def obmc_boot_test_py(loc_boot_stack=None, 1003 loc_stack_mode=None, 1004 loc_quiet=None): 1005 r""" 1006 Do main program processing. 1007 """ 1008 1009 global save_stack 1010 1011 gp.dprintn() 1012 # Process function parms. 1013 for parm_name in main_func_parm_list: 1014 # Get parm's value. 1015 parm_value = eval("loc_" + parm_name) 1016 gp.dpvars(parm_name, parm_value) 1017 1018 if parm_value is not None: 1019 # Save the global value on a stack. 1020 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1021 parm_name + "}\"), \"" + parm_name + "\")" 1022 gp.dpissuing(cmd_buf) 1023 exec(cmd_buf) 1024 1025 # Set the global value to the passed value. 1026 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1027 "}\", loc_" + parm_name + ")" 1028 gp.dpissuing(cmd_buf) 1029 exec(cmd_buf) 1030 1031 gp.dprintn(save_stack.sprint_obj()) 1032 1033 setup() 1034 1035 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1036 1037 if ffdc_only: 1038 gp.qprint_timen("Caller requested ffdc_only.") 1039 pre_boot_plug_in_setup() 1040 grk.run_key_u("my_ffdc") 1041 return 1042 1043 # Process caller's boot_stack. 1044 while (len(boot_stack) > 0): 1045 test_loop_body() 1046 1047 gp.qprint_timen("Finished processing stack.") 1048 1049 post_stack() 1050 1051 # Process caller's boot_list. 1052 if len(boot_list) > 0: 1053 for ix in range(1, max_num_tests + 1): 1054 test_loop_body() 1055 1056 gp.qprint_timen("Completed all requested boot tests.") 1057 1058 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1059 new_fail = boot_fail - init_boot_fail 1060 if new_fail > boot_fail_threshold: 1061 error_message = "Boot failures exceed the boot failure" +\ 1062 " threshold:\n" +\ 1063 gp.sprint_var(new_fail) +\ 1064 gp.sprint_var(boot_fail_threshold) 1065 BuiltIn().fail(gp.sprint_error(error_message)) 1066