1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13try: 14 import cPickle as pickle 15except ImportError: 16 import pickle 17import socket 18 19from robot.utils import DotDict 20from robot.libraries.BuiltIn import BuiltIn 21 22from boot_data import * 23import gen_print as gp 24import gen_robot_print as grp 25import gen_robot_plug_in as grpi 26import gen_robot_valid as grv 27import gen_misc as gm 28import gen_cmd as gc 29import gen_robot_keyword as grk 30import state as st 31import var_stack as vs 32 33base_path = os.path.dirname(os.path.dirname( 34 imp.find_module("gen_robot_print")[1])) +\ 35 os.sep 36sys.path.append(base_path + "extended/") 37import run_keyword as rk 38 39# Setting master_pid correctly influences the behavior of plug-ins like 40# DB_Logging 41program_pid = os.getpid() 42master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 43pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 44 45# Set up boot data structures. 46boot_table = create_boot_table() 47valid_boot_types = create_valid_boot_list(boot_table) 48 49boot_lists = read_boot_lists() 50last_ten = [] 51 52state = st.return_state_constant('default_state') 53cp_setup_called = 0 54next_boot = "" 55base_tool_dir_path = os.path.normpath(os.environ.get( 56 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 57 58ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 59boot_success = 0 60status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 61if status_dir_path != "": 62 status_dir_path = os.path.normpath(status_dir_path) + os.sep 63default_power_on = "REST Power On" 64default_power_off = "REST Power Off" 65boot_count = 0 66 67LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 68ffdc_prefix = "" 69boot_start_time = "" 70boot_end_time = "" 71save_stack = vs.var_stack('save_stack') 72main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 73 74 75def dump_ffdc_rc(): 76 r""" 77 Return the constant dump ffdc test return code value. 78 79 When a plug-in call point program returns this value, it indicates that 80 this program should collect FFDC. 81 """ 82 83 return 0x00000200 84 85 86def stop_test_rc(): 87 r""" 88 Return the constant stop test return code value. 89 90 When a plug-in call point program returns this value, it indicates that 91 this program should stop running. 92 """ 93 94 return 0x00000200 95 96 97def process_host(host, 98 host_var_name=""): 99 r""" 100 Process a host by getting the associated host name and IP address and 101 setting them in global variables. 102 103 If the caller does not pass the host_var_name, this function will try to 104 figure out the name of the variable used by the caller for the host parm. 105 Callers are advised to explicitly specify the host_var_name when calling 106 with an exec command. In such cases, the get_arg_name cannot figure out 107 the host variable name. 108 109 This function will then create similar global variable names by 110 removing "_host" and appending "_host_name" or "_ip" to the host variable 111 name. 112 113 Example: 114 115 If a call is made like this: 116 process_host(openbmc_host) 117 118 Global variables openbmc_host_name and openbmc_ip will be set. 119 120 Description of argument(s): 121 host A host name or IP. The name of the variable used should 122 have a suffix of "_host". 123 host_var_name The name of the variable being used as the host parm. 124 """ 125 126 if host_var_name == "": 127 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 128 129 host_name_var_name = re.sub("host", "host_name", host_var_name) 130 ip_var_name = re.sub("host", "ip", host_var_name) 131 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 132 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 133 host + "')" 134 exec(cmd_buf) 135 136 137def process_pgm_parms(): 138 r""" 139 Process the program parameters by assigning them all to corresponding 140 globals. Also, set some global values that depend on program parameters. 141 """ 142 143 # Program parameter processing. 144 # Assign all program parms to python variables which are global to this 145 # module. 146 147 global parm_list 148 parm_list = BuiltIn().get_variable_value("${parm_list}") 149 # The following subset of parms should be processed as integers. 150 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 151 'boot_fail_threshold', 'delete_errlogs', 152 'call_post_stack_plug', 'quiet', 'test_mode', 'debug'] 153 for parm in parm_list: 154 if parm in int_list: 155 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 156 "}\", \"0\"))" 157 else: 158 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 159 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 160 gp.dpissuing(cmd_buf) 161 exec(cmd_buf) 162 if re.match(r".*_host$", parm): 163 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 164 exec(cmd_buf) 165 if re.match(r".*_password$", parm): 166 # Register the value of any parm whose name ends in _password. 167 # This will cause the print functions to replace passwords with 168 # asterisks in the output. 169 cmd_buf = "gp.register_passwords(" + parm + ")" 170 exec(cmd_buf) 171 172 global ffdc_dir_path_style 173 global boot_list 174 global boot_stack 175 global boot_results_file_path 176 global boot_results 177 global last_ten 178 global ffdc_list_file_path 179 global ffdc_report_list_path 180 global ffdc_summary_list_path 181 182 if ffdc_dir_path_style == "": 183 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 184 185 # Convert these program parms to lists for easier processing.. 186 boot_list = filter(None, boot_list.split(":")) 187 boot_stack = filter(None, boot_stack.split(":")) 188 189 cleanup_boot_results_file() 190 boot_results_file_path = create_boot_results_file_path(pgm_name, 191 openbmc_nickname, 192 master_pid) 193 194 if os.path.isfile(boot_results_file_path): 195 # We've been called before in this run so we'll load the saved 196 # boot_results and last_ten objects. 197 boot_results, last_ten =\ 198 pickle.load(open(boot_results_file_path, 'rb')) 199 else: 200 boot_results = boot_results(boot_table, boot_pass, boot_fail) 201 202 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 203 "/FFDC_FILE_LIST" 204 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 205 "/FFDC_REPORT_FILE_LIST" 206 207 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 208 "/FFDC_SUMMARY_FILE_LIST" 209 210 211def initial_plug_in_setup(): 212 r""" 213 Initialize all plug-in environment variables which do not change for the 214 duration of the program. 215 216 """ 217 218 global LOG_LEVEL 219 BuiltIn().set_log_level("NONE") 220 221 BuiltIn().set_global_variable("${master_pid}", master_pid) 222 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 223 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 224 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 225 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 226 ffdc_list_file_path) 227 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 228 ffdc_report_list_path) 229 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 230 ffdc_summary_list_path) 231 232 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 233 ffdc_dir_path_style) 234 BuiltIn().set_global_variable("${FFDC_CHECK}", 235 ffdc_check) 236 237 # For each program parameter, set the corresponding AUTOBOOT_ environment 238 # variable value. Also, set an AUTOBOOT_ environment variable for every 239 # element in additional_values. 240 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 241 "status_dir_path", "base_tool_dir_path", 242 "ffdc_list_file_path", "ffdc_report_list_path", 243 "ffdc_summary_list_path"] 244 245 plug_in_vars = parm_list + additional_values 246 247 for var_name in plug_in_vars: 248 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 249 var_name = var_name.upper() 250 if var_value is None: 251 var_value = "" 252 os.environ["AUTOBOOT_" + var_name] = str(var_value) 253 254 BuiltIn().set_log_level(LOG_LEVEL) 255 256 # Make sure the ffdc list directory exists. 257 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 258 if not os.path.exists(ffdc_list_dir_path): 259 os.makedirs(ffdc_list_dir_path) 260 261 262def plug_in_setup(): 263 r""" 264 Initialize all changing plug-in environment variables for use by the 265 plug-in programs. 266 """ 267 268 global LOG_LEVEL 269 global test_really_running 270 271 BuiltIn().set_log_level("NONE") 272 273 boot_pass, boot_fail = boot_results.return_total_pass_fail() 274 if boot_pass > 1: 275 test_really_running = 1 276 else: 277 test_really_running = 0 278 279 BuiltIn().set_global_variable("${test_really_running}", 280 test_really_running) 281 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 282 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 283 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 284 BuiltIn().set_global_variable("${boot_success}", boot_success) 285 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 286 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 287 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 288 289 # For each program parameter, set the corresponding AUTOBOOT_ environment 290 # variable value. Also, set an AUTOBOOT_ environment variable for every 291 # element in additional_values. 292 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 293 "boot_fail", "test_really_running", "ffdc_prefix", 294 "boot_start_time", "boot_end_time"] 295 296 plug_in_vars = additional_values 297 298 for var_name in plug_in_vars: 299 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 300 var_name = var_name.upper() 301 if var_value is None: 302 var_value = "" 303 os.environ["AUTOBOOT_" + var_name] = str(var_value) 304 305 if debug: 306 shell_rc, out_buf = \ 307 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 308 309 BuiltIn().set_log_level(LOG_LEVEL) 310 311 312def pre_boot_plug_in_setup(): 313 314 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 315 try: 316 os.remove(ffdc_list_file_path) 317 except OSError: 318 pass 319 320 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 321 try: 322 os.remove(ffdc_report_list_path) 323 except OSError: 324 pass 325 326 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 327 try: 328 os.remove(ffdc_summary_list_path) 329 except OSError: 330 pass 331 332 global ffdc_prefix 333 334 seconds = time.time() 335 loc_time = time.localtime(seconds) 336 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 337 338 ffdc_prefix = openbmc_nickname + "." + time_string 339 340 341def setup(): 342 r""" 343 Do general program setup tasks. 344 """ 345 346 global cp_setup_called 347 global transitional_boot_selected 348 349 gp.qprintn() 350 351 transitional_boot_selected = False 352 353 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 354 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 355 # If we can't find process_plug_in_packages.py, ssh_pw or 356 # validate_plug_ins.py, then we don't have our repo bin in PATH. 357 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 358 + " ssh_pw validate_plug_ins.py", quiet=1, 359 print_output=0, show_err=0) 360 if shell_rc != 0: 361 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 362 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 363 if robot_pgm_dir_path not in sys.path: 364 sys.path.append(robot_pgm_dir_path) 365 PYTHONPATH = os.environ.get("PYTHONPATH", "") 366 if PYTHONPATH == "": 367 os.environ['PYTHONPATH'] = robot_pgm_dir_path 368 else: 369 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 370 371 validate_parms() 372 373 grp.rqprint_pgm_header() 374 375 grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF") 376 377 initial_plug_in_setup() 378 379 plug_in_setup() 380 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 381 call_point='setup') 382 if rc != 0: 383 error_message = "Plug-in setup failed.\n" 384 grp.rprint_error_report(error_message) 385 BuiltIn().fail(error_message) 386 # Setting cp_setup_called lets our Teardown know that it needs to call 387 # the cleanup plug-in call point. 388 cp_setup_called = 1 389 390 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 391 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 392 # FFDC_LOG_PATH is used by "FFDC" keyword. 393 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 394 395 # Also printed by FFDC. 396 global host_name 397 global host_ip 398 host = socket.gethostname() 399 host_name, host_ip = gm.get_host_name_ip(host) 400 401 gp.dprint_var(boot_table, 1) 402 gp.dprint_var(boot_lists) 403 404 405def validate_parms(): 406 r""" 407 Validate all program parameters. 408 """ 409 410 process_pgm_parms() 411 412 gp.qprintn() 413 414 global openbmc_model 415 grv.rvalid_value("openbmc_host") 416 grv.rvalid_value("openbmc_username") 417 grv.rvalid_value("openbmc_password") 418 if os_host != "": 419 grv.rvalid_value("os_username") 420 grv.rvalid_value("os_password") 421 422 if pdu_host != "": 423 grv.rvalid_value("pdu_username") 424 grv.rvalid_value("pdu_password") 425 grv.rvalid_integer("pdu_slot_no") 426 if openbmc_serial_host != "": 427 grv.rvalid_integer("openbmc_serial_port") 428 if openbmc_model == "": 429 status, ret_values =\ 430 grk.run_key_u("Get BMC System Model") 431 openbmc_model = ret_values 432 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 433 grv.rvalid_value("openbmc_model") 434 grv.rvalid_integer("max_num_tests") 435 grv.rvalid_integer("boot_pass") 436 grv.rvalid_integer("boot_fail") 437 438 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 439 BuiltIn().set_global_variable("${plug_in_packages_list}", 440 plug_in_packages_list) 441 442 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) 443 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 444 error_message = "You must provide either a value for either the" +\ 445 " boot_list or the boot_stack parm.\n" 446 BuiltIn().fail(gp.sprint_error(error_message)) 447 448 valid_boot_list(boot_list, valid_boot_types) 449 valid_boot_list(boot_stack, valid_boot_types) 450 451 selected_PDU_boots = list(set(boot_list + boot_stack) 452 & set(boot_lists['PDU_reboot'])) 453 454 if len(selected_PDU_boots) > 0 and pdu_host == "": 455 error_message = "You have selected the following boots which" +\ 456 " require a PDU host but no value for pdu_host:\n" 457 error_message += gp.sprint_var(selected_PDU_boots) 458 error_message += gp.sprint_var(pdu_host, 2) 459 BuiltIn().fail(gp.sprint_error(error_message)) 460 461 return 462 463 464def my_get_state(): 465 r""" 466 Get the system state plus a little bit of wrapping. 467 """ 468 469 global state 470 471 req_states = ['epoch_seconds'] + st.default_req_states 472 473 gp.qprint_timen("Getting system state.") 474 if test_mode: 475 state['epoch_seconds'] = int(time.time()) 476 else: 477 state = st.get_state(req_states=req_states, quiet=quiet) 478 gp.qprint_var(state) 479 480 481def valid_state(): 482 r""" 483 Verify that our state dictionary contains no blank values. If we don't get 484 valid state data, we cannot continue to work. 485 """ 486 487 if st.compare_states(state, st.invalid_state_match, 'or'): 488 error_message = "The state dictionary contains blank fields which" +\ 489 " is illegal.\n" + gp.sprint_var(state) 490 BuiltIn().fail(gp.sprint_error(error_message)) 491 492 493def select_boot(): 494 r""" 495 Select a boot test to be run based on our current state and return the 496 chosen boot type. 497 498 Description of arguments: 499 state The state of the machine. 500 """ 501 502 global transitional_boot_selected 503 global boot_stack 504 505 gp.qprint_timen("Selecting a boot test.") 506 507 if transitional_boot_selected and not boot_success: 508 prior_boot = next_boot 509 boot_candidate = boot_stack.pop() 510 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 511 + " transition to a valid state for '" + boot_candidate 512 + "' which was at the top of the boot_stack. Since" 513 + " the '" + next_boot + "' failed, the '" 514 + boot_candidate + "' has been removed from the stack" 515 + " to avoid and endless failure loop.") 516 if len(boot_stack) == 0: 517 return "" 518 519 my_get_state() 520 valid_state() 521 522 transitional_boot_selected = False 523 stack_popped = 0 524 if len(boot_stack) > 0: 525 stack_popped = 1 526 gp.qprint_dashes() 527 gp.qprint_var(boot_stack) 528 gp.qprint_dashes() 529 skip_boot_printed = 0 530 while len(boot_stack) > 0: 531 boot_candidate = boot_stack.pop() 532 if stack_mode == 'normal': 533 break 534 else: 535 if st.compare_states(state, boot_table[boot_candidate]['end']): 536 if not skip_boot_printed: 537 gp.qprint_var(stack_mode) 538 gp.qprintn() 539 gp.qprint_timen("Skipping the following boot tests" 540 + " which are unnecessary since their" 541 + " required end states match the" 542 + " current machine state:") 543 skip_boot_printed = 1 544 gp.qprint_var(boot_candidate) 545 boot_candidate = "" 546 if boot_candidate == "": 547 gp.qprint_dashes() 548 gp.qprint_var(boot_stack) 549 gp.qprint_dashes() 550 return boot_candidate 551 if st.compare_states(state, boot_table[boot_candidate]['start']): 552 gp.qprint_timen("The machine state is valid for a '" 553 + boot_candidate + "' boot test.") 554 gp.qprint_dashes() 555 gp.qprint_var(boot_stack) 556 gp.qprint_dashes() 557 return boot_candidate 558 else: 559 gp.qprint_timen("The machine state does not match the required" 560 + " starting state for a '" + boot_candidate 561 + "' boot test:") 562 gp.qprint_varx("boot_table[" + boot_candidate + "][start]", 563 boot_table[boot_candidate]['start'], 1) 564 boot_stack.append(boot_candidate) 565 transitional_boot_selected = True 566 popped_boot = boot_candidate 567 568 # Loop through your list selecting a boot_candidates 569 boot_candidates = [] 570 for boot_candidate in boot_list: 571 if st.compare_states(state, boot_table[boot_candidate]['start']): 572 if stack_popped: 573 if st.compare_states(boot_table[boot_candidate]['end'], 574 boot_table[popped_boot]['start']): 575 boot_candidates.append(boot_candidate) 576 else: 577 boot_candidates.append(boot_candidate) 578 579 if len(boot_candidates) == 0: 580 gp.qprint_timen("The user's boot list contained no boot tests" 581 + " which are valid for the current machine state.") 582 boot_candidate = default_power_on 583 if not st.compare_states(state, boot_table[default_power_on]['start']): 584 boot_candidate = default_power_off 585 boot_candidates.append(boot_candidate) 586 gp.qprint_timen("Using default '" + boot_candidate 587 + "' boot type to transition to valid state.") 588 589 gp.dprint_var(boot_candidates) 590 591 # Randomly select a boot from the candidate list. 592 boot = random.choice(boot_candidates) 593 594 return boot 595 596 597def print_last_boots(): 598 r""" 599 Print the last ten boots done with their time stamps. 600 """ 601 602 # indent 0, 90 chars wide, linefeed, char is "=" 603 gp.qprint_dashes(0, 90) 604 gp.qprintn("Last 10 boots:\n") 605 606 for boot_entry in last_ten: 607 grp.rqprint(boot_entry) 608 gp.qprint_dashes(0, 90) 609 610 611def print_defect_report(ffdc_file_list): 612 r""" 613 Print a defect report. 614 615 Description of argument(s): 616 ffdc_file_list A list of files which were collected by our ffdc functions. 617 """ 618 619 # Making deliberate choice to NOT run plug_in_setup(). We don't want 620 # ffdc_prefix updated. 621 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 622 call_point='ffdc_report', stop_on_plug_in_failure=0) 623 624 # Get additional header data which may have been created by ffdc plug-ins. 625 # Also, delete the individual header files to cleanup. 626 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 627 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 628 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 629 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 630 show_err=0) 631 632 # Get additional summary data which may have been created by ffdc plug-ins. 633 # Also, delete the individual header files to cleanup. 634 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 635 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 636 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 637 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 638 show_err=0) 639 640 # ffdc_list_file_path contains a list of any ffdc files created by plug- 641 # ins, etc. Read that data into a list. 642 try: 643 plug_in_ffdc_list = \ 644 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 645 plug_in_ffdc_list = filter(None, plug_in_ffdc_list) 646 except IOError: 647 plug_in_ffdc_list = [] 648 649 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 650 # in. Eliminate duplicates and sort the list. 651 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 652 653 if status_file_path != "": 654 ffdc_file_list.insert(0, status_file_path) 655 656 # Convert the list to a printable list. 657 printable_ffdc_file_list = "\n".join(ffdc_file_list) 658 659 # Open ffdc_file_list for writing. We will write a complete list of 660 # FFDC files to it for possible use by plug-ins like cp_stop_check. 661 ffdc_list_file = open(ffdc_list_file_path, 'w') 662 ffdc_list_file.write(printable_ffdc_file_list + "\n") 663 ffdc_list_file.close() 664 665 indent = 0 666 width = 90 667 linefeed = 1 668 char = "=" 669 670 gp.qprintn() 671 gp.qprint_dashes(indent, width, linefeed, char) 672 gp.qprintn("Copy this data to the defect:\n") 673 674 if len(more_header_info) > 0: 675 gp.qprintn(more_header_info) 676 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 677 openbmc_host_name, openbmc_ip, openbmc_username, 678 openbmc_password, os_host, os_host_name, os_ip, os_username, 679 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 680 pdu_password, pdu_slot_no, openbmc_serial_host, 681 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 682 683 gp.qprintn() 684 print_last_boots() 685 gp.qprintn() 686 gp.qprint_var(state) 687 gp.qprintn() 688 gp.qprintn("FFDC data files:") 689 gp.qprintn(printable_ffdc_file_list) 690 gp.qprintn() 691 692 if len(ffdc_summary_info) > 0: 693 gp.qprintn(ffdc_summary_info) 694 695 gp.qprint_dashes(indent, width, linefeed, char) 696 697 698def my_ffdc(): 699 r""" 700 Collect FFDC data. 701 """ 702 703 global state 704 705 plug_in_setup() 706 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 707 call_point='ffdc', stop_on_plug_in_failure=0) 708 709 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 710 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 711 + AUTOBOOT_FFDC_PREFIX 712 + " ffdc_function_list=" 713 + ffdc_function_list, ignore=1) 714 if status != 'PASS': 715 gp.qprint_error("Call to ffdc failed.\n") 716 717 my_get_state() 718 719 print_defect_report(ffdc_file_list) 720 721 722def print_test_start_message(boot_keyword): 723 r""" 724 Print a message indicating what boot test is about to run. 725 726 Description of arguments: 727 boot_keyword The name of the boot which is to be run 728 (e.g. "BMC Power On"). 729 """ 730 731 global last_ten 732 global boot_start_time 733 734 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 735 736 # Set boot_start_time for use by plug-ins. 737 boot_start_time = doing_msg[1:33] 738 gp.qprint_var(boot_start_time) 739 740 gp.qprint(doing_msg) 741 742 last_ten.append(doing_msg) 743 744 if len(last_ten) > 10: 745 del last_ten[0] 746 747 748def run_boot(boot): 749 r""" 750 Run the specified boot. 751 752 Description of arguments: 753 boot The name of the boot test to be performed. 754 """ 755 756 global state 757 758 print_test_start_message(boot) 759 760 plug_in_setup() 761 rc, shell_rc, failed_plug_in_name = \ 762 grpi.rprocess_plug_in_packages(call_point="pre_boot") 763 if rc != 0: 764 error_message = "Plug-in failed with non-zero return code.\n" +\ 765 gp.sprint_var(rc, 1) 766 BuiltIn().fail(gp.sprint_error(error_message)) 767 768 if test_mode: 769 # In test mode, we'll pretend the boot worked by assigning its 770 # required end state to the default state value. 771 state = st.strip_anchor_state(boot_table[boot]['end']) 772 else: 773 # Assertion: We trust that the state data was made fresh by the 774 # caller. 775 776 gp.qprintn() 777 778 if boot_table[boot]['method_type'] == "keyword": 779 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 780 boot_table[boot]['method'], 781 quiet=quiet) 782 783 if boot_table[boot]['bmc_reboot']: 784 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 785 plug_in_setup() 786 rc, shell_rc, failed_plug_in_name = \ 787 grpi.rprocess_plug_in_packages(call_point="post_reboot") 788 if rc != 0: 789 error_message = "Plug-in failed with non-zero return code.\n" 790 error_message += gp.sprint_var(rc, 1) 791 BuiltIn().fail(gp.sprint_error(error_message)) 792 else: 793 match_state = st.anchor_state(state) 794 del match_state['epoch_seconds'] 795 # Wait for the state to change in any way. 796 st.wait_state(match_state, wait_time=state_change_timeout, 797 interval="10 seconds", invert=1) 798 799 gp.qprintn() 800 if boot_table[boot]['end']['chassis'] == "Off": 801 boot_timeout = power_off_timeout 802 else: 803 boot_timeout = power_on_timeout 804 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 805 interval="10 seconds") 806 807 plug_in_setup() 808 rc, shell_rc, failed_plug_in_name = \ 809 grpi.rprocess_plug_in_packages(call_point="post_boot") 810 if rc != 0: 811 error_message = "Plug-in failed with non-zero return code.\n" +\ 812 gp.sprint_var(rc, 1) 813 BuiltIn().fail(gp.sprint_error(error_message)) 814 815 816def test_loop_body(): 817 r""" 818 The main loop body for the loop in main_py. 819 820 Description of arguments: 821 boot_count The iteration number (starts at 1). 822 """ 823 824 global boot_count 825 global state 826 global next_boot 827 global boot_success 828 global boot_end_time 829 830 gp.qprintn() 831 832 next_boot = select_boot() 833 if next_boot == "": 834 return True 835 836 boot_count += 1 837 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 838 839 pre_boot_plug_in_setup() 840 841 cmd_buf = ["run_boot", next_boot] 842 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 843 if boot_status == "FAIL": 844 gp.qprint(msg) 845 846 gp.qprintn() 847 if boot_status == "PASS": 848 boot_success = 1 849 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 850 + "\" succeeded.") 851 else: 852 boot_success = 0 853 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 854 + "\" failed.") 855 856 # Set boot_end_time for use by plug-ins. 857 boot_end_time = completion_msg[1:33] 858 gp.qprint_var(boot_end_time) 859 860 gp.qprint(completion_msg) 861 862 boot_results.update(next_boot, boot_status) 863 864 plug_in_setup() 865 # NOTE: A post_test_case call point failure is NOT counted as a boot 866 # failure. 867 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 868 call_point='post_test_case', stop_on_plug_in_failure=0) 869 870 plug_in_setup() 871 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 872 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 873 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 874 if boot_status != "PASS" or ffdc_check == "All" or\ 875 shell_rc == dump_ffdc_rc(): 876 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 877 if status != 'PASS': 878 gp.qprint_error("Call to my_ffdc failed.\n") 879 880 if delete_errlogs: 881 # We need to purge error logs between boots or they build up. 882 grk.run_key("Delete Error logs", ignore=1) 883 884 boot_results.print_report() 885 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 886 887 plug_in_setup() 888 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 889 call_point='stop_check', shell_rc=stop_test_rc(), 890 stop_on_non_zero_rc=1) 891 if shell_rc == stop_test_rc(): 892 message = "Stopping as requested by user.\n" 893 gp.print_time(message) 894 BuiltIn().fail(message) 895 896 # This should help prevent ConnectionErrors. 897 grk.run_key_u("Close All Connections") 898 899 return True 900 901 902def obmc_boot_test_teardown(): 903 r""" 904 Clean up after the Main keyword. 905 """ 906 907 if cp_setup_called: 908 plug_in_setup() 909 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 910 call_point='cleanup', stop_on_plug_in_failure=0) 911 912 if 'boot_results_file_path' in globals(): 913 # Save boot_results and last_ten objects to a file in case they are 914 # needed again. 915 gp.qprint_timen("Saving boot_results to the following path.") 916 gp.qprint_var(boot_results_file_path) 917 pickle.dump((boot_results, last_ten), 918 open(boot_results_file_path, 'wb'), 919 pickle.HIGHEST_PROTOCOL) 920 921 global save_stack 922 # Restore any global values saved on the save_stack. 923 for parm_name in main_func_parm_list: 924 # Get the parm_value if it was saved on the stack. 925 try: 926 parm_value = save_stack.pop(parm_name) 927 except BaseException: 928 # If it was not saved, no further action is required. 929 continue 930 931 # Restore the saved value. 932 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 933 "}\", parm_value)" 934 gp.dpissuing(cmd_buf) 935 exec(cmd_buf) 936 937 gp.dprintn(save_stack.sprint_obj()) 938 939 940def test_teardown(): 941 r""" 942 Clean up after this test case. 943 """ 944 945 gp.qprintn() 946 cmd_buf = ["Print Error", 947 "A keyword timeout occurred ending this program.\n"] 948 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 949 950 grp.rqprint_pgm_footer() 951 952 953def post_stack(): 954 r""" 955 Process post_stack plug-in programs. 956 """ 957 958 if not call_post_stack_plug: 959 # The caller does not wish to have post_stack plug-in processing done. 960 return 961 962 global boot_success 963 964 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 965 pre_boot_plug_in_setup() 966 # For the purposes of the following plug-ins, mark the "boot" as a success. 967 boot_success = 1 968 plug_in_setup() 969 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 970 call_point='post_stack', stop_on_plug_in_failure=0) 971 972 plug_in_setup() 973 if rc == 0: 974 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 975 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 976 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 977 if rc != 0 or shell_rc == dump_ffdc_rc(): 978 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 979 if status != 'PASS': 980 gp.qprint_error("Call to my_ffdc failed.\n") 981 982 plug_in_setup() 983 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 984 call_point='stop_check', shell_rc=stop_test_rc(), 985 stop_on_non_zero_rc=1) 986 if shell_rc == stop_test_rc(): 987 message = "Stopping as requested by user.\n" 988 gp.print_time(message) 989 BuiltIn().fail(message) 990 991 992def obmc_boot_test_py(loc_boot_stack=None, 993 loc_stack_mode=None, 994 loc_quiet=None): 995 r""" 996 Do main program processing. 997 """ 998 999 global save_stack 1000 1001 # Process function parms. 1002 for parm_name in main_func_parm_list: 1003 # Get parm's value. 1004 cmd_buf = "parm_value = loc_" + parm_name 1005 exec(cmd_buf) 1006 gp.dpvar(parm_name) 1007 gp.dpvar(parm_value) 1008 1009 if parm_value is None: 1010 # Parm was not specified by the calling function so set it to its 1011 # corresponding global value. 1012 cmd_buf = "loc_" + parm_name + " = BuiltIn().get_variable_value" +\ 1013 "(\"${" + parm_name + "}\")" 1014 gp.dpissuing(cmd_buf) 1015 exec(cmd_buf) 1016 else: 1017 # Save the global value on a stack. 1018 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1019 parm_name + "}\"), \"" + parm_name + "\")" 1020 gp.dpissuing(cmd_buf) 1021 exec(cmd_buf) 1022 1023 # Set the global value to the passed value. 1024 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1025 "}\", loc_" + parm_name + ")" 1026 gp.dpissuing(cmd_buf) 1027 exec(cmd_buf) 1028 1029 gp.dprintn(save_stack.sprint_obj()) 1030 1031 setup() 1032 1033 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1034 1035 if ffdc_only: 1036 gp.qprint_timen("Caller requested ffdc_only.") 1037 pre_boot_plug_in_setup() 1038 grk.run_key_u("my_ffdc") 1039 return 1040 1041 # Process caller's boot_stack. 1042 while (len(boot_stack) > 0): 1043 test_loop_body() 1044 1045 gp.qprint_timen("Finished processing stack.") 1046 1047 post_stack() 1048 1049 # Process caller's boot_list. 1050 if len(boot_list) > 0: 1051 for ix in range(1, max_num_tests + 1): 1052 test_loop_body() 1053 1054 gp.qprint_timen("Completed all requested boot tests.") 1055 1056 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1057 new_fail = boot_fail - init_boot_fail 1058 if new_fail > boot_fail_threshold: 1059 error_message = "Boot failures exceed the boot failure" +\ 1060 " threshold:\n" +\ 1061 gp.sprint_var(new_fail) +\ 1062 gp.sprint_var(boot_fail_threshold) 1063 BuiltIn().fail(gp.sprint_error(error_message)) 1064