1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13try: 14 import cPickle as pickle 15except ImportError: 16 import pickle 17import socket 18 19from robot.utils import DotDict 20from robot.libraries.BuiltIn import BuiltIn 21 22from boot_data import * 23import gen_print as gp 24import gen_robot_print as grp 25import gen_robot_plug_in as grpi 26import gen_robot_valid as grv 27import gen_misc as gm 28import gen_cmd as gc 29import gen_robot_keyword as grk 30import state as st 31import var_stack as vs 32 33base_path = os.path.dirname(os.path.dirname( 34 imp.find_module("gen_robot_print")[1])) +\ 35 os.sep 36sys.path.append(base_path + "extended/") 37import run_keyword as rk 38 39# Setting master_pid correctly influences the behavior of plug-ins like 40# DB_Logging 41program_pid = os.getpid() 42master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 43pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 44 45# Set up boot data structures. 46boot_table = create_boot_table() 47valid_boot_types = create_valid_boot_list(boot_table) 48 49boot_lists = read_boot_lists() 50# The maximum number of entries that can be in the last_ten global variable. 51max_boot_history = 10 52last_ten = [] 53 54state = st.return_state_constant('default_state') 55cp_setup_called = 0 56next_boot = "" 57base_tool_dir_path = os.path.normpath(os.environ.get( 58 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 59 60ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 61boot_success = 0 62status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 63if status_dir_path != "": 64 status_dir_path = os.path.normpath(status_dir_path) + os.sep 65default_power_on = "REST Power On" 66default_power_off = "REST Power Off" 67boot_count = 0 68 69LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 70ffdc_prefix = "" 71boot_start_time = "" 72boot_end_time = "" 73save_stack = vs.var_stack('save_stack') 74main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 75 76 77def dump_ffdc_rc(): 78 r""" 79 Return the constant dump ffdc test return code value. 80 81 When a plug-in call point program returns this value, it indicates that 82 this program should collect FFDC. 83 """ 84 85 return 0x00000200 86 87 88def stop_test_rc(): 89 r""" 90 Return the constant stop test return code value. 91 92 When a plug-in call point program returns this value, it indicates that 93 this program should stop running. 94 """ 95 96 return 0x00000200 97 98 99def process_host(host, 100 host_var_name=""): 101 r""" 102 Process a host by getting the associated host name and IP address and 103 setting them in global variables. 104 105 If the caller does not pass the host_var_name, this function will try to 106 figure out the name of the variable used by the caller for the host parm. 107 Callers are advised to explicitly specify the host_var_name when calling 108 with an exec command. In such cases, the get_arg_name cannot figure out 109 the host variable name. 110 111 This function will then create similar global variable names by 112 removing "_host" and appending "_host_name" or "_ip" to the host variable 113 name. 114 115 Example: 116 117 If a call is made like this: 118 process_host(openbmc_host) 119 120 Global variables openbmc_host_name and openbmc_ip will be set. 121 122 Description of argument(s): 123 host A host name or IP. The name of the variable used should 124 have a suffix of "_host". 125 host_var_name The name of the variable being used as the host parm. 126 """ 127 128 if host_var_name == "": 129 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 130 131 host_name_var_name = re.sub("host", "host_name", host_var_name) 132 ip_var_name = re.sub("host", "ip", host_var_name) 133 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 134 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 135 host + "')" 136 exec(cmd_buf) 137 138 139def process_pgm_parms(): 140 r""" 141 Process the program parameters by assigning them all to corresponding 142 globals. Also, set some global values that depend on program parameters. 143 """ 144 145 # Program parameter processing. 146 # Assign all program parms to python variables which are global to this 147 # module. 148 149 global parm_list 150 parm_list = BuiltIn().get_variable_value("${parm_list}") 151 # The following subset of parms should be processed as integers. 152 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 153 'boot_fail_threshold', 'delete_errlogs', 154 'call_post_stack_plug', 'quiet', 'test_mode', 'debug'] 155 for parm in parm_list: 156 if parm in int_list: 157 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 158 "}\", \"0\"))" 159 else: 160 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 161 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 162 gp.dpissuing(cmd_buf) 163 exec(cmd_buf) 164 if re.match(r".*_host$", parm): 165 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 166 exec(cmd_buf) 167 if re.match(r".*_password$", parm): 168 # Register the value of any parm whose name ends in _password. 169 # This will cause the print functions to replace passwords with 170 # asterisks in the output. 171 cmd_buf = "gp.register_passwords(" + parm + ")" 172 exec(cmd_buf) 173 174 global ffdc_dir_path_style 175 global boot_list 176 global boot_stack 177 global boot_results_file_path 178 global boot_results 179 global last_ten 180 global ffdc_list_file_path 181 global ffdc_report_list_path 182 global ffdc_summary_list_path 183 184 if ffdc_dir_path_style == "": 185 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 186 187 # Convert these program parms to lists for easier processing.. 188 boot_list = list(filter(None, boot_list.split(":"))) 189 boot_stack = list(filter(None, boot_stack.split(":"))) 190 191 cleanup_boot_results_file() 192 boot_results_file_path = create_boot_results_file_path(pgm_name, 193 openbmc_nickname, 194 master_pid) 195 196 if os.path.isfile(boot_results_file_path): 197 # We've been called before in this run so we'll load the saved 198 # boot_results and last_ten objects. 199 boot_results, last_ten =\ 200 pickle.load(open(boot_results_file_path, 'rb')) 201 else: 202 boot_results = boot_results(boot_table, boot_pass, boot_fail) 203 204 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 205 "/FFDC_FILE_LIST" 206 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 207 "/FFDC_REPORT_FILE_LIST" 208 209 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 210 "/FFDC_SUMMARY_FILE_LIST" 211 212 213def initial_plug_in_setup(): 214 r""" 215 Initialize all plug-in environment variables which do not change for the 216 duration of the program. 217 218 """ 219 220 global LOG_LEVEL 221 BuiltIn().set_log_level("NONE") 222 223 BuiltIn().set_global_variable("${master_pid}", master_pid) 224 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 225 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 226 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 227 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 228 ffdc_list_file_path) 229 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 230 ffdc_report_list_path) 231 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 232 ffdc_summary_list_path) 233 234 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 235 ffdc_dir_path_style) 236 BuiltIn().set_global_variable("${FFDC_CHECK}", 237 ffdc_check) 238 239 # For each program parameter, set the corresponding AUTOBOOT_ environment 240 # variable value. Also, set an AUTOBOOT_ environment variable for every 241 # element in additional_values. 242 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 243 "status_dir_path", "base_tool_dir_path", 244 "ffdc_list_file_path", "ffdc_report_list_path", 245 "ffdc_summary_list_path", "execdir"] 246 247 plug_in_vars = parm_list + additional_values 248 249 for var_name in plug_in_vars: 250 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 251 var_name = var_name.upper() 252 if var_value is None: 253 var_value = "" 254 os.environ["AUTOBOOT_" + var_name] = str(var_value) 255 256 BuiltIn().set_log_level(LOG_LEVEL) 257 258 # Make sure the ffdc list directory exists. 259 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 260 if not os.path.exists(ffdc_list_dir_path): 261 os.makedirs(ffdc_list_dir_path) 262 263 264def plug_in_setup(): 265 r""" 266 Initialize all changing plug-in environment variables for use by the 267 plug-in programs. 268 """ 269 270 global LOG_LEVEL 271 global test_really_running 272 273 BuiltIn().set_log_level("NONE") 274 275 boot_pass, boot_fail = boot_results.return_total_pass_fail() 276 if boot_pass > 1: 277 test_really_running = 1 278 else: 279 test_really_running = 0 280 281 BuiltIn().set_global_variable("${test_really_running}", 282 test_really_running) 283 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 284 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 285 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 286 BuiltIn().set_global_variable("${boot_success}", boot_success) 287 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 288 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 289 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 290 291 # For each program parameter, set the corresponding AUTOBOOT_ environment 292 # variable value. Also, set an AUTOBOOT_ environment variable for every 293 # element in additional_values. 294 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 295 "boot_fail", "test_really_running", "ffdc_prefix", 296 "boot_start_time", "boot_end_time"] 297 298 plug_in_vars = additional_values 299 300 for var_name in plug_in_vars: 301 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 302 var_name = var_name.upper() 303 if var_value is None: 304 var_value = "" 305 os.environ["AUTOBOOT_" + var_name] = str(var_value) 306 307 if debug: 308 shell_rc, out_buf = \ 309 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 310 311 BuiltIn().set_log_level(LOG_LEVEL) 312 313 314def pre_boot_plug_in_setup(): 315 316 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 317 try: 318 os.remove(ffdc_list_file_path) 319 except OSError: 320 pass 321 322 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 323 try: 324 os.remove(ffdc_report_list_path) 325 except OSError: 326 pass 327 328 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 329 try: 330 os.remove(ffdc_summary_list_path) 331 except OSError: 332 pass 333 334 global ffdc_prefix 335 336 seconds = time.time() 337 loc_time = time.localtime(seconds) 338 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 339 340 ffdc_prefix = openbmc_nickname + "." + time_string 341 342 343def setup(): 344 r""" 345 Do general program setup tasks. 346 """ 347 348 global cp_setup_called 349 global transitional_boot_selected 350 351 gp.qprintn() 352 353 transitional_boot_selected = False 354 355 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 356 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 357 # If we can't find process_plug_in_packages.py, ssh_pw or 358 # validate_plug_ins.py, then we don't have our repo bin in PATH. 359 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 360 + " ssh_pw validate_plug_ins.py", quiet=1, 361 print_output=0, show_err=0) 362 if shell_rc != 0: 363 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 364 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 365 if robot_pgm_dir_path not in sys.path: 366 sys.path.append(robot_pgm_dir_path) 367 PYTHONPATH = os.environ.get("PYTHONPATH", "") 368 if PYTHONPATH == "": 369 os.environ['PYTHONPATH'] = robot_pgm_dir_path 370 else: 371 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 372 373 validate_parms() 374 375 grp.rqprint_pgm_header() 376 377 grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF") 378 379 initial_plug_in_setup() 380 381 plug_in_setup() 382 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 383 call_point='setup') 384 if rc != 0: 385 error_message = "Plug-in setup failed.\n" 386 grp.rprint_error_report(error_message) 387 BuiltIn().fail(error_message) 388 # Setting cp_setup_called lets our Teardown know that it needs to call 389 # the cleanup plug-in call point. 390 cp_setup_called = 1 391 392 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 393 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 394 # FFDC_LOG_PATH is used by "FFDC" keyword. 395 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 396 397 # Also printed by FFDC. 398 global host_name 399 global host_ip 400 host = socket.gethostname() 401 host_name, host_ip = gm.get_host_name_ip(host) 402 403 gp.dprint_var(boot_table, 1) 404 gp.dprint_var(boot_lists) 405 406 407def validate_parms(): 408 r""" 409 Validate all program parameters. 410 """ 411 412 process_pgm_parms() 413 414 gp.qprintn() 415 416 global openbmc_model 417 grv.rvalid_value("openbmc_host") 418 grv.rvalid_value("openbmc_username") 419 grv.rvalid_value("openbmc_password") 420 grv.rvalid_value("rest_username") 421 grv.rvalid_value("rest_password") 422 grv.rvalid_value("ipmi_username") 423 grv.rvalid_value("ipmi_password") 424 if os_host != "": 425 grv.rvalid_value("os_username") 426 grv.rvalid_value("os_password") 427 428 if pdu_host != "": 429 grv.rvalid_value("pdu_username") 430 grv.rvalid_value("pdu_password") 431 grv.rvalid_integer("pdu_slot_no") 432 if openbmc_serial_host != "": 433 grv.rvalid_integer("openbmc_serial_port") 434 if openbmc_model == "": 435 status, ret_values =\ 436 grk.run_key_u("Get BMC System Model") 437 openbmc_model = ret_values 438 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 439 grv.rvalid_value("openbmc_model") 440 grv.rvalid_integer("max_num_tests") 441 grv.rvalid_integer("boot_pass") 442 grv.rvalid_integer("boot_fail") 443 444 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 445 BuiltIn().set_global_variable("${plug_in_packages_list}", 446 plug_in_packages_list) 447 448 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) 449 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 450 error_message = "You must provide either a value for either the" +\ 451 " boot_list or the boot_stack parm.\n" 452 BuiltIn().fail(gp.sprint_error(error_message)) 453 454 valid_boot_list(boot_list, valid_boot_types) 455 valid_boot_list(boot_stack, valid_boot_types) 456 457 selected_PDU_boots = list(set(boot_list + boot_stack) 458 & set(boot_lists['PDU_reboot'])) 459 460 if len(selected_PDU_boots) > 0 and pdu_host == "": 461 error_message = "You have selected the following boots which" +\ 462 " require a PDU host but no value for pdu_host:\n" 463 error_message += gp.sprint_var(selected_PDU_boots) 464 error_message += gp.sprint_var(pdu_host, 2) 465 BuiltIn().fail(gp.sprint_error(error_message)) 466 467 return 468 469 470def my_get_state(): 471 r""" 472 Get the system state plus a little bit of wrapping. 473 """ 474 475 global state 476 477 req_states = ['epoch_seconds'] + st.default_req_states 478 479 gp.qprint_timen("Getting system state.") 480 if test_mode: 481 state['epoch_seconds'] = int(time.time()) 482 else: 483 state = st.get_state(req_states=req_states, quiet=quiet) 484 gp.qprint_var(state) 485 486 487def valid_state(): 488 r""" 489 Verify that our state dictionary contains no blank values. If we don't get 490 valid state data, we cannot continue to work. 491 """ 492 493 if st.compare_states(state, st.invalid_state_match, 'or'): 494 error_message = "The state dictionary contains blank fields which" +\ 495 " is illegal.\n" + gp.sprint_var(state) 496 BuiltIn().fail(gp.sprint_error(error_message)) 497 498 499def select_boot(): 500 r""" 501 Select a boot test to be run based on our current state and return the 502 chosen boot type. 503 504 Description of arguments: 505 state The state of the machine. 506 """ 507 508 global transitional_boot_selected 509 global boot_stack 510 511 gp.qprint_timen("Selecting a boot test.") 512 513 if transitional_boot_selected and not boot_success: 514 prior_boot = next_boot 515 boot_candidate = boot_stack.pop() 516 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 517 + " transition to a valid state for '" + boot_candidate 518 + "' which was at the top of the boot_stack. Since" 519 + " the '" + next_boot + "' failed, the '" 520 + boot_candidate + "' has been removed from the stack" 521 + " to avoid and endless failure loop.") 522 if len(boot_stack) == 0: 523 return "" 524 525 my_get_state() 526 valid_state() 527 528 transitional_boot_selected = False 529 stack_popped = 0 530 if len(boot_stack) > 0: 531 stack_popped = 1 532 gp.qprint_dashes() 533 gp.qprint_var(boot_stack) 534 gp.qprint_dashes() 535 skip_boot_printed = 0 536 while len(boot_stack) > 0: 537 boot_candidate = boot_stack.pop() 538 if stack_mode == 'normal': 539 break 540 else: 541 if st.compare_states(state, boot_table[boot_candidate]['end']): 542 if not skip_boot_printed: 543 gp.qprint_var(stack_mode) 544 gp.qprintn() 545 gp.qprint_timen("Skipping the following boot tests" 546 + " which are unnecessary since their" 547 + " required end states match the" 548 + " current machine state:") 549 skip_boot_printed = 1 550 gp.qprint_var(boot_candidate) 551 boot_candidate = "" 552 if boot_candidate == "": 553 gp.qprint_dashes() 554 gp.qprint_var(boot_stack) 555 gp.qprint_dashes() 556 return boot_candidate 557 if st.compare_states(state, boot_table[boot_candidate]['start']): 558 gp.qprint_timen("The machine state is valid for a '" 559 + boot_candidate + "' boot test.") 560 gp.qprint_dashes() 561 gp.qprint_var(boot_stack) 562 gp.qprint_dashes() 563 return boot_candidate 564 else: 565 gp.qprint_timen("The machine state does not match the required" 566 + " starting state for a '" + boot_candidate 567 + "' boot test:") 568 gp.qprint_varx("boot_table[" + boot_candidate + "][start]", 569 boot_table[boot_candidate]['start'], 1) 570 boot_stack.append(boot_candidate) 571 transitional_boot_selected = True 572 popped_boot = boot_candidate 573 574 # Loop through your list selecting a boot_candidates 575 boot_candidates = [] 576 for boot_candidate in boot_list: 577 if st.compare_states(state, boot_table[boot_candidate]['start']): 578 if stack_popped: 579 if st.compare_states(boot_table[boot_candidate]['end'], 580 boot_table[popped_boot]['start']): 581 boot_candidates.append(boot_candidate) 582 else: 583 boot_candidates.append(boot_candidate) 584 585 if len(boot_candidates) == 0: 586 gp.qprint_timen("The user's boot list contained no boot tests" 587 + " which are valid for the current machine state.") 588 boot_candidate = default_power_on 589 if not st.compare_states(state, boot_table[default_power_on]['start']): 590 boot_candidate = default_power_off 591 boot_candidates.append(boot_candidate) 592 gp.qprint_timen("Using default '" + boot_candidate 593 + "' boot type to transition to valid state.") 594 595 gp.dprint_var(boot_candidates) 596 597 # Randomly select a boot from the candidate list. 598 boot = random.choice(boot_candidates) 599 600 return boot 601 602 603def print_last_boots(): 604 r""" 605 Print the last ten boots done with their time stamps. 606 """ 607 608 # indent 0, 90 chars wide, linefeed, char is "=" 609 gp.qprint_dashes(0, 90) 610 gp.qprintn("Last 10 boots:\n") 611 612 for boot_entry in last_ten: 613 grp.rqprint(boot_entry) 614 gp.qprint_dashes(0, 90) 615 616 617def print_defect_report(ffdc_file_list): 618 r""" 619 Print a defect report. 620 621 Description of argument(s): 622 ffdc_file_list A list of files which were collected by our ffdc functions. 623 """ 624 625 # Making deliberate choice to NOT run plug_in_setup(). We don't want 626 # ffdc_prefix updated. 627 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 628 call_point='ffdc_report', stop_on_plug_in_failure=0) 629 630 # Get additional header data which may have been created by ffdc plug-ins. 631 # Also, delete the individual header files to cleanup. 632 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 633 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 634 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 635 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 636 show_err=0) 637 638 # Get additional summary data which may have been created by ffdc plug-ins. 639 # Also, delete the individual header files to cleanup. 640 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 641 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 642 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 643 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 644 show_err=0) 645 646 # ffdc_list_file_path contains a list of any ffdc files created by plug- 647 # ins, etc. Read that data into a list. 648 try: 649 plug_in_ffdc_list = \ 650 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 651 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 652 except IOError: 653 plug_in_ffdc_list = [] 654 655 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 656 # in. Eliminate duplicates and sort the list. 657 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 658 659 if status_file_path != "": 660 ffdc_file_list.insert(0, status_file_path) 661 662 # Convert the list to a printable list. 663 printable_ffdc_file_list = "\n".join(ffdc_file_list) 664 665 # Open ffdc_file_list for writing. We will write a complete list of 666 # FFDC files to it for possible use by plug-ins like cp_stop_check. 667 ffdc_list_file = open(ffdc_list_file_path, 'w') 668 ffdc_list_file.write(printable_ffdc_file_list + "\n") 669 ffdc_list_file.close() 670 671 indent = 0 672 width = 90 673 linefeed = 1 674 char = "=" 675 676 gp.qprintn() 677 gp.qprint_dashes(indent, width, linefeed, char) 678 gp.qprintn("Copy this data to the defect:\n") 679 680 if len(more_header_info) > 0: 681 gp.qprintn(more_header_info) 682 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 683 openbmc_host_name, openbmc_ip, openbmc_username, 684 openbmc_password, rest_username, rest_password, ipmi_username, 685 ipmi_password, os_host, os_host_name, os_ip, os_username, 686 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 687 pdu_password, pdu_slot_no, openbmc_serial_host, 688 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 689 690 gp.qprintn() 691 print_last_boots() 692 gp.qprintn() 693 gp.qprint_var(state) 694 gp.qprintn() 695 gp.qprintn("FFDC data files:") 696 gp.qprintn(printable_ffdc_file_list) 697 gp.qprintn() 698 699 if len(ffdc_summary_info) > 0: 700 gp.qprintn(ffdc_summary_info) 701 702 gp.qprint_dashes(indent, width, linefeed, char) 703 704 705def my_ffdc(): 706 r""" 707 Collect FFDC data. 708 """ 709 710 global state 711 712 plug_in_setup() 713 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 714 call_point='ffdc', stop_on_plug_in_failure=0) 715 716 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 717 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 718 + AUTOBOOT_FFDC_PREFIX 719 + " ffdc_function_list=" 720 + ffdc_function_list, ignore=1) 721 if status != 'PASS': 722 gp.qprint_error("Call to ffdc failed.\n") 723 724 my_get_state() 725 726 print_defect_report(ffdc_file_list) 727 728 729def print_test_start_message(boot_keyword): 730 r""" 731 Print a message indicating what boot test is about to run. 732 733 Description of arguments: 734 boot_keyword The name of the boot which is to be run 735 (e.g. "BMC Power On"). 736 """ 737 738 global last_ten 739 global boot_start_time 740 741 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 742 743 # Set boot_start_time for use by plug-ins. 744 boot_start_time = doing_msg[1:33] 745 gp.qprint_var(boot_start_time) 746 747 gp.qprint(doing_msg) 748 749 last_ten.append(doing_msg) 750 751 # Trim list to max number of entries. 752 del last_ten[:max(0, len(last_ten) - max_boot_history)] 753 754 755def run_boot(boot): 756 r""" 757 Run the specified boot. 758 759 Description of arguments: 760 boot The name of the boot test to be performed. 761 """ 762 763 global state 764 765 print_test_start_message(boot) 766 767 plug_in_setup() 768 rc, shell_rc, failed_plug_in_name = \ 769 grpi.rprocess_plug_in_packages(call_point="pre_boot") 770 if rc != 0: 771 error_message = "Plug-in failed with non-zero return code.\n" +\ 772 gp.sprint_var(rc, 1) 773 BuiltIn().fail(gp.sprint_error(error_message)) 774 775 if test_mode: 776 # In test mode, we'll pretend the boot worked by assigning its 777 # required end state to the default state value. 778 state = st.strip_anchor_state(boot_table[boot]['end']) 779 else: 780 # Assertion: We trust that the state data was made fresh by the 781 # caller. 782 783 gp.qprintn() 784 785 if boot_table[boot]['method_type'] == "keyword": 786 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 787 boot_table[boot]['method'], 788 quiet=quiet) 789 790 if boot_table[boot]['bmc_reboot']: 791 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 792 plug_in_setup() 793 rc, shell_rc, failed_plug_in_name = \ 794 grpi.rprocess_plug_in_packages(call_point="post_reboot") 795 if rc != 0: 796 error_message = "Plug-in failed with non-zero return code.\n" 797 error_message += gp.sprint_var(rc, 1) 798 BuiltIn().fail(gp.sprint_error(error_message)) 799 else: 800 match_state = st.anchor_state(state) 801 del match_state['epoch_seconds'] 802 # Wait for the state to change in any way. 803 st.wait_state(match_state, wait_time=state_change_timeout, 804 interval="10 seconds", invert=1) 805 806 gp.qprintn() 807 if boot_table[boot]['end']['chassis'] == "Off": 808 boot_timeout = power_off_timeout 809 else: 810 boot_timeout = power_on_timeout 811 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 812 interval="10 seconds") 813 814 plug_in_setup() 815 rc, shell_rc, failed_plug_in_name = \ 816 grpi.rprocess_plug_in_packages(call_point="post_boot") 817 if rc != 0: 818 error_message = "Plug-in failed with non-zero return code.\n" +\ 819 gp.sprint_var(rc, 1) 820 BuiltIn().fail(gp.sprint_error(error_message)) 821 822 823def test_loop_body(): 824 r""" 825 The main loop body for the loop in main_py. 826 827 Description of arguments: 828 boot_count The iteration number (starts at 1). 829 """ 830 831 global boot_count 832 global state 833 global next_boot 834 global boot_success 835 global boot_end_time 836 837 gp.qprintn() 838 839 next_boot = select_boot() 840 if next_boot == "": 841 return True 842 843 boot_count += 1 844 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 845 846 pre_boot_plug_in_setup() 847 848 cmd_buf = ["run_boot", next_boot] 849 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 850 if boot_status == "FAIL": 851 gp.qprint(msg) 852 853 gp.qprintn() 854 if boot_status == "PASS": 855 boot_success = 1 856 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 857 + "\" succeeded.") 858 else: 859 boot_success = 0 860 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 861 + "\" failed.") 862 863 # Set boot_end_time for use by plug-ins. 864 boot_end_time = completion_msg[1:33] 865 gp.qprint_var(boot_end_time) 866 867 gp.qprint(completion_msg) 868 869 boot_results.update(next_boot, boot_status) 870 871 plug_in_setup() 872 # NOTE: A post_test_case call point failure is NOT counted as a boot 873 # failure. 874 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 875 call_point='post_test_case', stop_on_plug_in_failure=0) 876 877 plug_in_setup() 878 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 879 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 880 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 881 if boot_status != "PASS" or ffdc_check == "All" or\ 882 shell_rc == dump_ffdc_rc(): 883 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 884 if status != 'PASS': 885 gp.qprint_error("Call to my_ffdc failed.\n") 886 887 if delete_errlogs: 888 # We need to purge error logs between boots or they build up. 889 grk.run_key("Delete Error logs", ignore=1) 890 891 boot_results.print_report() 892 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 893 894 plug_in_setup() 895 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 896 call_point='stop_check', shell_rc=stop_test_rc(), 897 stop_on_non_zero_rc=1) 898 if shell_rc == stop_test_rc(): 899 message = "Stopping as requested by user.\n" 900 gp.print_time(message) 901 BuiltIn().fail(message) 902 903 # This should help prevent ConnectionErrors. 904 grk.run_key_u("Close All Connections") 905 906 return True 907 908 909def obmc_boot_test_teardown(): 910 r""" 911 Clean up after the Main keyword. 912 """ 913 914 if cp_setup_called: 915 plug_in_setup() 916 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 917 call_point='cleanup', stop_on_plug_in_failure=0) 918 919 if 'boot_results_file_path' in globals(): 920 # Save boot_results and last_ten objects to a file in case they are 921 # needed again. 922 gp.qprint_timen("Saving boot_results to the following path.") 923 gp.qprint_var(boot_results_file_path) 924 pickle.dump((boot_results, last_ten), 925 open(boot_results_file_path, 'wb'), 926 pickle.HIGHEST_PROTOCOL) 927 928 global save_stack 929 # Restore any global values saved on the save_stack. 930 for parm_name in main_func_parm_list: 931 # Get the parm_value if it was saved on the stack. 932 try: 933 parm_value = save_stack.pop(parm_name) 934 except BaseException: 935 # If it was not saved, no further action is required. 936 continue 937 938 # Restore the saved value. 939 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 940 "}\", parm_value)" 941 gp.dpissuing(cmd_buf) 942 exec(cmd_buf) 943 944 gp.dprintn(save_stack.sprint_obj()) 945 946 947def test_teardown(): 948 r""" 949 Clean up after this test case. 950 """ 951 952 gp.qprintn() 953 cmd_buf = ["Print Error", 954 "A keyword timeout occurred ending this program.\n"] 955 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 956 957 grp.rqprint_pgm_footer() 958 959 960def post_stack(): 961 r""" 962 Process post_stack plug-in programs. 963 """ 964 965 if not call_post_stack_plug: 966 # The caller does not wish to have post_stack plug-in processing done. 967 return 968 969 global boot_success 970 971 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 972 pre_boot_plug_in_setup() 973 # For the purposes of the following plug-ins, mark the "boot" as a success. 974 boot_success = 1 975 plug_in_setup() 976 rc, shell_rc, failed_plug_in_name, history =\ 977 grpi.rprocess_plug_in_packages(call_point='post_stack', 978 stop_on_plug_in_failure=0, 979 return_history=True) 980 last_ten.extend(history) 981 # Trim list to max number of entries. 982 del last_ten[:max(0, len(last_ten) - max_boot_history)] 983 if rc != 0: 984 boot_success = 0 985 986 plug_in_setup() 987 rc, shell_rc, failed_plug_in_name =\ 988 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 989 shell_rc=dump_ffdc_rc(), 990 stop_on_plug_in_failure=1, 991 stop_on_non_zero_rc=1) 992 if shell_rc == dump_ffdc_rc(): 993 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 994 if status != 'PASS': 995 gp.qprint_error("Call to my_ffdc failed.\n") 996 997 plug_in_setup() 998 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 999 call_point='stop_check', shell_rc=stop_test_rc(), 1000 stop_on_non_zero_rc=1) 1001 if shell_rc == stop_test_rc(): 1002 message = "Stopping as requested by user.\n" 1003 gp.print_time(message) 1004 BuiltIn().fail(message) 1005 1006 1007def obmc_boot_test_py(loc_boot_stack=None, 1008 loc_stack_mode=None, 1009 loc_quiet=None): 1010 r""" 1011 Do main program processing. 1012 """ 1013 1014 global save_stack 1015 1016 gp.dprintn() 1017 # Process function parms. 1018 for parm_name in main_func_parm_list: 1019 # Get parm's value. 1020 parm_value = eval("loc_" + parm_name) 1021 gp.dpvars(parm_name, parm_value) 1022 1023 if parm_value is not None: 1024 # Save the global value on a stack. 1025 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1026 parm_name + "}\"), \"" + parm_name + "\")" 1027 gp.dpissuing(cmd_buf) 1028 exec(cmd_buf) 1029 1030 # Set the global value to the passed value. 1031 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1032 "}\", loc_" + parm_name + ")" 1033 gp.dpissuing(cmd_buf) 1034 exec(cmd_buf) 1035 1036 gp.dprintn(save_stack.sprint_obj()) 1037 1038 setup() 1039 1040 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1041 1042 if ffdc_only: 1043 gp.qprint_timen("Caller requested ffdc_only.") 1044 pre_boot_plug_in_setup() 1045 grk.run_key_u("my_ffdc") 1046 return 1047 1048 # Process caller's boot_stack. 1049 while (len(boot_stack) > 0): 1050 test_loop_body() 1051 1052 gp.qprint_timen("Finished processing stack.") 1053 1054 post_stack() 1055 1056 # Process caller's boot_list. 1057 if len(boot_list) > 0: 1058 for ix in range(1, max_num_tests + 1): 1059 test_loop_body() 1060 1061 gp.qprint_timen("Completed all requested boot tests.") 1062 1063 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1064 new_fail = boot_fail - init_boot_fail 1065 if new_fail > boot_fail_threshold: 1066 error_message = "Boot failures exceed the boot failure" +\ 1067 " threshold:\n" +\ 1068 gp.sprint_var(new_fail) +\ 1069 gp.sprint_var(boot_fail_threshold) 1070 BuiltIn().fail(gp.sprint_error(error_message)) 1071