1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import cPickle as pickle 14import socket 15 16from robot.utils import DotDict 17from robot.libraries.BuiltIn import BuiltIn 18 19from boot_data import * 20import gen_print as gp 21import gen_robot_print as grp 22import gen_robot_plug_in as grpi 23import gen_robot_valid as grv 24import gen_misc as gm 25import gen_cmd as gc 26import gen_robot_keyword as grk 27import state as st 28import var_stack as vs 29 30base_path = os.path.dirname(os.path.dirname( 31 imp.find_module("gen_robot_print")[1])) +\ 32 os.sep 33sys.path.append(base_path + "extended/") 34import run_keyword as rk 35 36# Setting master_pid correctly influences the behavior of plug-ins like 37# DB_Logging 38program_pid = os.getpid() 39master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 40pgm_name = re.sub('\.py$', '', os.path.basename(__file__)) 41 42# Set up boot data structures. 43boot_table = create_boot_table() 44valid_boot_types = create_valid_boot_list(boot_table) 45 46boot_lists = read_boot_lists() 47last_ten = [] 48 49state = st.return_default_state() 50cp_setup_called = 0 51next_boot = "" 52base_tool_dir_path = os.path.normpath(os.environ.get( 53 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 54 55ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 56boot_success = 0 57status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 58if status_dir_path != "": 59 status_dir_path = os.path.normpath(status_dir_path) + os.sep 60default_power_on = "REST Power On" 61default_power_off = "REST Power Off" 62boot_count = 0 63 64LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 65ffdc_prefix = "" 66boot_start_time = "" 67boot_end_time = "" 68save_stack = vs.var_stack('save_stack') 69main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 70 71 72############################################################################### 73def process_host(host, 74 host_var_name=""): 75 76 r""" 77 Process a host by getting the associated host name and IP address and 78 setting them in global variables. 79 80 If the caller does not pass the host_var_name, this function will try to 81 figure out the name of the variable used by the caller for the host parm. 82 Callers are advised to explicitly specify the host_var_name when calling 83 with an exec command. In such cases, the get_arg_name cannot figure out 84 the host variable name. 85 86 This function will then create similar global variable names by 87 removing "_host" and appending "_host_name" or "_ip" to the host variable 88 name. 89 90 Example: 91 92 If a call is made like this: 93 process_host(openbmc_host) 94 95 Global variables openbmc_host_name and openbmc_ip will be set. 96 97 Description of argument(s): 98 host A host name or IP. The name of the variable used should 99 have a suffix of "_host". 100 host_var_name The name of the variable being used as the host parm. 101 """ 102 103 if host_var_name == "": 104 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 105 106 host_name_var_name = re.sub("host", "host_name", host_var_name) 107 ip_var_name = re.sub("host", "ip", host_var_name) 108 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 109 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 110 host + "')" 111 exec(cmd_buf) 112 113############################################################################### 114 115 116############################################################################### 117def process_pgm_parms(): 118 119 r""" 120 Process the program parameters by assigning them all to corresponding 121 globals. Also, set some global values that depend on program parameters. 122 """ 123 124 # Program parameter processing. 125 # Assign all program parms to python variables which are global to this 126 # module. 127 128 global parm_list 129 parm_list = BuiltIn().get_variable_value("${parm_list}") 130 # The following subset of parms should be processed as integers. 131 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 132 'boot_fail_threshold', 'delete_errlogs', 'quiet', 'test_mode', 133 'debug'] 134 for parm in parm_list: 135 if parm in int_list: 136 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 137 "}\", \"0\"))" 138 else: 139 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 140 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 141 gp.dpissuing(cmd_buf) 142 exec(cmd_buf) 143 if re.match(r".*_host$", parm): 144 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 145 exec(cmd_buf) 146 if re.match(r".*_password$", parm): 147 # Register the value of any parm whose name ends in _password. 148 # This will cause the print functions to replace passwords with 149 # asterisks in the output. 150 cmd_buf = "gp.register_passwords(" + parm + ")" 151 exec(cmd_buf) 152 153 global ffdc_dir_path_style 154 global boot_list 155 global boot_stack 156 global boot_results_file_path 157 global boot_results 158 global ffdc_list_file_path 159 global ffdc_report_list_path 160 global ffdc_summary_list_path 161 162 if ffdc_dir_path_style == "": 163 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 164 165 # Convert these program parms to lists for easier processing.. 166 boot_list = filter(None, boot_list.split(":")) 167 boot_stack = filter(None, boot_stack.split(":")) 168 169 cleanup_boot_results_file() 170 boot_results_file_path = create_boot_results_file_path(pgm_name, 171 openbmc_nickname, 172 master_pid) 173 174 if os.path.isfile(boot_results_file_path): 175 # We've been called before in this run so we'll load the saved 176 # boot_results object. 177 boot_results = pickle.load(open(boot_results_file_path, 'rb')) 178 else: 179 boot_results = boot_results(boot_table, boot_pass, boot_fail) 180 181 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 182 "/FFDC_FILE_LIST" 183 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 184 "/FFDC_REPORT_FILE_LIST" 185 186 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 187 "/FFDC_SUMMARY_FILE_LIST" 188 189############################################################################### 190 191 192############################################################################### 193def initial_plug_in_setup(): 194 195 r""" 196 Initialize all plug-in environment variables which do not change for the 197 duration of the program. 198 199 """ 200 201 global LOG_LEVEL 202 BuiltIn().set_log_level("NONE") 203 204 BuiltIn().set_global_variable("${master_pid}", master_pid) 205 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 206 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 207 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 208 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 209 ffdc_list_file_path) 210 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 211 ffdc_report_list_path) 212 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 213 ffdc_summary_list_path) 214 215 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 216 ffdc_dir_path_style) 217 BuiltIn().set_global_variable("${FFDC_CHECK}", 218 ffdc_check) 219 220 # For each program parameter, set the corresponding AUTOBOOT_ environment 221 # variable value. Also, set an AUTOBOOT_ environment variable for every 222 # element in additional_values. 223 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 224 "status_dir_path", "base_tool_dir_path", 225 "ffdc_list_file_path", "ffdc_report_list_path", 226 "ffdc_summary_list_path"] 227 228 plug_in_vars = parm_list + additional_values 229 230 for var_name in plug_in_vars: 231 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 232 var_name = var_name.upper() 233 if var_value is None: 234 var_value = "" 235 os.environ["AUTOBOOT_" + var_name] = str(var_value) 236 237 BuiltIn().set_log_level(LOG_LEVEL) 238 239 # Make sure the ffdc list directory exists. 240 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 241 if not os.path.exists(ffdc_list_dir_path): 242 os.makedirs(ffdc_list_dir_path) 243 244############################################################################### 245 246 247############################################################################### 248def plug_in_setup(): 249 250 r""" 251 Initialize all changing plug-in environment variables for use by the 252 plug-in programs. 253 """ 254 255 global LOG_LEVEL 256 global test_really_running 257 258 BuiltIn().set_log_level("NONE") 259 260 boot_pass, boot_fail = boot_results.return_total_pass_fail() 261 if boot_pass > 1: 262 test_really_running = 1 263 else: 264 test_really_running = 0 265 266 BuiltIn().set_global_variable("${test_really_running}", 267 test_really_running) 268 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 269 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 270 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 271 BuiltIn().set_global_variable("${boot_success}", boot_success) 272 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 273 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 274 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 275 276 # For each program parameter, set the corresponding AUTOBOOT_ environment 277 # variable value. Also, set an AUTOBOOT_ environment variable for every 278 # element in additional_values. 279 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 280 "boot_fail", "test_really_running", "ffdc_prefix", 281 "boot_start_time", "boot_end_time"] 282 283 plug_in_vars = additional_values 284 285 for var_name in plug_in_vars: 286 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 287 var_name = var_name.upper() 288 if var_value is None: 289 var_value = "" 290 os.environ["AUTOBOOT_" + var_name] = str(var_value) 291 292 if debug: 293 shell_rc, out_buf = \ 294 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 295 296 BuiltIn().set_log_level(LOG_LEVEL) 297 298############################################################################### 299 300 301############################################################################### 302def pre_boot_plug_in_setup(): 303 304 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 305 try: 306 os.remove(ffdc_list_file_path) 307 except OSError: 308 pass 309 310 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 311 try: 312 os.remove(ffdc_report_list_path) 313 except OSError: 314 pass 315 316 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 317 try: 318 os.remove(ffdc_summary_list_path) 319 except OSError: 320 pass 321 322 global ffdc_prefix 323 324 seconds = time.time() 325 loc_time = time.localtime(seconds) 326 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 327 328 ffdc_prefix = openbmc_nickname + "." + time_string 329 330############################################################################### 331 332 333############################################################################### 334def setup(): 335 336 r""" 337 Do general program setup tasks. 338 """ 339 340 global cp_setup_called 341 global transitional_boot_selected 342 343 gp.qprintn() 344 345 transitional_boot_selected = False 346 347 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 348 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 349 # If we can't find process_plug_in_packages.py, ssh_pw or 350 # validate_plug_ins.py, then we don't have our repo bin in PATH. 351 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" + 352 " ssh_pw validate_plug_ins.py", quiet=1, 353 print_output=0, show_err=0) 354 if shell_rc != 0: 355 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 356 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 357 if robot_pgm_dir_path not in sys.path: 358 sys.path.append(robot_pgm_dir_path) 359 PYTHONPATH = os.environ.get("PYTHONPATH", "") 360 if PYTHONPATH == "": 361 os.environ['PYTHONPATH'] = robot_pgm_dir_path 362 else: 363 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 364 365 validate_parms() 366 367 grp.rqprint_pgm_header() 368 369 grk.run_key("Set BMC Power Policy RESTORE_LAST_STATE") 370 371 initial_plug_in_setup() 372 373 plug_in_setup() 374 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 375 call_point='setup') 376 if rc != 0: 377 error_message = "Plug-in setup failed.\n" 378 grp.rprint_error_report(error_message) 379 BuiltIn().fail(error_message) 380 # Setting cp_setup_called lets our Teardown know that it needs to call 381 # the cleanup plug-in call point. 382 cp_setup_called = 1 383 384 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 385 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 386 # FFDC_LOG_PATH is used by "FFDC" keyword. 387 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 388 389 # Also printed by FFDC. 390 global host_name 391 global host_ip 392 host = socket.gethostname() 393 host_name, host_ip = gm.get_host_name_ip(host) 394 395 gp.dprint_var(boot_table, 1) 396 gp.dprint_var(boot_lists) 397 398############################################################################### 399 400 401############################################################################### 402def validate_parms(): 403 404 r""" 405 Validate all program parameters. 406 """ 407 408 process_pgm_parms() 409 410 gp.qprintn() 411 412 global openbmc_model 413 grv.rvalid_value("openbmc_host") 414 grv.rvalid_value("openbmc_username") 415 grv.rvalid_value("openbmc_password") 416 if os_host != "": 417 grv.rvalid_value("os_username") 418 grv.rvalid_value("os_password") 419 420 if pdu_host != "": 421 grv.rvalid_value("pdu_username") 422 grv.rvalid_value("pdu_password") 423 grv.rvalid_integer("pdu_slot_no") 424 if openbmc_serial_host != "": 425 grv.rvalid_integer("openbmc_serial_port") 426 if openbmc_model == "": 427 status, ret_values =\ 428 grk.run_key_u("Get BMC System Model") 429 openbmc_model = ret_values 430 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 431 grv.rvalid_value("openbmc_model") 432 grv.rvalid_integer("max_num_tests") 433 grv.rvalid_integer("boot_pass") 434 grv.rvalid_integer("boot_fail") 435 436 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 437 BuiltIn().set_global_variable("${plug_in_packages_list}", 438 plug_in_packages_list) 439 440 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) 441 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 442 error_message = "You must provide either a value for either the" +\ 443 " boot_list or the boot_stack parm.\n" 444 BuiltIn().fail(gp.sprint_error(error_message)) 445 446 valid_boot_list(boot_list, valid_boot_types) 447 valid_boot_list(boot_stack, valid_boot_types) 448 449 selected_PDU_boots = list(set(boot_list + boot_stack) & 450 set(boot_lists['PDU_reboot'])) 451 452 if len(selected_PDU_boots) > 0 and pdu_host == "": 453 error_message = "You have selected the following boots which" +\ 454 " require a PDU host but no value for pdu_host:\n" 455 error_message += gp.sprint_var(selected_PDU_boots) 456 error_message += gp.sprint_var(pdu_host, 2) 457 BuiltIn().fail(gp.sprint_error(error_message)) 458 459 return 460 461############################################################################### 462 463 464############################################################################### 465def my_get_state(): 466 467 r""" 468 Get the system state plus a little bit of wrapping. 469 """ 470 471 global state 472 473 req_states = ['epoch_seconds'] + st.default_req_states 474 475 gp.qprint_timen("Getting system state.") 476 if test_mode: 477 state['epoch_seconds'] = int(time.time()) 478 else: 479 state = st.get_state(req_states=req_states, quiet=quiet) 480 gp.qprint_var(state) 481 482############################################################################### 483 484 485############################################################################### 486def valid_state(): 487 488 r""" 489 Verify that our state dictionary contains no blank values. If we don't get 490 valid state data, we cannot continue to work. 491 """ 492 493 if st.compare_states(state, st.invalid_state_match, 'or'): 494 error_message = "The state dictionary contains blank fields which" +\ 495 " is illegal.\n" + gp.sprint_var(state) 496 BuiltIn().fail(gp.sprint_error(error_message)) 497 498############################################################################### 499 500 501############################################################################### 502def select_boot(): 503 504 r""" 505 Select a boot test to be run based on our current state and return the 506 chosen boot type. 507 508 Description of arguments: 509 state The state of the machine. 510 """ 511 512 global transitional_boot_selected 513 global boot_stack 514 515 gp.qprint_timen("Selecting a boot test.") 516 517 if transitional_boot_selected and not boot_success: 518 prior_boot = next_boot 519 boot_candidate = boot_stack.pop() 520 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" + 521 " transition to a valid state for '" + boot_candidate + 522 "' which was at the top of the boot_stack. Since" + 523 " the '" + next_boot + "' failed, the '" + 524 boot_candidate + "' has been removed from the stack" + 525 " to avoid and endless failure loop.") 526 if len(boot_stack) == 0: 527 return "" 528 529 my_get_state() 530 valid_state() 531 532 transitional_boot_selected = False 533 stack_popped = 0 534 if len(boot_stack) > 0: 535 stack_popped = 1 536 gp.qprint_dashes() 537 gp.qprint_var(boot_stack) 538 gp.qprint_dashes() 539 skip_boot_printed = 0 540 while len(boot_stack) > 0: 541 boot_candidate = boot_stack.pop() 542 if stack_mode == 'normal': 543 break 544 else: 545 if st.compare_states(state, boot_table[boot_candidate]['end']): 546 if not skip_boot_printed: 547 gp.qprint_var(stack_mode) 548 gp.qprintn() 549 gp.qprint_timen("Skipping the following boot tests" + 550 " which are unnecessary since their" + 551 " required end states match the" + 552 " current machine state:") 553 skip_boot_printed = 1 554 gp.qprint_var(boot_candidate) 555 boot_candidate = "" 556 if boot_candidate == "": 557 gp.qprint_dashes() 558 gp.qprint_var(boot_stack) 559 gp.qprint_dashes() 560 return boot_candidate 561 if st.compare_states(state, boot_table[boot_candidate]['start']): 562 gp.qprint_timen("The machine state is valid for a '" + 563 boot_candidate + "' boot test.") 564 gp.qprint_dashes() 565 gp.qprint_var(boot_stack) 566 gp.qprint_dashes() 567 return boot_candidate 568 else: 569 gp.qprint_timen("The machine state does not match the required" + 570 " starting state for a '" + boot_candidate + 571 "' boot test:") 572 gp.qprint_varx("boot_table[" + boot_candidate + "][start]", 573 boot_table[boot_candidate]['start'], 1) 574 boot_stack.append(boot_candidate) 575 transitional_boot_selected = True 576 popped_boot = boot_candidate 577 578 # Loop through your list selecting a boot_candidates 579 boot_candidates = [] 580 for boot_candidate in boot_list: 581 if st.compare_states(state, boot_table[boot_candidate]['start']): 582 if stack_popped: 583 if st.compare_states(boot_table[boot_candidate]['end'], 584 boot_table[popped_boot]['start']): 585 boot_candidates.append(boot_candidate) 586 else: 587 boot_candidates.append(boot_candidate) 588 589 if len(boot_candidates) == 0: 590 gp.qprint_timen("The user's boot list contained no boot tests" + 591 " which are valid for the current machine state.") 592 boot_candidate = default_power_on 593 if not st.compare_states(state, boot_table[default_power_on]['start']): 594 boot_candidate = default_power_off 595 boot_candidates.append(boot_candidate) 596 gp.qprint_timen("Using default '" + boot_candidate + 597 "' boot type to transition to valid state.") 598 599 gp.dprint_var(boot_candidates) 600 601 # Randomly select a boot from the candidate list. 602 boot = random.choice(boot_candidates) 603 604 return boot 605 606############################################################################### 607 608 609############################################################################### 610def print_last_boots(): 611 612 r""" 613 Print the last ten boots done with their time stamps. 614 """ 615 616 # indent 0, 90 chars wide, linefeed, char is "=" 617 gp.qprint_dashes(0, 90) 618 gp.qprintn("Last 10 boots:\n") 619 620 for boot_entry in last_ten: 621 grp.rqprint(boot_entry) 622 gp.qprint_dashes(0, 90) 623 624############################################################################### 625 626 627############################################################################### 628def print_defect_report(): 629 630 r""" 631 Print a defect report. 632 """ 633 634 # Making deliberate choice to NOT run plug_in_setup(). We don't want 635 # ffdc_prefix updated. 636 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 637 call_point='ffdc_report', stop_on_plug_in_failure=0) 638 639 # At some point I'd like to have the 'Call FFDC Methods' return a list 640 # of files it has collected. In that case, the following "ls" command 641 # would no longer be needed. For now, however, glob shows the files 642 # named in FFDC_LIST_FILE_PATH so I will refrain from printing those 643 # out (so we don't see duplicates in the list). 644 645 # Get additional header data which may have been created by ffdc plug-ins. 646 # Also, delete the individual header files to cleanup. 647 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 648 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 649 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 650 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 651 show_err=0) 652 653 # Get additional header data which may have been created by ffdc plug-ins. 654 # Also, delete the individual header files to cleanup. 655 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 656 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 657 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 658 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 659 show_err=0) 660 661 LOG_PREFIX = BuiltIn().get_variable_value("${LOG_PREFIX}") 662 663 output = '\n'.join(sorted(glob.glob(LOG_PREFIX + '*'))) 664 try: 665 ffdc_list = open(ffdc_list_file_path, 'r') 666 except IOError: 667 ffdc_list = "" 668 669 # Open ffdc_file_list for writing. We will write a complete list of 670 # FFDC files to it for possible use by plug-ins like cp_stop_check. 671 ffdc_list_file = open(ffdc_list_file_path, 'w') 672 673 gp.qprintn() 674 # indent=0, width=90, linefeed=1, char="=" 675 gp.qprint_dashes(0, 90, 1, "=") 676 gp.qprintn("Copy this data to the defect:\n") 677 678 if len(more_header_info) > 0: 679 gp.qprintn(more_header_info) 680 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 681 openbmc_host_name, openbmc_ip, openbmc_username, 682 openbmc_password, os_host, os_host_name, os_ip, os_username, 683 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 684 pdu_password, pdu_slot_no, openbmc_serial_host, 685 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 686 687 gp.qprintn() 688 689 print_last_boots() 690 gp.qprintn() 691 gp.qprint_var(state) 692 693 gp.qprintn() 694 gp.qprintn("FFDC data files:") 695 if status_file_path != "": 696 gp.qprintn(status_file_path) 697 ffdc_list_file.write(status_file_path + "\n") 698 699 gp.qprintn(output) 700 # gp.qprintn(ffdc_list) 701 gp.qprintn() 702 703 if len(ffdc_summary_info) > 0: 704 gp.qprintn(ffdc_summary_info) 705 706 gp.qprint_dashes(0, 90, 1, "=") 707 708 ffdc_list_file.write(output + "\n") 709 ffdc_list_file.close() 710 711############################################################################### 712 713 714############################################################################### 715def my_ffdc(): 716 717 r""" 718 Collect FFDC data. 719 """ 720 721 global state 722 723 plug_in_setup() 724 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 725 call_point='ffdc', stop_on_plug_in_failure=0) 726 727 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 728 status, ret_values = grk.run_key_u("FFDC ffdc_prefix=" + 729 AUTOBOOT_FFDC_PREFIX + 730 " ffdc_function_list=" + 731 ffdc_function_list, ignore=1) 732 if status != 'PASS': 733 gp.qprint_error("Call to ffdc failed.\n") 734 735 my_get_state() 736 737 print_defect_report() 738 739############################################################################### 740 741 742############################################################################### 743def print_test_start_message(boot_keyword): 744 745 r""" 746 Print a message indicating what boot test is about to run. 747 748 Description of arguments: 749 boot_keyword The name of the boot which is to be run 750 (e.g. "BMC Power On"). 751 """ 752 753 global last_ten 754 global boot_start_time 755 756 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 757 758 # Set boot_start_time for use by plug-ins. 759 boot_start_time = doing_msg[1:33] 760 gp.qprint_var(boot_start_time) 761 762 gp.qprint(doing_msg) 763 764 last_ten.append(doing_msg) 765 766 if len(last_ten) > 10: 767 del last_ten[0] 768 769############################################################################### 770 771 772############################################################################### 773def run_boot(boot): 774 775 r""" 776 Run the specified boot. 777 778 Description of arguments: 779 boot The name of the boot test to be performed. 780 """ 781 782 global state 783 784 print_test_start_message(boot) 785 786 plug_in_setup() 787 rc, shell_rc, failed_plug_in_name = \ 788 grpi.rprocess_plug_in_packages(call_point="pre_boot") 789 if rc != 0: 790 error_message = "Plug-in failed with non-zero return code.\n" +\ 791 gp.sprint_var(rc, 1) 792 BuiltIn().fail(gp.sprint_error(error_message)) 793 794 if test_mode: 795 # In test mode, we'll pretend the boot worked by assigning its 796 # required end state to the default state value. 797 state = st.strip_anchor_state(boot_table[boot]['end']) 798 else: 799 # Assertion: We trust that the state data was made fresh by the 800 # caller. 801 802 gp.qprintn() 803 804 if boot_table[boot]['method_type'] == "keyword": 805 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 806 boot_table[boot]['method'], 807 quiet=quiet) 808 809 if boot_table[boot]['bmc_reboot']: 810 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 811 plug_in_setup() 812 rc, shell_rc, failed_plug_in_name = \ 813 grpi.rprocess_plug_in_packages(call_point="post_reboot") 814 if rc != 0: 815 error_message = "Plug-in failed with non-zero return code.\n" 816 error_message += gp.sprint_var(rc, 1) 817 BuiltIn().fail(gp.sprint_error(error_message)) 818 else: 819 match_state = st.anchor_state(state) 820 del match_state['epoch_seconds'] 821 # Wait for the state to change in any way. 822 st.wait_state(match_state, wait_time=state_change_timeout, 823 interval="10 seconds", invert=1) 824 825 gp.qprintn() 826 if boot_table[boot]['end']['chassis'] == "Off": 827 boot_timeout = power_off_timeout 828 else: 829 boot_timeout = power_on_timeout 830 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 831 interval="10 seconds") 832 833 plug_in_setup() 834 rc, shell_rc, failed_plug_in_name = \ 835 grpi.rprocess_plug_in_packages(call_point="post_boot") 836 if rc != 0: 837 error_message = "Plug-in failed with non-zero return code.\n" +\ 838 gp.sprint_var(rc, 1) 839 BuiltIn().fail(gp.sprint_error(error_message)) 840 841############################################################################### 842 843 844############################################################################### 845def test_loop_body(): 846 847 r""" 848 The main loop body for the loop in main_py. 849 850 Description of arguments: 851 boot_count The iteration number (starts at 1). 852 """ 853 854 global boot_count 855 global state 856 global next_boot 857 global boot_success 858 global boot_end_time 859 860 gp.qprintn() 861 862 next_boot = select_boot() 863 if next_boot == "": 864 return True 865 866 boot_count += 1 867 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 868 869 pre_boot_plug_in_setup() 870 871 cmd_buf = ["run_boot", next_boot] 872 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 873 if boot_status == "FAIL": 874 gp.qprint(msg) 875 876 gp.qprintn() 877 if boot_status == "PASS": 878 boot_success = 1 879 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot + 880 "\" succeeded.") 881 else: 882 boot_success = 0 883 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot + 884 "\" failed.") 885 886 # Set boot_end_time for use by plug-ins. 887 boot_end_time = completion_msg[1:33] 888 gp.qprint_var(boot_end_time) 889 890 gp.qprint(completion_msg) 891 892 boot_results.update(next_boot, boot_status) 893 894 plug_in_setup() 895 # NOTE: A post_test_case call point failure is NOT counted as a boot 896 # failure. 897 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 898 call_point='post_test_case', stop_on_plug_in_failure=0) 899 900 plug_in_setup() 901 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 902 call_point='ffdc_check', shell_rc=0x00000200, 903 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 904 if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200: 905 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 906 if status != 'PASS': 907 gp.qprint_error("Call to my_ffdc failed.\n") 908 909 if delete_errlogs: 910 # We need to purge error logs between boots or they build up. 911 grk.run_key("Delete Error logs", ignore=1) 912 913 boot_results.print_report() 914 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 915 916 plug_in_setup() 917 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 918 call_point='stop_check') 919 if rc != 0: 920 error_message = "Stopping as requested by user.\n" 921 grp.rprint_error_report(error_message) 922 BuiltIn().fail(error_message) 923 924 # This should help prevent ConnectionErrors. 925 grk.run_key_u("Close All Connections") 926 927 return True 928 929############################################################################### 930 931 932############################################################################### 933def obmc_boot_test_teardown(): 934 935 r""" 936 Clean up after the Main keyword. 937 """ 938 939 if cp_setup_called: 940 plug_in_setup() 941 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 942 call_point='cleanup', stop_on_plug_in_failure=0) 943 944 if 'boot_results_file_path' in globals(): 945 # Save boot_results object to a file in case it is needed again. 946 gp.qprint_timen("Saving boot_results to the following path.") 947 gp.qprint_var(boot_results_file_path) 948 pickle.dump(boot_results, open(boot_results_file_path, 'wb'), 949 pickle.HIGHEST_PROTOCOL) 950 951 global save_stack 952 # Restore any global values saved on the save_stack. 953 for parm_name in main_func_parm_list: 954 # Get the parm_value if it was saved on the stack. 955 try: 956 parm_value = save_stack.pop(parm_name) 957 except: 958 # If it was not saved, no further action is required. 959 continue 960 961 # Restore the saved value. 962 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 963 "}\", parm_value)" 964 gp.dpissuing(cmd_buf) 965 exec(cmd_buf) 966 967 gp.dprintn(save_stack.sprint_obj()) 968 969############################################################################### 970 971 972############################################################################### 973def test_teardown(): 974 975 r""" 976 Clean up after this test case. 977 """ 978 979 gp.qprintn() 980 cmd_buf = ["Print Error", 981 "A keyword timeout occurred ending this program.\n"] 982 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 983 984 grp.rqprint_pgm_footer() 985 986############################################################################### 987 988 989############################################################################### 990def obmc_boot_test_py(loc_boot_stack=None, 991 loc_stack_mode=None, 992 loc_quiet=None): 993 994 r""" 995 Do main program processing. 996 """ 997 998 global save_stack 999 1000 # Process function parms. 1001 for parm_name in main_func_parm_list: 1002 # Get parm's value. 1003 cmd_buf = "parm_value = loc_" + parm_name 1004 exec(cmd_buf) 1005 gp.dpvar(parm_name) 1006 gp.dpvar(parm_value) 1007 1008 if parm_value is None: 1009 # Parm was not specified by the calling function so set it to its 1010 # corresponding global value. 1011 cmd_buf = "loc_" + parm_name + " = BuiltIn().get_variable_value" +\ 1012 "(\"${" + parm_name + "}\")" 1013 gp.dpissuing(cmd_buf) 1014 exec(cmd_buf) 1015 else: 1016 # Save the global value on a stack. 1017 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1018 parm_name + "}\"), \"" + parm_name + "\")" 1019 gp.dpissuing(cmd_buf) 1020 exec(cmd_buf) 1021 1022 # Set the global value to the passed value. 1023 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1024 "}\", loc_" + parm_name + ")" 1025 gp.dpissuing(cmd_buf) 1026 exec(cmd_buf) 1027 1028 gp.dprintn(save_stack.sprint_obj()) 1029 1030 setup() 1031 1032 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1033 1034 if ffdc_only: 1035 gp.qprint_timen("Caller requested ffdc_only.") 1036 pre_boot_plug_in_setup() 1037 grk.run_key_u("my_ffdc") 1038 return 1039 1040 # Process caller's boot_stack. 1041 while (len(boot_stack) > 0): 1042 test_loop_body() 1043 1044 gp.qprint_timen("Finished processing stack.") 1045 1046 # Process caller's boot_list. 1047 if len(boot_list) > 0: 1048 for ix in range(1, max_num_tests + 1): 1049 test_loop_body() 1050 1051 gp.qprint_timen("Completed all requested boot tests.") 1052 1053 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1054 new_fail = boot_fail - init_boot_fail 1055 if new_fail > boot_fail_threshold: 1056 error_message = "Boot failures exceed the boot failure" +\ 1057 " threshold:\n" +\ 1058 gp.sprint_var(new_fail) +\ 1059 gp.sprint_var(boot_fail_threshold) 1060 BuiltIn().fail(gp.sprint_error(error_message)) 1061 1062############################################################################### 1063