1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13try: 14 import cPickle as pickle 15except ImportError: 16 import pickle 17import socket 18 19from robot.utils import DotDict 20from robot.libraries.BuiltIn import BuiltIn 21 22from boot_data import * 23import gen_print as gp 24import gen_robot_print as grp 25import gen_robot_plug_in as grpi 26import gen_robot_valid as grv 27import gen_misc as gm 28import gen_cmd as gc 29import gen_robot_keyword as grk 30import state as st 31import var_stack as vs 32 33base_path = os.path.dirname(os.path.dirname( 34 imp.find_module("gen_robot_print")[1])) +\ 35 os.sep 36sys.path.append(base_path + "extended/") 37import run_keyword as rk 38 39# Setting master_pid correctly influences the behavior of plug-ins like 40# DB_Logging 41program_pid = os.getpid() 42master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 43pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 44 45# Set up boot data structures. 46boot_table = create_boot_table() 47valid_boot_types = create_valid_boot_list(boot_table) 48 49boot_lists = read_boot_lists() 50last_ten = [] 51 52state = st.return_state_constant('default_state') 53cp_setup_called = 0 54next_boot = "" 55base_tool_dir_path = os.path.normpath(os.environ.get( 56 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 57 58ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 59boot_success = 0 60status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 61if status_dir_path != "": 62 status_dir_path = os.path.normpath(status_dir_path) + os.sep 63default_power_on = "REST Power On" 64default_power_off = "REST Power Off" 65boot_count = 0 66 67LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 68ffdc_prefix = "" 69boot_start_time = "" 70boot_end_time = "" 71save_stack = vs.var_stack('save_stack') 72main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 73 74 75def process_host(host, 76 host_var_name=""): 77 r""" 78 Process a host by getting the associated host name and IP address and 79 setting them in global variables. 80 81 If the caller does not pass the host_var_name, this function will try to 82 figure out the name of the variable used by the caller for the host parm. 83 Callers are advised to explicitly specify the host_var_name when calling 84 with an exec command. In such cases, the get_arg_name cannot figure out 85 the host variable name. 86 87 This function will then create similar global variable names by 88 removing "_host" and appending "_host_name" or "_ip" to the host variable 89 name. 90 91 Example: 92 93 If a call is made like this: 94 process_host(openbmc_host) 95 96 Global variables openbmc_host_name and openbmc_ip will be set. 97 98 Description of argument(s): 99 host A host name or IP. The name of the variable used should 100 have a suffix of "_host". 101 host_var_name The name of the variable being used as the host parm. 102 """ 103 104 if host_var_name == "": 105 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 106 107 host_name_var_name = re.sub("host", "host_name", host_var_name) 108 ip_var_name = re.sub("host", "ip", host_var_name) 109 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 110 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 111 host + "')" 112 exec(cmd_buf) 113 114 115def process_pgm_parms(): 116 r""" 117 Process the program parameters by assigning them all to corresponding 118 globals. Also, set some global values that depend on program parameters. 119 """ 120 121 # Program parameter processing. 122 # Assign all program parms to python variables which are global to this 123 # module. 124 125 global parm_list 126 parm_list = BuiltIn().get_variable_value("${parm_list}") 127 # The following subset of parms should be processed as integers. 128 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 129 'boot_fail_threshold', 'delete_errlogs', 'quiet', 'test_mode', 130 'debug'] 131 for parm in parm_list: 132 if parm in int_list: 133 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 134 "}\", \"0\"))" 135 else: 136 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 137 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 138 gp.dpissuing(cmd_buf) 139 exec(cmd_buf) 140 if re.match(r".*_host$", parm): 141 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 142 exec(cmd_buf) 143 if re.match(r".*_password$", parm): 144 # Register the value of any parm whose name ends in _password. 145 # This will cause the print functions to replace passwords with 146 # asterisks in the output. 147 cmd_buf = "gp.register_passwords(" + parm + ")" 148 exec(cmd_buf) 149 150 global ffdc_dir_path_style 151 global boot_list 152 global boot_stack 153 global boot_results_file_path 154 global boot_results 155 global last_ten 156 global ffdc_list_file_path 157 global ffdc_report_list_path 158 global ffdc_summary_list_path 159 160 if ffdc_dir_path_style == "": 161 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 162 163 # Convert these program parms to lists for easier processing.. 164 boot_list = filter(None, boot_list.split(":")) 165 boot_stack = filter(None, boot_stack.split(":")) 166 167 cleanup_boot_results_file() 168 boot_results_file_path = create_boot_results_file_path(pgm_name, 169 openbmc_nickname, 170 master_pid) 171 172 if os.path.isfile(boot_results_file_path): 173 # We've been called before in this run so we'll load the saved 174 # boot_results and last_ten objects. 175 boot_results, last_ten =\ 176 pickle.load(open(boot_results_file_path, 'rb')) 177 else: 178 boot_results = boot_results(boot_table, boot_pass, boot_fail) 179 180 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 181 "/FFDC_FILE_LIST" 182 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 183 "/FFDC_REPORT_FILE_LIST" 184 185 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 186 "/FFDC_SUMMARY_FILE_LIST" 187 188 189def initial_plug_in_setup(): 190 r""" 191 Initialize all plug-in environment variables which do not change for the 192 duration of the program. 193 194 """ 195 196 global LOG_LEVEL 197 BuiltIn().set_log_level("NONE") 198 199 BuiltIn().set_global_variable("${master_pid}", master_pid) 200 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 201 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 202 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 203 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 204 ffdc_list_file_path) 205 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 206 ffdc_report_list_path) 207 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 208 ffdc_summary_list_path) 209 210 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 211 ffdc_dir_path_style) 212 BuiltIn().set_global_variable("${FFDC_CHECK}", 213 ffdc_check) 214 215 # For each program parameter, set the corresponding AUTOBOOT_ environment 216 # variable value. Also, set an AUTOBOOT_ environment variable for every 217 # element in additional_values. 218 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 219 "status_dir_path", "base_tool_dir_path", 220 "ffdc_list_file_path", "ffdc_report_list_path", 221 "ffdc_summary_list_path"] 222 223 plug_in_vars = parm_list + additional_values 224 225 for var_name in plug_in_vars: 226 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 227 var_name = var_name.upper() 228 if var_value is None: 229 var_value = "" 230 os.environ["AUTOBOOT_" + var_name] = str(var_value) 231 232 BuiltIn().set_log_level(LOG_LEVEL) 233 234 # Make sure the ffdc list directory exists. 235 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 236 if not os.path.exists(ffdc_list_dir_path): 237 os.makedirs(ffdc_list_dir_path) 238 239 240def plug_in_setup(): 241 r""" 242 Initialize all changing plug-in environment variables for use by the 243 plug-in programs. 244 """ 245 246 global LOG_LEVEL 247 global test_really_running 248 249 BuiltIn().set_log_level("NONE") 250 251 boot_pass, boot_fail = boot_results.return_total_pass_fail() 252 if boot_pass > 1: 253 test_really_running = 1 254 else: 255 test_really_running = 0 256 257 BuiltIn().set_global_variable("${test_really_running}", 258 test_really_running) 259 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 260 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 261 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 262 BuiltIn().set_global_variable("${boot_success}", boot_success) 263 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 264 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 265 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 266 267 # For each program parameter, set the corresponding AUTOBOOT_ environment 268 # variable value. Also, set an AUTOBOOT_ environment variable for every 269 # element in additional_values. 270 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 271 "boot_fail", "test_really_running", "ffdc_prefix", 272 "boot_start_time", "boot_end_time"] 273 274 plug_in_vars = additional_values 275 276 for var_name in plug_in_vars: 277 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 278 var_name = var_name.upper() 279 if var_value is None: 280 var_value = "" 281 os.environ["AUTOBOOT_" + var_name] = str(var_value) 282 283 if debug: 284 shell_rc, out_buf = \ 285 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 286 287 BuiltIn().set_log_level(LOG_LEVEL) 288 289 290def pre_boot_plug_in_setup(): 291 292 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 293 try: 294 os.remove(ffdc_list_file_path) 295 except OSError: 296 pass 297 298 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 299 try: 300 os.remove(ffdc_report_list_path) 301 except OSError: 302 pass 303 304 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 305 try: 306 os.remove(ffdc_summary_list_path) 307 except OSError: 308 pass 309 310 global ffdc_prefix 311 312 seconds = time.time() 313 loc_time = time.localtime(seconds) 314 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 315 316 ffdc_prefix = openbmc_nickname + "." + time_string 317 318 319def setup(): 320 r""" 321 Do general program setup tasks. 322 """ 323 324 global cp_setup_called 325 global transitional_boot_selected 326 327 gp.qprintn() 328 329 transitional_boot_selected = False 330 331 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 332 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 333 # If we can't find process_plug_in_packages.py, ssh_pw or 334 # validate_plug_ins.py, then we don't have our repo bin in PATH. 335 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 336 + " ssh_pw validate_plug_ins.py", quiet=1, 337 print_output=0, show_err=0) 338 if shell_rc != 0: 339 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 340 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 341 if robot_pgm_dir_path not in sys.path: 342 sys.path.append(robot_pgm_dir_path) 343 PYTHONPATH = os.environ.get("PYTHONPATH", "") 344 if PYTHONPATH == "": 345 os.environ['PYTHONPATH'] = robot_pgm_dir_path 346 else: 347 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 348 349 validate_parms() 350 351 grp.rqprint_pgm_header() 352 353 grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF") 354 355 initial_plug_in_setup() 356 357 plug_in_setup() 358 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 359 call_point='setup') 360 if rc != 0: 361 error_message = "Plug-in setup failed.\n" 362 grp.rprint_error_report(error_message) 363 BuiltIn().fail(error_message) 364 # Setting cp_setup_called lets our Teardown know that it needs to call 365 # the cleanup plug-in call point. 366 cp_setup_called = 1 367 368 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 369 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 370 # FFDC_LOG_PATH is used by "FFDC" keyword. 371 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 372 373 # Also printed by FFDC. 374 global host_name 375 global host_ip 376 host = socket.gethostname() 377 host_name, host_ip = gm.get_host_name_ip(host) 378 379 gp.dprint_var(boot_table, 1) 380 gp.dprint_var(boot_lists) 381 382 383def validate_parms(): 384 r""" 385 Validate all program parameters. 386 """ 387 388 process_pgm_parms() 389 390 gp.qprintn() 391 392 global openbmc_model 393 grv.rvalid_value("openbmc_host") 394 grv.rvalid_value("openbmc_username") 395 grv.rvalid_value("openbmc_password") 396 if os_host != "": 397 grv.rvalid_value("os_username") 398 grv.rvalid_value("os_password") 399 400 if pdu_host != "": 401 grv.rvalid_value("pdu_username") 402 grv.rvalid_value("pdu_password") 403 grv.rvalid_integer("pdu_slot_no") 404 if openbmc_serial_host != "": 405 grv.rvalid_integer("openbmc_serial_port") 406 if openbmc_model == "": 407 status, ret_values =\ 408 grk.run_key_u("Get BMC System Model") 409 openbmc_model = ret_values 410 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 411 grv.rvalid_value("openbmc_model") 412 grv.rvalid_integer("max_num_tests") 413 grv.rvalid_integer("boot_pass") 414 grv.rvalid_integer("boot_fail") 415 416 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 417 BuiltIn().set_global_variable("${plug_in_packages_list}", 418 plug_in_packages_list) 419 420 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) 421 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 422 error_message = "You must provide either a value for either the" +\ 423 " boot_list or the boot_stack parm.\n" 424 BuiltIn().fail(gp.sprint_error(error_message)) 425 426 valid_boot_list(boot_list, valid_boot_types) 427 valid_boot_list(boot_stack, valid_boot_types) 428 429 selected_PDU_boots = list(set(boot_list + boot_stack) 430 & set(boot_lists['PDU_reboot'])) 431 432 if len(selected_PDU_boots) > 0 and pdu_host == "": 433 error_message = "You have selected the following boots which" +\ 434 " require a PDU host but no value for pdu_host:\n" 435 error_message += gp.sprint_var(selected_PDU_boots) 436 error_message += gp.sprint_var(pdu_host, 2) 437 BuiltIn().fail(gp.sprint_error(error_message)) 438 439 return 440 441 442def my_get_state(): 443 r""" 444 Get the system state plus a little bit of wrapping. 445 """ 446 447 global state 448 449 req_states = ['epoch_seconds'] + st.default_req_states 450 451 gp.qprint_timen("Getting system state.") 452 if test_mode: 453 state['epoch_seconds'] = int(time.time()) 454 else: 455 state = st.get_state(req_states=req_states, quiet=quiet) 456 gp.qprint_var(state) 457 458 459def valid_state(): 460 r""" 461 Verify that our state dictionary contains no blank values. If we don't get 462 valid state data, we cannot continue to work. 463 """ 464 465 if st.compare_states(state, st.invalid_state_match, 'or'): 466 error_message = "The state dictionary contains blank fields which" +\ 467 " is illegal.\n" + gp.sprint_var(state) 468 BuiltIn().fail(gp.sprint_error(error_message)) 469 470 471def select_boot(): 472 r""" 473 Select a boot test to be run based on our current state and return the 474 chosen boot type. 475 476 Description of arguments: 477 state The state of the machine. 478 """ 479 480 global transitional_boot_selected 481 global boot_stack 482 483 gp.qprint_timen("Selecting a boot test.") 484 485 if transitional_boot_selected and not boot_success: 486 prior_boot = next_boot 487 boot_candidate = boot_stack.pop() 488 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 489 + " transition to a valid state for '" + boot_candidate 490 + "' which was at the top of the boot_stack. Since" 491 + " the '" + next_boot + "' failed, the '" 492 + boot_candidate + "' has been removed from the stack" 493 + " to avoid and endless failure loop.") 494 if len(boot_stack) == 0: 495 return "" 496 497 my_get_state() 498 valid_state() 499 500 transitional_boot_selected = False 501 stack_popped = 0 502 if len(boot_stack) > 0: 503 stack_popped = 1 504 gp.qprint_dashes() 505 gp.qprint_var(boot_stack) 506 gp.qprint_dashes() 507 skip_boot_printed = 0 508 while len(boot_stack) > 0: 509 boot_candidate = boot_stack.pop() 510 if stack_mode == 'normal': 511 break 512 else: 513 if st.compare_states(state, boot_table[boot_candidate]['end']): 514 if not skip_boot_printed: 515 gp.qprint_var(stack_mode) 516 gp.qprintn() 517 gp.qprint_timen("Skipping the following boot tests" 518 + " which are unnecessary since their" 519 + " required end states match the" 520 + " current machine state:") 521 skip_boot_printed = 1 522 gp.qprint_var(boot_candidate) 523 boot_candidate = "" 524 if boot_candidate == "": 525 gp.qprint_dashes() 526 gp.qprint_var(boot_stack) 527 gp.qprint_dashes() 528 return boot_candidate 529 if st.compare_states(state, boot_table[boot_candidate]['start']): 530 gp.qprint_timen("The machine state is valid for a '" 531 + boot_candidate + "' boot test.") 532 gp.qprint_dashes() 533 gp.qprint_var(boot_stack) 534 gp.qprint_dashes() 535 return boot_candidate 536 else: 537 gp.qprint_timen("The machine state does not match the required" 538 + " starting state for a '" + boot_candidate 539 + "' boot test:") 540 gp.qprint_varx("boot_table[" + boot_candidate + "][start]", 541 boot_table[boot_candidate]['start'], 1) 542 boot_stack.append(boot_candidate) 543 transitional_boot_selected = True 544 popped_boot = boot_candidate 545 546 # Loop through your list selecting a boot_candidates 547 boot_candidates = [] 548 for boot_candidate in boot_list: 549 if st.compare_states(state, boot_table[boot_candidate]['start']): 550 if stack_popped: 551 if st.compare_states(boot_table[boot_candidate]['end'], 552 boot_table[popped_boot]['start']): 553 boot_candidates.append(boot_candidate) 554 else: 555 boot_candidates.append(boot_candidate) 556 557 if len(boot_candidates) == 0: 558 gp.qprint_timen("The user's boot list contained no boot tests" 559 + " which are valid for the current machine state.") 560 boot_candidate = default_power_on 561 if not st.compare_states(state, boot_table[default_power_on]['start']): 562 boot_candidate = default_power_off 563 boot_candidates.append(boot_candidate) 564 gp.qprint_timen("Using default '" + boot_candidate 565 + "' boot type to transition to valid state.") 566 567 gp.dprint_var(boot_candidates) 568 569 # Randomly select a boot from the candidate list. 570 boot = random.choice(boot_candidates) 571 572 return boot 573 574 575def print_last_boots(): 576 r""" 577 Print the last ten boots done with their time stamps. 578 """ 579 580 # indent 0, 90 chars wide, linefeed, char is "=" 581 gp.qprint_dashes(0, 90) 582 gp.qprintn("Last 10 boots:\n") 583 584 for boot_entry in last_ten: 585 grp.rqprint(boot_entry) 586 gp.qprint_dashes(0, 90) 587 588 589def print_defect_report(ffdc_file_list): 590 r""" 591 Print a defect report. 592 593 Description of argument(s): 594 ffdc_file_list A list of files which were collected by our ffdc functions. 595 """ 596 597 # Making deliberate choice to NOT run plug_in_setup(). We don't want 598 # ffdc_prefix updated. 599 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 600 call_point='ffdc_report', stop_on_plug_in_failure=0) 601 602 # Get additional header data which may have been created by ffdc plug-ins. 603 # Also, delete the individual header files to cleanup. 604 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 605 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 606 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 607 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 608 show_err=0) 609 610 # Get additional summary data which may have been created by ffdc plug-ins. 611 # Also, delete the individual header files to cleanup. 612 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 613 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 614 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 615 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 616 show_err=0) 617 618 # ffdc_list_file_path contains a list of any ffdc files created by plug- 619 # ins, etc. Read that data into a list. 620 try: 621 plug_in_ffdc_list = \ 622 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 623 plug_in_ffdc_list = filter(None, plug_in_ffdc_list) 624 except IOError: 625 plug_in_ffdc_list = [] 626 627 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 628 # in. Eliminate duplicates and sort the list. 629 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 630 631 if status_file_path != "": 632 ffdc_file_list.insert(0, status_file_path) 633 634 # Convert the list to a printable list. 635 printable_ffdc_file_list = "\n".join(ffdc_file_list) 636 637 # Open ffdc_file_list for writing. We will write a complete list of 638 # FFDC files to it for possible use by plug-ins like cp_stop_check. 639 ffdc_list_file = open(ffdc_list_file_path, 'w') 640 ffdc_list_file.write(printable_ffdc_file_list + "\n") 641 ffdc_list_file.close() 642 643 indent = 0 644 width = 90 645 linefeed = 1 646 char = "=" 647 648 gp.qprintn() 649 gp.qprint_dashes(indent, width, linefeed, char) 650 gp.qprintn("Copy this data to the defect:\n") 651 652 if len(more_header_info) > 0: 653 gp.qprintn(more_header_info) 654 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 655 openbmc_host_name, openbmc_ip, openbmc_username, 656 openbmc_password, os_host, os_host_name, os_ip, os_username, 657 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 658 pdu_password, pdu_slot_no, openbmc_serial_host, 659 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 660 661 gp.qprintn() 662 print_last_boots() 663 gp.qprintn() 664 gp.qprint_var(state) 665 gp.qprintn() 666 gp.qprintn("FFDC data files:") 667 gp.qprintn(printable_ffdc_file_list) 668 gp.qprintn() 669 670 if len(ffdc_summary_info) > 0: 671 gp.qprintn(ffdc_summary_info) 672 673 gp.qprint_dashes(indent, width, linefeed, char) 674 675 676def my_ffdc(): 677 r""" 678 Collect FFDC data. 679 """ 680 681 global state 682 683 plug_in_setup() 684 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 685 call_point='ffdc', stop_on_plug_in_failure=0) 686 687 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 688 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 689 + AUTOBOOT_FFDC_PREFIX 690 + " ffdc_function_list=" 691 + ffdc_function_list, ignore=1) 692 if status != 'PASS': 693 gp.qprint_error("Call to ffdc failed.\n") 694 695 my_get_state() 696 697 print_defect_report(ffdc_file_list) 698 699 700def print_test_start_message(boot_keyword): 701 r""" 702 Print a message indicating what boot test is about to run. 703 704 Description of arguments: 705 boot_keyword The name of the boot which is to be run 706 (e.g. "BMC Power On"). 707 """ 708 709 global last_ten 710 global boot_start_time 711 712 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 713 714 # Set boot_start_time for use by plug-ins. 715 boot_start_time = doing_msg[1:33] 716 gp.qprint_var(boot_start_time) 717 718 gp.qprint(doing_msg) 719 720 last_ten.append(doing_msg) 721 722 if len(last_ten) > 10: 723 del last_ten[0] 724 725 726def run_boot(boot): 727 r""" 728 Run the specified boot. 729 730 Description of arguments: 731 boot The name of the boot test to be performed. 732 """ 733 734 global state 735 736 print_test_start_message(boot) 737 738 plug_in_setup() 739 rc, shell_rc, failed_plug_in_name = \ 740 grpi.rprocess_plug_in_packages(call_point="pre_boot") 741 if rc != 0: 742 error_message = "Plug-in failed with non-zero return code.\n" +\ 743 gp.sprint_var(rc, 1) 744 BuiltIn().fail(gp.sprint_error(error_message)) 745 746 if test_mode: 747 # In test mode, we'll pretend the boot worked by assigning its 748 # required end state to the default state value. 749 state = st.strip_anchor_state(boot_table[boot]['end']) 750 else: 751 # Assertion: We trust that the state data was made fresh by the 752 # caller. 753 754 gp.qprintn() 755 756 if boot_table[boot]['method_type'] == "keyword": 757 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 758 boot_table[boot]['method'], 759 quiet=quiet) 760 761 if boot_table[boot]['bmc_reboot']: 762 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 763 plug_in_setup() 764 rc, shell_rc, failed_plug_in_name = \ 765 grpi.rprocess_plug_in_packages(call_point="post_reboot") 766 if rc != 0: 767 error_message = "Plug-in failed with non-zero return code.\n" 768 error_message += gp.sprint_var(rc, 1) 769 BuiltIn().fail(gp.sprint_error(error_message)) 770 else: 771 match_state = st.anchor_state(state) 772 del match_state['epoch_seconds'] 773 # Wait for the state to change in any way. 774 st.wait_state(match_state, wait_time=state_change_timeout, 775 interval="10 seconds", invert=1) 776 777 gp.qprintn() 778 if boot_table[boot]['end']['chassis'] == "Off": 779 boot_timeout = power_off_timeout 780 else: 781 boot_timeout = power_on_timeout 782 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 783 interval="10 seconds") 784 785 plug_in_setup() 786 rc, shell_rc, failed_plug_in_name = \ 787 grpi.rprocess_plug_in_packages(call_point="post_boot") 788 if rc != 0: 789 error_message = "Plug-in failed with non-zero return code.\n" +\ 790 gp.sprint_var(rc, 1) 791 BuiltIn().fail(gp.sprint_error(error_message)) 792 793 794def test_loop_body(): 795 r""" 796 The main loop body for the loop in main_py. 797 798 Description of arguments: 799 boot_count The iteration number (starts at 1). 800 """ 801 802 global boot_count 803 global state 804 global next_boot 805 global boot_success 806 global boot_end_time 807 808 gp.qprintn() 809 810 next_boot = select_boot() 811 if next_boot == "": 812 return True 813 814 boot_count += 1 815 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 816 817 pre_boot_plug_in_setup() 818 819 cmd_buf = ["run_boot", next_boot] 820 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 821 if boot_status == "FAIL": 822 gp.qprint(msg) 823 824 gp.qprintn() 825 if boot_status == "PASS": 826 boot_success = 1 827 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 828 + "\" succeeded.") 829 else: 830 boot_success = 0 831 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 832 + "\" failed.") 833 834 # Set boot_end_time for use by plug-ins. 835 boot_end_time = completion_msg[1:33] 836 gp.qprint_var(boot_end_time) 837 838 gp.qprint(completion_msg) 839 840 boot_results.update(next_boot, boot_status) 841 842 plug_in_setup() 843 # NOTE: A post_test_case call point failure is NOT counted as a boot 844 # failure. 845 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 846 call_point='post_test_case', stop_on_plug_in_failure=0) 847 848 plug_in_setup() 849 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 850 call_point='ffdc_check', shell_rc=0x00000200, 851 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 852 if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200: 853 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 854 if status != 'PASS': 855 gp.qprint_error("Call to my_ffdc failed.\n") 856 857 if delete_errlogs: 858 # We need to purge error logs between boots or they build up. 859 grk.run_key("Delete Error logs", ignore=1) 860 861 boot_results.print_report() 862 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 863 864 plug_in_setup() 865 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 866 call_point='stop_check', shell_rc=0x00000200, stop_on_non_zero_rc=1) 867 if shell_rc == 0x00000200: 868 message = "Stopping as requested by user.\n" 869 gp.print_time(message) 870 BuiltIn().fail(message) 871 872 # This should help prevent ConnectionErrors. 873 grk.run_key_u("Close All Connections") 874 875 return True 876 877 878def obmc_boot_test_teardown(): 879 r""" 880 Clean up after the Main keyword. 881 """ 882 883 if cp_setup_called: 884 plug_in_setup() 885 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 886 call_point='cleanup', stop_on_plug_in_failure=0) 887 888 if 'boot_results_file_path' in globals(): 889 # Save boot_results and last_ten objects to a file in case they are 890 # needed again. 891 gp.qprint_timen("Saving boot_results to the following path.") 892 gp.qprint_var(boot_results_file_path) 893 pickle.dump((boot_results, last_ten), 894 open(boot_results_file_path, 'wb'), 895 pickle.HIGHEST_PROTOCOL) 896 897 global save_stack 898 # Restore any global values saved on the save_stack. 899 for parm_name in main_func_parm_list: 900 # Get the parm_value if it was saved on the stack. 901 try: 902 parm_value = save_stack.pop(parm_name) 903 except BaseException: 904 # If it was not saved, no further action is required. 905 continue 906 907 # Restore the saved value. 908 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 909 "}\", parm_value)" 910 gp.dpissuing(cmd_buf) 911 exec(cmd_buf) 912 913 gp.dprintn(save_stack.sprint_obj()) 914 915 916def test_teardown(): 917 r""" 918 Clean up after this test case. 919 """ 920 921 gp.qprintn() 922 cmd_buf = ["Print Error", 923 "A keyword timeout occurred ending this program.\n"] 924 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 925 926 grp.rqprint_pgm_footer() 927 928 929def obmc_boot_test_py(loc_boot_stack=None, 930 loc_stack_mode=None, 931 loc_quiet=None): 932 r""" 933 Do main program processing. 934 """ 935 936 global save_stack 937 938 # Process function parms. 939 for parm_name in main_func_parm_list: 940 # Get parm's value. 941 cmd_buf = "parm_value = loc_" + parm_name 942 exec(cmd_buf) 943 gp.dpvar(parm_name) 944 gp.dpvar(parm_value) 945 946 if parm_value is None: 947 # Parm was not specified by the calling function so set it to its 948 # corresponding global value. 949 cmd_buf = "loc_" + parm_name + " = BuiltIn().get_variable_value" +\ 950 "(\"${" + parm_name + "}\")" 951 gp.dpissuing(cmd_buf) 952 exec(cmd_buf) 953 else: 954 # Save the global value on a stack. 955 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 956 parm_name + "}\"), \"" + parm_name + "\")" 957 gp.dpissuing(cmd_buf) 958 exec(cmd_buf) 959 960 # Set the global value to the passed value. 961 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 962 "}\", loc_" + parm_name + ")" 963 gp.dpissuing(cmd_buf) 964 exec(cmd_buf) 965 966 gp.dprintn(save_stack.sprint_obj()) 967 968 setup() 969 970 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 971 972 if ffdc_only: 973 gp.qprint_timen("Caller requested ffdc_only.") 974 pre_boot_plug_in_setup() 975 grk.run_key_u("my_ffdc") 976 return 977 978 # Process caller's boot_stack. 979 while (len(boot_stack) > 0): 980 test_loop_body() 981 982 gp.qprint_timen("Finished processing stack.") 983 984 # Process caller's boot_list. 985 if len(boot_list) > 0: 986 for ix in range(1, max_num_tests + 1): 987 test_loop_body() 988 989 gp.qprint_timen("Completed all requested boot tests.") 990 991 boot_pass, boot_fail = boot_results.return_total_pass_fail() 992 new_fail = boot_fail - init_boot_fail 993 if new_fail > boot_fail_threshold: 994 error_message = "Boot failures exceed the boot failure" +\ 995 " threshold:\n" +\ 996 gp.sprint_var(new_fail) +\ 997 gp.sprint_var(boot_fail_threshold) 998 BuiltIn().fail(gp.sprint_error(error_message)) 999