1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import cPickle as pickle 14import socket 15 16from robot.utils import DotDict 17from robot.libraries.BuiltIn import BuiltIn 18 19from boot_data import * 20import gen_print as gp 21import gen_robot_print as grp 22import gen_robot_plug_in as grpi 23import gen_robot_valid as grv 24import gen_misc as gm 25import gen_cmd as gc 26import gen_robot_keyword as grk 27import state as st 28import var_stack as vs 29 30base_path = os.path.dirname(os.path.dirname( 31 imp.find_module("gen_robot_print")[1])) +\ 32 os.sep 33sys.path.append(base_path + "extended/") 34import run_keyword as rk 35 36# Setting master_pid correctly influences the behavior of plug-ins like 37# DB_Logging 38program_pid = os.getpid() 39master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 40pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 41 42# Set up boot data structures. 43boot_table = create_boot_table() 44valid_boot_types = create_valid_boot_list(boot_table) 45 46boot_lists = read_boot_lists() 47last_ten = [] 48 49state = st.return_state_constant('default_state') 50cp_setup_called = 0 51next_boot = "" 52base_tool_dir_path = os.path.normpath(os.environ.get( 53 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 54 55ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 56boot_success = 0 57status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 58if status_dir_path != "": 59 status_dir_path = os.path.normpath(status_dir_path) + os.sep 60default_power_on = "REST Power On" 61default_power_off = "REST Power Off" 62boot_count = 0 63 64LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 65ffdc_prefix = "" 66boot_start_time = "" 67boot_end_time = "" 68save_stack = vs.var_stack('save_stack') 69main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 70 71 72def process_host(host, 73 host_var_name=""): 74 r""" 75 Process a host by getting the associated host name and IP address and 76 setting them in global variables. 77 78 If the caller does not pass the host_var_name, this function will try to 79 figure out the name of the variable used by the caller for the host parm. 80 Callers are advised to explicitly specify the host_var_name when calling 81 with an exec command. In such cases, the get_arg_name cannot figure out 82 the host variable name. 83 84 This function will then create similar global variable names by 85 removing "_host" and appending "_host_name" or "_ip" to the host variable 86 name. 87 88 Example: 89 90 If a call is made like this: 91 process_host(openbmc_host) 92 93 Global variables openbmc_host_name and openbmc_ip will be set. 94 95 Description of argument(s): 96 host A host name or IP. The name of the variable used should 97 have a suffix of "_host". 98 host_var_name The name of the variable being used as the host parm. 99 """ 100 101 if host_var_name == "": 102 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 103 104 host_name_var_name = re.sub("host", "host_name", host_var_name) 105 ip_var_name = re.sub("host", "ip", host_var_name) 106 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 107 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 108 host + "')" 109 exec(cmd_buf) 110 111 112def process_pgm_parms(): 113 r""" 114 Process the program parameters by assigning them all to corresponding 115 globals. Also, set some global values that depend on program parameters. 116 """ 117 118 # Program parameter processing. 119 # Assign all program parms to python variables which are global to this 120 # module. 121 122 global parm_list 123 parm_list = BuiltIn().get_variable_value("${parm_list}") 124 # The following subset of parms should be processed as integers. 125 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 126 'boot_fail_threshold', 'delete_errlogs', 'quiet', 'test_mode', 127 'debug'] 128 for parm in parm_list: 129 if parm in int_list: 130 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 131 "}\", \"0\"))" 132 else: 133 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 134 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 135 gp.dpissuing(cmd_buf) 136 exec(cmd_buf) 137 if re.match(r".*_host$", parm): 138 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 139 exec(cmd_buf) 140 if re.match(r".*_password$", parm): 141 # Register the value of any parm whose name ends in _password. 142 # This will cause the print functions to replace passwords with 143 # asterisks in the output. 144 cmd_buf = "gp.register_passwords(" + parm + ")" 145 exec(cmd_buf) 146 147 global ffdc_dir_path_style 148 global boot_list 149 global boot_stack 150 global boot_results_file_path 151 global boot_results 152 global ffdc_list_file_path 153 global ffdc_report_list_path 154 global ffdc_summary_list_path 155 156 if ffdc_dir_path_style == "": 157 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 158 159 # Convert these program parms to lists for easier processing.. 160 boot_list = filter(None, boot_list.split(":")) 161 boot_stack = filter(None, boot_stack.split(":")) 162 163 cleanup_boot_results_file() 164 boot_results_file_path = create_boot_results_file_path(pgm_name, 165 openbmc_nickname, 166 master_pid) 167 168 if os.path.isfile(boot_results_file_path): 169 # We've been called before in this run so we'll load the saved 170 # boot_results object. 171 boot_results = pickle.load(open(boot_results_file_path, 'rb')) 172 else: 173 boot_results = boot_results(boot_table, boot_pass, boot_fail) 174 175 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 176 "/FFDC_FILE_LIST" 177 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 178 "/FFDC_REPORT_FILE_LIST" 179 180 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 181 "/FFDC_SUMMARY_FILE_LIST" 182 183 184def initial_plug_in_setup(): 185 r""" 186 Initialize all plug-in environment variables which do not change for the 187 duration of the program. 188 189 """ 190 191 global LOG_LEVEL 192 BuiltIn().set_log_level("NONE") 193 194 BuiltIn().set_global_variable("${master_pid}", master_pid) 195 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 196 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 197 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 198 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 199 ffdc_list_file_path) 200 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 201 ffdc_report_list_path) 202 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 203 ffdc_summary_list_path) 204 205 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 206 ffdc_dir_path_style) 207 BuiltIn().set_global_variable("${FFDC_CHECK}", 208 ffdc_check) 209 210 # For each program parameter, set the corresponding AUTOBOOT_ environment 211 # variable value. Also, set an AUTOBOOT_ environment variable for every 212 # element in additional_values. 213 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 214 "status_dir_path", "base_tool_dir_path", 215 "ffdc_list_file_path", "ffdc_report_list_path", 216 "ffdc_summary_list_path"] 217 218 plug_in_vars = parm_list + additional_values 219 220 for var_name in plug_in_vars: 221 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 222 var_name = var_name.upper() 223 if var_value is None: 224 var_value = "" 225 os.environ["AUTOBOOT_" + var_name] = str(var_value) 226 227 BuiltIn().set_log_level(LOG_LEVEL) 228 229 # Make sure the ffdc list directory exists. 230 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 231 if not os.path.exists(ffdc_list_dir_path): 232 os.makedirs(ffdc_list_dir_path) 233 234 235def plug_in_setup(): 236 r""" 237 Initialize all changing plug-in environment variables for use by the 238 plug-in programs. 239 """ 240 241 global LOG_LEVEL 242 global test_really_running 243 244 BuiltIn().set_log_level("NONE") 245 246 boot_pass, boot_fail = boot_results.return_total_pass_fail() 247 if boot_pass > 1: 248 test_really_running = 1 249 else: 250 test_really_running = 0 251 252 BuiltIn().set_global_variable("${test_really_running}", 253 test_really_running) 254 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 255 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 256 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 257 BuiltIn().set_global_variable("${boot_success}", boot_success) 258 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 259 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 260 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 261 262 # For each program parameter, set the corresponding AUTOBOOT_ environment 263 # variable value. Also, set an AUTOBOOT_ environment variable for every 264 # element in additional_values. 265 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 266 "boot_fail", "test_really_running", "ffdc_prefix", 267 "boot_start_time", "boot_end_time"] 268 269 plug_in_vars = additional_values 270 271 for var_name in plug_in_vars: 272 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 273 var_name = var_name.upper() 274 if var_value is None: 275 var_value = "" 276 os.environ["AUTOBOOT_" + var_name] = str(var_value) 277 278 if debug: 279 shell_rc, out_buf = \ 280 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 281 282 BuiltIn().set_log_level(LOG_LEVEL) 283 284 285def pre_boot_plug_in_setup(): 286 287 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 288 try: 289 os.remove(ffdc_list_file_path) 290 except OSError: 291 pass 292 293 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 294 try: 295 os.remove(ffdc_report_list_path) 296 except OSError: 297 pass 298 299 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 300 try: 301 os.remove(ffdc_summary_list_path) 302 except OSError: 303 pass 304 305 global ffdc_prefix 306 307 seconds = time.time() 308 loc_time = time.localtime(seconds) 309 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 310 311 ffdc_prefix = openbmc_nickname + "." + time_string 312 313 314def setup(): 315 r""" 316 Do general program setup tasks. 317 """ 318 319 global cp_setup_called 320 global transitional_boot_selected 321 322 gp.qprintn() 323 324 transitional_boot_selected = False 325 326 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 327 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 328 # If we can't find process_plug_in_packages.py, ssh_pw or 329 # validate_plug_ins.py, then we don't have our repo bin in PATH. 330 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 331 + " ssh_pw validate_plug_ins.py", quiet=1, 332 print_output=0, show_err=0) 333 if shell_rc != 0: 334 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 335 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 336 if robot_pgm_dir_path not in sys.path: 337 sys.path.append(robot_pgm_dir_path) 338 PYTHONPATH = os.environ.get("PYTHONPATH", "") 339 if PYTHONPATH == "": 340 os.environ['PYTHONPATH'] = robot_pgm_dir_path 341 else: 342 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 343 344 validate_parms() 345 346 grp.rqprint_pgm_header() 347 348 grk.run_key("Set BMC Power Policy ALWAYS_POWER_OFF") 349 350 initial_plug_in_setup() 351 352 plug_in_setup() 353 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 354 call_point='setup') 355 if rc != 0: 356 error_message = "Plug-in setup failed.\n" 357 grp.rprint_error_report(error_message) 358 BuiltIn().fail(error_message) 359 # Setting cp_setup_called lets our Teardown know that it needs to call 360 # the cleanup plug-in call point. 361 cp_setup_called = 1 362 363 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 364 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 365 # FFDC_LOG_PATH is used by "FFDC" keyword. 366 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 367 368 # Also printed by FFDC. 369 global host_name 370 global host_ip 371 host = socket.gethostname() 372 host_name, host_ip = gm.get_host_name_ip(host) 373 374 gp.dprint_var(boot_table, 1) 375 gp.dprint_var(boot_lists) 376 377 378def validate_parms(): 379 r""" 380 Validate all program parameters. 381 """ 382 383 process_pgm_parms() 384 385 gp.qprintn() 386 387 global openbmc_model 388 grv.rvalid_value("openbmc_host") 389 grv.rvalid_value("openbmc_username") 390 grv.rvalid_value("openbmc_password") 391 if os_host != "": 392 grv.rvalid_value("os_username") 393 grv.rvalid_value("os_password") 394 395 if pdu_host != "": 396 grv.rvalid_value("pdu_username") 397 grv.rvalid_value("pdu_password") 398 grv.rvalid_integer("pdu_slot_no") 399 if openbmc_serial_host != "": 400 grv.rvalid_integer("openbmc_serial_port") 401 if openbmc_model == "": 402 status, ret_values =\ 403 grk.run_key_u("Get BMC System Model") 404 openbmc_model = ret_values 405 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 406 grv.rvalid_value("openbmc_model") 407 grv.rvalid_integer("max_num_tests") 408 grv.rvalid_integer("boot_pass") 409 grv.rvalid_integer("boot_fail") 410 411 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 412 BuiltIn().set_global_variable("${plug_in_packages_list}", 413 plug_in_packages_list) 414 415 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) 416 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 417 error_message = "You must provide either a value for either the" +\ 418 " boot_list or the boot_stack parm.\n" 419 BuiltIn().fail(gp.sprint_error(error_message)) 420 421 valid_boot_list(boot_list, valid_boot_types) 422 valid_boot_list(boot_stack, valid_boot_types) 423 424 selected_PDU_boots = list(set(boot_list + boot_stack) 425 & set(boot_lists['PDU_reboot'])) 426 427 if len(selected_PDU_boots) > 0 and pdu_host == "": 428 error_message = "You have selected the following boots which" +\ 429 " require a PDU host but no value for pdu_host:\n" 430 error_message += gp.sprint_var(selected_PDU_boots) 431 error_message += gp.sprint_var(pdu_host, 2) 432 BuiltIn().fail(gp.sprint_error(error_message)) 433 434 return 435 436 437def my_get_state(): 438 r""" 439 Get the system state plus a little bit of wrapping. 440 """ 441 442 global state 443 444 req_states = ['epoch_seconds'] + st.default_req_states 445 446 gp.qprint_timen("Getting system state.") 447 if test_mode: 448 state['epoch_seconds'] = int(time.time()) 449 else: 450 state = st.get_state(req_states=req_states, quiet=quiet) 451 gp.qprint_var(state) 452 453 454def valid_state(): 455 r""" 456 Verify that our state dictionary contains no blank values. If we don't get 457 valid state data, we cannot continue to work. 458 """ 459 460 if st.compare_states(state, st.invalid_state_match, 'or'): 461 error_message = "The state dictionary contains blank fields which" +\ 462 " is illegal.\n" + gp.sprint_var(state) 463 BuiltIn().fail(gp.sprint_error(error_message)) 464 465 466def select_boot(): 467 r""" 468 Select a boot test to be run based on our current state and return the 469 chosen boot type. 470 471 Description of arguments: 472 state The state of the machine. 473 """ 474 475 global transitional_boot_selected 476 global boot_stack 477 478 gp.qprint_timen("Selecting a boot test.") 479 480 if transitional_boot_selected and not boot_success: 481 prior_boot = next_boot 482 boot_candidate = boot_stack.pop() 483 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 484 + " transition to a valid state for '" + boot_candidate 485 + "' which was at the top of the boot_stack. Since" 486 + " the '" + next_boot + "' failed, the '" 487 + boot_candidate + "' has been removed from the stack" 488 + " to avoid and endless failure loop.") 489 if len(boot_stack) == 0: 490 return "" 491 492 my_get_state() 493 valid_state() 494 495 transitional_boot_selected = False 496 stack_popped = 0 497 if len(boot_stack) > 0: 498 stack_popped = 1 499 gp.qprint_dashes() 500 gp.qprint_var(boot_stack) 501 gp.qprint_dashes() 502 skip_boot_printed = 0 503 while len(boot_stack) > 0: 504 boot_candidate = boot_stack.pop() 505 if stack_mode == 'normal': 506 break 507 else: 508 if st.compare_states(state, boot_table[boot_candidate]['end']): 509 if not skip_boot_printed: 510 gp.qprint_var(stack_mode) 511 gp.qprintn() 512 gp.qprint_timen("Skipping the following boot tests" 513 + " which are unnecessary since their" 514 + " required end states match the" 515 + " current machine state:") 516 skip_boot_printed = 1 517 gp.qprint_var(boot_candidate) 518 boot_candidate = "" 519 if boot_candidate == "": 520 gp.qprint_dashes() 521 gp.qprint_var(boot_stack) 522 gp.qprint_dashes() 523 return boot_candidate 524 if st.compare_states(state, boot_table[boot_candidate]['start']): 525 gp.qprint_timen("The machine state is valid for a '" 526 + boot_candidate + "' boot test.") 527 gp.qprint_dashes() 528 gp.qprint_var(boot_stack) 529 gp.qprint_dashes() 530 return boot_candidate 531 else: 532 gp.qprint_timen("The machine state does not match the required" 533 + " starting state for a '" + boot_candidate 534 + "' boot test:") 535 gp.qprint_varx("boot_table[" + boot_candidate + "][start]", 536 boot_table[boot_candidate]['start'], 1) 537 boot_stack.append(boot_candidate) 538 transitional_boot_selected = True 539 popped_boot = boot_candidate 540 541 # Loop through your list selecting a boot_candidates 542 boot_candidates = [] 543 for boot_candidate in boot_list: 544 if st.compare_states(state, boot_table[boot_candidate]['start']): 545 if stack_popped: 546 if st.compare_states(boot_table[boot_candidate]['end'], 547 boot_table[popped_boot]['start']): 548 boot_candidates.append(boot_candidate) 549 else: 550 boot_candidates.append(boot_candidate) 551 552 if len(boot_candidates) == 0: 553 gp.qprint_timen("The user's boot list contained no boot tests" 554 + " which are valid for the current machine state.") 555 boot_candidate = default_power_on 556 if not st.compare_states(state, boot_table[default_power_on]['start']): 557 boot_candidate = default_power_off 558 boot_candidates.append(boot_candidate) 559 gp.qprint_timen("Using default '" + boot_candidate 560 + "' boot type to transition to valid state.") 561 562 gp.dprint_var(boot_candidates) 563 564 # Randomly select a boot from the candidate list. 565 boot = random.choice(boot_candidates) 566 567 return boot 568 569 570def print_last_boots(): 571 r""" 572 Print the last ten boots done with their time stamps. 573 """ 574 575 # indent 0, 90 chars wide, linefeed, char is "=" 576 gp.qprint_dashes(0, 90) 577 gp.qprintn("Last 10 boots:\n") 578 579 for boot_entry in last_ten: 580 grp.rqprint(boot_entry) 581 gp.qprint_dashes(0, 90) 582 583 584def print_defect_report(ffdc_file_list): 585 r""" 586 Print a defect report. 587 588 Description of argument(s): 589 ffdc_file_list A list of files which were collected by our ffdc functions. 590 """ 591 592 # Making deliberate choice to NOT run plug_in_setup(). We don't want 593 # ffdc_prefix updated. 594 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 595 call_point='ffdc_report', stop_on_plug_in_failure=0) 596 597 # Get additional header data which may have been created by ffdc plug-ins. 598 # Also, delete the individual header files to cleanup. 599 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 600 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 601 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 602 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 603 show_err=0) 604 605 # Get additional summary data which may have been created by ffdc plug-ins. 606 # Also, delete the individual header files to cleanup. 607 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 608 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 609 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 610 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 611 show_err=0) 612 613 # ffdc_list_file_path contains a list of any ffdc files created by plug- 614 # ins, etc. Read that data into a list. 615 try: 616 plug_in_ffdc_list = \ 617 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 618 plug_in_ffdc_list = filter(None, plug_in_ffdc_list) 619 except IOError: 620 plug_in_ffdc_list = [] 621 622 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 623 # in. Eliminate duplicates and sort the list. 624 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 625 626 if status_file_path != "": 627 ffdc_file_list.insert(0, status_file_path) 628 629 # Convert the list to a printable list. 630 printable_ffdc_file_list = "\n".join(ffdc_file_list) 631 632 # Open ffdc_file_list for writing. We will write a complete list of 633 # FFDC files to it for possible use by plug-ins like cp_stop_check. 634 ffdc_list_file = open(ffdc_list_file_path, 'w') 635 ffdc_list_file.write(printable_ffdc_file_list + "\n") 636 ffdc_list_file.close() 637 638 indent = 0 639 width = 90 640 linefeed = 1 641 char = "=" 642 643 gp.qprintn() 644 gp.qprint_dashes(indent, width, linefeed, char) 645 gp.qprintn("Copy this data to the defect:\n") 646 647 if len(more_header_info) > 0: 648 gp.qprintn(more_header_info) 649 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 650 openbmc_host_name, openbmc_ip, openbmc_username, 651 openbmc_password, os_host, os_host_name, os_ip, os_username, 652 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 653 pdu_password, pdu_slot_no, openbmc_serial_host, 654 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 655 656 gp.qprintn() 657 print_last_boots() 658 gp.qprintn() 659 gp.qprint_var(state) 660 gp.qprintn() 661 gp.qprintn("FFDC data files:") 662 gp.qprintn(printable_ffdc_file_list) 663 gp.qprintn() 664 665 if len(ffdc_summary_info) > 0: 666 gp.qprintn(ffdc_summary_info) 667 668 gp.qprint_dashes(indent, width, linefeed, char) 669 670 671def my_ffdc(): 672 r""" 673 Collect FFDC data. 674 """ 675 676 global state 677 678 plug_in_setup() 679 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 680 call_point='ffdc', stop_on_plug_in_failure=0) 681 682 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 683 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 684 + AUTOBOOT_FFDC_PREFIX 685 + " ffdc_function_list=" 686 + ffdc_function_list, ignore=1) 687 if status != 'PASS': 688 gp.qprint_error("Call to ffdc failed.\n") 689 690 my_get_state() 691 692 print_defect_report(ffdc_file_list) 693 694 695def print_test_start_message(boot_keyword): 696 r""" 697 Print a message indicating what boot test is about to run. 698 699 Description of arguments: 700 boot_keyword The name of the boot which is to be run 701 (e.g. "BMC Power On"). 702 """ 703 704 global last_ten 705 global boot_start_time 706 707 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 708 709 # Set boot_start_time for use by plug-ins. 710 boot_start_time = doing_msg[1:33] 711 gp.qprint_var(boot_start_time) 712 713 gp.qprint(doing_msg) 714 715 last_ten.append(doing_msg) 716 717 if len(last_ten) > 10: 718 del last_ten[0] 719 720 721def run_boot(boot): 722 r""" 723 Run the specified boot. 724 725 Description of arguments: 726 boot The name of the boot test to be performed. 727 """ 728 729 global state 730 731 print_test_start_message(boot) 732 733 plug_in_setup() 734 rc, shell_rc, failed_plug_in_name = \ 735 grpi.rprocess_plug_in_packages(call_point="pre_boot") 736 if rc != 0: 737 error_message = "Plug-in failed with non-zero return code.\n" +\ 738 gp.sprint_var(rc, 1) 739 BuiltIn().fail(gp.sprint_error(error_message)) 740 741 if test_mode: 742 # In test mode, we'll pretend the boot worked by assigning its 743 # required end state to the default state value. 744 state = st.strip_anchor_state(boot_table[boot]['end']) 745 else: 746 # Assertion: We trust that the state data was made fresh by the 747 # caller. 748 749 gp.qprintn() 750 751 if boot_table[boot]['method_type'] == "keyword": 752 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 753 boot_table[boot]['method'], 754 quiet=quiet) 755 756 if boot_table[boot]['bmc_reboot']: 757 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 758 plug_in_setup() 759 rc, shell_rc, failed_plug_in_name = \ 760 grpi.rprocess_plug_in_packages(call_point="post_reboot") 761 if rc != 0: 762 error_message = "Plug-in failed with non-zero return code.\n" 763 error_message += gp.sprint_var(rc, 1) 764 BuiltIn().fail(gp.sprint_error(error_message)) 765 else: 766 match_state = st.anchor_state(state) 767 del match_state['epoch_seconds'] 768 # Wait for the state to change in any way. 769 st.wait_state(match_state, wait_time=state_change_timeout, 770 interval="10 seconds", invert=1) 771 772 gp.qprintn() 773 if boot_table[boot]['end']['chassis'] == "Off": 774 boot_timeout = power_off_timeout 775 else: 776 boot_timeout = power_on_timeout 777 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 778 interval="10 seconds") 779 780 plug_in_setup() 781 rc, shell_rc, failed_plug_in_name = \ 782 grpi.rprocess_plug_in_packages(call_point="post_boot") 783 if rc != 0: 784 error_message = "Plug-in failed with non-zero return code.\n" +\ 785 gp.sprint_var(rc, 1) 786 BuiltIn().fail(gp.sprint_error(error_message)) 787 788 789def test_loop_body(): 790 r""" 791 The main loop body for the loop in main_py. 792 793 Description of arguments: 794 boot_count The iteration number (starts at 1). 795 """ 796 797 global boot_count 798 global state 799 global next_boot 800 global boot_success 801 global boot_end_time 802 803 gp.qprintn() 804 805 next_boot = select_boot() 806 if next_boot == "": 807 return True 808 809 boot_count += 1 810 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 811 812 pre_boot_plug_in_setup() 813 814 cmd_buf = ["run_boot", next_boot] 815 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 816 if boot_status == "FAIL": 817 gp.qprint(msg) 818 819 gp.qprintn() 820 if boot_status == "PASS": 821 boot_success = 1 822 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 823 + "\" succeeded.") 824 else: 825 boot_success = 0 826 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 827 + "\" failed.") 828 829 # Set boot_end_time for use by plug-ins. 830 boot_end_time = completion_msg[1:33] 831 gp.qprint_var(boot_end_time) 832 833 gp.qprint(completion_msg) 834 835 boot_results.update(next_boot, boot_status) 836 837 plug_in_setup() 838 # NOTE: A post_test_case call point failure is NOT counted as a boot 839 # failure. 840 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 841 call_point='post_test_case', stop_on_plug_in_failure=0) 842 843 plug_in_setup() 844 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 845 call_point='ffdc_check', shell_rc=0x00000200, 846 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 847 if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200: 848 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 849 if status != 'PASS': 850 gp.qprint_error("Call to my_ffdc failed.\n") 851 852 if delete_errlogs: 853 # We need to purge error logs between boots or they build up. 854 grk.run_key("Delete Error logs", ignore=1) 855 856 boot_results.print_report() 857 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 858 859 plug_in_setup() 860 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 861 call_point='stop_check', shell_rc=0x00000200, stop_on_non_zero_rc=1) 862 if shell_rc == 0x00000200: 863 message = "Stopping as requested by user.\n" 864 gp.print_time(message) 865 BuiltIn().fail(message) 866 867 # This should help prevent ConnectionErrors. 868 grk.run_key_u("Close All Connections") 869 870 return True 871 872 873def obmc_boot_test_teardown(): 874 r""" 875 Clean up after the Main keyword. 876 """ 877 878 if cp_setup_called: 879 plug_in_setup() 880 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 881 call_point='cleanup', stop_on_plug_in_failure=0) 882 883 if 'boot_results_file_path' in globals(): 884 # Save boot_results object to a file in case it is needed again. 885 gp.qprint_timen("Saving boot_results to the following path.") 886 gp.qprint_var(boot_results_file_path) 887 pickle.dump(boot_results, open(boot_results_file_path, 'wb'), 888 pickle.HIGHEST_PROTOCOL) 889 890 global save_stack 891 # Restore any global values saved on the save_stack. 892 for parm_name in main_func_parm_list: 893 # Get the parm_value if it was saved on the stack. 894 try: 895 parm_value = save_stack.pop(parm_name) 896 except BaseException: 897 # If it was not saved, no further action is required. 898 continue 899 900 # Restore the saved value. 901 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 902 "}\", parm_value)" 903 gp.dpissuing(cmd_buf) 904 exec(cmd_buf) 905 906 gp.dprintn(save_stack.sprint_obj()) 907 908 909def test_teardown(): 910 r""" 911 Clean up after this test case. 912 """ 913 914 gp.qprintn() 915 cmd_buf = ["Print Error", 916 "A keyword timeout occurred ending this program.\n"] 917 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 918 919 grp.rqprint_pgm_footer() 920 921 922def obmc_boot_test_py(loc_boot_stack=None, 923 loc_stack_mode=None, 924 loc_quiet=None): 925 r""" 926 Do main program processing. 927 """ 928 929 global save_stack 930 931 # Process function parms. 932 for parm_name in main_func_parm_list: 933 # Get parm's value. 934 cmd_buf = "parm_value = loc_" + parm_name 935 exec(cmd_buf) 936 gp.dpvar(parm_name) 937 gp.dpvar(parm_value) 938 939 if parm_value is None: 940 # Parm was not specified by the calling function so set it to its 941 # corresponding global value. 942 cmd_buf = "loc_" + parm_name + " = BuiltIn().get_variable_value" +\ 943 "(\"${" + parm_name + "}\")" 944 gp.dpissuing(cmd_buf) 945 exec(cmd_buf) 946 else: 947 # Save the global value on a stack. 948 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 949 parm_name + "}\"), \"" + parm_name + "\")" 950 gp.dpissuing(cmd_buf) 951 exec(cmd_buf) 952 953 # Set the global value to the passed value. 954 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 955 "}\", loc_" + parm_name + ")" 956 gp.dpissuing(cmd_buf) 957 exec(cmd_buf) 958 959 gp.dprintn(save_stack.sprint_obj()) 960 961 setup() 962 963 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 964 965 if ffdc_only: 966 gp.qprint_timen("Caller requested ffdc_only.") 967 pre_boot_plug_in_setup() 968 grk.run_key_u("my_ffdc") 969 return 970 971 # Process caller's boot_stack. 972 while (len(boot_stack) > 0): 973 test_loop_body() 974 975 gp.qprint_timen("Finished processing stack.") 976 977 # Process caller's boot_list. 978 if len(boot_list) > 0: 979 for ix in range(1, max_num_tests + 1): 980 test_loop_body() 981 982 gp.qprint_timen("Completed all requested boot tests.") 983 984 boot_pass, boot_fail = boot_results.return_total_pass_fail() 985 new_fail = boot_fail - init_boot_fail 986 if new_fail > boot_fail_threshold: 987 error_message = "Boot failures exceed the boot failure" +\ 988 " threshold:\n" +\ 989 gp.sprint_var(new_fail) +\ 990 gp.sprint_var(boot_fail_threshold) 991 BuiltIn().fail(gp.sprint_error(error_message)) 992