1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import cPickle as pickle 14import socket 15 16from robot.utils import DotDict 17from robot.libraries.BuiltIn import BuiltIn 18 19from boot_data import * 20import gen_print as gp 21import gen_robot_print as grp 22import gen_robot_plug_in as grpi 23import gen_robot_valid as grv 24import gen_misc as gm 25import gen_cmd as gc 26import gen_robot_keyword as grk 27import state as st 28import var_stack as vs 29 30base_path = os.path.dirname(os.path.dirname( 31 imp.find_module("gen_robot_print")[1])) +\ 32 os.sep 33sys.path.append(base_path + "extended/") 34import run_keyword as rk 35 36# Setting master_pid correctly influences the behavior of plug-ins like 37# DB_Logging 38program_pid = os.getpid() 39master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 40pgm_name = re.sub('\.py$', '', os.path.basename(__file__)) 41 42# Set up boot data structures. 43boot_table = create_boot_table() 44valid_boot_types = create_valid_boot_list(boot_table) 45 46boot_lists = read_boot_lists() 47last_ten = [] 48 49state = st.return_default_state() 50cp_setup_called = 0 51next_boot = "" 52base_tool_dir_path = os.path.normpath(os.environ.get( 53 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 54 55ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 56boot_success = 0 57status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 58if status_dir_path != "": 59 status_dir_path = os.path.normpath(status_dir_path) + os.sep 60default_power_on = "REST Power On" 61default_power_off = "REST Power Off" 62boot_count = 0 63 64LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 65ffdc_prefix = "" 66boot_start_time = "" 67boot_end_time = "" 68save_stack = vs.var_stack('save_stack') 69main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 70 71 72def process_host(host, 73 host_var_name=""): 74 75 r""" 76 Process a host by getting the associated host name and IP address and 77 setting them in global variables. 78 79 If the caller does not pass the host_var_name, this function will try to 80 figure out the name of the variable used by the caller for the host parm. 81 Callers are advised to explicitly specify the host_var_name when calling 82 with an exec command. In such cases, the get_arg_name cannot figure out 83 the host variable name. 84 85 This function will then create similar global variable names by 86 removing "_host" and appending "_host_name" or "_ip" to the host variable 87 name. 88 89 Example: 90 91 If a call is made like this: 92 process_host(openbmc_host) 93 94 Global variables openbmc_host_name and openbmc_ip will be set. 95 96 Description of argument(s): 97 host A host name or IP. The name of the variable used should 98 have a suffix of "_host". 99 host_var_name The name of the variable being used as the host parm. 100 """ 101 102 if host_var_name == "": 103 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 104 105 host_name_var_name = re.sub("host", "host_name", host_var_name) 106 ip_var_name = re.sub("host", "ip", host_var_name) 107 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 108 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 109 host + "')" 110 exec(cmd_buf) 111 112 113def process_pgm_parms(): 114 115 r""" 116 Process the program parameters by assigning them all to corresponding 117 globals. Also, set some global values that depend on program parameters. 118 """ 119 120 # Program parameter processing. 121 # Assign all program parms to python variables which are global to this 122 # module. 123 124 global parm_list 125 parm_list = BuiltIn().get_variable_value("${parm_list}") 126 # The following subset of parms should be processed as integers. 127 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 128 'boot_fail_threshold', 'delete_errlogs', 'quiet', 'test_mode', 129 'debug'] 130 for parm in parm_list: 131 if parm in int_list: 132 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 133 "}\", \"0\"))" 134 else: 135 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 136 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 137 gp.dpissuing(cmd_buf) 138 exec(cmd_buf) 139 if re.match(r".*_host$", parm): 140 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 141 exec(cmd_buf) 142 if re.match(r".*_password$", parm): 143 # Register the value of any parm whose name ends in _password. 144 # This will cause the print functions to replace passwords with 145 # asterisks in the output. 146 cmd_buf = "gp.register_passwords(" + parm + ")" 147 exec(cmd_buf) 148 149 global ffdc_dir_path_style 150 global boot_list 151 global boot_stack 152 global boot_results_file_path 153 global boot_results 154 global ffdc_list_file_path 155 global ffdc_report_list_path 156 global ffdc_summary_list_path 157 158 if ffdc_dir_path_style == "": 159 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 160 161 # Convert these program parms to lists for easier processing.. 162 boot_list = filter(None, boot_list.split(":")) 163 boot_stack = filter(None, boot_stack.split(":")) 164 165 cleanup_boot_results_file() 166 boot_results_file_path = create_boot_results_file_path(pgm_name, 167 openbmc_nickname, 168 master_pid) 169 170 if os.path.isfile(boot_results_file_path): 171 # We've been called before in this run so we'll load the saved 172 # boot_results object. 173 boot_results = pickle.load(open(boot_results_file_path, 'rb')) 174 else: 175 boot_results = boot_results(boot_table, boot_pass, boot_fail) 176 177 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 178 "/FFDC_FILE_LIST" 179 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 180 "/FFDC_REPORT_FILE_LIST" 181 182 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 183 "/FFDC_SUMMARY_FILE_LIST" 184 185 186def initial_plug_in_setup(): 187 188 r""" 189 Initialize all plug-in environment variables which do not change for the 190 duration of the program. 191 192 """ 193 194 global LOG_LEVEL 195 BuiltIn().set_log_level("NONE") 196 197 BuiltIn().set_global_variable("${master_pid}", master_pid) 198 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 199 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 200 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 201 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 202 ffdc_list_file_path) 203 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 204 ffdc_report_list_path) 205 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 206 ffdc_summary_list_path) 207 208 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 209 ffdc_dir_path_style) 210 BuiltIn().set_global_variable("${FFDC_CHECK}", 211 ffdc_check) 212 213 # For each program parameter, set the corresponding AUTOBOOT_ environment 214 # variable value. Also, set an AUTOBOOT_ environment variable for every 215 # element in additional_values. 216 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 217 "status_dir_path", "base_tool_dir_path", 218 "ffdc_list_file_path", "ffdc_report_list_path", 219 "ffdc_summary_list_path"] 220 221 plug_in_vars = parm_list + additional_values 222 223 for var_name in plug_in_vars: 224 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 225 var_name = var_name.upper() 226 if var_value is None: 227 var_value = "" 228 os.environ["AUTOBOOT_" + var_name] = str(var_value) 229 230 BuiltIn().set_log_level(LOG_LEVEL) 231 232 # Make sure the ffdc list directory exists. 233 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 234 if not os.path.exists(ffdc_list_dir_path): 235 os.makedirs(ffdc_list_dir_path) 236 237 238def plug_in_setup(): 239 240 r""" 241 Initialize all changing plug-in environment variables for use by the 242 plug-in programs. 243 """ 244 245 global LOG_LEVEL 246 global test_really_running 247 248 BuiltIn().set_log_level("NONE") 249 250 boot_pass, boot_fail = boot_results.return_total_pass_fail() 251 if boot_pass > 1: 252 test_really_running = 1 253 else: 254 test_really_running = 0 255 256 BuiltIn().set_global_variable("${test_really_running}", 257 test_really_running) 258 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 259 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 260 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 261 BuiltIn().set_global_variable("${boot_success}", boot_success) 262 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 263 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 264 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 265 266 # For each program parameter, set the corresponding AUTOBOOT_ environment 267 # variable value. Also, set an AUTOBOOT_ environment variable for every 268 # element in additional_values. 269 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 270 "boot_fail", "test_really_running", "ffdc_prefix", 271 "boot_start_time", "boot_end_time"] 272 273 plug_in_vars = additional_values 274 275 for var_name in plug_in_vars: 276 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 277 var_name = var_name.upper() 278 if var_value is None: 279 var_value = "" 280 os.environ["AUTOBOOT_" + var_name] = str(var_value) 281 282 if debug: 283 shell_rc, out_buf = \ 284 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 285 286 BuiltIn().set_log_level(LOG_LEVEL) 287 288 289def pre_boot_plug_in_setup(): 290 291 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 292 try: 293 os.remove(ffdc_list_file_path) 294 except OSError: 295 pass 296 297 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 298 try: 299 os.remove(ffdc_report_list_path) 300 except OSError: 301 pass 302 303 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 304 try: 305 os.remove(ffdc_summary_list_path) 306 except OSError: 307 pass 308 309 global ffdc_prefix 310 311 seconds = time.time() 312 loc_time = time.localtime(seconds) 313 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 314 315 ffdc_prefix = openbmc_nickname + "." + time_string 316 317 318def setup(): 319 320 r""" 321 Do general program setup tasks. 322 """ 323 324 global cp_setup_called 325 global transitional_boot_selected 326 327 gp.qprintn() 328 329 transitional_boot_selected = False 330 331 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 332 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 333 # If we can't find process_plug_in_packages.py, ssh_pw or 334 # validate_plug_ins.py, then we don't have our repo bin in PATH. 335 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" + 336 " ssh_pw validate_plug_ins.py", quiet=1, 337 print_output=0, show_err=0) 338 if shell_rc != 0: 339 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 340 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 341 if robot_pgm_dir_path not in sys.path: 342 sys.path.append(robot_pgm_dir_path) 343 PYTHONPATH = os.environ.get("PYTHONPATH", "") 344 if PYTHONPATH == "": 345 os.environ['PYTHONPATH'] = robot_pgm_dir_path 346 else: 347 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 348 349 validate_parms() 350 351 grp.rqprint_pgm_header() 352 353 grk.run_key("Set BMC Power Policy RESTORE_LAST_STATE") 354 355 initial_plug_in_setup() 356 357 plug_in_setup() 358 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 359 call_point='setup') 360 if rc != 0: 361 error_message = "Plug-in setup failed.\n" 362 grp.rprint_error_report(error_message) 363 BuiltIn().fail(error_message) 364 # Setting cp_setup_called lets our Teardown know that it needs to call 365 # the cleanup plug-in call point. 366 cp_setup_called = 1 367 368 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 369 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 370 # FFDC_LOG_PATH is used by "FFDC" keyword. 371 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 372 373 # Also printed by FFDC. 374 global host_name 375 global host_ip 376 host = socket.gethostname() 377 host_name, host_ip = gm.get_host_name_ip(host) 378 379 gp.dprint_var(boot_table, 1) 380 gp.dprint_var(boot_lists) 381 382 383def validate_parms(): 384 385 r""" 386 Validate all program parameters. 387 """ 388 389 process_pgm_parms() 390 391 gp.qprintn() 392 393 global openbmc_model 394 grv.rvalid_value("openbmc_host") 395 grv.rvalid_value("openbmc_username") 396 grv.rvalid_value("openbmc_password") 397 if os_host != "": 398 grv.rvalid_value("os_username") 399 grv.rvalid_value("os_password") 400 401 if pdu_host != "": 402 grv.rvalid_value("pdu_username") 403 grv.rvalid_value("pdu_password") 404 grv.rvalid_integer("pdu_slot_no") 405 if openbmc_serial_host != "": 406 grv.rvalid_integer("openbmc_serial_port") 407 if openbmc_model == "": 408 status, ret_values =\ 409 grk.run_key_u("Get BMC System Model") 410 openbmc_model = ret_values 411 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 412 grv.rvalid_value("openbmc_model") 413 grv.rvalid_integer("max_num_tests") 414 grv.rvalid_integer("boot_pass") 415 grv.rvalid_integer("boot_fail") 416 417 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 418 BuiltIn().set_global_variable("${plug_in_packages_list}", 419 plug_in_packages_list) 420 421 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) 422 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 423 error_message = "You must provide either a value for either the" +\ 424 " boot_list or the boot_stack parm.\n" 425 BuiltIn().fail(gp.sprint_error(error_message)) 426 427 valid_boot_list(boot_list, valid_boot_types) 428 valid_boot_list(boot_stack, valid_boot_types) 429 430 selected_PDU_boots = list(set(boot_list + boot_stack) & 431 set(boot_lists['PDU_reboot'])) 432 433 if len(selected_PDU_boots) > 0 and pdu_host == "": 434 error_message = "You have selected the following boots which" +\ 435 " require a PDU host but no value for pdu_host:\n" 436 error_message += gp.sprint_var(selected_PDU_boots) 437 error_message += gp.sprint_var(pdu_host, 2) 438 BuiltIn().fail(gp.sprint_error(error_message)) 439 440 return 441 442 443def my_get_state(): 444 445 r""" 446 Get the system state plus a little bit of wrapping. 447 """ 448 449 global state 450 451 req_states = ['epoch_seconds'] + st.default_req_states 452 453 gp.qprint_timen("Getting system state.") 454 if test_mode: 455 state['epoch_seconds'] = int(time.time()) 456 else: 457 state = st.get_state(req_states=req_states, quiet=quiet) 458 gp.qprint_var(state) 459 460 461def valid_state(): 462 463 r""" 464 Verify that our state dictionary contains no blank values. If we don't get 465 valid state data, we cannot continue to work. 466 """ 467 468 if st.compare_states(state, st.invalid_state_match, 'or'): 469 error_message = "The state dictionary contains blank fields which" +\ 470 " is illegal.\n" + gp.sprint_var(state) 471 BuiltIn().fail(gp.sprint_error(error_message)) 472 473 474def select_boot(): 475 476 r""" 477 Select a boot test to be run based on our current state and return the 478 chosen boot type. 479 480 Description of arguments: 481 state The state of the machine. 482 """ 483 484 global transitional_boot_selected 485 global boot_stack 486 487 gp.qprint_timen("Selecting a boot test.") 488 489 if transitional_boot_selected and not boot_success: 490 prior_boot = next_boot 491 boot_candidate = boot_stack.pop() 492 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" + 493 " transition to a valid state for '" + boot_candidate + 494 "' which was at the top of the boot_stack. Since" + 495 " the '" + next_boot + "' failed, the '" + 496 boot_candidate + "' has been removed from the stack" + 497 " to avoid and endless failure loop.") 498 if len(boot_stack) == 0: 499 return "" 500 501 my_get_state() 502 valid_state() 503 504 transitional_boot_selected = False 505 stack_popped = 0 506 if len(boot_stack) > 0: 507 stack_popped = 1 508 gp.qprint_dashes() 509 gp.qprint_var(boot_stack) 510 gp.qprint_dashes() 511 skip_boot_printed = 0 512 while len(boot_stack) > 0: 513 boot_candidate = boot_stack.pop() 514 if stack_mode == 'normal': 515 break 516 else: 517 if st.compare_states(state, boot_table[boot_candidate]['end']): 518 if not skip_boot_printed: 519 gp.qprint_var(stack_mode) 520 gp.qprintn() 521 gp.qprint_timen("Skipping the following boot tests" + 522 " which are unnecessary since their" + 523 " required end states match the" + 524 " current machine state:") 525 skip_boot_printed = 1 526 gp.qprint_var(boot_candidate) 527 boot_candidate = "" 528 if boot_candidate == "": 529 gp.qprint_dashes() 530 gp.qprint_var(boot_stack) 531 gp.qprint_dashes() 532 return boot_candidate 533 if st.compare_states(state, boot_table[boot_candidate]['start']): 534 gp.qprint_timen("The machine state is valid for a '" + 535 boot_candidate + "' boot test.") 536 gp.qprint_dashes() 537 gp.qprint_var(boot_stack) 538 gp.qprint_dashes() 539 return boot_candidate 540 else: 541 gp.qprint_timen("The machine state does not match the required" + 542 " starting state for a '" + boot_candidate + 543 "' boot test:") 544 gp.qprint_varx("boot_table[" + boot_candidate + "][start]", 545 boot_table[boot_candidate]['start'], 1) 546 boot_stack.append(boot_candidate) 547 transitional_boot_selected = True 548 popped_boot = boot_candidate 549 550 # Loop through your list selecting a boot_candidates 551 boot_candidates = [] 552 for boot_candidate in boot_list: 553 if st.compare_states(state, boot_table[boot_candidate]['start']): 554 if stack_popped: 555 if st.compare_states(boot_table[boot_candidate]['end'], 556 boot_table[popped_boot]['start']): 557 boot_candidates.append(boot_candidate) 558 else: 559 boot_candidates.append(boot_candidate) 560 561 if len(boot_candidates) == 0: 562 gp.qprint_timen("The user's boot list contained no boot tests" + 563 " which are valid for the current machine state.") 564 boot_candidate = default_power_on 565 if not st.compare_states(state, boot_table[default_power_on]['start']): 566 boot_candidate = default_power_off 567 boot_candidates.append(boot_candidate) 568 gp.qprint_timen("Using default '" + boot_candidate + 569 "' boot type to transition to valid state.") 570 571 gp.dprint_var(boot_candidates) 572 573 # Randomly select a boot from the candidate list. 574 boot = random.choice(boot_candidates) 575 576 return boot 577 578 579def print_last_boots(): 580 581 r""" 582 Print the last ten boots done with their time stamps. 583 """ 584 585 # indent 0, 90 chars wide, linefeed, char is "=" 586 gp.qprint_dashes(0, 90) 587 gp.qprintn("Last 10 boots:\n") 588 589 for boot_entry in last_ten: 590 grp.rqprint(boot_entry) 591 gp.qprint_dashes(0, 90) 592 593 594def print_defect_report(ffdc_file_list): 595 596 r""" 597 Print a defect report. 598 599 Description of argument(s): 600 ffdc_file_list A list of files which were collected by our ffdc functions. 601 """ 602 603 # Making deliberate choice to NOT run plug_in_setup(). We don't want 604 # ffdc_prefix updated. 605 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 606 call_point='ffdc_report', stop_on_plug_in_failure=0) 607 608 # Get additional header data which may have been created by ffdc plug-ins. 609 # Also, delete the individual header files to cleanup. 610 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 611 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 612 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 613 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 614 show_err=0) 615 616 # Get additional summary data which may have been created by ffdc plug-ins. 617 # Also, delete the individual header files to cleanup. 618 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 619 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 620 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 621 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 622 show_err=0) 623 624 # ffdc_list_file_path contains a list of any ffdc files created by plug- 625 # ins, etc. Read that data into a list. 626 try: 627 plug_in_ffdc_list = \ 628 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 629 plug_in_ffdc_list = filter(None, plug_in_ffdc_list) 630 except IOError: 631 plug_in_ffdc_list = [] 632 633 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 634 # in. Eliminate duplicates and sort the list. 635 ffdc_file_list = list(set(ffdc_file_list + plug_in_ffdc_list)) 636 ffdc_file_list.sort() 637 638 if status_file_path != "": 639 ffdc_file_list.insert(0, status_file_path) 640 641 # Convert the list to a printable list. 642 printable_ffdc_file_list = "\n".join(ffdc_file_list) 643 644 # Open ffdc_file_list for writing. We will write a complete list of 645 # FFDC files to it for possible use by plug-ins like cp_stop_check. 646 ffdc_list_file = open(ffdc_list_file_path, 'w') 647 ffdc_list_file.write(printable_ffdc_file_list + "\n") 648 ffdc_list_file.close() 649 650 indent = 0 651 width = 90 652 linefeed = 1 653 char = "=" 654 655 gp.qprintn() 656 gp.qprint_dashes(indent, width, linefeed, char) 657 gp.qprintn("Copy this data to the defect:\n") 658 659 if len(more_header_info) > 0: 660 gp.qprintn(more_header_info) 661 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 662 openbmc_host_name, openbmc_ip, openbmc_username, 663 openbmc_password, os_host, os_host_name, os_ip, os_username, 664 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 665 pdu_password, pdu_slot_no, openbmc_serial_host, 666 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 667 668 gp.qprintn() 669 print_last_boots() 670 gp.qprintn() 671 gp.qprint_var(state) 672 gp.qprintn() 673 gp.qprintn("FFDC data files:") 674 gp.qprintn(printable_ffdc_file_list) 675 gp.qprintn() 676 677 if len(ffdc_summary_info) > 0: 678 gp.qprintn(ffdc_summary_info) 679 680 gp.qprint_dashes(indent, width, linefeed, char) 681 682 683def my_ffdc(): 684 685 r""" 686 Collect FFDC data. 687 """ 688 689 global state 690 691 plug_in_setup() 692 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 693 call_point='ffdc', stop_on_plug_in_failure=0) 694 695 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 696 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" + 697 AUTOBOOT_FFDC_PREFIX + 698 " ffdc_function_list=" + 699 ffdc_function_list, ignore=1) 700 if status != 'PASS': 701 gp.qprint_error("Call to ffdc failed.\n") 702 703 my_get_state() 704 705 print_defect_report(ffdc_file_list) 706 707 708def print_test_start_message(boot_keyword): 709 710 r""" 711 Print a message indicating what boot test is about to run. 712 713 Description of arguments: 714 boot_keyword The name of the boot which is to be run 715 (e.g. "BMC Power On"). 716 """ 717 718 global last_ten 719 global boot_start_time 720 721 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 722 723 # Set boot_start_time for use by plug-ins. 724 boot_start_time = doing_msg[1:33] 725 gp.qprint_var(boot_start_time) 726 727 gp.qprint(doing_msg) 728 729 last_ten.append(doing_msg) 730 731 if len(last_ten) > 10: 732 del last_ten[0] 733 734 735def run_boot(boot): 736 737 r""" 738 Run the specified boot. 739 740 Description of arguments: 741 boot The name of the boot test to be performed. 742 """ 743 744 global state 745 746 print_test_start_message(boot) 747 748 plug_in_setup() 749 rc, shell_rc, failed_plug_in_name = \ 750 grpi.rprocess_plug_in_packages(call_point="pre_boot") 751 if rc != 0: 752 error_message = "Plug-in failed with non-zero return code.\n" +\ 753 gp.sprint_var(rc, 1) 754 BuiltIn().fail(gp.sprint_error(error_message)) 755 756 if test_mode: 757 # In test mode, we'll pretend the boot worked by assigning its 758 # required end state to the default state value. 759 state = st.strip_anchor_state(boot_table[boot]['end']) 760 else: 761 # Assertion: We trust that the state data was made fresh by the 762 # caller. 763 764 gp.qprintn() 765 766 if boot_table[boot]['method_type'] == "keyword": 767 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 768 boot_table[boot]['method'], 769 quiet=quiet) 770 771 if boot_table[boot]['bmc_reboot']: 772 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 773 plug_in_setup() 774 rc, shell_rc, failed_plug_in_name = \ 775 grpi.rprocess_plug_in_packages(call_point="post_reboot") 776 if rc != 0: 777 error_message = "Plug-in failed with non-zero return code.\n" 778 error_message += gp.sprint_var(rc, 1) 779 BuiltIn().fail(gp.sprint_error(error_message)) 780 else: 781 match_state = st.anchor_state(state) 782 del match_state['epoch_seconds'] 783 # Wait for the state to change in any way. 784 st.wait_state(match_state, wait_time=state_change_timeout, 785 interval="10 seconds", invert=1) 786 787 gp.qprintn() 788 if boot_table[boot]['end']['chassis'] == "Off": 789 boot_timeout = power_off_timeout 790 else: 791 boot_timeout = power_on_timeout 792 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 793 interval="10 seconds") 794 795 plug_in_setup() 796 rc, shell_rc, failed_plug_in_name = \ 797 grpi.rprocess_plug_in_packages(call_point="post_boot") 798 if rc != 0: 799 error_message = "Plug-in failed with non-zero return code.\n" +\ 800 gp.sprint_var(rc, 1) 801 BuiltIn().fail(gp.sprint_error(error_message)) 802 803 804def test_loop_body(): 805 806 r""" 807 The main loop body for the loop in main_py. 808 809 Description of arguments: 810 boot_count The iteration number (starts at 1). 811 """ 812 813 global boot_count 814 global state 815 global next_boot 816 global boot_success 817 global boot_end_time 818 819 gp.qprintn() 820 821 next_boot = select_boot() 822 if next_boot == "": 823 return True 824 825 boot_count += 1 826 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 827 828 pre_boot_plug_in_setup() 829 830 cmd_buf = ["run_boot", next_boot] 831 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 832 if boot_status == "FAIL": 833 gp.qprint(msg) 834 835 gp.qprintn() 836 if boot_status == "PASS": 837 boot_success = 1 838 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot + 839 "\" succeeded.") 840 else: 841 boot_success = 0 842 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot + 843 "\" failed.") 844 845 # Set boot_end_time for use by plug-ins. 846 boot_end_time = completion_msg[1:33] 847 gp.qprint_var(boot_end_time) 848 849 gp.qprint(completion_msg) 850 851 boot_results.update(next_boot, boot_status) 852 853 plug_in_setup() 854 # NOTE: A post_test_case call point failure is NOT counted as a boot 855 # failure. 856 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 857 call_point='post_test_case', stop_on_plug_in_failure=0) 858 859 plug_in_setup() 860 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 861 call_point='ffdc_check', shell_rc=0x00000200, 862 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 863 if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200: 864 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 865 if status != 'PASS': 866 gp.qprint_error("Call to my_ffdc failed.\n") 867 868 if delete_errlogs: 869 # We need to purge error logs between boots or they build up. 870 grk.run_key("Delete Error logs", ignore=1) 871 872 boot_results.print_report() 873 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 874 875 plug_in_setup() 876 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 877 call_point='stop_check') 878 if rc != 0: 879 error_message = "Stopping as requested by user.\n" 880 grp.rprint_error_report(error_message) 881 BuiltIn().fail(error_message) 882 883 # This should help prevent ConnectionErrors. 884 grk.run_key_u("Close All Connections") 885 886 return True 887 888 889def obmc_boot_test_teardown(): 890 891 r""" 892 Clean up after the Main keyword. 893 """ 894 895 if cp_setup_called: 896 plug_in_setup() 897 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 898 call_point='cleanup', stop_on_plug_in_failure=0) 899 900 if 'boot_results_file_path' in globals(): 901 # Save boot_results object to a file in case it is needed again. 902 gp.qprint_timen("Saving boot_results to the following path.") 903 gp.qprint_var(boot_results_file_path) 904 pickle.dump(boot_results, open(boot_results_file_path, 'wb'), 905 pickle.HIGHEST_PROTOCOL) 906 907 global save_stack 908 # Restore any global values saved on the save_stack. 909 for parm_name in main_func_parm_list: 910 # Get the parm_value if it was saved on the stack. 911 try: 912 parm_value = save_stack.pop(parm_name) 913 except: 914 # If it was not saved, no further action is required. 915 continue 916 917 # Restore the saved value. 918 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 919 "}\", parm_value)" 920 gp.dpissuing(cmd_buf) 921 exec(cmd_buf) 922 923 gp.dprintn(save_stack.sprint_obj()) 924 925 926def test_teardown(): 927 928 r""" 929 Clean up after this test case. 930 """ 931 932 gp.qprintn() 933 cmd_buf = ["Print Error", 934 "A keyword timeout occurred ending this program.\n"] 935 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 936 937 grp.rqprint_pgm_footer() 938 939 940def obmc_boot_test_py(loc_boot_stack=None, 941 loc_stack_mode=None, 942 loc_quiet=None): 943 944 r""" 945 Do main program processing. 946 """ 947 948 global save_stack 949 950 # Process function parms. 951 for parm_name in main_func_parm_list: 952 # Get parm's value. 953 cmd_buf = "parm_value = loc_" + parm_name 954 exec(cmd_buf) 955 gp.dpvar(parm_name) 956 gp.dpvar(parm_value) 957 958 if parm_value is None: 959 # Parm was not specified by the calling function so set it to its 960 # corresponding global value. 961 cmd_buf = "loc_" + parm_name + " = BuiltIn().get_variable_value" +\ 962 "(\"${" + parm_name + "}\")" 963 gp.dpissuing(cmd_buf) 964 exec(cmd_buf) 965 else: 966 # Save the global value on a stack. 967 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 968 parm_name + "}\"), \"" + parm_name + "\")" 969 gp.dpissuing(cmd_buf) 970 exec(cmd_buf) 971 972 # Set the global value to the passed value. 973 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 974 "}\", loc_" + parm_name + ")" 975 gp.dpissuing(cmd_buf) 976 exec(cmd_buf) 977 978 gp.dprintn(save_stack.sprint_obj()) 979 980 setup() 981 982 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 983 984 if ffdc_only: 985 gp.qprint_timen("Caller requested ffdc_only.") 986 pre_boot_plug_in_setup() 987 grk.run_key_u("my_ffdc") 988 return 989 990 # Process caller's boot_stack. 991 while (len(boot_stack) > 0): 992 test_loop_body() 993 994 gp.qprint_timen("Finished processing stack.") 995 996 # Process caller's boot_list. 997 if len(boot_list) > 0: 998 for ix in range(1, max_num_tests + 1): 999 test_loop_body() 1000 1001 gp.qprint_timen("Completed all requested boot tests.") 1002 1003 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1004 new_fail = boot_fail - init_boot_fail 1005 if new_fail > boot_fail_threshold: 1006 error_message = "Boot failures exceed the boot failure" +\ 1007 " threshold:\n" +\ 1008 gp.sprint_var(new_fail) +\ 1009 gp.sprint_var(boot_fail_threshold) 1010 BuiltIn().fail(gp.sprint_error(error_message)) 1011 1012