1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import cPickle as pickle 14import socket 15 16from robot.utils import DotDict 17from robot.libraries.BuiltIn import BuiltIn 18 19from boot_data import * 20import gen_print as gp 21import gen_robot_print as grp 22import gen_robot_plug_in as grpi 23import gen_robot_valid as grv 24import gen_misc as gm 25import gen_cmd as gc 26import gen_robot_keyword as grk 27import state as st 28import var_stack as vs 29 30base_path = os.path.dirname(os.path.dirname( 31 imp.find_module("gen_robot_print")[1])) +\ 32 os.sep 33sys.path.append(base_path + "extended/") 34import run_keyword as rk 35 36# Setting master_pid correctly influences the behavior of plug-ins like 37# DB_Logging 38program_pid = os.getpid() 39master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 40pgm_name = re.sub('\.py$', '', os.path.basename(__file__)) 41 42# Set up boot data structures. 43boot_table = create_boot_table() 44valid_boot_types = create_valid_boot_list(boot_table) 45 46boot_lists = read_boot_lists() 47last_ten = [] 48 49state = st.return_default_state() 50cp_setup_called = 0 51next_boot = "" 52base_tool_dir_path = os.path.normpath(os.environ.get( 53 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 54 55ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 56boot_success = 0 57status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 58if status_dir_path != "": 59 status_dir_path = os.path.normpath(status_dir_path) + os.sep 60default_power_on = "REST Power On" 61default_power_off = "REST Power Off" 62boot_count = 0 63 64LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 65ffdc_prefix = "" 66boot_start_time = "" 67boot_end_time = "" 68save_stack = vs.var_stack('save_stack') 69main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 70 71 72############################################################################### 73def process_host(host, 74 host_var_name=""): 75 76 r""" 77 Process a host by getting the associated host name and IP address and 78 setting them in global variables. 79 80 If the caller does not pass the host_var_name, this function will try to 81 figure out the name of the variable used by the caller for the host parm. 82 Callers are advised to explicitly specify the host_var_name when calling 83 with an exec command. In such cases, the get_arg_name cannot figure out 84 the host variable name. 85 86 This function will then create similar global variable names by 87 removing "_host" and appending "_host_name" or "_ip" to the host variable 88 name. 89 90 Example: 91 92 If a call is made like this: 93 process_host(openbmc_host) 94 95 Global variables openbmc_host_name and openbmc_ip will be set. 96 97 Description of argument(s): 98 host A host name or IP. The name of the variable used should 99 have a suffix of "_host". 100 host_var_name The name of the variable being used as the host parm. 101 """ 102 103 if host_var_name == "": 104 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 105 106 host_name_var_name = re.sub("host", "host_name", host_var_name) 107 ip_var_name = re.sub("host", "ip", host_var_name) 108 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 109 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 110 host + "')" 111 exec(cmd_buf) 112 113############################################################################### 114 115 116############################################################################### 117def process_pgm_parms(): 118 119 r""" 120 Process the program parameters by assigning them all to corresponding 121 globals. Also, set some global values that depend on program parameters. 122 """ 123 124 # Program parameter processing. 125 # Assign all program parms to python variables which are global to this 126 # module. 127 128 global parm_list 129 parm_list = BuiltIn().get_variable_value("${parm_list}") 130 # The following subset of parms should be processed as integers. 131 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 132 'boot_fail_threshold', 'delete_errlogs', 'quiet', 'test_mode', 133 'debug'] 134 for parm in parm_list: 135 if parm in int_list: 136 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 137 "}\", \"0\"))" 138 else: 139 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 140 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 141 gp.dpissuing(cmd_buf) 142 exec(cmd_buf) 143 if re.match(r".*_host$", parm): 144 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 145 exec(cmd_buf) 146 if re.match(r".*_password$", parm): 147 # Register the value of any parm whose name ends in _password. 148 # This will cause the print functions to replace passwords with 149 # asterisks in the output. 150 cmd_buf = "gp.register_passwords(" + parm + ")" 151 exec(cmd_buf) 152 153 global ffdc_dir_path_style 154 global boot_list 155 global boot_stack 156 global boot_results_file_path 157 global boot_results 158 global ffdc_list_file_path 159 global ffdc_report_list_path 160 global ffdc_summary_list_path 161 162 if ffdc_dir_path_style == "": 163 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 164 165 # Convert these program parms to lists for easier processing.. 166 boot_list = filter(None, boot_list.split(":")) 167 boot_stack = filter(None, boot_stack.split(":")) 168 169 cleanup_boot_results_file() 170 boot_results_file_path = create_boot_results_file_path(pgm_name, 171 openbmc_nickname, 172 master_pid) 173 174 if os.path.isfile(boot_results_file_path): 175 # We've been called before in this run so we'll load the saved 176 # boot_results object. 177 boot_results = pickle.load(open(boot_results_file_path, 'rb')) 178 else: 179 boot_results = boot_results(boot_table, boot_pass, boot_fail) 180 181 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 182 "/FFDC_FILE_LIST" 183 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 184 "/FFDC_REPORT_FILE_LIST" 185 186 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 187 "/FFDC_SUMMARY_FILE_LIST" 188 189############################################################################### 190 191 192############################################################################### 193def initial_plug_in_setup(): 194 195 r""" 196 Initialize all plug-in environment variables which do not change for the 197 duration of the program. 198 199 """ 200 201 global LOG_LEVEL 202 BuiltIn().set_log_level("NONE") 203 204 BuiltIn().set_global_variable("${master_pid}", master_pid) 205 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 206 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 207 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 208 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 209 ffdc_list_file_path) 210 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 211 ffdc_report_list_path) 212 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 213 ffdc_summary_list_path) 214 215 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 216 ffdc_dir_path_style) 217 BuiltIn().set_global_variable("${FFDC_CHECK}", 218 ffdc_check) 219 220 # For each program parameter, set the corresponding AUTOBOOT_ environment 221 # variable value. Also, set an AUTOBOOT_ environment variable for every 222 # element in additional_values. 223 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 224 "status_dir_path", "base_tool_dir_path", 225 "ffdc_list_file_path", "ffdc_report_list_path", 226 "ffdc_summary_list_path"] 227 228 plug_in_vars = parm_list + additional_values 229 230 for var_name in plug_in_vars: 231 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 232 var_name = var_name.upper() 233 if var_value is None: 234 var_value = "" 235 os.environ["AUTOBOOT_" + var_name] = str(var_value) 236 237 BuiltIn().set_log_level(LOG_LEVEL) 238 239 # Make sure the ffdc list directory exists. 240 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 241 if not os.path.exists(ffdc_list_dir_path): 242 os.makedirs(ffdc_list_dir_path) 243 244############################################################################### 245 246 247############################################################################### 248def plug_in_setup(): 249 250 r""" 251 Initialize all changing plug-in environment variables for use by the 252 plug-in programs. 253 """ 254 255 global LOG_LEVEL 256 global test_really_running 257 258 BuiltIn().set_log_level("NONE") 259 260 boot_pass, boot_fail = boot_results.return_total_pass_fail() 261 if boot_pass > 1: 262 test_really_running = 1 263 else: 264 test_really_running = 0 265 266 BuiltIn().set_global_variable("${test_really_running}", 267 test_really_running) 268 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 269 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 270 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 271 BuiltIn().set_global_variable("${boot_success}", boot_success) 272 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 273 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 274 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 275 276 # For each program parameter, set the corresponding AUTOBOOT_ environment 277 # variable value. Also, set an AUTOBOOT_ environment variable for every 278 # element in additional_values. 279 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 280 "boot_fail", "test_really_running", "ffdc_prefix", 281 "boot_start_time", "boot_end_time"] 282 283 plug_in_vars = additional_values 284 285 for var_name in plug_in_vars: 286 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 287 var_name = var_name.upper() 288 if var_value is None: 289 var_value = "" 290 os.environ["AUTOBOOT_" + var_name] = str(var_value) 291 292 if debug: 293 shell_rc, out_buf = \ 294 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 295 296 BuiltIn().set_log_level(LOG_LEVEL) 297 298############################################################################### 299 300 301############################################################################### 302def pre_boot_plug_in_setup(): 303 304 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 305 try: 306 os.remove(ffdc_list_file_path) 307 except OSError: 308 pass 309 310 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 311 try: 312 os.remove(ffdc_report_list_path) 313 except OSError: 314 pass 315 316 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 317 try: 318 os.remove(ffdc_summary_list_path) 319 except OSError: 320 pass 321 322 global ffdc_prefix 323 324 seconds = time.time() 325 loc_time = time.localtime(seconds) 326 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 327 328 ffdc_prefix = openbmc_nickname + "." + time_string 329 330############################################################################### 331 332 333############################################################################### 334def setup(): 335 336 r""" 337 Do general program setup tasks. 338 """ 339 340 global cp_setup_called 341 global transitional_boot_selected 342 343 gp.qprintn() 344 345 transitional_boot_selected = False 346 347 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 348 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 349 # If we can't find process_plug_in_packages.py, ssh_pw or 350 # validate_plug_ins.py, then we don't have our repo bin in PATH. 351 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" + 352 " ssh_pw validate_plug_ins.py", quiet=1, 353 print_output=0, show_err=0) 354 if shell_rc != 0: 355 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 356 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 357 if robot_pgm_dir_path not in sys.path: 358 sys.path.append(robot_pgm_dir_path) 359 PYTHONPATH = os.environ.get("PYTHONPATH", "") 360 if PYTHONPATH == "": 361 os.environ['PYTHONPATH'] = robot_pgm_dir_path 362 else: 363 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 364 365 validate_parms() 366 367 grp.rqprint_pgm_header() 368 369 grk.run_key("Set BMC Power Policy RESTORE_LAST_STATE") 370 371 initial_plug_in_setup() 372 373 plug_in_setup() 374 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 375 call_point='setup') 376 if rc != 0: 377 error_message = "Plug-in setup failed.\n" 378 grp.rprint_error_report(error_message) 379 BuiltIn().fail(error_message) 380 # Setting cp_setup_called lets our Teardown know that it needs to call 381 # the cleanup plug-in call point. 382 cp_setup_called = 1 383 384 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 385 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 386 # FFDC_LOG_PATH is used by "FFDC" keyword. 387 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 388 389 # Also printed by FFDC. 390 global host_name 391 global host_ip 392 host = socket.gethostname() 393 host_name, host_ip = gm.get_host_name_ip(host) 394 395 gp.dprint_var(boot_table, 1) 396 gp.dprint_var(boot_lists) 397 398############################################################################### 399 400 401############################################################################### 402def validate_parms(): 403 404 r""" 405 Validate all program parameters. 406 """ 407 408 process_pgm_parms() 409 410 gp.qprintn() 411 412 global openbmc_model 413 grv.rvalid_value("openbmc_host") 414 grv.rvalid_value("openbmc_username") 415 grv.rvalid_value("openbmc_password") 416 if os_host != "": 417 grv.rvalid_value("os_username") 418 grv.rvalid_value("os_password") 419 420 if pdu_host != "": 421 grv.rvalid_value("pdu_username") 422 grv.rvalid_value("pdu_password") 423 grv.rvalid_integer("pdu_slot_no") 424 if openbmc_serial_host != "": 425 grv.rvalid_integer("openbmc_serial_port") 426 if openbmc_model == "": 427 status, ret_values =\ 428 grk.run_key_u("Get BMC System Model") 429 openbmc_model = ret_values 430 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 431 grv.rvalid_value("openbmc_model") 432 grv.rvalid_integer("max_num_tests") 433 grv.rvalid_integer("boot_pass") 434 grv.rvalid_integer("boot_fail") 435 436 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 437 BuiltIn().set_global_variable("${plug_in_packages_list}", 438 plug_in_packages_list) 439 440 grv.rvalid_value("stack_mode", valid_values=['normal', 'skip']) 441 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 442 error_message = "You must provide either a value for either the" +\ 443 " boot_list or the boot_stack parm.\n" 444 BuiltIn().fail(gp.sprint_error(error_message)) 445 446 valid_boot_list(boot_list, valid_boot_types) 447 valid_boot_list(boot_stack, valid_boot_types) 448 449 selected_PDU_boots = list(set(boot_list + boot_stack) & 450 set(boot_lists['PDU_reboot'])) 451 452 if len(selected_PDU_boots) > 0 and pdu_host == "": 453 error_message = "You have selected the following boots which" +\ 454 " require a PDU host but no value for pdu_host:\n" 455 error_message += gp.sprint_var(selected_PDU_boots) 456 error_message += gp.sprint_var(pdu_host, 2) 457 BuiltIn().fail(gp.sprint_error(error_message)) 458 459 return 460 461############################################################################### 462 463 464############################################################################### 465def my_get_state(): 466 467 r""" 468 Get the system state plus a little bit of wrapping. 469 """ 470 471 global state 472 473 req_states = ['epoch_seconds'] + st.default_req_states 474 475 gp.qprint_timen("Getting system state.") 476 if test_mode: 477 state['epoch_seconds'] = int(time.time()) 478 else: 479 state = st.get_state(req_states=req_states, quiet=quiet) 480 gp.qprint_var(state) 481 482############################################################################### 483 484 485############################################################################### 486def valid_state(): 487 488 r""" 489 Verify that our state dictionary contains no blank values. If we don't get 490 valid state data, we cannot continue to work. 491 """ 492 493 if st.compare_states(state, st.invalid_state_match, 'or'): 494 error_message = "The state dictionary contains blank fields which" +\ 495 " is illegal.\n" + gp.sprint_var(state) 496 BuiltIn().fail(gp.sprint_error(error_message)) 497 498############################################################################### 499 500 501############################################################################### 502def select_boot(): 503 504 r""" 505 Select a boot test to be run based on our current state and return the 506 chosen boot type. 507 508 Description of arguments: 509 state The state of the machine. 510 """ 511 512 global transitional_boot_selected 513 global boot_stack 514 515 gp.qprint_timen("Selecting a boot test.") 516 517 if transitional_boot_selected and not boot_success: 518 prior_boot = next_boot 519 boot_candidate = boot_stack.pop() 520 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" + 521 " transition to a valid state for '" + boot_candidate + 522 "' which was at the top of the boot_stack. Since" + 523 " the '" + next_boot + "' failed, the '" + 524 boot_candidate + "' has been removed from the stack" + 525 " to avoid and endless failure loop.") 526 if len(boot_stack) == 0: 527 return "" 528 529 my_get_state() 530 valid_state() 531 532 transitional_boot_selected = False 533 stack_popped = 0 534 if len(boot_stack) > 0: 535 stack_popped = 1 536 gp.qprint_dashes() 537 gp.qprint_var(boot_stack) 538 gp.qprint_dashes() 539 skip_boot_printed = 0 540 while len(boot_stack) > 0: 541 boot_candidate = boot_stack.pop() 542 if stack_mode == 'normal': 543 break 544 else: 545 if st.compare_states(state, boot_table[boot_candidate]['end']): 546 if not skip_boot_printed: 547 gp.qprint_var(stack_mode) 548 gp.qprintn() 549 gp.qprint_timen("Skipping the following boot tests" + 550 " which are unnecessary since their" + 551 " required end states match the" + 552 " current machine state:") 553 skip_boot_printed = 1 554 gp.qprint_var(boot_candidate) 555 boot_candidate = "" 556 if boot_candidate == "": 557 gp.qprint_dashes() 558 gp.qprint_var(boot_stack) 559 gp.qprint_dashes() 560 return boot_candidate 561 if st.compare_states(state, boot_table[boot_candidate]['start']): 562 gp.qprint_timen("The machine state is valid for a '" + 563 boot_candidate + "' boot test.") 564 gp.qprint_dashes() 565 gp.qprint_var(boot_stack) 566 gp.qprint_dashes() 567 return boot_candidate 568 else: 569 gp.qprint_timen("The machine state does not match the required" + 570 " starting state for a '" + boot_candidate + 571 "' boot test:") 572 gp.qprint_varx("boot_table[" + boot_candidate + "][start]", 573 boot_table[boot_candidate]['start'], 1) 574 boot_stack.append(boot_candidate) 575 transitional_boot_selected = True 576 popped_boot = boot_candidate 577 578 # Loop through your list selecting a boot_candidates 579 boot_candidates = [] 580 for boot_candidate in boot_list: 581 if st.compare_states(state, boot_table[boot_candidate]['start']): 582 if stack_popped: 583 if st.compare_states(boot_table[boot_candidate]['end'], 584 boot_table[popped_boot]['start']): 585 boot_candidates.append(boot_candidate) 586 else: 587 boot_candidates.append(boot_candidate) 588 589 if len(boot_candidates) == 0: 590 gp.qprint_timen("The user's boot list contained no boot tests" + 591 " which are valid for the current machine state.") 592 boot_candidate = default_power_on 593 if not st.compare_states(state, boot_table[default_power_on]['start']): 594 boot_candidate = default_power_off 595 boot_candidates.append(boot_candidate) 596 gp.qprint_timen("Using default '" + boot_candidate + 597 "' boot type to transition to valid state.") 598 599 gp.dprint_var(boot_candidates) 600 601 # Randomly select a boot from the candidate list. 602 boot = random.choice(boot_candidates) 603 604 return boot 605 606############################################################################### 607 608 609############################################################################### 610def print_last_boots(): 611 612 r""" 613 Print the last ten boots done with their time stamps. 614 """ 615 616 # indent 0, 90 chars wide, linefeed, char is "=" 617 gp.qprint_dashes(0, 90) 618 gp.qprintn("Last 10 boots:\n") 619 620 for boot_entry in last_ten: 621 grp.rqprint(boot_entry) 622 gp.qprint_dashes(0, 90) 623 624############################################################################### 625 626 627############################################################################### 628def print_defect_report(ffdc_file_list): 629 630 r""" 631 Print a defect report. 632 633 Description of argument(s): 634 ffdc_file_list A list of files which were collected by our ffdc functions. 635 """ 636 637 # Making deliberate choice to NOT run plug_in_setup(). We don't want 638 # ffdc_prefix updated. 639 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 640 call_point='ffdc_report', stop_on_plug_in_failure=0) 641 642 # Get additional header data which may have been created by ffdc plug-ins. 643 # Also, delete the individual header files to cleanup. 644 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 645 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 646 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 647 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 648 show_err=0) 649 650 # Get additional summary data which may have been created by ffdc plug-ins. 651 # Also, delete the individual header files to cleanup. 652 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 653 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 654 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 655 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 656 show_err=0) 657 658 # ffdc_list_file_path contains a list of any ffdc files created by plug- 659 # ins, etc. Read that data into a list. 660 try: 661 plug_in_ffdc_list = \ 662 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 663 plug_in_ffdc_list = filter(None, plug_in_ffdc_list) 664 except IOError: 665 plug_in_ffdc_list = [] 666 667 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 668 # in. Eliminate duplicates and sort the list. 669 ffdc_file_list = list(set(ffdc_file_list + plug_in_ffdc_list)) 670 ffdc_file_list.sort() 671 672 if status_file_path != "": 673 ffdc_file_list.insert(0, status_file_path) 674 675 # Convert the list to a printable list. 676 printable_ffdc_file_list = "\n".join(ffdc_file_list) 677 678 # Open ffdc_file_list for writing. We will write a complete list of 679 # FFDC files to it for possible use by plug-ins like cp_stop_check. 680 ffdc_list_file = open(ffdc_list_file_path, 'w') 681 ffdc_list_file.write(printable_ffdc_file_list + "\n") 682 ffdc_list_file.close() 683 684 indent = 0 685 width = 90 686 linefeed = 1 687 char = "=" 688 689 gp.qprintn() 690 gp.qprint_dashes(indent, width, linefeed, char) 691 gp.qprintn("Copy this data to the defect:\n") 692 693 if len(more_header_info) > 0: 694 gp.qprintn(more_header_info) 695 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 696 openbmc_host_name, openbmc_ip, openbmc_username, 697 openbmc_password, os_host, os_host_name, os_ip, os_username, 698 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 699 pdu_password, pdu_slot_no, openbmc_serial_host, 700 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 701 702 gp.qprintn() 703 print_last_boots() 704 gp.qprintn() 705 gp.qprint_var(state) 706 gp.qprintn() 707 gp.qprintn("FFDC data files:") 708 gp.qprintn(printable_ffdc_file_list) 709 gp.qprintn() 710 711 if len(ffdc_summary_info) > 0: 712 gp.qprintn(ffdc_summary_info) 713 714 gp.qprint_dashes(indent, width, linefeed, char) 715 716############################################################################### 717 718 719############################################################################### 720def my_ffdc(): 721 722 r""" 723 Collect FFDC data. 724 """ 725 726 global state 727 728 plug_in_setup() 729 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 730 call_point='ffdc', stop_on_plug_in_failure=0) 731 732 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 733 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" + 734 AUTOBOOT_FFDC_PREFIX + 735 " ffdc_function_list=" + 736 ffdc_function_list, ignore=1) 737 if status != 'PASS': 738 gp.qprint_error("Call to ffdc failed.\n") 739 740 my_get_state() 741 742 print_defect_report(ffdc_file_list) 743 744############################################################################### 745 746 747############################################################################### 748def print_test_start_message(boot_keyword): 749 750 r""" 751 Print a message indicating what boot test is about to run. 752 753 Description of arguments: 754 boot_keyword The name of the boot which is to be run 755 (e.g. "BMC Power On"). 756 """ 757 758 global last_ten 759 global boot_start_time 760 761 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 762 763 # Set boot_start_time for use by plug-ins. 764 boot_start_time = doing_msg[1:33] 765 gp.qprint_var(boot_start_time) 766 767 gp.qprint(doing_msg) 768 769 last_ten.append(doing_msg) 770 771 if len(last_ten) > 10: 772 del last_ten[0] 773 774############################################################################### 775 776 777############################################################################### 778def run_boot(boot): 779 780 r""" 781 Run the specified boot. 782 783 Description of arguments: 784 boot The name of the boot test to be performed. 785 """ 786 787 global state 788 789 print_test_start_message(boot) 790 791 plug_in_setup() 792 rc, shell_rc, failed_plug_in_name = \ 793 grpi.rprocess_plug_in_packages(call_point="pre_boot") 794 if rc != 0: 795 error_message = "Plug-in failed with non-zero return code.\n" +\ 796 gp.sprint_var(rc, 1) 797 BuiltIn().fail(gp.sprint_error(error_message)) 798 799 if test_mode: 800 # In test mode, we'll pretend the boot worked by assigning its 801 # required end state to the default state value. 802 state = st.strip_anchor_state(boot_table[boot]['end']) 803 else: 804 # Assertion: We trust that the state data was made fresh by the 805 # caller. 806 807 gp.qprintn() 808 809 if boot_table[boot]['method_type'] == "keyword": 810 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 811 boot_table[boot]['method'], 812 quiet=quiet) 813 814 if boot_table[boot]['bmc_reboot']: 815 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 816 plug_in_setup() 817 rc, shell_rc, failed_plug_in_name = \ 818 grpi.rprocess_plug_in_packages(call_point="post_reboot") 819 if rc != 0: 820 error_message = "Plug-in failed with non-zero return code.\n" 821 error_message += gp.sprint_var(rc, 1) 822 BuiltIn().fail(gp.sprint_error(error_message)) 823 else: 824 match_state = st.anchor_state(state) 825 del match_state['epoch_seconds'] 826 # Wait for the state to change in any way. 827 st.wait_state(match_state, wait_time=state_change_timeout, 828 interval="10 seconds", invert=1) 829 830 gp.qprintn() 831 if boot_table[boot]['end']['chassis'] == "Off": 832 boot_timeout = power_off_timeout 833 else: 834 boot_timeout = power_on_timeout 835 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 836 interval="10 seconds") 837 838 plug_in_setup() 839 rc, shell_rc, failed_plug_in_name = \ 840 grpi.rprocess_plug_in_packages(call_point="post_boot") 841 if rc != 0: 842 error_message = "Plug-in failed with non-zero return code.\n" +\ 843 gp.sprint_var(rc, 1) 844 BuiltIn().fail(gp.sprint_error(error_message)) 845 846############################################################################### 847 848 849############################################################################### 850def test_loop_body(): 851 852 r""" 853 The main loop body for the loop in main_py. 854 855 Description of arguments: 856 boot_count The iteration number (starts at 1). 857 """ 858 859 global boot_count 860 global state 861 global next_boot 862 global boot_success 863 global boot_end_time 864 865 gp.qprintn() 866 867 next_boot = select_boot() 868 if next_boot == "": 869 return True 870 871 boot_count += 1 872 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 873 874 pre_boot_plug_in_setup() 875 876 cmd_buf = ["run_boot", next_boot] 877 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 878 if boot_status == "FAIL": 879 gp.qprint(msg) 880 881 gp.qprintn() 882 if boot_status == "PASS": 883 boot_success = 1 884 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot + 885 "\" succeeded.") 886 else: 887 boot_success = 0 888 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot + 889 "\" failed.") 890 891 # Set boot_end_time for use by plug-ins. 892 boot_end_time = completion_msg[1:33] 893 gp.qprint_var(boot_end_time) 894 895 gp.qprint(completion_msg) 896 897 boot_results.update(next_boot, boot_status) 898 899 plug_in_setup() 900 # NOTE: A post_test_case call point failure is NOT counted as a boot 901 # failure. 902 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 903 call_point='post_test_case', stop_on_plug_in_failure=0) 904 905 plug_in_setup() 906 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 907 call_point='ffdc_check', shell_rc=0x00000200, 908 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 909 if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200: 910 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 911 if status != 'PASS': 912 gp.qprint_error("Call to my_ffdc failed.\n") 913 914 if delete_errlogs: 915 # We need to purge error logs between boots or they build up. 916 grk.run_key("Delete Error logs", ignore=1) 917 918 boot_results.print_report() 919 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 920 921 plug_in_setup() 922 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 923 call_point='stop_check') 924 if rc != 0: 925 error_message = "Stopping as requested by user.\n" 926 grp.rprint_error_report(error_message) 927 BuiltIn().fail(error_message) 928 929 # This should help prevent ConnectionErrors. 930 grk.run_key_u("Close All Connections") 931 932 return True 933 934############################################################################### 935 936 937############################################################################### 938def obmc_boot_test_teardown(): 939 940 r""" 941 Clean up after the Main keyword. 942 """ 943 944 if cp_setup_called: 945 plug_in_setup() 946 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 947 call_point='cleanup', stop_on_plug_in_failure=0) 948 949 if 'boot_results_file_path' in globals(): 950 # Save boot_results object to a file in case it is needed again. 951 gp.qprint_timen("Saving boot_results to the following path.") 952 gp.qprint_var(boot_results_file_path) 953 pickle.dump(boot_results, open(boot_results_file_path, 'wb'), 954 pickle.HIGHEST_PROTOCOL) 955 956 global save_stack 957 # Restore any global values saved on the save_stack. 958 for parm_name in main_func_parm_list: 959 # Get the parm_value if it was saved on the stack. 960 try: 961 parm_value = save_stack.pop(parm_name) 962 except: 963 # If it was not saved, no further action is required. 964 continue 965 966 # Restore the saved value. 967 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 968 "}\", parm_value)" 969 gp.dpissuing(cmd_buf) 970 exec(cmd_buf) 971 972 gp.dprintn(save_stack.sprint_obj()) 973 974############################################################################### 975 976 977############################################################################### 978def test_teardown(): 979 980 r""" 981 Clean up after this test case. 982 """ 983 984 gp.qprintn() 985 cmd_buf = ["Print Error", 986 "A keyword timeout occurred ending this program.\n"] 987 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 988 989 grp.rqprint_pgm_footer() 990 991############################################################################### 992 993 994############################################################################### 995def obmc_boot_test_py(loc_boot_stack=None, 996 loc_stack_mode=None, 997 loc_quiet=None): 998 999 r""" 1000 Do main program processing. 1001 """ 1002 1003 global save_stack 1004 1005 # Process function parms. 1006 for parm_name in main_func_parm_list: 1007 # Get parm's value. 1008 cmd_buf = "parm_value = loc_" + parm_name 1009 exec(cmd_buf) 1010 gp.dpvar(parm_name) 1011 gp.dpvar(parm_value) 1012 1013 if parm_value is None: 1014 # Parm was not specified by the calling function so set it to its 1015 # corresponding global value. 1016 cmd_buf = "loc_" + parm_name + " = BuiltIn().get_variable_value" +\ 1017 "(\"${" + parm_name + "}\")" 1018 gp.dpissuing(cmd_buf) 1019 exec(cmd_buf) 1020 else: 1021 # Save the global value on a stack. 1022 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1023 parm_name + "}\"), \"" + parm_name + "\")" 1024 gp.dpissuing(cmd_buf) 1025 exec(cmd_buf) 1026 1027 # Set the global value to the passed value. 1028 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1029 "}\", loc_" + parm_name + ")" 1030 gp.dpissuing(cmd_buf) 1031 exec(cmd_buf) 1032 1033 gp.dprintn(save_stack.sprint_obj()) 1034 1035 setup() 1036 1037 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1038 1039 if ffdc_only: 1040 gp.qprint_timen("Caller requested ffdc_only.") 1041 pre_boot_plug_in_setup() 1042 grk.run_key_u("my_ffdc") 1043 return 1044 1045 # Process caller's boot_stack. 1046 while (len(boot_stack) > 0): 1047 test_loop_body() 1048 1049 gp.qprint_timen("Finished processing stack.") 1050 1051 # Process caller's boot_list. 1052 if len(boot_list) > 0: 1053 for ix in range(1, max_num_tests + 1): 1054 test_loop_body() 1055 1056 gp.qprint_timen("Completed all requested boot tests.") 1057 1058 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1059 new_fail = boot_fail - init_boot_fail 1060 if new_fail > boot_fail_threshold: 1061 error_message = "Boot failures exceed the boot failure" +\ 1062 " threshold:\n" +\ 1063 gp.sprint_var(new_fail) +\ 1064 gp.sprint_var(boot_fail_threshold) 1065 BuiltIn().fail(gp.sprint_error(error_message)) 1066 1067############################################################################### 1068