1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import re 13import signal 14try: 15 import cPickle as pickle 16except ImportError: 17 import pickle 18import socket 19 20from robot.utils import DotDict 21from robot.libraries.BuiltIn import BuiltIn 22 23from boot_data import * 24import gen_print as gp 25import gen_robot_plug_in as grpi 26import gen_arg as ga 27import gen_valid as gv 28import gen_misc as gm 29import gen_cmd as gc 30import gen_robot_keyword as grk 31import state as st 32import var_stack as vs 33import gen_plug_in_utils as gpu 34import pel_utils as pel 35import logging_utils as log 36 37base_path = os.path.dirname(os.path.dirname( 38 imp.find_module("gen_robot_print")[1])) +\ 39 os.sep 40sys.path.append(base_path + "extended/") 41import run_keyword as rk 42 43# Setting master_pid correctly influences the behavior of plug-ins like 44# DB_Logging 45program_pid = os.getpid() 46master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 47pgm_name = re.sub('\\.py$', '', os.path.basename(__file__)) 48 49# Set up boot data structures. 50os_host = BuiltIn().get_variable_value("${OS_HOST}", default="") 51 52boot_lists = read_boot_lists() 53 54# The maximum number of entries that can be in the boot_history global variable. 55max_boot_history = 10 56boot_history = [] 57 58state = st.return_state_constant('default_state') 59cp_setup_called = 0 60next_boot = "" 61base_tool_dir_path = os.path.normpath(os.environ.get( 62 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 63 64ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 65boot_success = 0 66status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 67if status_dir_path != "": 68 status_dir_path = os.path.normpath(status_dir_path) + os.sep 69redfish_support_trans_state = int(os.environ.get('REDFISH_SUPPORT_TRANS_STATE', 0)) or \ 70 int(BuiltIn().get_variable_value("${REDFISH_SUPPORT_TRANS_STATE}", default=0)) 71redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False) 72redfish_rest_supported = BuiltIn().get_variable_value("${REDFISH_REST_SUPPORTED}", default=False) 73redfish_delete_sessions = int(BuiltIn().get_variable_value("${REDFISH_DELETE_SESSIONS}", default=1)) 74if redfish_supported: 75 redfish = BuiltIn().get_library_instance('redfish') 76 default_power_on = "Redfish Power On" 77 default_power_off = "Redfish Power Off" 78 if not redfish_support_trans_state: 79 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 80 delete_bmcdump_cmd = "Delete All BMC Dump" 81 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 82 else: 83 delete_errlogs_cmd = "Redfish Purge Event Log" 84 delete_bmcdump_cmd = "Redfish Delete All BMC Dumps" 85 default_set_power_policy = "Redfish Set Power Restore Policy AlwaysOff" 86else: 87 default_power_on = "REST Power On" 88 default_power_off = "REST Power Off" 89 delete_errlogs_cmd = "Delete Error Logs ${quiet}=${1}" 90 delete_bmcdump_cmd = "Delete All BMC Dump" 91 default_set_power_policy = "Set BMC Power Policy ALWAYS_POWER_OFF" 92boot_count = 0 93 94LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 95AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '') 96ffdc_prefix = AUTOBOOT_FFDC_PREFIX 97boot_start_time = "" 98boot_end_time = "" 99save_stack = vs.var_stack('save_stack') 100main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet'] 101 102 103def dump_ffdc_rc(): 104 r""" 105 Return the constant dump ffdc test return code value. 106 107 When a plug-in call point program returns this value, it indicates that 108 this program should collect FFDC. 109 """ 110 111 return 0x00000200 112 113 114def stop_test_rc(): 115 r""" 116 Return the constant stop test return code value. 117 118 When a plug-in call point program returns this value, it indicates that 119 this program should stop running. 120 """ 121 122 return 0x00000200 123 124 125def process_host(host, 126 host_var_name=""): 127 r""" 128 Process a host by getting the associated host name and IP address and 129 setting them in global variables. 130 131 If the caller does not pass the host_var_name, this function will try to 132 figure out the name of the variable used by the caller for the host parm. 133 Callers are advised to explicitly specify the host_var_name when calling 134 with an exec command. In such cases, the get_arg_name cannot figure out 135 the host variable name. 136 137 This function will then create similar global variable names by 138 removing "_host" and appending "_host_name" or "_ip" to the host variable 139 name. 140 141 Example: 142 143 If a call is made like this: 144 process_host(openbmc_host) 145 146 Global variables openbmc_host_name and openbmc_ip will be set. 147 148 Description of argument(s): 149 host A host name or IP. The name of the variable used should 150 have a suffix of "_host". 151 host_var_name The name of the variable being used as the host parm. 152 """ 153 154 if host_var_name == "": 155 host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2) 156 157 host_name_var_name = re.sub("host", "host_name", host_var_name) 158 ip_var_name = re.sub("host", "ip", host_var_name) 159 cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\ 160 host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\ 161 host + "')" 162 exec(cmd_buf) 163 164 165def process_pgm_parms(): 166 r""" 167 Process the program parameters by assigning them all to corresponding 168 globals. Also, set some global values that depend on program parameters. 169 """ 170 171 # Program parameter processing. 172 # Assign all program parms to python variables which are global to this 173 # module. 174 175 global parm_list 176 parm_list = BuiltIn().get_variable_value("${parm_list}") 177 # The following subset of parms should be processed as integers. 178 int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 179 'boot_fail_threshold', 'delete_errlogs', 180 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet', 181 'test_mode', 'debug'] 182 for parm in parm_list: 183 if parm in int_list: 184 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 185 "}\", \"0\"))" 186 else: 187 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 188 cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd 189 gp.dpissuing(cmd_buf) 190 exec(cmd_buf) 191 if re.match(r".*_host$", parm): 192 cmd_buf = "process_host(" + parm + ", '" + parm + "')" 193 exec(cmd_buf) 194 if re.match(r".*_password$", parm): 195 # Register the value of any parm whose name ends in _password. 196 # This will cause the print functions to replace passwords with 197 # asterisks in the output. 198 cmd_buf = "gp.register_passwords(" + parm + ")" 199 exec(cmd_buf) 200 201 global ffdc_dir_path_style 202 global boot_list 203 global boot_stack 204 global boot_results_file_path 205 global boot_results 206 global boot_history 207 global ffdc_list_file_path 208 global ffdc_report_list_path 209 global ffdc_summary_list_path 210 global boot_table 211 global valid_boot_types 212 213 if ffdc_dir_path_style == "": 214 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 215 216 # Convert these program parms to lists for easier processing.. 217 boot_list = list(filter(None, boot_list.split(":"))) 218 boot_stack = list(filter(None, boot_stack.split(":"))) 219 220 boot_table = create_boot_table(boot_table_path, os_host=os_host) 221 valid_boot_types = create_valid_boot_list(boot_table) 222 223 cleanup_boot_results_file() 224 boot_results_file_path = create_boot_results_file_path(pgm_name, 225 openbmc_nickname, 226 master_pid) 227 228 if os.path.isfile(boot_results_file_path): 229 # We've been called before in this run so we'll load the saved 230 # boot_results and boot_history objects. 231 boot_results, boot_history =\ 232 pickle.load(open(boot_results_file_path, 'rb')) 233 else: 234 boot_results = boot_results(boot_table, boot_pass, boot_fail) 235 236 ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\ 237 "/FFDC_FILE_LIST" 238 ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\ 239 "/FFDC_REPORT_FILE_LIST" 240 241 ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\ 242 "/FFDC_SUMMARY_FILE_LIST" 243 244 245def initial_plug_in_setup(): 246 r""" 247 Initialize all plug-in environment variables which do not change for the 248 duration of the program. 249 250 """ 251 252 global LOG_LEVEL 253 BuiltIn().set_log_level("NONE") 254 255 BuiltIn().set_global_variable("${master_pid}", master_pid) 256 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 257 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 258 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 259 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 260 ffdc_list_file_path) 261 BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}", 262 ffdc_report_list_path) 263 BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}", 264 ffdc_summary_list_path) 265 266 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 267 ffdc_dir_path_style) 268 BuiltIn().set_global_variable("${FFDC_CHECK}", 269 ffdc_check) 270 271 # For each program parameter, set the corresponding AUTOBOOT_ environment 272 # variable value. Also, set an AUTOBOOT_ environment variable for every 273 # element in additional_values. 274 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 275 "status_dir_path", "base_tool_dir_path", 276 "ffdc_list_file_path", "ffdc_report_list_path", 277 "ffdc_summary_list_path", "execdir", "redfish_supported", 278 "redfish_rest_supported", "redfish_support_trans_state"] 279 280 plug_in_vars = parm_list + additional_values 281 282 for var_name in plug_in_vars: 283 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 284 var_name = var_name.upper() 285 if var_value is None: 286 var_value = "" 287 os.environ["AUTOBOOT_" + var_name] = str(var_value) 288 289 BuiltIn().set_log_level(LOG_LEVEL) 290 291 # Make sure the ffdc list directory exists. 292 ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep 293 if not os.path.exists(ffdc_list_dir_path): 294 os.makedirs(ffdc_list_dir_path) 295 296 297def plug_in_setup(): 298 r""" 299 Initialize all changing plug-in environment variables for use by the 300 plug-in programs. 301 """ 302 303 global LOG_LEVEL 304 global test_really_running 305 306 BuiltIn().set_log_level("NONE") 307 308 boot_pass, boot_fail = boot_results.return_total_pass_fail() 309 if boot_pass > 1: 310 test_really_running = 1 311 else: 312 test_really_running = 0 313 314 BuiltIn().set_global_variable("${test_really_running}", 315 test_really_running) 316 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 317 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 318 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 319 BuiltIn().set_global_variable("${boot_success}", boot_success) 320 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 321 BuiltIn().set_global_variable("${boot_start_time}", boot_start_time) 322 BuiltIn().set_global_variable("${boot_end_time}", boot_end_time) 323 324 # For each program parameter, set the corresponding AUTOBOOT_ environment 325 # variable value. Also, set an AUTOBOOT_ environment variable for every 326 # element in additional_values. 327 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 328 "boot_fail", "test_really_running", "ffdc_prefix", 329 "boot_start_time", "boot_end_time"] 330 331 plug_in_vars = additional_values 332 333 for var_name in plug_in_vars: 334 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 335 var_name = var_name.upper() 336 if var_value is None: 337 var_value = "" 338 os.environ["AUTOBOOT_" + var_name] = str(var_value) 339 340 if debug: 341 shell_rc, out_buf = \ 342 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 343 344 BuiltIn().set_log_level(LOG_LEVEL) 345 346 347def pre_boot_plug_in_setup(): 348 349 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 350 try: 351 os.remove(ffdc_list_file_path) 352 except OSError: 353 pass 354 355 # Clear the ffdc_report_list_path file. Plug-ins may now write to it. 356 try: 357 os.remove(ffdc_report_list_path) 358 except OSError: 359 pass 360 361 # Clear the ffdc_summary_list_path file. Plug-ins may now write to it. 362 try: 363 os.remove(ffdc_summary_list_path) 364 except OSError: 365 pass 366 367 global ffdc_prefix 368 369 seconds = time.time() 370 loc_time = time.localtime(seconds) 371 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 372 373 ffdc_prefix = openbmc_nickname + "." + time_string 374 375 376def default_sigusr1(signal_number=0, 377 frame=None): 378 r""" 379 Handle SIGUSR1 by doing nothing. 380 381 This function assists in debugging SIGUSR1 processing by printing messages 382 to stdout and to the log.html file. 383 384 Description of argument(s): 385 signal_number The signal number (should always be 10 for SIGUSR1). 386 frame The frame data. 387 """ 388 389 gp.qprintn() 390 gp.qprint_executing() 391 gp.lprint_executing() 392 393 394def set_default_siguser1(): 395 r""" 396 Set the default_sigusr1 function to be the SIGUSR1 handler. 397 """ 398 399 gp.qprintn() 400 gp.qprint_executing() 401 gp.lprint_executing() 402 signal.signal(signal.SIGUSR1, default_sigusr1) 403 404 405def setup(): 406 r""" 407 Do general program setup tasks. 408 """ 409 410 global cp_setup_called 411 global transitional_boot_selected 412 413 gp.qprintn() 414 415 if redfish_supported: 416 redfish.login() 417 418 set_default_siguser1() 419 transitional_boot_selected = False 420 421 robot_pgm_dir_path = os.path.dirname(__file__) + os.sep 422 repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/") 423 # If we can't find process_plug_in_packages.py, ssh_pw or 424 # validate_plug_ins.py, then we don't have our repo bin in PATH. 425 shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" 426 + " ssh_pw validate_plug_ins.py", quiet=1, 427 print_output=0, show_err=0) 428 if shell_rc != 0: 429 os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "") 430 # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH. 431 if robot_pgm_dir_path not in sys.path: 432 sys.path.append(robot_pgm_dir_path) 433 PYTHONPATH = os.environ.get("PYTHONPATH", "") 434 if PYTHONPATH == "": 435 os.environ['PYTHONPATH'] = robot_pgm_dir_path 436 else: 437 os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH 438 439 validate_parms() 440 441 gp.qprint_pgm_header() 442 443 grk.run_key_u(default_set_power_policy) 444 445 initial_plug_in_setup() 446 447 plug_in_setup() 448 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 449 call_point='setup') 450 if rc != 0: 451 error_message = "Plug-in setup failed.\n" 452 gp.print_error_report(error_message) 453 BuiltIn().fail(error_message) 454 # Setting cp_setup_called lets our Teardown know that it needs to call 455 # the cleanup plug-in call point. 456 cp_setup_called = 1 457 458 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 459 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 460 # FFDC_LOG_PATH is used by "FFDC" keyword. 461 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 462 463 # Also printed by FFDC. 464 global host_name 465 global host_ip 466 host = socket.gethostname() 467 host_name, host_ip = gm.get_host_name_ip(host) 468 469 gp.dprint_var(boot_table) 470 gp.dprint_var(boot_lists) 471 472 473def validate_parms(): 474 r""" 475 Validate all program parameters. 476 """ 477 478 process_pgm_parms() 479 480 gp.qprintn() 481 482 global openbmc_model 483 if openbmc_model == "": 484 status, ret_values =\ 485 grk.run_key_u("Get BMC System Model") 486 openbmc_model = ret_values 487 BuiltIn().set_global_variable("${openbmc_model}", openbmc_model) 488 gv.set_exit_on_error(True) 489 gv.valid_value(openbmc_host) 490 gv.valid_value(openbmc_username) 491 gv.valid_value(openbmc_password) 492 gv.valid_value(rest_username) 493 gv.valid_value(rest_password) 494 gv.valid_value(ipmi_username) 495 gv.valid_value(ipmi_password) 496 if os_host != "": 497 gv.valid_value(os_username) 498 gv.valid_value(os_password) 499 if pdu_host != "": 500 gv.valid_value(pdu_username) 501 gv.valid_value(pdu_password) 502 gv.valid_integer(pdu_slot_no) 503 if openbmc_serial_host != "": 504 gv.valid_integer(openbmc_serial_port) 505 gv.valid_value(openbmc_model) 506 gv.valid_integer(max_num_tests) 507 gv.valid_integer(boot_pass) 508 gv.valid_integer(boot_fail) 509 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 510 BuiltIn().set_global_variable("${plug_in_packages_list}", 511 plug_in_packages_list) 512 gv.valid_value(stack_mode, valid_values=['normal', 'skip']) 513 gv.set_exit_on_error(False) 514 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 515 error_message = "You must provide either a value for either the" +\ 516 " boot_list or the boot_stack parm.\n" 517 BuiltIn().fail(gp.sprint_error(error_message)) 518 valid_boot_list(boot_list, valid_boot_types) 519 valid_boot_list(boot_stack, valid_boot_types) 520 selected_PDU_boots = list(set(boot_list + boot_stack) 521 & set(boot_lists['PDU_reboot'])) 522 if len(selected_PDU_boots) > 0 and pdu_host == "": 523 error_message = "You have selected the following boots which" +\ 524 " require a PDU host but no value for pdu_host:\n" 525 error_message += gp.sprint_var(selected_PDU_boots) 526 error_message += gp.sprint_var(pdu_host, fmt=gp.blank()) 527 BuiltIn().fail(gp.sprint_error(error_message)) 528 529 return 530 531 532def my_get_state(): 533 r""" 534 Get the system state plus a little bit of wrapping. 535 """ 536 537 global state 538 539 req_states = ['epoch_seconds'] + st.default_req_states 540 541 gp.qprint_timen("Getting system state.") 542 if test_mode: 543 state['epoch_seconds'] = int(time.time()) 544 else: 545 state = st.get_state(req_states=req_states, quiet=quiet) 546 gp.qprint_var(state) 547 548 549def valid_state(): 550 r""" 551 Verify that our state dictionary contains no blank values. If we don't get 552 valid state data, we cannot continue to work. 553 """ 554 555 if st.compare_states(state, st.invalid_state_match, 'or'): 556 error_message = "The state dictionary contains blank fields which" +\ 557 " is illegal.\n" + gp.sprint_var(state) 558 BuiltIn().fail(gp.sprint_error(error_message)) 559 560 561def select_boot(): 562 r""" 563 Select a boot test to be run based on our current state and return the 564 chosen boot type. 565 566 Description of arguments: 567 state The state of the machine. 568 """ 569 570 global transitional_boot_selected 571 global boot_stack 572 573 gp.qprint_timen("Selecting a boot test.") 574 575 if transitional_boot_selected and not boot_success: 576 prior_boot = next_boot 577 boot_candidate = boot_stack.pop() 578 gp.qprint_timen("The prior '" + next_boot + "' was chosen to" 579 + " transition to a valid state for '" + boot_candidate 580 + "' which was at the top of the boot_stack. Since" 581 + " the '" + next_boot + "' failed, the '" 582 + boot_candidate + "' has been removed from the stack" 583 + " to avoid and endless failure loop.") 584 if len(boot_stack) == 0: 585 return "" 586 587 my_get_state() 588 valid_state() 589 590 transitional_boot_selected = False 591 stack_popped = 0 592 if len(boot_stack) > 0: 593 stack_popped = 1 594 gp.qprint_dashes() 595 gp.qprint_var(boot_stack) 596 gp.qprint_dashes() 597 skip_boot_printed = 0 598 while len(boot_stack) > 0: 599 boot_candidate = boot_stack.pop() 600 if stack_mode == 'normal': 601 break 602 else: 603 if st.compare_states(state, boot_table[boot_candidate]['end']): 604 if not skip_boot_printed: 605 gp.qprint_var(stack_mode) 606 gp.qprintn() 607 gp.qprint_timen("Skipping the following boot tests" 608 + " which are unnecessary since their" 609 + " required end states match the" 610 + " current machine state:") 611 skip_boot_printed = 1 612 gp.qprint_var(boot_candidate) 613 boot_candidate = "" 614 if boot_candidate == "": 615 gp.qprint_dashes() 616 gp.qprint_var(boot_stack) 617 gp.qprint_dashes() 618 return boot_candidate 619 if st.compare_states(state, boot_table[boot_candidate]['start']): 620 gp.qprint_timen("The machine state is valid for a '" 621 + boot_candidate + "' boot test.") 622 gp.qprint_dashes() 623 gp.qprint_var(boot_stack) 624 gp.qprint_dashes() 625 return boot_candidate 626 else: 627 gp.qprint_timen("The machine state does not match the required" 628 + " starting state for a '" + boot_candidate 629 + "' boot test:") 630 gp.qprint_varx("boot_table_start_entry", 631 boot_table[boot_candidate]['start']) 632 boot_stack.append(boot_candidate) 633 transitional_boot_selected = True 634 popped_boot = boot_candidate 635 636 # Loop through your list selecting a boot_candidates 637 boot_candidates = [] 638 for boot_candidate in boot_list: 639 if st.compare_states(state, boot_table[boot_candidate]['start']): 640 if stack_popped: 641 if st.compare_states(boot_table[boot_candidate]['end'], 642 boot_table[popped_boot]['start']): 643 boot_candidates.append(boot_candidate) 644 else: 645 boot_candidates.append(boot_candidate) 646 647 if len(boot_candidates) == 0: 648 gp.qprint_timen("The user's boot list contained no boot tests" 649 + " which are valid for the current machine state.") 650 boot_candidate = default_power_on 651 if not st.compare_states(state, boot_table[default_power_on]['start']): 652 boot_candidate = default_power_off 653 boot_candidates.append(boot_candidate) 654 gp.qprint_timen("Using default '" + boot_candidate 655 + "' boot type to transition to valid state.") 656 657 gp.dprint_var(boot_candidates) 658 659 # Randomly select a boot from the candidate list. 660 boot = random.choice(boot_candidates) 661 662 return boot 663 664 665def print_defect_report(ffdc_file_list): 666 r""" 667 Print a defect report. 668 669 Description of argument(s): 670 ffdc_file_list A list of files which were collected by our ffdc functions. 671 """ 672 673 # Making deliberate choice to NOT run plug_in_setup(). We don't want 674 # ffdc_prefix updated. 675 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 676 call_point='ffdc_report', stop_on_plug_in_failure=0) 677 678 # Get additional header data which may have been created by ffdc plug-ins. 679 # Also, delete the individual header files to cleanup. 680 cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\ 681 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 682 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 683 shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 684 show_err=0) 685 686 # Get additional summary data which may have been created by ffdc plug-ins. 687 # Also, delete the individual header files to cleanup. 688 cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\ 689 " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\ 690 " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :" 691 shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0, 692 show_err=0) 693 694 # ffdc_list_file_path contains a list of any ffdc files created by plug- 695 # ins, etc. Read that data into a list. 696 try: 697 plug_in_ffdc_list = \ 698 open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n") 699 plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list)) 700 except IOError: 701 plug_in_ffdc_list = [] 702 703 # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed 704 # in. Eliminate duplicates and sort the list. 705 ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list)) 706 707 if status_file_path != "": 708 ffdc_file_list.insert(0, status_file_path) 709 710 # Convert the list to a printable list. 711 printable_ffdc_file_list = "\n".join(ffdc_file_list) 712 713 # Open ffdc_file_list for writing. We will write a complete list of 714 # FFDC files to it for possible use by plug-ins like cp_stop_check. 715 ffdc_list_file = open(ffdc_list_file_path, 'w') 716 ffdc_list_file.write(printable_ffdc_file_list + "\n") 717 ffdc_list_file.close() 718 719 indent = 0 720 width = 90 721 linefeed = 1 722 char = "=" 723 724 gp.qprintn() 725 gp.qprint_dashes(indent, width, linefeed, char) 726 gp.qprintn("Copy this data to the defect:\n") 727 728 if len(more_header_info) > 0: 729 gp.qprintn(more_header_info) 730 gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host, 731 openbmc_host_name, openbmc_ip, openbmc_username, 732 openbmc_password, rest_username, rest_password, ipmi_username, 733 ipmi_password, os_host, os_host_name, os_ip, os_username, 734 os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username, 735 pdu_password, pdu_slot_no, openbmc_serial_host, 736 openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port) 737 738 gp.qprintn() 739 print_boot_history(boot_history) 740 gp.qprintn() 741 gp.qprint_var(state) 742 gp.qprintn() 743 gp.qprintn("FFDC data files:") 744 gp.qprintn(printable_ffdc_file_list) 745 gp.qprintn() 746 747 if len(ffdc_summary_info) > 0: 748 gp.qprintn(ffdc_summary_info) 749 750 gp.qprint_dashes(indent, width, linefeed, char) 751 752 753def my_ffdc(): 754 r""" 755 Collect FFDC data. 756 """ 757 758 global state 759 760 plug_in_setup() 761 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 762 call_point='ffdc', stop_on_plug_in_failure=0) 763 764 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 765 status, ffdc_file_list = grk.run_key_u("FFDC ffdc_prefix=" 766 + AUTOBOOT_FFDC_PREFIX 767 + " ffdc_function_list=" 768 + ffdc_function_list, ignore=1) 769 if status != 'PASS': 770 gp.qprint_error("Call to ffdc failed.\n") 771 if type(ffdc_file_list) is not list: 772 ffdc_file_list = [] 773 # Leave a record for caller that "soft" errors occurred. 774 soft_errors = 1 775 gpu.save_plug_in_value(soft_errors, pgm_name) 776 777 my_get_state() 778 779 print_defect_report(ffdc_file_list) 780 781 782def print_test_start_message(boot_keyword): 783 r""" 784 Print a message indicating what boot test is about to run. 785 786 Description of arguments: 787 boot_keyword The name of the boot which is to be run 788 (e.g. "BMC Power On"). 789 """ 790 791 global boot_history 792 global boot_start_time 793 794 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 795 796 # Set boot_start_time for use by plug-ins. 797 boot_start_time = doing_msg[1:33] 798 gp.qprint_var(boot_start_time) 799 800 gp.qprint(doing_msg) 801 802 update_boot_history(boot_history, doing_msg, max_boot_history) 803 804 805def stop_boot_test(signal_number=0, 806 frame=None): 807 r""" 808 Handle SIGUSR1 by aborting the boot test that is running. 809 810 Description of argument(s): 811 signal_number The signal number (should always be 10 for SIGUSR1). 812 frame The frame data. 813 """ 814 815 gp.qprintn() 816 gp.qprint_executing() 817 gp.lprint_executing() 818 819 # Restore original sigusr1 handler. 820 set_default_siguser1() 821 822 message = "The caller has asked that the boot test be stopped and marked" 823 message += " as a failure." 824 825 function_stack = gm.get_function_stack() 826 if "wait_state" in function_stack: 827 st.set_exit_wait_early_message(message) 828 else: 829 BuiltIn().fail(gp.sprint_error(message)) 830 831 832def run_boot(boot): 833 r""" 834 Run the specified boot. 835 836 Description of arguments: 837 boot The name of the boot test to be performed. 838 """ 839 840 global state 841 842 signal.signal(signal.SIGUSR1, stop_boot_test) 843 gp.qprint_timen("stop_boot_test is armed.") 844 845 print_test_start_message(boot) 846 847 plug_in_setup() 848 rc, shell_rc, failed_plug_in_name = \ 849 grpi.rprocess_plug_in_packages(call_point="pre_boot") 850 if rc != 0: 851 error_message = "Plug-in failed with non-zero return code.\n" +\ 852 gp.sprint_var(rc, fmt=gp.hexa()) 853 set_default_siguser1() 854 BuiltIn().fail(gp.sprint_error(error_message)) 855 856 if test_mode: 857 # In test mode, we'll pretend the boot worked by assigning its 858 # required end state to the default state value. 859 state = st.strip_anchor_state(boot_table[boot]['end']) 860 else: 861 # Assertion: We trust that the state data was made fresh by the 862 # caller. 863 864 gp.qprintn() 865 866 if boot_table[boot]['method_type'] == "keyword": 867 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 868 boot_table[boot]['method'], 869 quiet=quiet) 870 871 if boot_table[boot]['bmc_reboot']: 872 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 873 plug_in_setup() 874 rc, shell_rc, failed_plug_in_name = \ 875 grpi.rprocess_plug_in_packages(call_point="post_reboot") 876 if rc != 0: 877 error_message = "Plug-in failed with non-zero return code.\n" 878 error_message += gp.sprint_var(rc, fmt=gp.hexa()) 879 set_default_siguser1() 880 BuiltIn().fail(gp.sprint_error(error_message)) 881 else: 882 match_state = st.anchor_state(state) 883 del match_state['epoch_seconds'] 884 # Wait for the state to change in any way. 885 st.wait_state(match_state, wait_time=state_change_timeout, 886 interval="10 seconds", invert=1) 887 888 gp.qprintn() 889 if boot_table[boot]['end']['chassis'] == "Off": 890 boot_timeout = power_off_timeout 891 else: 892 boot_timeout = power_on_timeout 893 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 894 interval="10 seconds") 895 896 plug_in_setup() 897 rc, shell_rc, failed_plug_in_name = \ 898 grpi.rprocess_plug_in_packages(call_point="post_boot") 899 if rc != 0: 900 error_message = "Plug-in failed with non-zero return code.\n" +\ 901 gp.sprint_var(rc, fmt=gp.hexa()) 902 set_default_siguser1() 903 BuiltIn().fail(gp.sprint_error(error_message)) 904 905 # Restore original sigusr1 handler. 906 set_default_siguser1() 907 908 909def test_loop_body(): 910 r""" 911 The main loop body for the loop in main_py. 912 913 Description of arguments: 914 boot_count The iteration number (starts at 1). 915 """ 916 917 global boot_count 918 global state 919 global next_boot 920 global boot_success 921 global boot_end_time 922 923 gp.qprintn() 924 925 next_boot = select_boot() 926 if next_boot == "": 927 return True 928 929 boot_count += 1 930 gp.qprint_timen("Starting boot " + str(boot_count) + ".") 931 932 pre_boot_plug_in_setup() 933 934 cmd_buf = ["run_boot", next_boot] 935 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 936 if boot_status == "FAIL": 937 gp.qprint(msg) 938 939 gp.qprintn() 940 if boot_status == "PASS": 941 boot_success = 1 942 completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot 943 + "\" succeeded.") 944 else: 945 boot_success = 0 946 completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot 947 + "\" failed.") 948 949 # Set boot_end_time for use by plug-ins. 950 boot_end_time = completion_msg[1:33] 951 gp.qprint_var(boot_end_time) 952 953 gp.qprint(completion_msg) 954 955 boot_results.update(next_boot, boot_status) 956 957 plug_in_setup() 958 # NOTE: A post_test_case call point failure is NOT counted as a boot 959 # failure. 960 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 961 call_point='post_test_case', stop_on_plug_in_failure=0) 962 963 plug_in_setup() 964 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 965 call_point='ffdc_check', shell_rc=dump_ffdc_rc(), 966 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 967 if ffdc_check == "All" or\ 968 shell_rc == dump_ffdc_rc(): 969 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 970 if status != 'PASS': 971 gp.qprint_error("Call to my_ffdc failed.\n") 972 # Leave a record for caller that "soft" errors occurred. 973 soft_errors = 1 974 gpu.save_plug_in_value(soft_errors, pgm_name) 975 976 if delete_errlogs: 977 # print error logs before delete 978 status, error_logs = grk.run_key_u("Get Error Logs") 979 pels = pel.peltool("-l", ignore_err=1) 980 log.print_error_logs(error_logs, "AdditionalData Message Severity") 981 gp.qprint_var(pels) 982 983 # We need to purge error logs between boots or they build up. 984 grk.run_key(delete_errlogs_cmd, ignore=1) 985 grk.run_key(delete_bmcdump_cmd, ignore=1) 986 987 boot_results.print_report() 988 gp.qprint_timen("Finished boot " + str(boot_count) + ".") 989 990 plug_in_setup() 991 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 992 call_point='stop_check', shell_rc=stop_test_rc(), 993 stop_on_non_zero_rc=1) 994 if shell_rc == stop_test_rc(): 995 message = "Stopping as requested by user.\n" 996 gp.qprint_time(message) 997 BuiltIn().fail(message) 998 999 # This should help prevent ConnectionErrors. 1000 # Purge all redfish and REST connection sessions. 1001 if redfish_delete_sessions: 1002 grk.run_key_u("Close All Connections", ignore=1) 1003 grk.run_key_u("Delete All Redfish Sessions", ignore=1) 1004 1005 return True 1006 1007 1008def obmc_boot_test_teardown(): 1009 r""" 1010 Clean up after the main keyword. 1011 """ 1012 gp.qprint_executing() 1013 1014 if ga.psutil_imported: 1015 ga.terminate_descendants() 1016 1017 if cp_setup_called: 1018 plug_in_setup() 1019 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1020 call_point='cleanup', stop_on_plug_in_failure=0) 1021 1022 if 'boot_results_file_path' in globals(): 1023 # Save boot_results and boot_history objects to a file in case they are 1024 # needed again. 1025 gp.qprint_timen("Saving boot_results to the following path.") 1026 gp.qprint_var(boot_results_file_path) 1027 pickle.dump((boot_results, boot_history), 1028 open(boot_results_file_path, 'wb'), 1029 pickle.HIGHEST_PROTOCOL) 1030 1031 global save_stack 1032 # Restore any global values saved on the save_stack. 1033 for parm_name in main_func_parm_list: 1034 # Get the parm_value if it was saved on the stack. 1035 try: 1036 parm_value = save_stack.pop(parm_name) 1037 except BaseException: 1038 # If it was not saved, no further action is required. 1039 continue 1040 1041 # Restore the saved value. 1042 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1043 "}\", parm_value)" 1044 gp.dpissuing(cmd_buf) 1045 exec(cmd_buf) 1046 1047 gp.dprintn(save_stack.sprint_obj()) 1048 1049 1050def test_teardown(): 1051 r""" 1052 Clean up after this test case. 1053 """ 1054 1055 gp.qprintn() 1056 gp.qprint_executing() 1057 1058 if ga.psutil_imported: 1059 ga.terminate_descendants() 1060 1061 cmd_buf = ["Print Error", 1062 "A keyword timeout occurred ending this program.\n"] 1063 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 1064 1065 if redfish_supported: 1066 redfish.logout() 1067 1068 gp.qprint_pgm_footer() 1069 1070 1071def post_stack(): 1072 r""" 1073 Process post_stack plug-in programs. 1074 """ 1075 1076 if not call_post_stack_plug: 1077 # The caller does not wish to have post_stack plug-in processing done. 1078 return 1079 1080 global boot_success 1081 1082 # NOTE: A post_stack call-point failure is NOT counted as a boot failure. 1083 pre_boot_plug_in_setup() 1084 # For the purposes of the following plug-ins, mark the "boot" as a success. 1085 boot_success = 1 1086 plug_in_setup() 1087 rc, shell_rc, failed_plug_in_name, history =\ 1088 grpi.rprocess_plug_in_packages(call_point='post_stack', 1089 stop_on_plug_in_failure=0, 1090 return_history=True) 1091 for doing_msg in history: 1092 update_boot_history(boot_history, doing_msg, max_boot_history) 1093 if rc != 0: 1094 boot_success = 0 1095 1096 plug_in_setup() 1097 rc, shell_rc, failed_plug_in_name =\ 1098 grpi.rprocess_plug_in_packages(call_point='ffdc_check', 1099 shell_rc=dump_ffdc_rc(), 1100 stop_on_plug_in_failure=1, 1101 stop_on_non_zero_rc=1) 1102 if shell_rc == dump_ffdc_rc(): 1103 status, ret_values = grk.run_key_u("my_ffdc", ignore=1) 1104 if status != 'PASS': 1105 gp.qprint_error("Call to my_ffdc failed.\n") 1106 # Leave a record for caller that "soft" errors occurred. 1107 soft_errors = 1 1108 gpu.save_plug_in_value(soft_errors, pgm_name) 1109 1110 plug_in_setup() 1111 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 1112 call_point='stop_check', shell_rc=stop_test_rc(), 1113 stop_on_non_zero_rc=1) 1114 if shell_rc == stop_test_rc(): 1115 message = "Stopping as requested by user.\n" 1116 gp.qprint_time(message) 1117 BuiltIn().fail(message) 1118 1119 1120def obmc_boot_test_py(loc_boot_stack=None, 1121 loc_stack_mode=None, 1122 loc_quiet=None): 1123 r""" 1124 Do main program processing. 1125 """ 1126 1127 global save_stack 1128 1129 ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']}) 1130 1131 gp.dprintn() 1132 # Process function parms. 1133 for parm_name in main_func_parm_list: 1134 # Get parm's value. 1135 parm_value = eval("loc_" + parm_name) 1136 gp.dpvars(parm_name, parm_value) 1137 1138 if parm_value is not None: 1139 # Save the global value on a stack. 1140 cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\ 1141 parm_name + "}\"), \"" + parm_name + "\")" 1142 gp.dpissuing(cmd_buf) 1143 exec(cmd_buf) 1144 1145 # Set the global value to the passed value. 1146 cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\ 1147 "}\", loc_" + parm_name + ")" 1148 gp.dpissuing(cmd_buf) 1149 exec(cmd_buf) 1150 1151 gp.dprintn(save_stack.sprint_obj()) 1152 1153 setup() 1154 1155 init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail() 1156 1157 if ffdc_only: 1158 gp.qprint_timen("Caller requested ffdc_only.") 1159 if do_pre_boot_plug_in_setup: 1160 pre_boot_plug_in_setup() 1161 grk.run_key_u("my_ffdc") 1162 return 1163 1164 if delete_errlogs: 1165 # print error logs before delete 1166 status, error_logs = grk.run_key_u("Get Error Logs") 1167 pels = pel.peltool("-l", ignore_err=1) 1168 log.print_error_logs(error_logs, "AdditionalData Message Severity") 1169 gp.qprint_var(pels) 1170 1171 # Delete errlogs prior to doing any boot tests. 1172 grk.run_key(delete_errlogs_cmd, ignore=1) 1173 grk.run_key(delete_bmcdump_cmd, ignore=1) 1174 1175 # Process caller's boot_stack. 1176 while (len(boot_stack) > 0): 1177 test_loop_body() 1178 1179 gp.qprint_timen("Finished processing stack.") 1180 1181 post_stack() 1182 1183 # Process caller's boot_list. 1184 if len(boot_list) > 0: 1185 for ix in range(1, max_num_tests + 1): 1186 test_loop_body() 1187 1188 gp.qprint_timen("Completed all requested boot tests.") 1189 1190 boot_pass, boot_fail = boot_results.return_total_pass_fail() 1191 new_fail = boot_fail - init_boot_fail 1192 if new_fail > boot_fail_threshold: 1193 error_message = "Boot failures exceed the boot failure" +\ 1194 " threshold:\n" +\ 1195 gp.sprint_var(new_fail) +\ 1196 gp.sprint_var(boot_fail_threshold) 1197 BuiltIn().fail(gp.sprint_error(error_message)) 1198