1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import cPickle as pickle 13 14from robot.utils import DotDict 15from robot.libraries.BuiltIn import BuiltIn 16 17from boot_data import * 18import gen_print as gp 19import gen_robot_print as grp 20import gen_robot_plug_in as grpi 21import gen_robot_valid as grv 22import gen_misc as gm 23import gen_cmd as gc 24import state as st 25 26base_path = os.path.dirname(os.path.dirname( 27 imp.find_module("gen_robot_print")[1])) +\ 28 os.sep 29sys.path.append(base_path + "extended/") 30import run_keyword as rk 31 32# Program parameter processing. 33# Assign all program parms to python variables which are global to this module. 34parm_list = BuiltIn().get_variable_value("${parm_list}") 35int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only', 'quiet', 36 'test_mode', 'debug'] 37for parm in parm_list: 38 if parm in int_list: 39 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 40 "}\", \"0\"))" 41 else: 42 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 43 cmd_buf = parm + " = " + sub_cmd 44 exec(cmd_buf) 45 46if ffdc_dir_path_style == "": 47 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 48 49# Set up boot data structures. 50boot_table = create_boot_table() 51valid_boot_types = create_valid_boot_list(boot_table) 52 53# Setting master_pid correctly influences the behavior of plug-ins like 54# DB_Logging 55program_pid = os.getpid() 56master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 57 58boot_results_file_path = "/tmp/" + openbmc_nickname + ":pid_" +\ 59 str(master_pid) + ":boot_results" 60if os.path.isfile(boot_results_file_path): 61 # We've been called before in this run so we'll load the saved 62 # boot_results object. 63 boot_results = pickle.load(open(boot_results_file_path, 'rb')) 64else: 65 boot_results = boot_results(boot_table, boot_pass, boot_fail) 66 67boot_lists = read_boot_lists() 68last_ten = [] 69# Convert these program parms to more useable lists. 70boot_list = filter(None, boot_list.split(":")) 71boot_stack = filter(None, boot_stack.split(":")) 72 73state = st.return_default_state() 74cp_setup_called = 0 75next_boot = "" 76base_tool_dir_path = os.path.normpath(os.environ.get( 77 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 78ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 79ffdc_list_file_path = base_tool_dir_path + openbmc_nickname + "/FFDC_FILE_LIST" 80boot_success = 0 81status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 82if status_dir_path != "": 83 status_dir_path = os.path.normpath(status_dir_path) + os.sep 84default_power_on = "REST Power On" 85default_power_off = "REST Power Off" 86boot_count = 0 87 88LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}") 89 90 91############################################################################### 92def initial_plug_in_setup(): 93 94 r""" 95 Initialize all plug-in environment variables which do not change for the 96 duration of the program. 97 98 """ 99 100 global LOG_LEVEL 101 BuiltIn().set_log_level("NONE") 102 103 BuiltIn().set_global_variable("${master_pid}", master_pid) 104 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 105 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 106 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 107 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 108 ffdc_list_file_path) 109 110 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 111 ffdc_dir_path_style) 112 BuiltIn().set_global_variable("${FFDC_CHECK}", 113 ffdc_check) 114 115 # For each program parameter, set the corresponding AUTOBOOT_ environment 116 # variable value. Also, set an AUTOBOOT_ environment variable for every 117 # element in additional_values. 118 additional_values = ["program_pid", "master_pid", "ffdc_dir_path", 119 "status_dir_path", "base_tool_dir_path", 120 "ffdc_list_file_path"] 121 122 plug_in_vars = parm_list + additional_values 123 124 for var_name in plug_in_vars: 125 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 126 var_name = var_name.upper() 127 if var_value is None: 128 var_value = "" 129 os.environ["AUTOBOOT_" + var_name] = str(var_value) 130 131 BuiltIn().set_log_level(LOG_LEVEL) 132 133 134############################################################################### 135 136 137############################################################################### 138def plug_in_setup(): 139 140 r""" 141 Initialize all changing plug-in environment variables for use by the 142 plug-in programs. 143 """ 144 145 global LOG_LEVEL 146 global test_really_running 147 148 BuiltIn().set_log_level("NONE") 149 150 boot_pass, boot_fail = boot_results.return_total_pass_fail() 151 if boot_pass > 1: 152 test_really_running = 1 153 else: 154 test_really_running = 0 155 156 seconds = time.time() 157 loc_time = time.localtime(seconds) 158 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 159 160 ffdc_prefix = openbmc_nickname + "." + time_string 161 162 BuiltIn().set_global_variable("${test_really_running}", 163 test_really_running) 164 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 165 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 166 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 167 BuiltIn().set_global_variable("${boot_success}", boot_success) 168 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 169 170 # For each program parameter, set the corresponding AUTOBOOT_ environment 171 # variable value. Also, set an AUTOBOOT_ environment variable for every 172 # element in additional_values. 173 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 174 "boot_fail", "test_really_running", "ffdc_prefix"] 175 176 plug_in_vars = additional_values 177 178 for var_name in plug_in_vars: 179 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 180 var_name = var_name.upper() 181 if var_value is None: 182 var_value = "" 183 os.environ["AUTOBOOT_" + var_name] = str(var_value) 184 185 if debug: 186 shell_rc, out_buf = \ 187 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 188 189 BuiltIn().set_log_level(LOG_LEVEL) 190 191############################################################################### 192 193 194############################################################################### 195def setup(): 196 197 r""" 198 Do general program setup tasks. 199 """ 200 201 global cp_setup_called 202 203 grp.rqprintn() 204 205 validate_parms() 206 207 grp.rqprint_pgm_header() 208 209 cmd_buf = ["Set BMC Power Policy", "RESTORE_LAST_STATE"] 210 grp.rpissuing_keyword(cmd_buf, test_mode) 211 if not test_mode: 212 BuiltIn().run_keyword(*cmd_buf) 213 214 initial_plug_in_setup() 215 216 plug_in_setup() 217 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 218 call_point='setup') 219 if rc != 0: 220 error_message = "Plug-in setup failed.\n" 221 grp.rprint_error_report(error_message) 222 BuiltIn().fail(error_message) 223 # Setting cp_setup_called lets our Teardown know that it needs to call 224 # the cleanup plug-in call point. 225 cp_setup_called = 1 226 227 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 228 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 229 # FFDC_LOG_PATH is used by "FFDC" keyword. 230 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 231 232 grp.rdprint_var(boot_table, 1) 233 grp.rdprint_var(boot_lists) 234 235############################################################################### 236 237 238############################################################################### 239def validate_parms(): 240 241 r""" 242 Validate all program parameters. 243 """ 244 245 grp.rqprintn() 246 247 grv.rvalid_value("openbmc_host") 248 grv.rvalid_value("openbmc_username") 249 grv.rvalid_value("openbmc_password") 250 if os_host != "": 251 grv.rvalid_value("os_username") 252 grv.rvalid_value("os_password") 253 254 if pdu_host != "": 255 grv.rvalid_value("pdu_username") 256 grv.rvalid_value("pdu_password") 257 grv.rvalid_integer("pdu_slot_no") 258 if openbmc_serial_host != "": 259 grv.rvalid_integer("openbmc_serial_port") 260 grv.rvalid_integer("max_num_tests") 261 grv.rvalid_value("openbmc_model") 262 grv.rvalid_integer("boot_pass") 263 grv.rvalid_integer("boot_fail") 264 265 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 266 BuiltIn().set_global_variable("${plug_in_packages_list}", 267 plug_in_packages_list) 268 269 if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only: 270 error_message = "You must provide either a value for either the" +\ 271 " boot_list or the boot_stack parm.\n" 272 BuiltIn().fail(gp.sprint_error(error_message)) 273 274 valid_boot_list(boot_list, valid_boot_types) 275 valid_boot_list(boot_stack, valid_boot_types) 276 277 selected_PDU_boots = list(set(boot_list + boot_stack) & 278 set(boot_lists['PDU_reboot'])) 279 280 if len(selected_PDU_boots) > 0 and pdu_host == "": 281 error_message = "You have selected the following boots which" +\ 282 " require a PDU host but no value for pdu_host:\n" 283 error_message += gp.sprint_var(selected_PDU_boots) 284 error_message += gp.sprint_var(pdu_host, 2) 285 BuiltIn().fail(gp.sprint_error(error_message)) 286 287 return 288 289############################################################################### 290 291 292############################################################################### 293def my_get_state(): 294 295 r""" 296 Get the system state plus a little bit of wrapping. 297 """ 298 299 global state 300 301 req_states = ['epoch_seconds'] + st.default_req_states 302 303 grp.rqprint_timen("Getting system state.") 304 if test_mode: 305 state['epoch_seconds'] = int(time.time()) 306 else: 307 state = st.get_state(req_states=req_states, quiet=0) 308 grp.rprint_var(state) 309 310############################################################################### 311 312 313############################################################################### 314def select_boot(): 315 316 r""" 317 Select a boot test to be run based on our current state and return the 318 chosen boot type. 319 320 Description of arguments: 321 state The state of the machine. 322 """ 323 324 global boot_stack 325 326 grp.rprint_timen("Selecting a boot test.") 327 328 my_get_state() 329 330 stack_popped = 0 331 if len(boot_stack) > 0: 332 stack_popped = 1 333 grp.rprint_dashes() 334 grp.rprint_var(boot_stack) 335 grp.rprint_dashes() 336 boot_candidate = boot_stack.pop() 337 if st.compare_states(state, boot_table[boot_candidate]['start']): 338 grp.rprint_timen("The machine state is valid for a '" + 339 boot_candidate + "' boot test.") 340 grp.rprint_dashes() 341 grp.rprint_var(boot_stack) 342 grp.rprint_dashes() 343 return boot_candidate 344 else: 345 grp.rprint_timen("The machine state is not valid for a '" + 346 boot_candidate + "' boot test.") 347 boot_stack.append(boot_candidate) 348 popped_boot = boot_candidate 349 350 # Loop through your list selecting a boot_candidates 351 boot_candidates = [] 352 for boot_candidate in boot_list: 353 if st.compare_states(state, boot_table[boot_candidate]['start']): 354 if stack_popped: 355 if st.compare_states(boot_table[boot_candidate]['end'], 356 boot_table[popped_boot]['start']): 357 boot_candidates.append(boot_candidate) 358 else: 359 boot_candidates.append(boot_candidate) 360 361 if len(boot_candidates) == 0: 362 grp.rprint_timen("The user's boot list contained no boot tests" + 363 " which are valid for the current machine state.") 364 boot_candidate = default_power_on 365 if not st.compare_states(state, boot_table[default_power_on]['start']): 366 boot_candidate = default_power_off 367 boot_candidates.append(boot_candidate) 368 grp.rprint_timen("Using default '" + boot_candidate + 369 "' boot type to transtion to valid state.") 370 371 grp.rdprint_var(boot_candidates) 372 373 # Randomly select a boot from the candidate list. 374 boot = random.choice(boot_candidates) 375 376 return boot 377 378############################################################################### 379 380 381############################################################################### 382def print_last_boots(): 383 384 r""" 385 Print the last ten boots done with their time stamps. 386 """ 387 388 # indent 0, 90 chars wide, linefeed, char is "=" 389 grp.rqprint_dashes(0, 90) 390 grp.rqprintn("Last 10 boots:\n") 391 392 for boot_entry in last_ten: 393 grp.rqprint(boot_entry) 394 grp.rqprint_dashes(0, 90) 395 396############################################################################### 397 398 399############################################################################### 400def print_defect_report(): 401 402 r""" 403 Print a defect report. 404 """ 405 406 grp.rqprintn() 407 # indent=0, width=90, linefeed=1, char="=" 408 grp.rqprint_dashes(0, 90, 1, "=") 409 grp.rqprintn("Copy this data to the defect:\n") 410 411 grp.rqpvars(*parm_list) 412 413 grp.rqprintn() 414 415 print_last_boots() 416 grp.rqprintn() 417 grp.rqpvar(state) 418 419 # At some point I'd like to have the 'Call FFDC Methods' return a list 420 # of files it has collected. In that case, the following "ls" command 421 # would no longer be needed. For now, however, glob shows the files 422 # named in FFDC_LIST_FILE_PATH so I will refrain from printing those 423 # out (so we don't see duplicates in the list). 424 425 LOG_PREFIX = BuiltIn().get_variable_value("${LOG_PREFIX}") 426 427 output = '\n'.join(glob.glob(LOG_PREFIX + '*')) 428 try: 429 ffdc_list = open(ffdc_list_file_path, 'r') 430 except IOError: 431 ffdc_list = "" 432 433 grp.rqprintn() 434 grp.rqprintn("FFDC data files:") 435 if status_file_path != "": 436 grp.rqprintn(status_file_path) 437 438 grp.rqprintn(output) 439 # grp.rqprintn(ffdc_list) 440 grp.rqprintn() 441 442 grp.rqprint_dashes(0, 90, 1, "=") 443 444############################################################################### 445 446 447############################################################################### 448def my_ffdc(): 449 450 r""" 451 Collect FFDC data. 452 """ 453 454 global state 455 456 plug_in_setup() 457 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 458 call_point='ffdc', stop_on_plug_in_failure=1) 459 460 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 461 462 cmd_buf = ["FFDC", "ffdc_prefix=" + AUTOBOOT_FFDC_PREFIX] 463 grp.rpissuing_keyword(cmd_buf) 464 try: 465 BuiltIn().run_keyword_and_continue_on_failure(*cmd_buf) 466 except: 467 gp.print_error("Call to ffdc failed.\n") 468 469 my_get_state() 470 471 print_defect_report() 472 473############################################################################### 474 475 476############################################################################### 477def print_test_start_message(boot_keyword): 478 479 r""" 480 Print a message indicating what boot test is about to run. 481 482 Description of arguments: 483 boot_keyword The name of the boot which is to be run 484 (e.g. "BMC Power On"). 485 """ 486 487 global last_ten 488 489 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 490 grp.rqprint(doing_msg) 491 492 last_ten.append(doing_msg) 493 494 if len(last_ten) > 10: 495 del last_ten[0] 496 497############################################################################### 498 499 500############################################################################### 501def run_boot(boot): 502 503 r""" 504 Run the specified boot. 505 506 Description of arguments: 507 boot The name of the boot test to be performed. 508 """ 509 510 global state 511 512 print_test_start_message(boot) 513 514 plug_in_setup() 515 rc, shell_rc, failed_plug_in_name = \ 516 grpi.rprocess_plug_in_packages(call_point="pre_boot") 517 if rc != 0: 518 error_message = "Plug-in failed with non-zero return code.\n" +\ 519 gp.sprint_var(rc, 1) 520 BuiltIn().fail(gp.sprint_error(error_message)) 521 522 if test_mode: 523 # In test mode, we'll pretend the boot worked by assigning its 524 # required end state to the default state value. 525 state = st.strip_anchor_state(boot_table[boot]['end']) 526 else: 527 # Assertion: We trust that the state data was made fresh by the 528 # caller. 529 530 grp.rprintn() 531 532 if boot_table[boot]['method_type'] == "keyword": 533 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 534 boot_table[boot]['method']) 535 536 if boot_table[boot]['bmc_reboot']: 537 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 538 plug_in_setup() 539 rc, shell_rc, failed_plug_in_name = \ 540 grpi.rprocess_plug_in_packages(call_point="post_reboot") 541 if rc != 0: 542 error_message = "Plug-in failed with non-zero return code.\n" 543 error_message += gp.sprint_var(rc, 1) 544 BuiltIn().fail(gp.sprint_error(error_message)) 545 else: 546 match_state = st.anchor_state(state) 547 del match_state['epoch_seconds'] 548 # Wait for the state to change in any way. 549 st.wait_state(match_state, wait_time=state_change_timeout, 550 interval="3 seconds", invert=1) 551 552 grp.rprintn() 553 if boot_table[boot]['end']['chassis'] == "Off": 554 boot_timeout = power_off_timeout 555 else: 556 boot_timeout = power_on_timeout 557 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 558 interval="3 seconds") 559 560 plug_in_setup() 561 rc, shell_rc, failed_plug_in_name = \ 562 grpi.rprocess_plug_in_packages(call_point="post_boot") 563 if rc != 0: 564 error_message = "Plug-in failed with non-zero return code.\n" +\ 565 gp.sprint_var(rc, 1) 566 BuiltIn().fail(gp.sprint_error(error_message)) 567 568############################################################################### 569 570 571############################################################################### 572def test_loop_body(): 573 574 r""" 575 The main loop body for the loop in main_py. 576 577 Description of arguments: 578 boot_count The iteration number (starts at 1). 579 """ 580 581 global boot_count 582 global state 583 global next_boot 584 global boot_success 585 586 grp.rqprintn() 587 588 boot_count += 1 589 590 next_boot = select_boot() 591 592 grp.rqprint_timen("Starting boot " + str(boot_count) + ".") 593 594 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 595 try: 596 os.remove(ffdc_list_file_path) 597 except OSError: 598 pass 599 600 cmd_buf = ["run_boot", next_boot] 601 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 602 if boot_status == "FAIL": 603 grp.rprint(msg) 604 605 grp.rqprintn() 606 if boot_status == "PASS": 607 boot_success = 1 608 grp.rqprint_timen("BOOT_SUCCESS: \"" + next_boot + "\" succeeded.") 609 else: 610 boot_success = 0 611 grp.rqprint_timen("BOOT_FAILED: \"" + next_boot + "\" failed.") 612 613 boot_results.update(next_boot, boot_status) 614 615 plug_in_setup() 616 # NOTE: A post_test_case call point failure is NOT counted as a boot 617 # failure. 618 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 619 call_point='post_test_case', stop_on_plug_in_failure=1) 620 621 plug_in_setup() 622 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 623 call_point='ffdc_check', shell_rc=0x00000200, 624 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 625 if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200: 626 cmd_buf = ["my_ffdc"] 627 grp.rpissuing_keyword(cmd_buf) 628 try: 629 BuiltIn().run_keyword_and_continue_on_failure(*cmd_buf) 630 except: 631 gp.print_error("Call to my_ffdc failed.\n") 632 633 # We need to purge error logs between boots or they build up. 634 cmd_buf = ["Delete Error logs"] 635 grp.rpissuing_keyword(cmd_buf, test_mode) 636 BuiltIn().run_keyword(*cmd_buf) 637 638 boot_results.print_report() 639 grp.rqprint_timen("Finished boot " + str(boot_count) + ".") 640 641 plug_in_setup() 642 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 643 call_point='stop_check') 644 if rc != 0: 645 error_message = "Stopping as requested by user.\n" 646 grp.rprint_error_report(error_message) 647 BuiltIn().fail(error_message) 648 649 # This should help prevent ConnectionErrors. 650 cmd_buf = ["Delete All Sessions"] 651 grp.rpissuing_keyword(cmd_buf, test_mode) 652 BuiltIn().run_keyword(*cmd_buf) 653 654 return True 655 656############################################################################### 657 658 659############################################################################### 660def main_keyword_teardown(): 661 662 r""" 663 Clean up after the Main keyword. 664 """ 665 666 if cp_setup_called: 667 plug_in_setup() 668 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 669 call_point='cleanup', stop_on_plug_in_failure=1) 670 671 # Save boot_results object to a file in case it is needed again. 672 grp.rprint_timen("Saving boot_results to the following path.") 673 grp.rprint_var(boot_results_file_path) 674 pickle.dump(boot_results, open(boot_results_file_path, 'wb'), 675 pickle.HIGHEST_PROTOCOL) 676 677############################################################################### 678 679 680############################################################################### 681def test_teardown(): 682 683 r""" 684 Clean up after this test case. 685 """ 686 687 gp.qprintn() 688 cmd_buf = ["Print Error", 689 "A keyword timeout occurred ending this program.\n"] 690 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 691 692############################################################################### 693 694 695############################################################################### 696def main_py(): 697 698 r""" 699 Do main program processing. 700 """ 701 702 setup() 703 704 if ffdc_only: 705 gp.qprint_timen("Caller requested ffdc_only.") 706 cmd_buf = ["my_ffdc"] 707 grp.rpissuing_keyword(cmd_buf) 708 try: 709 BuiltIn().run_keyword_and_continue_on_failure(*cmd_buf) 710 except: 711 gp.print_error("Call to my_ffdc failed.\n") 712 713 # Process caller's boot_stack. 714 while (len(boot_stack) > 0): 715 test_loop_body() 716 717 grp.rprint_timen("Finished processing stack.") 718 719 # Process caller's boot_list. 720 if len(boot_list) > 0: 721 for ix in range(1, max_num_tests + 1): 722 test_loop_body() 723 724 grp.rqprint_timen("Completed all requested boot tests.") 725 726############################################################################### 727