1#!/usr/bin/env python 2 3r""" 4This module is the python counterpart to obmc_boot_test. 5""" 6 7import os 8import imp 9import time 10import glob 11import random 12import cPickle as pickle 13 14from robot.utils import DotDict 15from robot.libraries.BuiltIn import BuiltIn 16 17from boot_data import * 18import gen_print as gp 19import gen_robot_print as grp 20import gen_robot_plug_in as grpi 21import gen_robot_valid as grv 22import gen_misc as gm 23import gen_cmd as gc 24import state as st 25 26base_path = os.path.dirname(os.path.dirname( 27 imp.find_module("gen_robot_print")[1])) +\ 28 os.sep 29sys.path.append(base_path + "extended/") 30import run_keyword as rk 31 32# Program parameter processing. 33# Assign all program parms to python variables which are global to this module. 34parm_list = BuiltIn().get_variable_value("${parm_list}") 35int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'quiet', 'test_mode', 36 'debug'] 37for parm in parm_list: 38 if parm in int_list: 39 sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\ 40 "}\", \"0\"))" 41 else: 42 sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")" 43 cmd_buf = parm + " = " + sub_cmd 44 exec(cmd_buf) 45 46if ffdc_dir_path_style == "": 47 ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0')) 48 49# Set up boot data structures. 50boot_table = create_boot_table() 51valid_boot_types = create_valid_boot_list(boot_table) 52 53# Setting master_pid correctly influences the behavior of plug-ins like 54# DB_Logging 55program_pid = os.getpid() 56master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid) 57 58boot_results_file_path = "/tmp/" + openbmc_nickname + ":pid_" +\ 59 str(master_pid) + ":boot_results" 60if os.path.isfile(boot_results_file_path): 61 # We've been called before in this run so we'll load the saved 62 # boot_results object. 63 boot_results = pickle.load(open(boot_results_file_path, 'rb')) 64else: 65 boot_results = boot_results(boot_table, boot_pass, boot_fail) 66 67boot_lists = read_boot_lists() 68last_ten = [] 69# Convert these program parms to more useable lists. 70boot_list = filter(None, boot_list.split(":")) 71boot_stack = filter(None, boot_stack.split(":")) 72 73state = st.return_default_state() 74cp_setup_called = 0 75next_boot = "" 76base_tool_dir_path = os.path.normpath(os.environ.get( 77 'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep 78ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep 79ffdc_list_file_path = base_tool_dir_path + openbmc_nickname + "/FFDC_FILE_LIST" 80boot_success = 0 81status_dir_path = os.environ.get('STATUS_DIR_PATH', "") 82if status_dir_path != "": 83 status_dir_path = os.path.normpath(status_dir_path) + os.sep 84default_power_on = "REST Power On" 85default_power_off = "REST Power Off" 86boot_count = 0 87 88 89############################################################################### 90def plug_in_setup(): 91 92 r""" 93 Initialize all plug-in environment variables for use by the plug-in 94 programs. 95 """ 96 97 boot_pass, boot_fail = boot_results.return_total_pass_fail() 98 if boot_pass > 1: 99 test_really_running = 1 100 else: 101 test_really_running = 0 102 103 seconds = time.time() 104 loc_time = time.localtime(seconds) 105 time_string = time.strftime("%y%m%d.%H%M%S.", loc_time) 106 107 ffdc_prefix = openbmc_nickname + "." + time_string 108 109 BuiltIn().set_global_variable("${test_really_running}", 110 test_really_running) 111 BuiltIn().set_global_variable("${boot_type_desc}", next_boot) 112 BuiltIn().set_global_variable("${master_pid}", master_pid) 113 BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path) 114 BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path) 115 BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path) 116 BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}", 117 ffdc_list_file_path) 118 BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}", 119 ffdc_dir_path_style) 120 BuiltIn().set_global_variable("${FFDC_CHECK}", 121 ffdc_check) 122 BuiltIn().set_global_variable("${boot_pass}", boot_pass) 123 BuiltIn().set_global_variable("${boot_fail}", boot_fail) 124 BuiltIn().set_global_variable("${boot_success}", boot_success) 125 BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix) 126 127 # For each program parameter, set the corresponding AUTOBOOT_ environment 128 # variable value. Also, set an AUTOBOOT_ environment variable for every 129 # element in additional_values. 130 additional_values = ["boot_type_desc", "boot_success", "boot_pass", 131 "boot_fail", "test_really_running", "program_pid", 132 "master_pid", "ffdc_prefix", "ffdc_dir_path", 133 "status_dir_path", "base_tool_dir_path", 134 "ffdc_list_file_path"] 135 136 plug_in_vars = parm_list + additional_values 137 138 for var_name in plug_in_vars: 139 var_value = BuiltIn().get_variable_value("${" + var_name + "}") 140 var_name = var_name.upper() 141 if var_value is None: 142 var_value = "" 143 os.environ["AUTOBOOT_" + var_name] = str(var_value) 144 145 if debug: 146 shell_rc, out_buf = \ 147 gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u") 148 149############################################################################### 150 151 152############################################################################### 153def setup(): 154 155 r""" 156 Do general program setup tasks. 157 """ 158 159 global cp_setup_called 160 161 grp.rqprintn() 162 163 validate_parms() 164 165 grp.rqprint_pgm_header() 166 167 plug_in_setup() 168 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 169 call_point='setup') 170 if rc != 0: 171 error_message = "Plug-in setup failed.\n" 172 grp.rprint_error_report(error_message) 173 BuiltIn().fail(error_message) 174 # Setting cp_setup_called lets our Teardown know that it needs to call 175 # the cleanup plug-in call point. 176 cp_setup_called = 1 177 178 # Keyword "FFDC" will fail if TEST_MESSAGE is not set. 179 BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}") 180 181 grp.rdprint_var(boot_table, 1) 182 grp.rdprint_var(boot_lists) 183 184############################################################################### 185 186 187############################################################################### 188def validate_parms(): 189 190 r""" 191 Validate all program parameters. 192 """ 193 194 grp.rqprintn() 195 196 grv.rvalid_value("openbmc_host") 197 grv.rvalid_value("openbmc_username") 198 grv.rvalid_value("openbmc_password") 199 if os_host != "": 200 grv.rvalid_value("os_username") 201 grv.rvalid_value("os_password") 202 203 if pdu_host != "": 204 grv.rvalid_value("pdu_username") 205 grv.rvalid_value("pdu_password") 206 grv.rvalid_integer("pdu_slot_no") 207 if openbmc_serial_host != "": 208 grv.rvalid_integer("openbmc_serial_port") 209 grv.rvalid_integer("max_num_tests") 210 grv.rvalid_value("openbmc_model") 211 grv.rvalid_integer("boot_pass") 212 grv.rvalid_integer("boot_fail") 213 214 plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths) 215 BuiltIn().set_global_variable("${plug_in_packages_list}", 216 plug_in_packages_list) 217 218 if len(boot_list) == 0 and len(boot_stack) == 0: 219 error_message = "You must provide either a value for either the" +\ 220 " boot_list or the boot_stack parm.\n" 221 BuiltIn().fail(gp.sprint_error(error_message)) 222 223 valid_boot_list(boot_list, valid_boot_types) 224 valid_boot_list(boot_stack, valid_boot_types) 225 226 return 227 228############################################################################### 229 230 231############################################################################### 232def my_get_state(): 233 234 r""" 235 Get the system state plus a little bit of wrapping. 236 """ 237 238 global state 239 240 req_states = ['epoch_seconds'] + st.default_req_states 241 242 grp.rqprint_timen("Getting system state.") 243 if test_mode: 244 state['epoch_seconds'] = int(time.time()) 245 else: 246 state = st.get_state(req_states=req_states, quiet=0) 247 grp.rprint_var(state) 248 249############################################################################### 250 251 252############################################################################### 253def select_boot(): 254 255 r""" 256 Select a boot test to be run based on our current state and return the 257 chosen boot type. 258 259 Description of arguments: 260 state The state of the machine. 261 """ 262 263 global boot_stack 264 265 grp.rprint_timen("Selecting a boot test.") 266 267 my_get_state() 268 269 stack_popped = 0 270 if len(boot_stack) > 0: 271 stack_popped = 1 272 grp.rprint_dashes() 273 grp.rprint_var(boot_stack) 274 grp.rprint_dashes() 275 boot_candidate = boot_stack.pop() 276 if st.compare_states(state, boot_table[boot_candidate]['start']): 277 grp.rprint_timen("The machine state is valid for a '" + 278 boot_candidate + "' boot test.") 279 grp.rprint_dashes() 280 grp.rprint_var(boot_stack) 281 grp.rprint_dashes() 282 return boot_candidate 283 else: 284 grp.rprint_timen("The machine state is not valid for a '" + 285 boot_candidate + "' boot test.") 286 boot_stack.append(boot_candidate) 287 popped_boot = boot_candidate 288 289 # Loop through your list selecting a boot_candidates 290 boot_candidates = [] 291 for boot_candidate in boot_list: 292 if st.compare_states(state, boot_table[boot_candidate]['start']): 293 if stack_popped: 294 if st.compare_states(boot_table[boot_candidate]['end'], 295 boot_table[popped_boot]['start']): 296 boot_candidates.append(boot_candidate) 297 else: 298 boot_candidates.append(boot_candidate) 299 300 if len(boot_candidates) == 0: 301 grp.rprint_timen("The user's boot list contained no boot tests" + 302 " which are valid for the current machine state.") 303 boot_candidate = default_power_on 304 if not st.compare_states(state, boot_table[default_power_on]['start']): 305 boot_candidate = default_power_off 306 boot_candidates.append(boot_candidate) 307 grp.rprint_timen("Using default '" + boot_candidate + 308 "' boot type to transtion to valid state.") 309 310 grp.rdprint_var(boot_candidates) 311 312 # Randomly select a boot from the candidate list. 313 boot = random.choice(boot_candidates) 314 315 return boot 316 317############################################################################### 318 319 320############################################################################### 321def print_last_boots(): 322 323 r""" 324 Print the last ten boots done with their time stamps. 325 """ 326 327 # indent 0, 90 chars wide, linefeed, char is "=" 328 grp.rqprint_dashes(0, 90) 329 grp.rqprintn("Last 10 boots:\n") 330 331 for boot_entry in last_ten: 332 grp.rqprint(boot_entry) 333 grp.rqprint_dashes(0, 90) 334 335############################################################################### 336 337 338############################################################################### 339def print_defect_report(): 340 341 r""" 342 Print a defect report. 343 """ 344 345 grp.rqprintn() 346 # indent=0, width=90, linefeed=1, char="=" 347 grp.rqprint_dashes(0, 90, 1, "=") 348 grp.rqprintn("Copy this data to the defect:\n") 349 350 grp.rqpvars(*parm_list) 351 352 grp.rqprintn() 353 354 print_last_boots() 355 grp.rqprintn() 356 grp.rqpvar(state) 357 358 # At some point I'd like to have the 'Call FFDC Methods' return a list 359 # of files it has collected. In that case, the following "ls" command 360 # would no longer be needed. For now, however, glob shows the files 361 # named in FFDC_LIST_FILE_PATH so I will refrain from printing those 362 # out (so we don't see duplicates in the list). 363 364 LOG_PREFIX = BuiltIn().get_variable_value("${LOG_PREFIX}") 365 366 output = '\n'.join(glob.glob(LOG_PREFIX + '*')) 367 try: 368 ffdc_list = open(ffdc_list_file_path, 'r') 369 except IOError: 370 ffdc_list = "" 371 372 grp.rqprintn() 373 grp.rqprintn("FFDC data files:") 374 if status_file_path != "": 375 grp.rqprintn(status_file_path) 376 377 grp.rqprintn(output) 378 # grp.rqprintn(ffdc_list) 379 grp.rqprintn() 380 381 grp.rqprint_dashes(0, 90, 1, "=") 382 383############################################################################### 384 385 386############################################################################### 387def my_ffdc(): 388 389 r""" 390 Collect FFDC data. 391 """ 392 393 global state 394 395 plug_in_setup() 396 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 397 call_point='ffdc', stop_on_plug_in_failure=1) 398 399 AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX'] 400 401 # FFDC_LOG_PATH is used by "FFDC" keyword. 402 BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path) 403 404 cmd_buf = ["FFDC", "ffdc_prefix=" + AUTOBOOT_FFDC_PREFIX] 405 grp.rpissuing_keyword(cmd_buf) 406 BuiltIn().run_keyword_and_continue_on_failure(*cmd_buf) 407 408 my_get_state() 409 410 print_defect_report() 411 412############################################################################### 413 414 415############################################################################### 416def print_test_start_message(boot_keyword): 417 418 r""" 419 Print a message indicating what boot test is about to run. 420 421 Description of arguments: 422 boot_keyword The name of the boot which is to be run 423 (e.g. "BMC Power On"). 424 """ 425 426 global last_ten 427 428 doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".") 429 grp.rqprint(doing_msg) 430 431 last_ten.append(doing_msg) 432 433 if len(last_ten) > 10: 434 del last_ten[0] 435 436############################################################################### 437 438 439############################################################################### 440def run_boot(boot): 441 442 r""" 443 Run the specified boot. 444 445 Description of arguments: 446 boot The name of the boot test to be performed. 447 """ 448 449 global state 450 451 print_test_start_message(boot) 452 453 plug_in_setup() 454 rc, shell_rc, failed_plug_in_name = \ 455 grpi.rprocess_plug_in_packages(call_point="pre_boot") 456 if rc != 0: 457 error_message = "Plug-in failed with non-zero return code.\n" +\ 458 gp.sprint_var(rc, 1) 459 BuiltIn().fail(gp.sprint_error(error_message)) 460 461 if test_mode: 462 # In test mode, we'll pretend the boot worked by assigning its 463 # required end state to the default state value. 464 state = st.strip_anchor_state(boot_table[boot]['end']) 465 else: 466 # Assertion: We trust that the state data was made fresh by the 467 # caller. 468 469 grp.rprintn() 470 471 if boot_table[boot]['method_type'] == "keyword": 472 rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''), 473 boot_table[boot]['method']) 474 475 if boot_table[boot]['bmc_reboot']: 476 st.wait_for_comm_cycle(int(state['epoch_seconds'])) 477 plug_in_setup() 478 rc, shell_rc, failed_plug_in_name = \ 479 grpi.rprocess_plug_in_packages(call_point="post_reboot") 480 if rc != 0: 481 error_message = "Plug-in failed with non-zero return code.\n" 482 error_message += gp.sprint_var(rc, 1) 483 BuiltIn().fail(gp.sprint_error(error_message)) 484 else: 485 match_state = st.anchor_state(state) 486 del match_state['epoch_seconds'] 487 # Wait for the state to change in any way. 488 st.wait_state(match_state, wait_time=state_change_timeout, 489 interval="3 seconds", invert=1) 490 491 grp.rprintn() 492 if boot_table[boot]['end']['chassis'] == "Off": 493 boot_timeout = power_off_timeout 494 else: 495 boot_timeout = power_on_timeout 496 st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout, 497 interval="3 seconds") 498 499 plug_in_setup() 500 rc, shell_rc, failed_plug_in_name = \ 501 grpi.rprocess_plug_in_packages(call_point="post_boot") 502 if rc != 0: 503 error_message = "Plug-in failed with non-zero return code.\n" +\ 504 gp.sprint_var(rc, 1) 505 BuiltIn().fail(gp.sprint_error(error_message)) 506 507############################################################################### 508 509 510############################################################################### 511def test_loop_body(): 512 513 r""" 514 The main loop body for the loop in main_py. 515 516 Description of arguments: 517 boot_count The iteration number (starts at 1). 518 """ 519 520 global boot_count 521 global state 522 global next_boot 523 global boot_success 524 525 grp.rqprintn() 526 527 boot_count += 1 528 529 next_boot = select_boot() 530 531 grp.rqprint_timen("Starting boot " + str(boot_count) + ".") 532 533 # Clear the ffdc_list_file_path file. Plug-ins may now write to it. 534 try: 535 os.remove(ffdc_list_file_path) 536 except OSError: 537 pass 538 539 cmd_buf = ["run_boot", next_boot] 540 boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf) 541 if boot_status == "FAIL": 542 grp.rprint(msg) 543 544 grp.rqprintn() 545 if boot_status == "PASS": 546 boot_success = 1 547 grp.rqprint_timen("BOOT_SUCCESS: \"" + next_boot + "\" succeeded.") 548 else: 549 boot_success = 0 550 grp.rqprint_timen("BOOT_FAILED: \"" + next_boot + "\" failed.") 551 552 boot_results.update(next_boot, boot_status) 553 554 plug_in_setup() 555 # NOTE: A post_test_case call point failure is NOT counted as a boot 556 # failure. 557 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 558 call_point='post_test_case', stop_on_plug_in_failure=1) 559 560 plug_in_setup() 561 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 562 call_point='ffdc_check', shell_rc=0x00000200, 563 stop_on_plug_in_failure=1, stop_on_non_zero_rc=1) 564 if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200: 565 cmd_buf = ["my_ffdc"] 566 grp.rpissuing_keyword(cmd_buf) 567 BuiltIn().run_keyword_and_continue_on_failure(*cmd_buf) 568 569 boot_results.print_report() 570 grp.rqprint_timen("Finished boot " + str(boot_count) + ".") 571 572 plug_in_setup() 573 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 574 call_point='stop_check') 575 if rc != 0: 576 error_message = "Stopping as requested by user.\n" 577 grp.rprint_error_report(error_message) 578 BuiltIn().fail(error_message) 579 580 return True 581 582############################################################################### 583 584 585############################################################################### 586def main_keyword_teardown(): 587 588 r""" 589 Clean up after the Main keyword. 590 """ 591 592 if cp_setup_called: 593 plug_in_setup() 594 rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages( 595 call_point='cleanup', stop_on_plug_in_failure=1) 596 597 # Save boot_results object to a file in case it is needed again. 598 grp.rprint_timen("Saving boot_results to the following path.") 599 grp.rprint_var(boot_results_file_path) 600 pickle.dump(boot_results, open(boot_results_file_path, 'wb'), 601 pickle.HIGHEST_PROTOCOL) 602 603############################################################################### 604 605 606############################################################################### 607def test_teardown(): 608 609 r""" 610 Clean up after this test case. 611 """ 612 613 gp.qprintn() 614 cmd_buf = ["Print Error", 615 "A keyword timeout occurred ending this program.\n"] 616 BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf) 617 618############################################################################### 619 620 621############################################################################### 622def main_py(): 623 624 r""" 625 Do main program processing. 626 """ 627 628 setup() 629 630 # Process caller's boot_stack. 631 while (len(boot_stack) > 0): 632 test_loop_body() 633 634 grp.rprint_timen("Finished processing stack.") 635 636 # Process caller's boot_list. 637 if len(boot_list) > 0: 638 for ix in range(1, max_num_tests + 1): 639 test_loop_body() 640 641 grp.rqprint_timen("Completed all requested boot tests.") 642 643############################################################################### 644