1#!/usr/bin/env python
2
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
7import os
8import imp
9import time
10import glob
11import random
12import cPickle as pickle
13
14from robot.utils import DotDict
15from robot.libraries.BuiltIn import BuiltIn
16
17from boot_data import *
18import gen_print as gp
19import gen_robot_print as grp
20import gen_robot_plug_in as grpi
21import gen_robot_valid as grv
22import gen_misc as gm
23import gen_cmd as gc
24import gen_robot_keyword as grk
25import state as st
26
27base_path = os.path.dirname(os.path.dirname(
28                            imp.find_module("gen_robot_print")[1])) +\
29    os.sep
30sys.path.append(base_path + "extended/")
31import run_keyword as rk
32
33# Setting master_pid correctly influences the behavior of plug-ins like
34# DB_Logging
35program_pid = os.getpid()
36master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid)
37
38# Set up boot data structures.
39boot_table = create_boot_table()
40valid_boot_types = create_valid_boot_list(boot_table)
41
42boot_lists = read_boot_lists()
43last_ten = []
44
45state = st.return_default_state()
46cp_setup_called = 0
47next_boot = ""
48base_tool_dir_path = os.path.normpath(os.environ.get(
49    'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep
50
51ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep
52boot_success = 0
53status_dir_path = os.environ.get('STATUS_DIR_PATH', "")
54if status_dir_path != "":
55    status_dir_path = os.path.normpath(status_dir_path) + os.sep
56default_power_on = "REST Power On"
57default_power_off = "REST Power Off"
58boot_count = 0
59
60LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
61
62
63###############################################################################
64def process_pgm_parms():
65
66    r"""
67    Process the program parameters by assigning them all to corresponding
68    globals.  Also, set some global values that depend on program parameters.
69    """
70
71    # Program parameter processing.
72    # Assign all program parms to python variables which are global to this
73    # module.
74
75    global parm_list
76    parm_list = BuiltIn().get_variable_value("${parm_list}")
77    # The following subset of parms should be processed as integers.
78    int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only',
79                'boot_fail_threshold', 'quiet', 'test_mode', 'debug']
80    for parm in parm_list:
81        if parm in int_list:
82            sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\
83                      "}\", \"0\"))"
84        else:
85            sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")"
86        cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
87        exec(cmd_buf)
88
89    global ffdc_dir_path_style
90    global boot_list
91    global boot_stack
92    global boot_results_file_path
93    global boot_results
94    global ffdc_list_file_path
95
96    if ffdc_dir_path_style == "":
97        ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0'))
98
99    # Convert these program parms to lists for easier processing..
100    boot_list = filter(None, boot_list.split(":"))
101    boot_stack = filter(None, boot_stack.split(":"))
102
103    boot_results_file_path = "/tmp/" + openbmc_nickname + ":pid_" +\
104                             str(master_pid) + ":boot_results"
105
106    if os.path.isfile(boot_results_file_path):
107        # We've been called before in this run so we'll load the saved
108        # boot_results object.
109        boot_results = pickle.load(open(boot_results_file_path, 'rb'))
110    else:
111        boot_results = boot_results(boot_table, boot_pass, boot_fail)
112
113    ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\
114        "/FFDC_FILE_LIST"
115
116###############################################################################
117
118
119###############################################################################
120def initial_plug_in_setup():
121
122    r"""
123    Initialize all plug-in environment variables which do not change for the
124    duration of the program.
125
126    """
127
128    global LOG_LEVEL
129    BuiltIn().set_log_level("NONE")
130
131    BuiltIn().set_global_variable("${master_pid}", master_pid)
132    BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
133    BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
134    BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
135    BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}",
136                                  ffdc_list_file_path)
137
138    BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}",
139                                  ffdc_dir_path_style)
140    BuiltIn().set_global_variable("${FFDC_CHECK}",
141                                  ffdc_check)
142
143    # For each program parameter, set the corresponding AUTOBOOT_ environment
144    # variable value.  Also, set an AUTOBOOT_ environment variable for every
145    # element in additional_values.
146    additional_values = ["program_pid", "master_pid", "ffdc_dir_path",
147                         "status_dir_path", "base_tool_dir_path",
148                         "ffdc_list_file_path"]
149
150    plug_in_vars = parm_list + additional_values
151
152    for var_name in plug_in_vars:
153        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
154        var_name = var_name.upper()
155        if var_value is None:
156            var_value = ""
157        os.environ["AUTOBOOT_" + var_name] = str(var_value)
158
159    BuiltIn().set_log_level(LOG_LEVEL)
160
161    # Make sure the ffdc list directory exists.
162    ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
163    if not os.path.exists(ffdc_list_dir_path):
164        os.makedirs(ffdc_list_dir_path)
165
166###############################################################################
167
168
169###############################################################################
170def plug_in_setup():
171
172    r"""
173    Initialize all changing plug-in environment variables for use by the
174    plug-in programs.
175    """
176
177    global LOG_LEVEL
178    global test_really_running
179
180    BuiltIn().set_log_level("NONE")
181
182    boot_pass, boot_fail = boot_results.return_total_pass_fail()
183    if boot_pass > 1:
184        test_really_running = 1
185    else:
186        test_really_running = 0
187
188    seconds = time.time()
189    loc_time = time.localtime(seconds)
190    time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
191
192    ffdc_prefix = openbmc_nickname + "." + time_string
193
194    BuiltIn().set_global_variable("${test_really_running}",
195                                  test_really_running)
196    BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
197    BuiltIn().set_global_variable("${boot_pass}", boot_pass)
198    BuiltIn().set_global_variable("${boot_fail}", boot_fail)
199    BuiltIn().set_global_variable("${boot_success}", boot_success)
200    BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
201
202    # For each program parameter, set the corresponding AUTOBOOT_ environment
203    # variable value.  Also, set an AUTOBOOT_ environment variable for every
204    # element in additional_values.
205    additional_values = ["boot_type_desc", "boot_success", "boot_pass",
206                         "boot_fail", "test_really_running", "ffdc_prefix"]
207
208    plug_in_vars = additional_values
209
210    for var_name in plug_in_vars:
211        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
212        var_name = var_name.upper()
213        if var_value is None:
214            var_value = ""
215        os.environ["AUTOBOOT_" + var_name] = str(var_value)
216
217    if debug:
218        shell_rc, out_buf = \
219            gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u")
220
221    BuiltIn().set_log_level(LOG_LEVEL)
222
223###############################################################################
224
225
226###############################################################################
227def setup():
228
229    r"""
230    Do general program setup tasks.
231    """
232
233    global cp_setup_called
234
235    gp.qprintn()
236
237    robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
238    repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
239    # If we can't find ssh_pw, then we don't have our repo bin in PATH.
240    shell_rc, out_buf = gc.cmd_fnc_u("which ssh_pw", quiet=1, print_output=0,
241                                     show_err=0)
242    if shell_rc != 0:
243        os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
244    # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
245    if robot_pgm_dir_path not in sys.path:
246        sys.path.append(robot_pgm_dir_path)
247        PYTHONPATH = os.environ.get("PYTHONPATH", "")
248        if PYTHONPATH == "":
249            os.environ['PYTHONPATH'] = robot_pgm_dir_path
250        else:
251            os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
252
253    validate_parms()
254
255    grp.rqprint_pgm_header()
256
257    grk.run_key("Set BMC Power Policy  RESTORE_LAST_STATE")
258
259    initial_plug_in_setup()
260
261    plug_in_setup()
262    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
263        call_point='setup')
264    if rc != 0:
265        error_message = "Plug-in setup failed.\n"
266        grp.rprint_error_report(error_message)
267        BuiltIn().fail(error_message)
268    # Setting cp_setup_called lets our Teardown know that it needs to call
269    # the cleanup plug-in call point.
270    cp_setup_called = 1
271
272    # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
273    BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
274    # FFDC_LOG_PATH is used by "FFDC" keyword.
275    BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
276
277    gp.dprint_var(boot_table, 1)
278    gp.dprint_var(boot_lists)
279
280###############################################################################
281
282
283###############################################################################
284def validate_parms():
285
286    r"""
287    Validate all program parameters.
288    """
289
290    process_pgm_parms()
291
292    gp.qprintn()
293
294    global openbmc_model
295    grv.rvalid_value("openbmc_host")
296    grv.rvalid_value("openbmc_username")
297    grv.rvalid_value("openbmc_password")
298    if os_host != "":
299        grv.rvalid_value("os_username")
300        grv.rvalid_value("os_password")
301
302    if pdu_host != "":
303        grv.rvalid_value("pdu_username")
304        grv.rvalid_value("pdu_password")
305        grv.rvalid_integer("pdu_slot_no")
306    if openbmc_serial_host != "":
307        grv.rvalid_integer("openbmc_serial_port")
308    if openbmc_model == "":
309        status, ret_values =\
310            grk.run_key_u("Get BMC System Model")
311        openbmc_model = ret_values
312        BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
313    grv.rvalid_value("openbmc_model")
314    grv.rvalid_integer("max_num_tests")
315    grv.rvalid_integer("boot_pass")
316    grv.rvalid_integer("boot_fail")
317
318    plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
319    BuiltIn().set_global_variable("${plug_in_packages_list}",
320                                  plug_in_packages_list)
321
322    grv.rvalid_value("stack_mode", valid_values=['normal', 'skip'])
323    if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
324        error_message = "You must provide either a value for either the" +\
325            " boot_list or the boot_stack parm.\n"
326        BuiltIn().fail(gp.sprint_error(error_message))
327
328    valid_boot_list(boot_list, valid_boot_types)
329    valid_boot_list(boot_stack, valid_boot_types)
330
331    selected_PDU_boots = list(set(boot_list + boot_stack) &
332                              set(boot_lists['PDU_reboot']))
333
334    if len(selected_PDU_boots) > 0 and pdu_host == "":
335        error_message = "You have selected the following boots which" +\
336                        " require a PDU host but no value for pdu_host:\n"
337        error_message += gp.sprint_var(selected_PDU_boots)
338        error_message += gp.sprint_var(pdu_host, 2)
339        BuiltIn().fail(gp.sprint_error(error_message))
340
341    return
342
343###############################################################################
344
345
346###############################################################################
347def my_get_state():
348
349    r"""
350    Get the system state plus a little bit of wrapping.
351    """
352
353    global state
354
355    req_states = ['epoch_seconds'] + st.default_req_states
356
357    gp.qprint_timen("Getting system state.")
358    if test_mode:
359        state['epoch_seconds'] = int(time.time())
360    else:
361        state = st.get_state(req_states=req_states, quiet=quiet)
362    gp.qprint_var(state)
363
364###############################################################################
365
366
367###############################################################################
368def select_boot():
369
370    r"""
371    Select a boot test to be run based on our current state and return the
372    chosen boot type.
373
374    Description of arguments:
375    state  The state of the machine.
376    """
377
378    global boot_stack
379
380    gp.qprint_timen("Selecting a boot test.")
381
382    my_get_state()
383
384    stack_popped = 0
385    if len(boot_stack) > 0:
386        stack_popped = 1
387        gp.qprint_dashes()
388        gp.qprint_var(boot_stack)
389        gp.qprint_dashes()
390        skip_boot_printed = 0
391        while len(boot_stack) > 0:
392            boot_candidate = boot_stack.pop()
393            if stack_mode == 'normal':
394                break
395            else:
396                if st.compare_states(state, boot_table[boot_candidate]['end']):
397                    if not skip_boot_printed:
398                        gp.print_var(stack_mode)
399                        gp.printn()
400                        gp.print_timen("Skipping the following boot tests" +
401                                       " which are unnecessary since their" +
402                                       " required end states match the" +
403                                       " current machine state:")
404                        skip_boot_printed = 1
405                    gp.print_var(boot_candidate)
406                    boot_candidate = ""
407        if boot_candidate == "":
408            gp.qprint_dashes()
409            gp.qprint_var(boot_stack)
410            gp.qprint_dashes()
411            return boot_candidate
412        if st.compare_states(state, boot_table[boot_candidate]['start']):
413            gp.qprint_timen("The machine state is valid for a '" +
414                            boot_candidate + "' boot test.")
415            gp.qprint_dashes()
416            gp.qprint_var(boot_stack)
417            gp.qprint_dashes()
418            return boot_candidate
419        else:
420            gp.qprint_timen("The machine state does not match the required" +
421                            " starting state for a '" + boot_candidate +
422                            "' boot test:")
423            gp.print_varx("boot_table[" + boot_candidate + "][start]",
424                          boot_table[boot_candidate]['start'], 1)
425            boot_stack.append(boot_candidate)
426            popped_boot = boot_candidate
427
428    # Loop through your list selecting a boot_candidates
429    boot_candidates = []
430    for boot_candidate in boot_list:
431        if st.compare_states(state, boot_table[boot_candidate]['start']):
432            if stack_popped:
433                if st.compare_states(boot_table[boot_candidate]['end'],
434                   boot_table[popped_boot]['start']):
435                    boot_candidates.append(boot_candidate)
436            else:
437                boot_candidates.append(boot_candidate)
438
439    if len(boot_candidates) == 0:
440        gp.qprint_timen("The user's boot list contained no boot tests" +
441                        " which are valid for the current machine state.")
442        boot_candidate = default_power_on
443        if not st.compare_states(state, boot_table[default_power_on]['start']):
444            boot_candidate = default_power_off
445        boot_candidates.append(boot_candidate)
446        gp.qprint_timen("Using default '" + boot_candidate +
447                        "' boot type to transition to valid state.")
448
449    gp.dprint_var(boot_candidates)
450
451    # Randomly select a boot from the candidate list.
452    boot = random.choice(boot_candidates)
453
454    return boot
455
456###############################################################################
457
458
459###############################################################################
460def print_last_boots():
461
462    r"""
463    Print the last ten boots done with their time stamps.
464    """
465
466    # indent 0, 90 chars wide, linefeed, char is "="
467    gp.qprint_dashes(0, 90)
468    gp.qprintn("Last 10 boots:\n")
469
470    for boot_entry in last_ten:
471        grp.rqprint(boot_entry)
472    gp.qprint_dashes(0, 90)
473
474###############################################################################
475
476
477###############################################################################
478def print_defect_report():
479
480    r"""
481    Print a defect report.
482    """
483
484    # At some point I'd like to have the 'Call FFDC Methods' return a list
485    # of files it has collected.  In that case, the following "ls" command
486    # would no longer be needed.  For now, however, glob shows the files
487    # named in FFDC_LIST_FILE_PATH so I will refrain from printing those
488    # out (so we don't see duplicates in the list).
489
490    LOG_PREFIX = BuiltIn().get_variable_value("${LOG_PREFIX}")
491
492    output = '\n'.join(glob.glob(LOG_PREFIX + '*'))
493    try:
494        ffdc_list = open(ffdc_list_file_path, 'r')
495    except IOError:
496        ffdc_list = ""
497
498    # Open ffdc_file_list for writing.  We will write a complete list of
499    # FFDC files to it for possible use by plug-ins like cp_stop_check.
500    ffdc_list_file = open(ffdc_list_file_path, 'w')
501
502    gp.qprintn()
503    # indent=0, width=90, linefeed=1, char="="
504    gp.qprint_dashes(0, 90, 1, "=")
505    gp.qprintn("Copy this data to the defect:\n")
506
507    grp.rqpvars(*parm_list)
508
509    gp.qprintn()
510
511    print_last_boots()
512    gp.qprintn()
513    gp.qprint_var(state)
514
515    gp.qprintn()
516    gp.qprintn("FFDC data files:")
517    if status_file_path != "":
518        gp.qprintn(status_file_path)
519        ffdc_list_file.write(status_file_path + "\n")
520
521    gp.qprintn(output)
522    # gp.qprintn(ffdc_list)
523    gp.qprintn()
524
525    gp.qprint_dashes(0, 90, 1, "=")
526
527    ffdc_list_file.write(output + "\n")
528    ffdc_list_file.close()
529
530###############################################################################
531
532
533###############################################################################
534def my_ffdc():
535
536    r"""
537    Collect FFDC data.
538    """
539
540    global state
541
542    plug_in_setup()
543    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
544        call_point='ffdc', stop_on_plug_in_failure=1)
545
546    AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX']
547    status, ret_values = grk.run_key_u("FFDC  ffdc_prefix=" +
548                                       AUTOBOOT_FFDC_PREFIX +
549                                       "  ffdc_function_list=" +
550                                       ffdc_function_list, ignore=1)
551    if status != 'PASS':
552        gp.print_error("Call to ffdc failed.\n")
553
554    my_get_state()
555
556    print_defect_report()
557
558###############################################################################
559
560
561###############################################################################
562def print_test_start_message(boot_keyword):
563
564    r"""
565    Print a message indicating what boot test is about to run.
566
567    Description of arguments:
568    boot_keyword  The name of the boot which is to be run
569                  (e.g. "BMC Power On").
570    """
571
572    global last_ten
573
574    doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".")
575    gp.qprint(doing_msg)
576
577    last_ten.append(doing_msg)
578
579    if len(last_ten) > 10:
580        del last_ten[0]
581
582###############################################################################
583
584
585###############################################################################
586def run_boot(boot):
587
588    r"""
589    Run the specified boot.
590
591    Description of arguments:
592    boot  The name of the boot test to be performed.
593    """
594
595    global state
596
597    print_test_start_message(boot)
598
599    plug_in_setup()
600    rc, shell_rc, failed_plug_in_name = \
601        grpi.rprocess_plug_in_packages(call_point="pre_boot")
602    if rc != 0:
603        error_message = "Plug-in failed with non-zero return code.\n" +\
604            gp.sprint_var(rc, 1)
605        BuiltIn().fail(gp.sprint_error(error_message))
606
607    if test_mode:
608        # In test mode, we'll pretend the boot worked by assigning its
609        # required end state to the default state value.
610        state = st.strip_anchor_state(boot_table[boot]['end'])
611    else:
612        # Assertion:  We trust that the state data was made fresh by the
613        # caller.
614
615        gp.qprintn()
616
617        if boot_table[boot]['method_type'] == "keyword":
618            rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''),
619                               boot_table[boot]['method'],
620                               quiet=quiet)
621
622        if boot_table[boot]['bmc_reboot']:
623            st.wait_for_comm_cycle(int(state['epoch_seconds']))
624            plug_in_setup()
625            rc, shell_rc, failed_plug_in_name = \
626                grpi.rprocess_plug_in_packages(call_point="post_reboot")
627            if rc != 0:
628                error_message = "Plug-in failed with non-zero return code.\n"
629                error_message += gp.sprint_var(rc, 1)
630                BuiltIn().fail(gp.sprint_error(error_message))
631        else:
632            match_state = st.anchor_state(state)
633            del match_state['epoch_seconds']
634            # Wait for the state to change in any way.
635            st.wait_state(match_state, wait_time=state_change_timeout,
636                          interval="3 seconds", invert=1)
637
638        gp.qprintn()
639        if boot_table[boot]['end']['chassis'] == "Off":
640            boot_timeout = power_off_timeout
641        else:
642            boot_timeout = power_on_timeout
643        st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout,
644                      interval="3 seconds")
645
646    plug_in_setup()
647    rc, shell_rc, failed_plug_in_name = \
648        grpi.rprocess_plug_in_packages(call_point="post_boot")
649    if rc != 0:
650        error_message = "Plug-in failed with non-zero return code.\n" +\
651            gp.sprint_var(rc, 1)
652        BuiltIn().fail(gp.sprint_error(error_message))
653
654###############################################################################
655
656
657###############################################################################
658def test_loop_body():
659
660    r"""
661    The main loop body for the loop in main_py.
662
663    Description of arguments:
664    boot_count  The iteration number (starts at 1).
665    """
666
667    global boot_count
668    global state
669    global next_boot
670    global boot_success
671
672    gp.qprintn()
673
674    next_boot = select_boot()
675    if next_boot == "":
676        return True
677
678    boot_count += 1
679    gp.qprint_timen("Starting boot " + str(boot_count) + ".")
680
681    # Clear the ffdc_list_file_path file.  Plug-ins may now write to it.
682    try:
683        os.remove(ffdc_list_file_path)
684    except OSError:
685        pass
686
687    cmd_buf = ["run_boot", next_boot]
688    boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
689    if boot_status == "FAIL":
690        gp.qprint(msg)
691
692    gp.qprintn()
693    if boot_status == "PASS":
694        boot_success = 1
695        gp.qprint_timen("BOOT_SUCCESS: \"" + next_boot + "\" succeeded.")
696    else:
697        boot_success = 0
698        gp.qprint_timen("BOOT_FAILED: \"" + next_boot + "\" failed.")
699
700    boot_results.update(next_boot, boot_status)
701
702    plug_in_setup()
703    # NOTE: A post_test_case call point failure is NOT counted as a boot
704    # failure.
705    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
706        call_point='post_test_case', stop_on_plug_in_failure=1)
707
708    plug_in_setup()
709    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
710        call_point='ffdc_check', shell_rc=0x00000200,
711        stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
712    if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200:
713        status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
714        if status != 'PASS':
715            gp.print_error("Call to my_ffdc failed.\n")
716
717    # We need to purge error logs between boots or they build up.
718    grk.run_key("Delete Error logs", ignore=1)
719
720    boot_results.print_report()
721    gp.qprint_timen("Finished boot " + str(boot_count) + ".")
722
723    plug_in_setup()
724    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
725        call_point='stop_check')
726    if rc != 0:
727        error_message = "Stopping as requested by user.\n"
728        grp.rprint_error_report(error_message)
729        BuiltIn().fail(error_message)
730
731    # This should help prevent ConnectionErrors.
732    grk.run_key_u("Delete All Sessions")
733
734    return True
735
736###############################################################################
737
738
739###############################################################################
740def obmc_boot_test_teardown():
741
742    r"""
743    Clean up after the Main keyword.
744    """
745
746    if cp_setup_called:
747        plug_in_setup()
748        rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
749            call_point='cleanup', stop_on_plug_in_failure=1)
750
751    # Save boot_results object to a file in case it is needed again.
752    gp.qprint_timen("Saving boot_results to the following path.")
753    gp.qprint_var(boot_results_file_path)
754    pickle.dump(boot_results, open(boot_results_file_path, 'wb'),
755                pickle.HIGHEST_PROTOCOL)
756
757###############################################################################
758
759
760###############################################################################
761def test_teardown():
762
763    r"""
764    Clean up after this test case.
765    """
766
767    gp.qprintn()
768    cmd_buf = ["Print Error",
769               "A keyword timeout occurred ending this program.\n"]
770    BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
771
772    grp.rqprint_pgm_footer()
773
774###############################################################################
775
776
777###############################################################################
778def obmc_boot_test_py(alt_boot_stack=None):
779
780    r"""
781    Do main program processing.
782    """
783
784    if alt_boot_stack is not None:
785        BuiltIn().set_global_variable("${boot_stack}", alt_boot_stack)
786
787    setup()
788
789    if ffdc_only:
790        gp.qprint_timen("Caller requested ffdc_only.")
791        grk.run_key_u("my_ffdc")
792        return
793
794    # Process caller's boot_stack.
795    while (len(boot_stack) > 0):
796        test_loop_body()
797
798    gp.qprint_timen("Finished processing stack.")
799
800    # Process caller's boot_list.
801    if len(boot_list) > 0:
802        for ix in range(1, max_num_tests + 1):
803            test_loop_body()
804
805    gp.qprint_timen("Completed all requested boot tests.")
806
807    boot_pass, boot_fail = boot_results.return_total_pass_fail()
808    if boot_fail > boot_fail_threshold:
809        error_message = "Boot failures exceed the boot failure" +\
810                        " threshold:\n" +\
811                        gp.sprint_var(boot_fail) +\
812                        gp.sprint_var(boot_fail_threshold)
813        BuiltIn().fail(gp.sprint_error(error_message))
814
815###############################################################################
816