1#!/usr/bin/env python
2
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
7import os
8import imp
9import time
10import glob
11import random
12import re
13import cPickle as pickle
14
15from robot.utils import DotDict
16from robot.libraries.BuiltIn import BuiltIn
17
18from boot_data import *
19import gen_print as gp
20import gen_robot_print as grp
21import gen_robot_plug_in as grpi
22import gen_robot_valid as grv
23import gen_misc as gm
24import gen_cmd as gc
25import gen_robot_keyword as grk
26import state as st
27
28base_path = os.path.dirname(os.path.dirname(
29                            imp.find_module("gen_robot_print")[1])) +\
30    os.sep
31sys.path.append(base_path + "extended/")
32import run_keyword as rk
33
34# Setting master_pid correctly influences the behavior of plug-ins like
35# DB_Logging
36program_pid = os.getpid()
37master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid)
38
39# Set up boot data structures.
40boot_table = create_boot_table()
41valid_boot_types = create_valid_boot_list(boot_table)
42
43boot_lists = read_boot_lists()
44last_ten = []
45
46state = st.return_default_state()
47cp_setup_called = 0
48next_boot = ""
49base_tool_dir_path = os.path.normpath(os.environ.get(
50    'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep
51
52ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep
53boot_success = 0
54status_dir_path = os.environ.get('STATUS_DIR_PATH', "")
55if status_dir_path != "":
56    status_dir_path = os.path.normpath(status_dir_path) + os.sep
57default_power_on = "REST Power On"
58default_power_off = "REST Power Off"
59boot_count = 0
60
61LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
62
63
64###############################################################################
65def process_host(host,
66                 host_var_name=""):
67
68    r"""
69    Process a host by getting the associated host name and IP address and
70    setting them in global variables.
71
72    If the caller does not pass the host_var_name, this function will try to
73    figure out the name of the variable used by the caller for the host parm.
74    Callers are advised to explicitly specify the host_var_name when calling
75    with an exec command.  In such cases, the get_arg_name cannot figure out
76    the host variable name.
77
78    This function will then create similar global variable names by
79    removing "_host" and appending "_host_name" or "_ip" to the host variable
80    name.
81
82    Example:
83
84    If a call is made like this:
85    process_host(openbmc_host)
86
87    Global variables openbmc_host_name and openbmc_ip will be set.
88
89    Description of argument(s):
90    host           A host name or IP.  The name of the variable used should
91                   have a suffix of "_host".
92    host_var_name  The name of the variable being used as the host parm.
93    """
94
95    if host_var_name == "":
96        host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
97
98    host_name_var_name = re.sub("host", "host_name", host_var_name)
99    ip_var_name = re.sub("host", "ip", host_var_name)
100    cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\
101        host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\
102        host + "')"
103    exec(cmd_buf)
104
105###############################################################################
106
107
108###############################################################################
109def process_pgm_parms():
110
111    r"""
112    Process the program parameters by assigning them all to corresponding
113    globals.  Also, set some global values that depend on program parameters.
114    """
115
116    # Program parameter processing.
117    # Assign all program parms to python variables which are global to this
118    # module.
119
120    global parm_list
121    parm_list = BuiltIn().get_variable_value("${parm_list}")
122    # The following subset of parms should be processed as integers.
123    int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only',
124                'boot_fail_threshold', 'quiet', 'test_mode', 'debug']
125    for parm in parm_list:
126        if parm in int_list:
127            sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\
128                      "}\", \"0\"))"
129        else:
130            sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")"
131        cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
132        exec(cmd_buf)
133        if re.match(r".*_host$", parm):
134            cmd_buf = "process_host(" + parm + ", '" + parm + "')"
135            exec(cmd_buf)
136        if re.match(r".*_password$", parm):
137            # Register the value of any parm whose name ends in _password.
138            # This will cause the print functions to replace passwords with
139            # asterisks in the output.
140            cmd_buf = "gp.register_passwords(" + parm + ")"
141            exec(cmd_buf)
142
143    global ffdc_dir_path_style
144    global boot_list
145    global boot_stack
146    global boot_results_file_path
147    global boot_results
148    global ffdc_list_file_path
149
150    if ffdc_dir_path_style == "":
151        ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0'))
152
153    # Convert these program parms to lists for easier processing..
154    boot_list = filter(None, boot_list.split(":"))
155    boot_stack = filter(None, boot_stack.split(":"))
156
157    boot_results_file_path = "/tmp/" + openbmc_nickname + ":pid_" +\
158                             str(master_pid) + ":boot_results"
159
160    if os.path.isfile(boot_results_file_path):
161        # We've been called before in this run so we'll load the saved
162        # boot_results object.
163        boot_results = pickle.load(open(boot_results_file_path, 'rb'))
164    else:
165        boot_results = boot_results(boot_table, boot_pass, boot_fail)
166
167    ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\
168        "/FFDC_FILE_LIST"
169
170###############################################################################
171
172
173###############################################################################
174def initial_plug_in_setup():
175
176    r"""
177    Initialize all plug-in environment variables which do not change for the
178    duration of the program.
179
180    """
181
182    global LOG_LEVEL
183    BuiltIn().set_log_level("NONE")
184
185    BuiltIn().set_global_variable("${master_pid}", master_pid)
186    BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
187    BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
188    BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
189    BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}",
190                                  ffdc_list_file_path)
191
192    BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}",
193                                  ffdc_dir_path_style)
194    BuiltIn().set_global_variable("${FFDC_CHECK}",
195                                  ffdc_check)
196
197    # For each program parameter, set the corresponding AUTOBOOT_ environment
198    # variable value.  Also, set an AUTOBOOT_ environment variable for every
199    # element in additional_values.
200    additional_values = ["program_pid", "master_pid", "ffdc_dir_path",
201                         "status_dir_path", "base_tool_dir_path",
202                         "ffdc_list_file_path"]
203
204    plug_in_vars = parm_list + additional_values
205
206    for var_name in plug_in_vars:
207        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
208        var_name = var_name.upper()
209        if var_value is None:
210            var_value = ""
211        os.environ["AUTOBOOT_" + var_name] = str(var_value)
212
213    BuiltIn().set_log_level(LOG_LEVEL)
214
215    # Make sure the ffdc list directory exists.
216    ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
217    if not os.path.exists(ffdc_list_dir_path):
218        os.makedirs(ffdc_list_dir_path)
219
220###############################################################################
221
222
223###############################################################################
224def plug_in_setup():
225
226    r"""
227    Initialize all changing plug-in environment variables for use by the
228    plug-in programs.
229    """
230
231    global LOG_LEVEL
232    global test_really_running
233
234    BuiltIn().set_log_level("NONE")
235
236    boot_pass, boot_fail = boot_results.return_total_pass_fail()
237    if boot_pass > 1:
238        test_really_running = 1
239    else:
240        test_really_running = 0
241
242    seconds = time.time()
243    loc_time = time.localtime(seconds)
244    time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
245
246    ffdc_prefix = openbmc_nickname + "." + time_string
247
248    BuiltIn().set_global_variable("${test_really_running}",
249                                  test_really_running)
250    BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
251    BuiltIn().set_global_variable("${boot_pass}", boot_pass)
252    BuiltIn().set_global_variable("${boot_fail}", boot_fail)
253    BuiltIn().set_global_variable("${boot_success}", boot_success)
254    BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
255
256    # For each program parameter, set the corresponding AUTOBOOT_ environment
257    # variable value.  Also, set an AUTOBOOT_ environment variable for every
258    # element in additional_values.
259    additional_values = ["boot_type_desc", "boot_success", "boot_pass",
260                         "boot_fail", "test_really_running", "ffdc_prefix"]
261
262    plug_in_vars = additional_values
263
264    for var_name in plug_in_vars:
265        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
266        var_name = var_name.upper()
267        if var_value is None:
268            var_value = ""
269        os.environ["AUTOBOOT_" + var_name] = str(var_value)
270
271    if debug:
272        shell_rc, out_buf = \
273            gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u")
274
275    BuiltIn().set_log_level(LOG_LEVEL)
276
277###############################################################################
278
279
280###############################################################################
281def setup():
282
283    r"""
284    Do general program setup tasks.
285    """
286
287    global cp_setup_called
288
289    gp.qprintn()
290
291    robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
292    repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
293    # If we can't find ssh_pw, then we don't have our repo bin in PATH.
294    shell_rc, out_buf = gc.cmd_fnc_u("which ssh_pw", quiet=1, print_output=0,
295                                     show_err=0)
296    if shell_rc != 0:
297        os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
298    # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
299    if robot_pgm_dir_path not in sys.path:
300        sys.path.append(robot_pgm_dir_path)
301        PYTHONPATH = os.environ.get("PYTHONPATH", "")
302        if PYTHONPATH == "":
303            os.environ['PYTHONPATH'] = robot_pgm_dir_path
304        else:
305            os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
306
307    validate_parms()
308
309    grp.rqprint_pgm_header()
310
311    grk.run_key("Set BMC Power Policy  RESTORE_LAST_STATE")
312
313    initial_plug_in_setup()
314
315    plug_in_setup()
316    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
317        call_point='setup')
318    if rc != 0:
319        error_message = "Plug-in setup failed.\n"
320        grp.rprint_error_report(error_message)
321        BuiltIn().fail(error_message)
322    # Setting cp_setup_called lets our Teardown know that it needs to call
323    # the cleanup plug-in call point.
324    cp_setup_called = 1
325
326    # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
327    BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
328    # FFDC_LOG_PATH is used by "FFDC" keyword.
329    BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
330
331    gp.dprint_var(boot_table, 1)
332    gp.dprint_var(boot_lists)
333
334###############################################################################
335
336
337###############################################################################
338def validate_parms():
339
340    r"""
341    Validate all program parameters.
342    """
343
344    process_pgm_parms()
345
346    gp.qprintn()
347
348    global openbmc_model
349    grv.rvalid_value("openbmc_host")
350    grv.rvalid_value("openbmc_username")
351    grv.rvalid_value("openbmc_password")
352    if os_host != "":
353        grv.rvalid_value("os_username")
354        grv.rvalid_value("os_password")
355
356    if pdu_host != "":
357        grv.rvalid_value("pdu_username")
358        grv.rvalid_value("pdu_password")
359        grv.rvalid_integer("pdu_slot_no")
360    if openbmc_serial_host != "":
361        grv.rvalid_integer("openbmc_serial_port")
362    if openbmc_model == "":
363        status, ret_values =\
364            grk.run_key_u("Get BMC System Model")
365        openbmc_model = ret_values
366        BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
367    grv.rvalid_value("openbmc_model")
368    grv.rvalid_integer("max_num_tests")
369    grv.rvalid_integer("boot_pass")
370    grv.rvalid_integer("boot_fail")
371
372    plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
373    BuiltIn().set_global_variable("${plug_in_packages_list}",
374                                  plug_in_packages_list)
375
376    grv.rvalid_value("stack_mode", valid_values=['normal', 'skip'])
377    if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
378        error_message = "You must provide either a value for either the" +\
379            " boot_list or the boot_stack parm.\n"
380        BuiltIn().fail(gp.sprint_error(error_message))
381
382    valid_boot_list(boot_list, valid_boot_types)
383    valid_boot_list(boot_stack, valid_boot_types)
384
385    selected_PDU_boots = list(set(boot_list + boot_stack) &
386                              set(boot_lists['PDU_reboot']))
387
388    if len(selected_PDU_boots) > 0 and pdu_host == "":
389        error_message = "You have selected the following boots which" +\
390                        " require a PDU host but no value for pdu_host:\n"
391        error_message += gp.sprint_var(selected_PDU_boots)
392        error_message += gp.sprint_var(pdu_host, 2)
393        BuiltIn().fail(gp.sprint_error(error_message))
394
395    return
396
397###############################################################################
398
399
400###############################################################################
401def my_get_state():
402
403    r"""
404    Get the system state plus a little bit of wrapping.
405    """
406
407    global state
408
409    req_states = ['epoch_seconds'] + st.default_req_states
410
411    gp.qprint_timen("Getting system state.")
412    if test_mode:
413        state['epoch_seconds'] = int(time.time())
414    else:
415        state = st.get_state(req_states=req_states, quiet=quiet)
416    gp.qprint_var(state)
417
418###############################################################################
419
420
421###############################################################################
422def select_boot():
423
424    r"""
425    Select a boot test to be run based on our current state and return the
426    chosen boot type.
427
428    Description of arguments:
429    state  The state of the machine.
430    """
431
432    global boot_stack
433
434    gp.qprint_timen("Selecting a boot test.")
435
436    my_get_state()
437
438    stack_popped = 0
439    if len(boot_stack) > 0:
440        stack_popped = 1
441        gp.qprint_dashes()
442        gp.qprint_var(boot_stack)
443        gp.qprint_dashes()
444        skip_boot_printed = 0
445        while len(boot_stack) > 0:
446            boot_candidate = boot_stack.pop()
447            if stack_mode == 'normal':
448                break
449            else:
450                if st.compare_states(state, boot_table[boot_candidate]['end']):
451                    if not skip_boot_printed:
452                        gp.print_var(stack_mode)
453                        gp.printn()
454                        gp.print_timen("Skipping the following boot tests" +
455                                       " which are unnecessary since their" +
456                                       " required end states match the" +
457                                       " current machine state:")
458                        skip_boot_printed = 1
459                    gp.print_var(boot_candidate)
460                    boot_candidate = ""
461        if boot_candidate == "":
462            gp.qprint_dashes()
463            gp.qprint_var(boot_stack)
464            gp.qprint_dashes()
465            return boot_candidate
466        if st.compare_states(state, boot_table[boot_candidate]['start']):
467            gp.qprint_timen("The machine state is valid for a '" +
468                            boot_candidate + "' boot test.")
469            gp.qprint_dashes()
470            gp.qprint_var(boot_stack)
471            gp.qprint_dashes()
472            return boot_candidate
473        else:
474            gp.qprint_timen("The machine state does not match the required" +
475                            " starting state for a '" + boot_candidate +
476                            "' boot test:")
477            gp.print_varx("boot_table[" + boot_candidate + "][start]",
478                          boot_table[boot_candidate]['start'], 1)
479            boot_stack.append(boot_candidate)
480            popped_boot = boot_candidate
481
482    # Loop through your list selecting a boot_candidates
483    boot_candidates = []
484    for boot_candidate in boot_list:
485        if st.compare_states(state, boot_table[boot_candidate]['start']):
486            if stack_popped:
487                if st.compare_states(boot_table[boot_candidate]['end'],
488                   boot_table[popped_boot]['start']):
489                    boot_candidates.append(boot_candidate)
490            else:
491                boot_candidates.append(boot_candidate)
492
493    if len(boot_candidates) == 0:
494        gp.qprint_timen("The user's boot list contained no boot tests" +
495                        " which are valid for the current machine state.")
496        boot_candidate = default_power_on
497        if not st.compare_states(state, boot_table[default_power_on]['start']):
498            boot_candidate = default_power_off
499        boot_candidates.append(boot_candidate)
500        gp.qprint_timen("Using default '" + boot_candidate +
501                        "' boot type to transition to valid state.")
502
503    gp.dprint_var(boot_candidates)
504
505    # Randomly select a boot from the candidate list.
506    boot = random.choice(boot_candidates)
507
508    return boot
509
510###############################################################################
511
512
513###############################################################################
514def print_last_boots():
515
516    r"""
517    Print the last ten boots done with their time stamps.
518    """
519
520    # indent 0, 90 chars wide, linefeed, char is "="
521    gp.qprint_dashes(0, 90)
522    gp.qprintn("Last 10 boots:\n")
523
524    for boot_entry in last_ten:
525        grp.rqprint(boot_entry)
526    gp.qprint_dashes(0, 90)
527
528###############################################################################
529
530
531###############################################################################
532def print_defect_report():
533
534    r"""
535    Print a defect report.
536    """
537
538    # At some point I'd like to have the 'Call FFDC Methods' return a list
539    # of files it has collected.  In that case, the following "ls" command
540    # would no longer be needed.  For now, however, glob shows the files
541    # named in FFDC_LIST_FILE_PATH so I will refrain from printing those
542    # out (so we don't see duplicates in the list).
543
544    LOG_PREFIX = BuiltIn().get_variable_value("${LOG_PREFIX}")
545
546    output = '\n'.join(glob.glob(LOG_PREFIX + '*'))
547    try:
548        ffdc_list = open(ffdc_list_file_path, 'r')
549    except IOError:
550        ffdc_list = ""
551
552    # Open ffdc_file_list for writing.  We will write a complete list of
553    # FFDC files to it for possible use by plug-ins like cp_stop_check.
554    ffdc_list_file = open(ffdc_list_file_path, 'w')
555
556    gp.qprintn()
557    # indent=0, width=90, linefeed=1, char="="
558    gp.qprint_dashes(0, 90, 1, "=")
559    gp.qprintn("Copy this data to the defect:\n")
560
561    gp.qpvars(openbmc_nickname, openbmc_host, openbmc_host_name, openbmc_ip,
562              openbmc_username, openbmc_password, os_host, os_host_name,
563              os_ip, os_username, os_password, pdu_host, pdu_host_name,
564              pdu_ip, pdu_username, pdu_password, pdu_slot_no,
565              openbmc_serial_host, openbmc_serial_host_name, openbmc_serial_ip,
566              openbmc_serial_port)
567
568    gp.qprintn()
569
570    print_last_boots()
571    gp.qprintn()
572    gp.qprint_var(state)
573
574    gp.qprintn()
575    gp.qprintn("FFDC data files:")
576    if status_file_path != "":
577        gp.qprintn(status_file_path)
578        ffdc_list_file.write(status_file_path + "\n")
579
580    gp.qprintn(output)
581    # gp.qprintn(ffdc_list)
582    gp.qprintn()
583
584    gp.qprint_dashes(0, 90, 1, "=")
585
586    ffdc_list_file.write(output + "\n")
587    ffdc_list_file.close()
588
589###############################################################################
590
591
592###############################################################################
593def my_ffdc():
594
595    r"""
596    Collect FFDC data.
597    """
598
599    global state
600
601    plug_in_setup()
602    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
603        call_point='ffdc', stop_on_plug_in_failure=1)
604
605    AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX']
606    status, ret_values = grk.run_key_u("FFDC  ffdc_prefix=" +
607                                       AUTOBOOT_FFDC_PREFIX +
608                                       "  ffdc_function_list=" +
609                                       ffdc_function_list, ignore=1)
610    if status != 'PASS':
611        gp.print_error("Call to ffdc failed.\n")
612
613    my_get_state()
614
615    print_defect_report()
616
617###############################################################################
618
619
620###############################################################################
621def print_test_start_message(boot_keyword):
622
623    r"""
624    Print a message indicating what boot test is about to run.
625
626    Description of arguments:
627    boot_keyword  The name of the boot which is to be run
628                  (e.g. "BMC Power On").
629    """
630
631    global last_ten
632
633    doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".")
634    gp.qprint(doing_msg)
635
636    last_ten.append(doing_msg)
637
638    if len(last_ten) > 10:
639        del last_ten[0]
640
641###############################################################################
642
643
644###############################################################################
645def run_boot(boot):
646
647    r"""
648    Run the specified boot.
649
650    Description of arguments:
651    boot  The name of the boot test to be performed.
652    """
653
654    global state
655
656    print_test_start_message(boot)
657
658    plug_in_setup()
659    rc, shell_rc, failed_plug_in_name = \
660        grpi.rprocess_plug_in_packages(call_point="pre_boot")
661    if rc != 0:
662        error_message = "Plug-in failed with non-zero return code.\n" +\
663            gp.sprint_var(rc, 1)
664        BuiltIn().fail(gp.sprint_error(error_message))
665
666    if test_mode:
667        # In test mode, we'll pretend the boot worked by assigning its
668        # required end state to the default state value.
669        state = st.strip_anchor_state(boot_table[boot]['end'])
670    else:
671        # Assertion:  We trust that the state data was made fresh by the
672        # caller.
673
674        gp.qprintn()
675
676        if boot_table[boot]['method_type'] == "keyword":
677            rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''),
678                               boot_table[boot]['method'],
679                               quiet=quiet)
680
681        if boot_table[boot]['bmc_reboot']:
682            st.wait_for_comm_cycle(int(state['epoch_seconds']))
683            plug_in_setup()
684            rc, shell_rc, failed_plug_in_name = \
685                grpi.rprocess_plug_in_packages(call_point="post_reboot")
686            if rc != 0:
687                error_message = "Plug-in failed with non-zero return code.\n"
688                error_message += gp.sprint_var(rc, 1)
689                BuiltIn().fail(gp.sprint_error(error_message))
690        else:
691            match_state = st.anchor_state(state)
692            del match_state['epoch_seconds']
693            # Wait for the state to change in any way.
694            st.wait_state(match_state, wait_time=state_change_timeout,
695                          interval="3 seconds", invert=1)
696
697        gp.qprintn()
698        if boot_table[boot]['end']['chassis'] == "Off":
699            boot_timeout = power_off_timeout
700        else:
701            boot_timeout = power_on_timeout
702        st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout,
703                      interval="3 seconds")
704
705    plug_in_setup()
706    rc, shell_rc, failed_plug_in_name = \
707        grpi.rprocess_plug_in_packages(call_point="post_boot")
708    if rc != 0:
709        error_message = "Plug-in failed with non-zero return code.\n" +\
710            gp.sprint_var(rc, 1)
711        BuiltIn().fail(gp.sprint_error(error_message))
712
713###############################################################################
714
715
716###############################################################################
717def test_loop_body():
718
719    r"""
720    The main loop body for the loop in main_py.
721
722    Description of arguments:
723    boot_count  The iteration number (starts at 1).
724    """
725
726    global boot_count
727    global state
728    global next_boot
729    global boot_success
730
731    gp.qprintn()
732
733    next_boot = select_boot()
734    if next_boot == "":
735        return True
736
737    boot_count += 1
738    gp.qprint_timen("Starting boot " + str(boot_count) + ".")
739
740    # Clear the ffdc_list_file_path file.  Plug-ins may now write to it.
741    try:
742        os.remove(ffdc_list_file_path)
743    except OSError:
744        pass
745
746    cmd_buf = ["run_boot", next_boot]
747    boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
748    if boot_status == "FAIL":
749        gp.qprint(msg)
750
751    gp.qprintn()
752    if boot_status == "PASS":
753        boot_success = 1
754        gp.qprint_timen("BOOT_SUCCESS: \"" + next_boot + "\" succeeded.")
755    else:
756        boot_success = 0
757        gp.qprint_timen("BOOT_FAILED: \"" + next_boot + "\" failed.")
758
759    boot_results.update(next_boot, boot_status)
760
761    plug_in_setup()
762    # NOTE: A post_test_case call point failure is NOT counted as a boot
763    # failure.
764    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
765        call_point='post_test_case', stop_on_plug_in_failure=1)
766
767    plug_in_setup()
768    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
769        call_point='ffdc_check', shell_rc=0x00000200,
770        stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
771    if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200:
772        status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
773        if status != 'PASS':
774            gp.print_error("Call to my_ffdc failed.\n")
775
776    # We need to purge error logs between boots or they build up.
777    grk.run_key("Delete Error logs", ignore=1)
778
779    boot_results.print_report()
780    gp.qprint_timen("Finished boot " + str(boot_count) + ".")
781
782    plug_in_setup()
783    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
784        call_point='stop_check')
785    if rc != 0:
786        error_message = "Stopping as requested by user.\n"
787        grp.rprint_error_report(error_message)
788        BuiltIn().fail(error_message)
789
790    # This should help prevent ConnectionErrors.
791    grk.run_key_u("Delete All Sessions")
792
793    return True
794
795###############################################################################
796
797
798###############################################################################
799def obmc_boot_test_teardown():
800
801    r"""
802    Clean up after the Main keyword.
803    """
804
805    if cp_setup_called:
806        plug_in_setup()
807        rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
808            call_point='cleanup', stop_on_plug_in_failure=1)
809
810    # Save boot_results object to a file in case it is needed again.
811    gp.qprint_timen("Saving boot_results to the following path.")
812    gp.qprint_var(boot_results_file_path)
813    pickle.dump(boot_results, open(boot_results_file_path, 'wb'),
814                pickle.HIGHEST_PROTOCOL)
815
816###############################################################################
817
818
819###############################################################################
820def test_teardown():
821
822    r"""
823    Clean up after this test case.
824    """
825
826    gp.qprintn()
827    cmd_buf = ["Print Error",
828               "A keyword timeout occurred ending this program.\n"]
829    BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
830
831    grp.rqprint_pgm_footer()
832
833###############################################################################
834
835
836###############################################################################
837def obmc_boot_test_py(alt_boot_stack=None):
838
839    r"""
840    Do main program processing.
841    """
842
843    if alt_boot_stack is not None:
844        BuiltIn().set_global_variable("${boot_stack}", alt_boot_stack)
845
846    setup()
847
848    if ffdc_only:
849        gp.qprint_timen("Caller requested ffdc_only.")
850        grk.run_key_u("my_ffdc")
851        return
852
853    # Process caller's boot_stack.
854    while (len(boot_stack) > 0):
855        test_loop_body()
856
857    gp.qprint_timen("Finished processing stack.")
858
859    # Process caller's boot_list.
860    if len(boot_list) > 0:
861        for ix in range(1, max_num_tests + 1):
862            test_loop_body()
863
864    gp.qprint_timen("Completed all requested boot tests.")
865
866    boot_pass, boot_fail = boot_results.return_total_pass_fail()
867    if boot_fail > boot_fail_threshold:
868        error_message = "Boot failures exceed the boot failure" +\
869                        " threshold:\n" +\
870                        gp.sprint_var(boot_fail) +\
871                        gp.sprint_var(boot_fail_threshold)
872        BuiltIn().fail(gp.sprint_error(error_message))
873
874###############################################################################
875