1#!/usr/bin/env python
2
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
7import os
8import imp
9import time
10import glob
11import random
12import re
13import cPickle as pickle
14import socket
15
16from robot.utils import DotDict
17from robot.libraries.BuiltIn import BuiltIn
18
19from boot_data import *
20import gen_print as gp
21import gen_robot_print as grp
22import gen_robot_plug_in as grpi
23import gen_robot_valid as grv
24import gen_misc as gm
25import gen_cmd as gc
26import gen_robot_keyword as grk
27import state as st
28
29base_path = os.path.dirname(os.path.dirname(
30                            imp.find_module("gen_robot_print")[1])) +\
31    os.sep
32sys.path.append(base_path + "extended/")
33import run_keyword as rk
34
35# Setting master_pid correctly influences the behavior of plug-ins like
36# DB_Logging
37program_pid = os.getpid()
38master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid)
39
40# Set up boot data structures.
41boot_table = create_boot_table()
42valid_boot_types = create_valid_boot_list(boot_table)
43
44boot_lists = read_boot_lists()
45last_ten = []
46
47state = st.return_default_state()
48cp_setup_called = 0
49next_boot = ""
50base_tool_dir_path = os.path.normpath(os.environ.get(
51    'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep
52
53ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep
54boot_success = 0
55status_dir_path = os.environ.get('STATUS_DIR_PATH', "")
56if status_dir_path != "":
57    status_dir_path = os.path.normpath(status_dir_path) + os.sep
58default_power_on = "REST Power On"
59default_power_off = "REST Power Off"
60boot_count = 0
61
62LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
63
64
65###############################################################################
66def process_host(host,
67                 host_var_name=""):
68
69    r"""
70    Process a host by getting the associated host name and IP address and
71    setting them in global variables.
72
73    If the caller does not pass the host_var_name, this function will try to
74    figure out the name of the variable used by the caller for the host parm.
75    Callers are advised to explicitly specify the host_var_name when calling
76    with an exec command.  In such cases, the get_arg_name cannot figure out
77    the host variable name.
78
79    This function will then create similar global variable names by
80    removing "_host" and appending "_host_name" or "_ip" to the host variable
81    name.
82
83    Example:
84
85    If a call is made like this:
86    process_host(openbmc_host)
87
88    Global variables openbmc_host_name and openbmc_ip will be set.
89
90    Description of argument(s):
91    host           A host name or IP.  The name of the variable used should
92                   have a suffix of "_host".
93    host_var_name  The name of the variable being used as the host parm.
94    """
95
96    if host_var_name == "":
97        host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
98
99    host_name_var_name = re.sub("host", "host_name", host_var_name)
100    ip_var_name = re.sub("host", "ip", host_var_name)
101    cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\
102        host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\
103        host + "')"
104    exec(cmd_buf)
105
106###############################################################################
107
108
109###############################################################################
110def process_pgm_parms():
111
112    r"""
113    Process the program parameters by assigning them all to corresponding
114    globals.  Also, set some global values that depend on program parameters.
115    """
116
117    # Program parameter processing.
118    # Assign all program parms to python variables which are global to this
119    # module.
120
121    global parm_list
122    parm_list = BuiltIn().get_variable_value("${parm_list}")
123    # The following subset of parms should be processed as integers.
124    int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only',
125                'boot_fail_threshold', 'quiet', 'test_mode', 'debug']
126    for parm in parm_list:
127        if parm in int_list:
128            sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\
129                      "}\", \"0\"))"
130        else:
131            sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")"
132        cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
133        exec(cmd_buf)
134        if re.match(r".*_host$", parm):
135            cmd_buf = "process_host(" + parm + ", '" + parm + "')"
136            exec(cmd_buf)
137        if re.match(r".*_password$", parm):
138            # Register the value of any parm whose name ends in _password.
139            # This will cause the print functions to replace passwords with
140            # asterisks in the output.
141            cmd_buf = "gp.register_passwords(" + parm + ")"
142            exec(cmd_buf)
143
144    global ffdc_dir_path_style
145    global boot_list
146    global boot_stack
147    global boot_results_file_path
148    global boot_results
149    global ffdc_list_file_path
150
151    if ffdc_dir_path_style == "":
152        ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0'))
153
154    # Convert these program parms to lists for easier processing..
155    boot_list = filter(None, boot_list.split(":"))
156    boot_stack = filter(None, boot_stack.split(":"))
157
158    boot_results_file_path = "/tmp/" + openbmc_nickname + ":pid_" +\
159                             str(master_pid) + ":boot_results"
160
161    if os.path.isfile(boot_results_file_path):
162        # We've been called before in this run so we'll load the saved
163        # boot_results object.
164        boot_results = pickle.load(open(boot_results_file_path, 'rb'))
165    else:
166        boot_results = boot_results(boot_table, boot_pass, boot_fail)
167
168    ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\
169        "/FFDC_FILE_LIST"
170
171###############################################################################
172
173
174###############################################################################
175def initial_plug_in_setup():
176
177    r"""
178    Initialize all plug-in environment variables which do not change for the
179    duration of the program.
180
181    """
182
183    global LOG_LEVEL
184    BuiltIn().set_log_level("NONE")
185
186    BuiltIn().set_global_variable("${master_pid}", master_pid)
187    BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
188    BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
189    BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
190    BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}",
191                                  ffdc_list_file_path)
192
193    BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}",
194                                  ffdc_dir_path_style)
195    BuiltIn().set_global_variable("${FFDC_CHECK}",
196                                  ffdc_check)
197
198    # For each program parameter, set the corresponding AUTOBOOT_ environment
199    # variable value.  Also, set an AUTOBOOT_ environment variable for every
200    # element in additional_values.
201    additional_values = ["program_pid", "master_pid", "ffdc_dir_path",
202                         "status_dir_path", "base_tool_dir_path",
203                         "ffdc_list_file_path"]
204
205    plug_in_vars = parm_list + additional_values
206
207    for var_name in plug_in_vars:
208        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
209        var_name = var_name.upper()
210        if var_value is None:
211            var_value = ""
212        os.environ["AUTOBOOT_" + var_name] = str(var_value)
213
214    BuiltIn().set_log_level(LOG_LEVEL)
215
216    # Make sure the ffdc list directory exists.
217    ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
218    if not os.path.exists(ffdc_list_dir_path):
219        os.makedirs(ffdc_list_dir_path)
220
221###############################################################################
222
223
224###############################################################################
225def plug_in_setup():
226
227    r"""
228    Initialize all changing plug-in environment variables for use by the
229    plug-in programs.
230    """
231
232    global LOG_LEVEL
233    global test_really_running
234
235    BuiltIn().set_log_level("NONE")
236
237    boot_pass, boot_fail = boot_results.return_total_pass_fail()
238    if boot_pass > 1:
239        test_really_running = 1
240    else:
241        test_really_running = 0
242
243    seconds = time.time()
244    loc_time = time.localtime(seconds)
245    time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
246
247    ffdc_prefix = openbmc_nickname + "." + time_string
248
249    BuiltIn().set_global_variable("${test_really_running}",
250                                  test_really_running)
251    BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
252    BuiltIn().set_global_variable("${boot_pass}", boot_pass)
253    BuiltIn().set_global_variable("${boot_fail}", boot_fail)
254    BuiltIn().set_global_variable("${boot_success}", boot_success)
255    BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
256
257    # For each program parameter, set the corresponding AUTOBOOT_ environment
258    # variable value.  Also, set an AUTOBOOT_ environment variable for every
259    # element in additional_values.
260    additional_values = ["boot_type_desc", "boot_success", "boot_pass",
261                         "boot_fail", "test_really_running", "ffdc_prefix"]
262
263    plug_in_vars = additional_values
264
265    for var_name in plug_in_vars:
266        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
267        var_name = var_name.upper()
268        if var_value is None:
269            var_value = ""
270        os.environ["AUTOBOOT_" + var_name] = str(var_value)
271
272    if debug:
273        shell_rc, out_buf = \
274            gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u")
275
276    BuiltIn().set_log_level(LOG_LEVEL)
277
278###############################################################################
279
280
281###############################################################################
282def setup():
283
284    r"""
285    Do general program setup tasks.
286    """
287
288    global cp_setup_called
289
290    gp.qprintn()
291
292    robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
293    repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
294    # If we can't find ssh_pw, then we don't have our repo bin in PATH.
295    shell_rc, out_buf = gc.cmd_fnc_u("which ssh_pw", quiet=1, print_output=0,
296                                     show_err=0)
297    if shell_rc != 0:
298        os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
299    # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
300    if robot_pgm_dir_path not in sys.path:
301        sys.path.append(robot_pgm_dir_path)
302        PYTHONPATH = os.environ.get("PYTHONPATH", "")
303        if PYTHONPATH == "":
304            os.environ['PYTHONPATH'] = robot_pgm_dir_path
305        else:
306            os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
307
308    validate_parms()
309
310    grp.rqprint_pgm_header()
311
312    grk.run_key("Set BMC Power Policy  RESTORE_LAST_STATE")
313
314    initial_plug_in_setup()
315
316    plug_in_setup()
317    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
318        call_point='setup')
319    if rc != 0:
320        error_message = "Plug-in setup failed.\n"
321        grp.rprint_error_report(error_message)
322        BuiltIn().fail(error_message)
323    # Setting cp_setup_called lets our Teardown know that it needs to call
324    # the cleanup plug-in call point.
325    cp_setup_called = 1
326
327    # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
328    BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
329    # FFDC_LOG_PATH is used by "FFDC" keyword.
330    BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
331
332    # Also printed by FFDC.
333    global host_name
334    global host_ip
335    host = socket.gethostname()
336    host_name, host_ip = gm.get_host_name_ip(host)
337
338    gp.dprint_var(boot_table, 1)
339    gp.dprint_var(boot_lists)
340
341###############################################################################
342
343
344###############################################################################
345def validate_parms():
346
347    r"""
348    Validate all program parameters.
349    """
350
351    process_pgm_parms()
352
353    gp.qprintn()
354
355    global openbmc_model
356    grv.rvalid_value("openbmc_host")
357    grv.rvalid_value("openbmc_username")
358    grv.rvalid_value("openbmc_password")
359    if os_host != "":
360        grv.rvalid_value("os_username")
361        grv.rvalid_value("os_password")
362
363    if pdu_host != "":
364        grv.rvalid_value("pdu_username")
365        grv.rvalid_value("pdu_password")
366        grv.rvalid_integer("pdu_slot_no")
367    if openbmc_serial_host != "":
368        grv.rvalid_integer("openbmc_serial_port")
369    if openbmc_model == "":
370        status, ret_values =\
371            grk.run_key_u("Get BMC System Model")
372        openbmc_model = ret_values
373        BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
374    grv.rvalid_value("openbmc_model")
375    grv.rvalid_integer("max_num_tests")
376    grv.rvalid_integer("boot_pass")
377    grv.rvalid_integer("boot_fail")
378
379    plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
380    BuiltIn().set_global_variable("${plug_in_packages_list}",
381                                  plug_in_packages_list)
382
383    grv.rvalid_value("stack_mode", valid_values=['normal', 'skip'])
384    if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
385        error_message = "You must provide either a value for either the" +\
386            " boot_list or the boot_stack parm.\n"
387        BuiltIn().fail(gp.sprint_error(error_message))
388
389    valid_boot_list(boot_list, valid_boot_types)
390    valid_boot_list(boot_stack, valid_boot_types)
391
392    selected_PDU_boots = list(set(boot_list + boot_stack) &
393                              set(boot_lists['PDU_reboot']))
394
395    if len(selected_PDU_boots) > 0 and pdu_host == "":
396        error_message = "You have selected the following boots which" +\
397                        " require a PDU host but no value for pdu_host:\n"
398        error_message += gp.sprint_var(selected_PDU_boots)
399        error_message += gp.sprint_var(pdu_host, 2)
400        BuiltIn().fail(gp.sprint_error(error_message))
401
402    return
403
404###############################################################################
405
406
407###############################################################################
408def my_get_state():
409
410    r"""
411    Get the system state plus a little bit of wrapping.
412    """
413
414    global state
415
416    req_states = ['epoch_seconds'] + st.default_req_states
417
418    gp.qprint_timen("Getting system state.")
419    if test_mode:
420        state['epoch_seconds'] = int(time.time())
421    else:
422        state = st.get_state(req_states=req_states, quiet=quiet)
423    gp.qprint_var(state)
424
425###############################################################################
426
427
428###############################################################################
429def select_boot():
430
431    r"""
432    Select a boot test to be run based on our current state and return the
433    chosen boot type.
434
435    Description of arguments:
436    state  The state of the machine.
437    """
438
439    global boot_stack
440
441    gp.qprint_timen("Selecting a boot test.")
442
443    my_get_state()
444
445    stack_popped = 0
446    if len(boot_stack) > 0:
447        stack_popped = 1
448        gp.qprint_dashes()
449        gp.qprint_var(boot_stack)
450        gp.qprint_dashes()
451        skip_boot_printed = 0
452        while len(boot_stack) > 0:
453            boot_candidate = boot_stack.pop()
454            if stack_mode == 'normal':
455                break
456            else:
457                if st.compare_states(state, boot_table[boot_candidate]['end']):
458                    if not skip_boot_printed:
459                        gp.print_var(stack_mode)
460                        gp.printn()
461                        gp.print_timen("Skipping the following boot tests" +
462                                       " which are unnecessary since their" +
463                                       " required end states match the" +
464                                       " current machine state:")
465                        skip_boot_printed = 1
466                    gp.print_var(boot_candidate)
467                    boot_candidate = ""
468        if boot_candidate == "":
469            gp.qprint_dashes()
470            gp.qprint_var(boot_stack)
471            gp.qprint_dashes()
472            return boot_candidate
473        if st.compare_states(state, boot_table[boot_candidate]['start']):
474            gp.qprint_timen("The machine state is valid for a '" +
475                            boot_candidate + "' boot test.")
476            gp.qprint_dashes()
477            gp.qprint_var(boot_stack)
478            gp.qprint_dashes()
479            return boot_candidate
480        else:
481            gp.qprint_timen("The machine state does not match the required" +
482                            " starting state for a '" + boot_candidate +
483                            "' boot test:")
484            gp.print_varx("boot_table[" + boot_candidate + "][start]",
485                          boot_table[boot_candidate]['start'], 1)
486            boot_stack.append(boot_candidate)
487            popped_boot = boot_candidate
488
489    # Loop through your list selecting a boot_candidates
490    boot_candidates = []
491    for boot_candidate in boot_list:
492        if st.compare_states(state, boot_table[boot_candidate]['start']):
493            if stack_popped:
494                if st.compare_states(boot_table[boot_candidate]['end'],
495                   boot_table[popped_boot]['start']):
496                    boot_candidates.append(boot_candidate)
497            else:
498                boot_candidates.append(boot_candidate)
499
500    if len(boot_candidates) == 0:
501        gp.qprint_timen("The user's boot list contained no boot tests" +
502                        " which are valid for the current machine state.")
503        boot_candidate = default_power_on
504        if not st.compare_states(state, boot_table[default_power_on]['start']):
505            boot_candidate = default_power_off
506        boot_candidates.append(boot_candidate)
507        gp.qprint_timen("Using default '" + boot_candidate +
508                        "' boot type to transition to valid state.")
509
510    gp.dprint_var(boot_candidates)
511
512    # Randomly select a boot from the candidate list.
513    boot = random.choice(boot_candidates)
514
515    return boot
516
517###############################################################################
518
519
520###############################################################################
521def print_last_boots():
522
523    r"""
524    Print the last ten boots done with their time stamps.
525    """
526
527    # indent 0, 90 chars wide, linefeed, char is "="
528    gp.qprint_dashes(0, 90)
529    gp.qprintn("Last 10 boots:\n")
530
531    for boot_entry in last_ten:
532        grp.rqprint(boot_entry)
533    gp.qprint_dashes(0, 90)
534
535###############################################################################
536
537
538###############################################################################
539def print_defect_report():
540
541    r"""
542    Print a defect report.
543    """
544
545    # At some point I'd like to have the 'Call FFDC Methods' return a list
546    # of files it has collected.  In that case, the following "ls" command
547    # would no longer be needed.  For now, however, glob shows the files
548    # named in FFDC_LIST_FILE_PATH so I will refrain from printing those
549    # out (so we don't see duplicates in the list).
550
551    LOG_PREFIX = BuiltIn().get_variable_value("${LOG_PREFIX}")
552
553    output = '\n'.join(glob.glob(LOG_PREFIX + '*'))
554    try:
555        ffdc_list = open(ffdc_list_file_path, 'r')
556    except IOError:
557        ffdc_list = ""
558
559    # Open ffdc_file_list for writing.  We will write a complete list of
560    # FFDC files to it for possible use by plug-ins like cp_stop_check.
561    ffdc_list_file = open(ffdc_list_file_path, 'w')
562
563    gp.qprintn()
564    # indent=0, width=90, linefeed=1, char="="
565    gp.qprint_dashes(0, 90, 1, "=")
566    gp.qprintn("Copy this data to the defect:\n")
567
568    gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host,
569              openbmc_host_name, openbmc_ip, openbmc_username,
570              openbmc_password, os_host, os_host_name, os_ip, os_username,
571              os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username,
572              pdu_password, pdu_slot_no, openbmc_serial_host,
573              openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port)
574
575    gp.qprintn()
576
577    print_last_boots()
578    gp.qprintn()
579    gp.qprint_var(state)
580
581    gp.qprintn()
582    gp.qprintn("FFDC data files:")
583    if status_file_path != "":
584        gp.qprintn(status_file_path)
585        ffdc_list_file.write(status_file_path + "\n")
586
587    gp.qprintn(output)
588    # gp.qprintn(ffdc_list)
589    gp.qprintn()
590
591    gp.qprint_dashes(0, 90, 1, "=")
592
593    ffdc_list_file.write(output + "\n")
594    ffdc_list_file.close()
595
596###############################################################################
597
598
599###############################################################################
600def my_ffdc():
601
602    r"""
603    Collect FFDC data.
604    """
605
606    global state
607
608    plug_in_setup()
609    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
610        call_point='ffdc', stop_on_plug_in_failure=1)
611
612    AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX']
613    status, ret_values = grk.run_key_u("FFDC  ffdc_prefix=" +
614                                       AUTOBOOT_FFDC_PREFIX +
615                                       "  ffdc_function_list=" +
616                                       ffdc_function_list, ignore=1)
617    if status != 'PASS':
618        gp.print_error("Call to ffdc failed.\n")
619
620    my_get_state()
621
622    print_defect_report()
623
624###############################################################################
625
626
627###############################################################################
628def print_test_start_message(boot_keyword):
629
630    r"""
631    Print a message indicating what boot test is about to run.
632
633    Description of arguments:
634    boot_keyword  The name of the boot which is to be run
635                  (e.g. "BMC Power On").
636    """
637
638    global last_ten
639
640    doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".")
641    gp.qprint(doing_msg)
642
643    last_ten.append(doing_msg)
644
645    if len(last_ten) > 10:
646        del last_ten[0]
647
648###############################################################################
649
650
651###############################################################################
652def run_boot(boot):
653
654    r"""
655    Run the specified boot.
656
657    Description of arguments:
658    boot  The name of the boot test to be performed.
659    """
660
661    global state
662
663    print_test_start_message(boot)
664
665    plug_in_setup()
666    rc, shell_rc, failed_plug_in_name = \
667        grpi.rprocess_plug_in_packages(call_point="pre_boot")
668    if rc != 0:
669        error_message = "Plug-in failed with non-zero return code.\n" +\
670            gp.sprint_var(rc, 1)
671        BuiltIn().fail(gp.sprint_error(error_message))
672
673    if test_mode:
674        # In test mode, we'll pretend the boot worked by assigning its
675        # required end state to the default state value.
676        state = st.strip_anchor_state(boot_table[boot]['end'])
677    else:
678        # Assertion:  We trust that the state data was made fresh by the
679        # caller.
680
681        gp.qprintn()
682
683        if boot_table[boot]['method_type'] == "keyword":
684            rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''),
685                               boot_table[boot]['method'],
686                               quiet=quiet)
687
688        if boot_table[boot]['bmc_reboot']:
689            st.wait_for_comm_cycle(int(state['epoch_seconds']))
690            plug_in_setup()
691            rc, shell_rc, failed_plug_in_name = \
692                grpi.rprocess_plug_in_packages(call_point="post_reboot")
693            if rc != 0:
694                error_message = "Plug-in failed with non-zero return code.\n"
695                error_message += gp.sprint_var(rc, 1)
696                BuiltIn().fail(gp.sprint_error(error_message))
697        else:
698            match_state = st.anchor_state(state)
699            del match_state['epoch_seconds']
700            # Wait for the state to change in any way.
701            st.wait_state(match_state, wait_time=state_change_timeout,
702                          interval="3 seconds", invert=1)
703
704        gp.qprintn()
705        if boot_table[boot]['end']['chassis'] == "Off":
706            boot_timeout = power_off_timeout
707        else:
708            boot_timeout = power_on_timeout
709        st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout,
710                      interval="3 seconds")
711
712    plug_in_setup()
713    rc, shell_rc, failed_plug_in_name = \
714        grpi.rprocess_plug_in_packages(call_point="post_boot")
715    if rc != 0:
716        error_message = "Plug-in failed with non-zero return code.\n" +\
717            gp.sprint_var(rc, 1)
718        BuiltIn().fail(gp.sprint_error(error_message))
719
720###############################################################################
721
722
723###############################################################################
724def test_loop_body():
725
726    r"""
727    The main loop body for the loop in main_py.
728
729    Description of arguments:
730    boot_count  The iteration number (starts at 1).
731    """
732
733    global boot_count
734    global state
735    global next_boot
736    global boot_success
737
738    gp.qprintn()
739
740    next_boot = select_boot()
741    if next_boot == "":
742        return True
743
744    boot_count += 1
745    gp.qprint_timen("Starting boot " + str(boot_count) + ".")
746
747    # Clear the ffdc_list_file_path file.  Plug-ins may now write to it.
748    try:
749        os.remove(ffdc_list_file_path)
750    except OSError:
751        pass
752
753    cmd_buf = ["run_boot", next_boot]
754    boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
755    if boot_status == "FAIL":
756        gp.qprint(msg)
757
758    gp.qprintn()
759    if boot_status == "PASS":
760        boot_success = 1
761        gp.qprint_timen("BOOT_SUCCESS: \"" + next_boot + "\" succeeded.")
762    else:
763        boot_success = 0
764        gp.qprint_timen("BOOT_FAILED: \"" + next_boot + "\" failed.")
765
766    boot_results.update(next_boot, boot_status)
767
768    plug_in_setup()
769    # NOTE: A post_test_case call point failure is NOT counted as a boot
770    # failure.
771    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
772        call_point='post_test_case', stop_on_plug_in_failure=1)
773
774    plug_in_setup()
775    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
776        call_point='ffdc_check', shell_rc=0x00000200,
777        stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
778    if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200:
779        status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
780        if status != 'PASS':
781            gp.print_error("Call to my_ffdc failed.\n")
782
783    # We need to purge error logs between boots or they build up.
784    grk.run_key("Delete Error logs", ignore=1)
785
786    boot_results.print_report()
787    gp.qprint_timen("Finished boot " + str(boot_count) + ".")
788
789    plug_in_setup()
790    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
791        call_point='stop_check')
792    if rc != 0:
793        error_message = "Stopping as requested by user.\n"
794        grp.rprint_error_report(error_message)
795        BuiltIn().fail(error_message)
796
797    # This should help prevent ConnectionErrors.
798    grk.run_key_u("Delete All Sessions")
799
800    return True
801
802###############################################################################
803
804
805###############################################################################
806def obmc_boot_test_teardown():
807
808    r"""
809    Clean up after the Main keyword.
810    """
811
812    if cp_setup_called:
813        plug_in_setup()
814        rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
815            call_point='cleanup', stop_on_plug_in_failure=1)
816
817    # Save boot_results object to a file in case it is needed again.
818    gp.qprint_timen("Saving boot_results to the following path.")
819    gp.qprint_var(boot_results_file_path)
820    pickle.dump(boot_results, open(boot_results_file_path, 'wb'),
821                pickle.HIGHEST_PROTOCOL)
822
823###############################################################################
824
825
826###############################################################################
827def test_teardown():
828
829    r"""
830    Clean up after this test case.
831    """
832
833    gp.qprintn()
834    cmd_buf = ["Print Error",
835               "A keyword timeout occurred ending this program.\n"]
836    BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
837
838    grp.rqprint_pgm_footer()
839
840###############################################################################
841
842
843###############################################################################
844def obmc_boot_test_py(alt_boot_stack=None):
845
846    r"""
847    Do main program processing.
848    """
849
850    if alt_boot_stack is not None:
851        BuiltIn().set_global_variable("${boot_stack}", alt_boot_stack)
852
853    setup()
854
855    if ffdc_only:
856        gp.qprint_timen("Caller requested ffdc_only.")
857        grk.run_key_u("my_ffdc")
858        return
859
860    # Process caller's boot_stack.
861    while (len(boot_stack) > 0):
862        test_loop_body()
863
864    gp.qprint_timen("Finished processing stack.")
865
866    # Process caller's boot_list.
867    if len(boot_list) > 0:
868        for ix in range(1, max_num_tests + 1):
869            test_loop_body()
870
871    gp.qprint_timen("Completed all requested boot tests.")
872
873    boot_pass, boot_fail = boot_results.return_total_pass_fail()
874    if boot_fail > boot_fail_threshold:
875        error_message = "Boot failures exceed the boot failure" +\
876                        " threshold:\n" +\
877                        gp.sprint_var(boot_fail) +\
878                        gp.sprint_var(boot_fail_threshold)
879        BuiltIn().fail(gp.sprint_error(error_message))
880
881###############################################################################
882