1#!/usr/bin/env python
2
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
7import os
8import imp
9import time
10import glob
11import random
12import re
13import signal
14try:
15    import cPickle as pickle
16except ImportError:
17    import pickle
18import socket
19
20from robot.utils import DotDict
21from robot.libraries.BuiltIn import BuiltIn
22
23from boot_data import *
24import gen_print as gp
25import gen_robot_plug_in as grpi
26import gen_arg as ga
27import gen_valid as gv
28import gen_misc as gm
29import gen_cmd as gc
30import gen_robot_keyword as grk
31import state as st
32import var_stack as vs
33import gen_plug_in_utils as gpu
34import pel_utils as pel
35import logging_utils as log
36
37base_path = os.path.dirname(os.path.dirname(
38                            imp.find_module("gen_robot_print")[1])) +\
39    os.sep
40sys.path.append(base_path + "extended/")
41import run_keyword as rk
42
43# Setting master_pid correctly influences the behavior of plug-ins like
44# DB_Logging
45program_pid = os.getpid()
46master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid)
47pgm_name = re.sub('\\.py$', '', os.path.basename(__file__))
48
49# Set up boot data structures.
50os_host = BuiltIn().get_variable_value("${OS_HOST}", default="")
51
52boot_lists = read_boot_lists()
53
54# The maximum number of entries that can be in the boot_history global variable.
55max_boot_history = 10
56boot_history = []
57
58state = st.return_state_constant('default_state')
59cp_setup_called = 0
60next_boot = ""
61base_tool_dir_path = os.path.normpath(os.environ.get(
62    'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep
63
64ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep
65boot_success = 0
66status_dir_path = os.environ.get('STATUS_DIR_PATH', "")
67if status_dir_path != "":
68    status_dir_path = os.path.normpath(status_dir_path) + os.sep
69redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False)
70redfish_rest_supported = BuiltIn().get_variable_value("${REDFISH_REST_SUPPORTED}", default=False)
71if redfish_supported:
72    redfish = BuiltIn().get_library_instance('redfish')
73    default_power_on = "Redfish Power On"
74    default_power_off = "Redfish Power Off"
75    if redfish_rest_supported:
76        delete_errlogs_cmd = "Delete Error Logs  ${quiet}=${1}"
77        delete_bmcdump_cmd = "Delete All BMC Dump"
78        default_set_power_policy = "Set BMC Power Policy  ALWAYS_POWER_OFF"
79    else:
80        delete_errlogs_cmd = "Redfish Purge Event Log"
81        delete_bmcdump_cmd = "Redfish Delete All BMC Dumps"
82        default_set_power_policy = "Redfish Set Power Restore Policy  AlwaysOff"
83else:
84    default_power_on = "REST Power On"
85    default_power_off = "REST Power Off"
86    delete_errlogs_cmd = "Delete Error Logs  ${quiet}=${1}"
87    delete_bmcdump_cmd = "Delete All BMC Dump"
88    default_set_power_policy = "Set BMC Power Policy  ALWAYS_POWER_OFF"
89boot_count = 0
90
91LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
92AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '')
93ffdc_prefix = AUTOBOOT_FFDC_PREFIX
94boot_start_time = ""
95boot_end_time = ""
96save_stack = vs.var_stack('save_stack')
97main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet']
98
99
100def dump_ffdc_rc():
101    r"""
102    Return the constant dump ffdc test return code value.
103
104    When a plug-in call point program returns this value, it indicates that
105    this program should collect FFDC.
106    """
107
108    return 0x00000200
109
110
111def stop_test_rc():
112    r"""
113    Return the constant stop test return code value.
114
115    When a plug-in call point program returns this value, it indicates that
116    this program should stop running.
117    """
118
119    return 0x00000200
120
121
122def process_host(host,
123                 host_var_name=""):
124    r"""
125    Process a host by getting the associated host name and IP address and
126    setting them in global variables.
127
128    If the caller does not pass the host_var_name, this function will try to
129    figure out the name of the variable used by the caller for the host parm.
130    Callers are advised to explicitly specify the host_var_name when calling
131    with an exec command.  In such cases, the get_arg_name cannot figure out
132    the host variable name.
133
134    This function will then create similar global variable names by
135    removing "_host" and appending "_host_name" or "_ip" to the host variable
136    name.
137
138    Example:
139
140    If a call is made like this:
141    process_host(openbmc_host)
142
143    Global variables openbmc_host_name and openbmc_ip will be set.
144
145    Description of argument(s):
146    host           A host name or IP.  The name of the variable used should
147                   have a suffix of "_host".
148    host_var_name  The name of the variable being used as the host parm.
149    """
150
151    if host_var_name == "":
152        host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
153
154    host_name_var_name = re.sub("host", "host_name", host_var_name)
155    ip_var_name = re.sub("host", "ip", host_var_name)
156    cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\
157        host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\
158        host + "')"
159    exec(cmd_buf)
160
161
162def process_pgm_parms():
163    r"""
164    Process the program parameters by assigning them all to corresponding
165    globals.  Also, set some global values that depend on program parameters.
166    """
167
168    # Program parameter processing.
169    # Assign all program parms to python variables which are global to this
170    # module.
171
172    global parm_list
173    parm_list = BuiltIn().get_variable_value("${parm_list}")
174    # The following subset of parms should be processed as integers.
175    int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only',
176                'boot_fail_threshold', 'delete_errlogs',
177                'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet',
178                'test_mode', 'debug']
179    for parm in parm_list:
180        if parm in int_list:
181            sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\
182                      "}\", \"0\"))"
183        else:
184            sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")"
185        cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
186        gp.dpissuing(cmd_buf)
187        exec(cmd_buf)
188        if re.match(r".*_host$", parm):
189            cmd_buf = "process_host(" + parm + ", '" + parm + "')"
190            exec(cmd_buf)
191        if re.match(r".*_password$", parm):
192            # Register the value of any parm whose name ends in _password.
193            # This will cause the print functions to replace passwords with
194            # asterisks in the output.
195            cmd_buf = "gp.register_passwords(" + parm + ")"
196            exec(cmd_buf)
197
198    global ffdc_dir_path_style
199    global boot_list
200    global boot_stack
201    global boot_results_file_path
202    global boot_results
203    global boot_history
204    global ffdc_list_file_path
205    global ffdc_report_list_path
206    global ffdc_summary_list_path
207    global boot_table
208    global valid_boot_types
209
210    if ffdc_dir_path_style == "":
211        ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0'))
212
213    # Convert these program parms to lists for easier processing..
214    boot_list = list(filter(None, boot_list.split(":")))
215    boot_stack = list(filter(None, boot_stack.split(":")))
216
217    boot_table = create_boot_table(boot_table_path, os_host=os_host)
218    valid_boot_types = create_valid_boot_list(boot_table)
219
220    cleanup_boot_results_file()
221    boot_results_file_path = create_boot_results_file_path(pgm_name,
222                                                           openbmc_nickname,
223                                                           master_pid)
224
225    if os.path.isfile(boot_results_file_path):
226        # We've been called before in this run so we'll load the saved
227        # boot_results and boot_history objects.
228        boot_results, boot_history =\
229            pickle.load(open(boot_results_file_path, 'rb'))
230    else:
231        boot_results = boot_results(boot_table, boot_pass, boot_fail)
232
233    ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\
234        "/FFDC_FILE_LIST"
235    ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\
236        "/FFDC_REPORT_FILE_LIST"
237
238    ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\
239        "/FFDC_SUMMARY_FILE_LIST"
240
241
242def initial_plug_in_setup():
243    r"""
244    Initialize all plug-in environment variables which do not change for the
245    duration of the program.
246
247    """
248
249    global LOG_LEVEL
250    BuiltIn().set_log_level("NONE")
251
252    BuiltIn().set_global_variable("${master_pid}", master_pid)
253    BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
254    BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
255    BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
256    BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}",
257                                  ffdc_list_file_path)
258    BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}",
259                                  ffdc_report_list_path)
260    BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}",
261                                  ffdc_summary_list_path)
262
263    BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}",
264                                  ffdc_dir_path_style)
265    BuiltIn().set_global_variable("${FFDC_CHECK}",
266                                  ffdc_check)
267
268    # For each program parameter, set the corresponding AUTOBOOT_ environment
269    # variable value.  Also, set an AUTOBOOT_ environment variable for every
270    # element in additional_values.
271    additional_values = ["program_pid", "master_pid", "ffdc_dir_path",
272                         "status_dir_path", "base_tool_dir_path",
273                         "ffdc_list_file_path", "ffdc_report_list_path",
274                         "ffdc_summary_list_path", "execdir", "redfish_supported",
275                         "redfish_rest_supported"]
276
277    plug_in_vars = parm_list + additional_values
278
279    for var_name in plug_in_vars:
280        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
281        var_name = var_name.upper()
282        if var_value is None:
283            var_value = ""
284        os.environ["AUTOBOOT_" + var_name] = str(var_value)
285
286    BuiltIn().set_log_level(LOG_LEVEL)
287
288    # Make sure the ffdc list directory exists.
289    ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
290    if not os.path.exists(ffdc_list_dir_path):
291        os.makedirs(ffdc_list_dir_path)
292
293
294def plug_in_setup():
295    r"""
296    Initialize all changing plug-in environment variables for use by the
297    plug-in programs.
298    """
299
300    global LOG_LEVEL
301    global test_really_running
302
303    BuiltIn().set_log_level("NONE")
304
305    boot_pass, boot_fail = boot_results.return_total_pass_fail()
306    if boot_pass > 1:
307        test_really_running = 1
308    else:
309        test_really_running = 0
310
311    BuiltIn().set_global_variable("${test_really_running}",
312                                  test_really_running)
313    BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
314    BuiltIn().set_global_variable("${boot_pass}", boot_pass)
315    BuiltIn().set_global_variable("${boot_fail}", boot_fail)
316    BuiltIn().set_global_variable("${boot_success}", boot_success)
317    BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
318    BuiltIn().set_global_variable("${boot_start_time}", boot_start_time)
319    BuiltIn().set_global_variable("${boot_end_time}", boot_end_time)
320
321    # For each program parameter, set the corresponding AUTOBOOT_ environment
322    # variable value.  Also, set an AUTOBOOT_ environment variable for every
323    # element in additional_values.
324    additional_values = ["boot_type_desc", "boot_success", "boot_pass",
325                         "boot_fail", "test_really_running", "ffdc_prefix",
326                         "boot_start_time", "boot_end_time"]
327
328    plug_in_vars = additional_values
329
330    for var_name in plug_in_vars:
331        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
332        var_name = var_name.upper()
333        if var_value is None:
334            var_value = ""
335        os.environ["AUTOBOOT_" + var_name] = str(var_value)
336
337    if debug:
338        shell_rc, out_buf = \
339            gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u")
340
341    BuiltIn().set_log_level(LOG_LEVEL)
342
343
344def pre_boot_plug_in_setup():
345
346    # Clear the ffdc_list_file_path file.  Plug-ins may now write to it.
347    try:
348        os.remove(ffdc_list_file_path)
349    except OSError:
350        pass
351
352    # Clear the ffdc_report_list_path file.  Plug-ins may now write to it.
353    try:
354        os.remove(ffdc_report_list_path)
355    except OSError:
356        pass
357
358    # Clear the ffdc_summary_list_path file.  Plug-ins may now write to it.
359    try:
360        os.remove(ffdc_summary_list_path)
361    except OSError:
362        pass
363
364    global ffdc_prefix
365
366    seconds = time.time()
367    loc_time = time.localtime(seconds)
368    time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
369
370    ffdc_prefix = openbmc_nickname + "." + time_string
371
372
373def default_sigusr1(signal_number=0,
374                    frame=None):
375    r"""
376    Handle SIGUSR1 by doing nothing.
377
378    This function assists in debugging SIGUSR1 processing by printing messages
379    to stdout and to the log.html file.
380
381    Description of argument(s):
382    signal_number  The signal number (should always be 10 for SIGUSR1).
383    frame          The frame data.
384    """
385
386    gp.qprintn()
387    gp.qprint_executing()
388    gp.lprint_executing()
389
390
391def set_default_siguser1():
392    r"""
393    Set the default_sigusr1 function to be the SIGUSR1 handler.
394    """
395
396    gp.qprintn()
397    gp.qprint_executing()
398    gp.lprint_executing()
399    signal.signal(signal.SIGUSR1, default_sigusr1)
400
401
402def setup():
403    r"""
404    Do general program setup tasks.
405    """
406
407    global cp_setup_called
408    global transitional_boot_selected
409
410    gp.qprintn()
411
412    if redfish_supported:
413        redfish.login()
414
415    set_default_siguser1()
416    transitional_boot_selected = False
417
418    robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
419    repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
420    # If we can't find process_plug_in_packages.py, ssh_pw or
421    # validate_plug_ins.py, then we don't have our repo bin in PATH.
422    shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py"
423                                     + " ssh_pw validate_plug_ins.py", quiet=1,
424                                     print_output=0, show_err=0)
425    if shell_rc != 0:
426        os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
427    # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
428    if robot_pgm_dir_path not in sys.path:
429        sys.path.append(robot_pgm_dir_path)
430        PYTHONPATH = os.environ.get("PYTHONPATH", "")
431        if PYTHONPATH == "":
432            os.environ['PYTHONPATH'] = robot_pgm_dir_path
433        else:
434            os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
435
436    validate_parms()
437
438    gp.qprint_pgm_header()
439
440    grk.run_key_u(default_set_power_policy)
441
442    initial_plug_in_setup()
443
444    plug_in_setup()
445    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
446        call_point='setup')
447    if rc != 0:
448        error_message = "Plug-in setup failed.\n"
449        gp.print_error_report(error_message)
450        BuiltIn().fail(error_message)
451    # Setting cp_setup_called lets our Teardown know that it needs to call
452    # the cleanup plug-in call point.
453    cp_setup_called = 1
454
455    # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
456    BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
457    # FFDC_LOG_PATH is used by "FFDC" keyword.
458    BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
459
460    # Also printed by FFDC.
461    global host_name
462    global host_ip
463    host = socket.gethostname()
464    host_name, host_ip = gm.get_host_name_ip(host)
465
466    gp.dprint_var(boot_table)
467    gp.dprint_var(boot_lists)
468
469
470def validate_parms():
471    r"""
472    Validate all program parameters.
473    """
474
475    process_pgm_parms()
476
477    gp.qprintn()
478
479    global openbmc_model
480    if openbmc_model == "":
481        status, ret_values =\
482            grk.run_key_u("Get BMC System Model")
483        openbmc_model = ret_values
484        BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
485    gv.set_exit_on_error(True)
486    gv.valid_value(openbmc_host)
487    gv.valid_value(openbmc_username)
488    gv.valid_value(openbmc_password)
489    gv.valid_value(rest_username)
490    gv.valid_value(rest_password)
491    gv.valid_value(ipmi_username)
492    gv.valid_value(ipmi_password)
493    if os_host != "":
494        gv.valid_value(os_username)
495        gv.valid_value(os_password)
496    if pdu_host != "":
497        gv.valid_value(pdu_username)
498        gv.valid_value(pdu_password)
499        gv.valid_integer(pdu_slot_no)
500    if openbmc_serial_host != "":
501        gv.valid_integer(openbmc_serial_port)
502    gv.valid_value(openbmc_model)
503    gv.valid_integer(max_num_tests)
504    gv.valid_integer(boot_pass)
505    gv.valid_integer(boot_fail)
506    plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
507    BuiltIn().set_global_variable("${plug_in_packages_list}",
508                                  plug_in_packages_list)
509    gv.valid_value(stack_mode, valid_values=['normal', 'skip'])
510    gv.set_exit_on_error(False)
511    if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
512        error_message = "You must provide either a value for either the" +\
513            " boot_list or the boot_stack parm.\n"
514        BuiltIn().fail(gp.sprint_error(error_message))
515    valid_boot_list(boot_list, valid_boot_types)
516    valid_boot_list(boot_stack, valid_boot_types)
517    selected_PDU_boots = list(set(boot_list + boot_stack)
518                              & set(boot_lists['PDU_reboot']))
519    if len(selected_PDU_boots) > 0 and pdu_host == "":
520        error_message = "You have selected the following boots which" +\
521                        " require a PDU host but no value for pdu_host:\n"
522        error_message += gp.sprint_var(selected_PDU_boots)
523        error_message += gp.sprint_var(pdu_host, fmt=gp.blank())
524        BuiltIn().fail(gp.sprint_error(error_message))
525
526    return
527
528
529def my_get_state():
530    r"""
531    Get the system state plus a little bit of wrapping.
532    """
533
534    global state
535
536    req_states = ['epoch_seconds'] + st.default_req_states
537
538    gp.qprint_timen("Getting system state.")
539    if test_mode:
540        state['epoch_seconds'] = int(time.time())
541    else:
542        state = st.get_state(req_states=req_states, quiet=quiet)
543    gp.qprint_var(state)
544
545
546def valid_state():
547    r"""
548    Verify that our state dictionary contains no blank values.  If we don't get
549    valid state data, we cannot continue to work.
550    """
551
552    if st.compare_states(state, st.invalid_state_match, 'or'):
553        error_message = "The state dictionary contains blank fields which" +\
554            " is illegal.\n" + gp.sprint_var(state)
555        BuiltIn().fail(gp.sprint_error(error_message))
556
557
558def select_boot():
559    r"""
560    Select a boot test to be run based on our current state and return the
561    chosen boot type.
562
563    Description of arguments:
564    state  The state of the machine.
565    """
566
567    global transitional_boot_selected
568    global boot_stack
569
570    gp.qprint_timen("Selecting a boot test.")
571
572    if transitional_boot_selected and not boot_success:
573        prior_boot = next_boot
574        boot_candidate = boot_stack.pop()
575        gp.qprint_timen("The prior '" + next_boot + "' was chosen to"
576                        + " transition to a valid state for '" + boot_candidate
577                        + "' which was at the top of the boot_stack.  Since"
578                        + " the '" + next_boot + "' failed, the '"
579                        + boot_candidate + "' has been removed from the stack"
580                        + " to avoid and endless failure loop.")
581        if len(boot_stack) == 0:
582            return ""
583
584    my_get_state()
585    valid_state()
586
587    transitional_boot_selected = False
588    stack_popped = 0
589    if len(boot_stack) > 0:
590        stack_popped = 1
591        gp.qprint_dashes()
592        gp.qprint_var(boot_stack)
593        gp.qprint_dashes()
594        skip_boot_printed = 0
595        while len(boot_stack) > 0:
596            boot_candidate = boot_stack.pop()
597            if stack_mode == 'normal':
598                break
599            else:
600                if st.compare_states(state, boot_table[boot_candidate]['end']):
601                    if not skip_boot_printed:
602                        gp.qprint_var(stack_mode)
603                        gp.qprintn()
604                        gp.qprint_timen("Skipping the following boot tests"
605                                        + " which are unnecessary since their"
606                                        + " required end states match the"
607                                        + " current machine state:")
608                        skip_boot_printed = 1
609                    gp.qprint_var(boot_candidate)
610                    boot_candidate = ""
611        if boot_candidate == "":
612            gp.qprint_dashes()
613            gp.qprint_var(boot_stack)
614            gp.qprint_dashes()
615            return boot_candidate
616        if st.compare_states(state, boot_table[boot_candidate]['start']):
617            gp.qprint_timen("The machine state is valid for a '"
618                            + boot_candidate + "' boot test.")
619            gp.qprint_dashes()
620            gp.qprint_var(boot_stack)
621            gp.qprint_dashes()
622            return boot_candidate
623        else:
624            gp.qprint_timen("The machine state does not match the required"
625                            + " starting state for a '" + boot_candidate
626                            + "' boot test:")
627            gp.qprint_varx("boot_table_start_entry",
628                           boot_table[boot_candidate]['start'])
629            boot_stack.append(boot_candidate)
630            transitional_boot_selected = True
631            popped_boot = boot_candidate
632
633    # Loop through your list selecting a boot_candidates
634    boot_candidates = []
635    for boot_candidate in boot_list:
636        if st.compare_states(state, boot_table[boot_candidate]['start']):
637            if stack_popped:
638                if st.compare_states(boot_table[boot_candidate]['end'],
639                                     boot_table[popped_boot]['start']):
640                    boot_candidates.append(boot_candidate)
641            else:
642                boot_candidates.append(boot_candidate)
643
644    if len(boot_candidates) == 0:
645        gp.qprint_timen("The user's boot list contained no boot tests"
646                        + " which are valid for the current machine state.")
647        boot_candidate = default_power_on
648        if not st.compare_states(state, boot_table[default_power_on]['start']):
649            boot_candidate = default_power_off
650        boot_candidates.append(boot_candidate)
651        gp.qprint_timen("Using default '" + boot_candidate
652                        + "' boot type to transition to valid state.")
653
654    gp.dprint_var(boot_candidates)
655
656    # Randomly select a boot from the candidate list.
657    boot = random.choice(boot_candidates)
658
659    return boot
660
661
662def print_defect_report(ffdc_file_list):
663    r"""
664    Print a defect report.
665
666    Description of argument(s):
667    ffdc_file_list  A list of files which were collected by our ffdc functions.
668    """
669
670    # Making deliberate choice to NOT run plug_in_setup().  We don't want
671    # ffdc_prefix updated.
672    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
673        call_point='ffdc_report', stop_on_plug_in_failure=0)
674
675    # Get additional header data which may have been created by ffdc plug-ins.
676    # Also, delete the individual header files to cleanup.
677    cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\
678              " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
679              " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
680    shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
681                                              show_err=0)
682
683    # Get additional summary data which may have been created by ffdc plug-ins.
684    # Also, delete the individual header files to cleanup.
685    cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\
686              " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
687              " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
688    shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
689                                               show_err=0)
690
691    # ffdc_list_file_path contains a list of any ffdc files created by plug-
692    # ins, etc.  Read that data into a list.
693    try:
694        plug_in_ffdc_list = \
695            open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n")
696        plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list))
697    except IOError:
698        plug_in_ffdc_list = []
699
700    # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed
701    # in.  Eliminate duplicates and sort the list.
702    ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list))
703
704    if status_file_path != "":
705        ffdc_file_list.insert(0, status_file_path)
706
707    # Convert the list to a printable list.
708    printable_ffdc_file_list = "\n".join(ffdc_file_list)
709
710    # Open ffdc_file_list for writing.  We will write a complete list of
711    # FFDC files to it for possible use by plug-ins like cp_stop_check.
712    ffdc_list_file = open(ffdc_list_file_path, 'w')
713    ffdc_list_file.write(printable_ffdc_file_list + "\n")
714    ffdc_list_file.close()
715
716    indent = 0
717    width = 90
718    linefeed = 1
719    char = "="
720
721    gp.qprintn()
722    gp.qprint_dashes(indent, width, linefeed, char)
723    gp.qprintn("Copy this data to the defect:\n")
724
725    if len(more_header_info) > 0:
726        gp.qprintn(more_header_info)
727    gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host,
728              openbmc_host_name, openbmc_ip, openbmc_username,
729              openbmc_password, rest_username, rest_password, ipmi_username,
730              ipmi_password, os_host, os_host_name, os_ip, os_username,
731              os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username,
732              pdu_password, pdu_slot_no, openbmc_serial_host,
733              openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port)
734
735    gp.qprintn()
736    print_boot_history(boot_history)
737    gp.qprintn()
738    gp.qprint_var(state)
739    gp.qprintn()
740    gp.qprintn("FFDC data files:")
741    gp.qprintn(printable_ffdc_file_list)
742    gp.qprintn()
743
744    if len(ffdc_summary_info) > 0:
745        gp.qprintn(ffdc_summary_info)
746
747    gp.qprint_dashes(indent, width, linefeed, char)
748
749
750def my_ffdc():
751    r"""
752    Collect FFDC data.
753    """
754
755    global state
756
757    plug_in_setup()
758    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
759        call_point='ffdc', stop_on_plug_in_failure=0)
760
761    AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX']
762    status, ffdc_file_list = grk.run_key_u("FFDC  ffdc_prefix="
763                                           + AUTOBOOT_FFDC_PREFIX
764                                           + "  ffdc_function_list="
765                                           + ffdc_function_list, ignore=1)
766    if status != 'PASS':
767        gp.qprint_error("Call to ffdc failed.\n")
768        if type(ffdc_file_list) is not list:
769            ffdc_file_list = []
770        # Leave a record for caller that "soft" errors occurred.
771        soft_errors = 1
772        gpu.save_plug_in_value(soft_errors, pgm_name)
773
774    my_get_state()
775
776    print_defect_report(ffdc_file_list)
777
778
779def print_test_start_message(boot_keyword):
780    r"""
781    Print a message indicating what boot test is about to run.
782
783    Description of arguments:
784    boot_keyword  The name of the boot which is to be run
785                  (e.g. "BMC Power On").
786    """
787
788    global boot_history
789    global boot_start_time
790
791    doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".")
792
793    # Set boot_start_time for use by plug-ins.
794    boot_start_time = doing_msg[1:33]
795    gp.qprint_var(boot_start_time)
796
797    gp.qprint(doing_msg)
798
799    update_boot_history(boot_history, doing_msg, max_boot_history)
800
801
802def stop_boot_test(signal_number=0,
803                   frame=None):
804    r"""
805    Handle SIGUSR1 by aborting the boot test that is running.
806
807    Description of argument(s):
808    signal_number  The signal number (should always be 10 for SIGUSR1).
809    frame          The frame data.
810    """
811
812    gp.qprintn()
813    gp.qprint_executing()
814    gp.lprint_executing()
815
816    # Restore original sigusr1 handler.
817    set_default_siguser1()
818
819    message = "The caller has asked that the boot test be stopped and marked"
820    message += " as a failure."
821
822    function_stack = gm.get_function_stack()
823    if "wait_state" in function_stack:
824        st.set_exit_wait_early_message(message)
825    else:
826        BuiltIn().fail(gp.sprint_error(message))
827
828
829def run_boot(boot):
830    r"""
831    Run the specified boot.
832
833    Description of arguments:
834    boot  The name of the boot test to be performed.
835    """
836
837    global state
838
839    signal.signal(signal.SIGUSR1, stop_boot_test)
840    gp.qprint_timen("stop_boot_test is armed.")
841
842    print_test_start_message(boot)
843
844    plug_in_setup()
845    rc, shell_rc, failed_plug_in_name = \
846        grpi.rprocess_plug_in_packages(call_point="pre_boot")
847    if rc != 0:
848        error_message = "Plug-in failed with non-zero return code.\n" +\
849            gp.sprint_var(rc, fmt=gp.hexa())
850        set_default_siguser1()
851        BuiltIn().fail(gp.sprint_error(error_message))
852
853    if test_mode:
854        # In test mode, we'll pretend the boot worked by assigning its
855        # required end state to the default state value.
856        state = st.strip_anchor_state(boot_table[boot]['end'])
857    else:
858        # Assertion:  We trust that the state data was made fresh by the
859        # caller.
860
861        gp.qprintn()
862
863        if boot_table[boot]['method_type'] == "keyword":
864            rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''),
865                               boot_table[boot]['method'],
866                               quiet=quiet)
867
868        if boot_table[boot]['bmc_reboot']:
869            st.wait_for_comm_cycle(int(state['epoch_seconds']))
870            plug_in_setup()
871            rc, shell_rc, failed_plug_in_name = \
872                grpi.rprocess_plug_in_packages(call_point="post_reboot")
873            if rc != 0:
874                error_message = "Plug-in failed with non-zero return code.\n"
875                error_message += gp.sprint_var(rc, fmt=gp.hexa())
876                set_default_siguser1()
877                BuiltIn().fail(gp.sprint_error(error_message))
878        else:
879            match_state = st.anchor_state(state)
880            del match_state['epoch_seconds']
881            # Wait for the state to change in any way.
882            st.wait_state(match_state, wait_time=state_change_timeout,
883                          interval="10 seconds", invert=1)
884
885        gp.qprintn()
886        if boot_table[boot]['end']['chassis'] == "Off":
887            boot_timeout = power_off_timeout
888        else:
889            boot_timeout = power_on_timeout
890        st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout,
891                      interval="10 seconds")
892
893    plug_in_setup()
894    rc, shell_rc, failed_plug_in_name = \
895        grpi.rprocess_plug_in_packages(call_point="post_boot")
896    if rc != 0:
897        error_message = "Plug-in failed with non-zero return code.\n" +\
898            gp.sprint_var(rc, fmt=gp.hexa())
899        set_default_siguser1()
900        BuiltIn().fail(gp.sprint_error(error_message))
901
902    # Restore original sigusr1 handler.
903    set_default_siguser1()
904
905
906def test_loop_body():
907    r"""
908    The main loop body for the loop in main_py.
909
910    Description of arguments:
911    boot_count  The iteration number (starts at 1).
912    """
913
914    global boot_count
915    global state
916    global next_boot
917    global boot_success
918    global boot_end_time
919
920    gp.qprintn()
921
922    next_boot = select_boot()
923    if next_boot == "":
924        return True
925
926    boot_count += 1
927    gp.qprint_timen("Starting boot " + str(boot_count) + ".")
928
929    pre_boot_plug_in_setup()
930
931    cmd_buf = ["run_boot", next_boot]
932    boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
933    if boot_status == "FAIL":
934        gp.qprint(msg)
935
936    gp.qprintn()
937    if boot_status == "PASS":
938        boot_success = 1
939        completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot
940                                         + "\" succeeded.")
941    else:
942        boot_success = 0
943        completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot
944                                         + "\" failed.")
945
946    # Set boot_end_time for use by plug-ins.
947    boot_end_time = completion_msg[1:33]
948    gp.qprint_var(boot_end_time)
949
950    gp.qprint(completion_msg)
951
952    boot_results.update(next_boot, boot_status)
953
954    plug_in_setup()
955    # NOTE: A post_test_case call point failure is NOT counted as a boot
956    # failure.
957    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
958        call_point='post_test_case', stop_on_plug_in_failure=0)
959
960    plug_in_setup()
961    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
962        call_point='ffdc_check', shell_rc=dump_ffdc_rc(),
963        stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
964    if ffdc_check == "All" or\
965       shell_rc == dump_ffdc_rc():
966        status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
967        if status != 'PASS':
968            gp.qprint_error("Call to my_ffdc failed.\n")
969            # Leave a record for caller that "soft" errors occurred.
970            soft_errors = 1
971            gpu.save_plug_in_value(soft_errors, pgm_name)
972
973    if delete_errlogs:
974        # print error logs before delete
975        status, error_logs = grk.run_key_u("Get Error Logs")
976        pels = pel.peltool("-l", ignore_err=1)
977        log.print_error_logs(error_logs, "AdditionalData Message Severity")
978        gp.qprint_var(pels)
979
980        # We need to purge error logs between boots or they build up.
981        grk.run_key(delete_errlogs_cmd, ignore=1)
982        grk.run_key(delete_bmcdump_cmd, ignore=1)
983
984    boot_results.print_report()
985    gp.qprint_timen("Finished boot " + str(boot_count) + ".")
986
987    plug_in_setup()
988    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
989        call_point='stop_check', shell_rc=stop_test_rc(),
990        stop_on_non_zero_rc=1)
991    if shell_rc == stop_test_rc():
992        message = "Stopping as requested by user.\n"
993        gp.qprint_time(message)
994        BuiltIn().fail(message)
995
996    # This should help prevent ConnectionErrors.
997    # Purge all redfish and REST connection sessions.
998    grk.run_key_u("Close All Connections", ignore=1)
999    grk.run_key_u("Delete All Redfish Sessions", ignore=1)
1000
1001    return True
1002
1003
1004def obmc_boot_test_teardown():
1005    r"""
1006    Clean up after the main keyword.
1007    """
1008    gp.qprint_executing()
1009
1010    if ga.psutil_imported:
1011        ga.terminate_descendants()
1012
1013    if cp_setup_called:
1014        plug_in_setup()
1015        rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1016            call_point='cleanup', stop_on_plug_in_failure=0)
1017
1018    if 'boot_results_file_path' in globals():
1019        # Save boot_results and boot_history objects to a file in case they are
1020        # needed again.
1021        gp.qprint_timen("Saving boot_results to the following path.")
1022        gp.qprint_var(boot_results_file_path)
1023        pickle.dump((boot_results, boot_history),
1024                    open(boot_results_file_path, 'wb'),
1025                    pickle.HIGHEST_PROTOCOL)
1026
1027    global save_stack
1028    # Restore any global values saved on the save_stack.
1029    for parm_name in main_func_parm_list:
1030        # Get the parm_value if it was saved on the stack.
1031        try:
1032            parm_value = save_stack.pop(parm_name)
1033        except BaseException:
1034            # If it was not saved, no further action is required.
1035            continue
1036
1037        # Restore the saved value.
1038        cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1039            "}\", parm_value)"
1040        gp.dpissuing(cmd_buf)
1041        exec(cmd_buf)
1042
1043    gp.dprintn(save_stack.sprint_obj())
1044
1045
1046def test_teardown():
1047    r"""
1048    Clean up after this test case.
1049    """
1050
1051    gp.qprintn()
1052    gp.qprint_executing()
1053
1054    if ga.psutil_imported:
1055        ga.terminate_descendants()
1056
1057    cmd_buf = ["Print Error",
1058               "A keyword timeout occurred ending this program.\n"]
1059    BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
1060
1061    if redfish_supported:
1062        redfish.logout()
1063
1064    gp.qprint_pgm_footer()
1065
1066
1067def post_stack():
1068    r"""
1069    Process post_stack plug-in programs.
1070    """
1071
1072    if not call_post_stack_plug:
1073        # The caller does not wish to have post_stack plug-in processing done.
1074        return
1075
1076    global boot_success
1077
1078    # NOTE: A post_stack call-point failure is NOT counted as a boot failure.
1079    pre_boot_plug_in_setup()
1080    # For the purposes of the following plug-ins, mark the "boot" as a success.
1081    boot_success = 1
1082    plug_in_setup()
1083    rc, shell_rc, failed_plug_in_name, history =\
1084        grpi.rprocess_plug_in_packages(call_point='post_stack',
1085                                       stop_on_plug_in_failure=0,
1086                                       return_history=True)
1087    for doing_msg in history:
1088        update_boot_history(boot_history, doing_msg, max_boot_history)
1089    if rc != 0:
1090        boot_success = 0
1091
1092    plug_in_setup()
1093    rc, shell_rc, failed_plug_in_name =\
1094        grpi.rprocess_plug_in_packages(call_point='ffdc_check',
1095                                       shell_rc=dump_ffdc_rc(),
1096                                       stop_on_plug_in_failure=1,
1097                                       stop_on_non_zero_rc=1)
1098    if shell_rc == dump_ffdc_rc():
1099        status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
1100        if status != 'PASS':
1101            gp.qprint_error("Call to my_ffdc failed.\n")
1102            # Leave a record for caller that "soft" errors occurred.
1103            soft_errors = 1
1104            gpu.save_plug_in_value(soft_errors, pgm_name)
1105
1106    plug_in_setup()
1107    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1108        call_point='stop_check', shell_rc=stop_test_rc(),
1109        stop_on_non_zero_rc=1)
1110    if shell_rc == stop_test_rc():
1111        message = "Stopping as requested by user.\n"
1112        gp.qprint_time(message)
1113        BuiltIn().fail(message)
1114
1115
1116def obmc_boot_test_py(loc_boot_stack=None,
1117                      loc_stack_mode=None,
1118                      loc_quiet=None):
1119    r"""
1120    Do main program processing.
1121    """
1122
1123    global save_stack
1124
1125    ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']})
1126
1127    gp.dprintn()
1128    # Process function parms.
1129    for parm_name in main_func_parm_list:
1130        # Get parm's value.
1131        parm_value = eval("loc_" + parm_name)
1132        gp.dpvars(parm_name, parm_value)
1133
1134        if parm_value is not None:
1135            # Save the global value on a stack.
1136            cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\
1137                parm_name + "}\"), \"" + parm_name + "\")"
1138            gp.dpissuing(cmd_buf)
1139            exec(cmd_buf)
1140
1141            # Set the global value to the passed value.
1142            cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1143                "}\", loc_" + parm_name + ")"
1144            gp.dpissuing(cmd_buf)
1145            exec(cmd_buf)
1146
1147    gp.dprintn(save_stack.sprint_obj())
1148
1149    setup()
1150
1151    init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail()
1152
1153    if ffdc_only:
1154        gp.qprint_timen("Caller requested ffdc_only.")
1155        if do_pre_boot_plug_in_setup:
1156            pre_boot_plug_in_setup()
1157        grk.run_key_u("my_ffdc")
1158        return
1159
1160    if delete_errlogs:
1161        # print error logs before delete
1162        status, error_logs = grk.run_key_u("Get Error Logs")
1163        pels = pel.peltool("-l", ignore_err=1)
1164        log.print_error_logs(error_logs, "AdditionalData Message Severity")
1165        gp.qprint_var(pels)
1166
1167        # Delete errlogs prior to doing any boot tests.
1168        grk.run_key(delete_errlogs_cmd, ignore=1)
1169        grk.run_key(delete_bmcdump_cmd, ignore=1)
1170
1171    # Process caller's boot_stack.
1172    while (len(boot_stack) > 0):
1173        test_loop_body()
1174
1175    gp.qprint_timen("Finished processing stack.")
1176
1177    post_stack()
1178
1179    # Process caller's boot_list.
1180    if len(boot_list) > 0:
1181        for ix in range(1, max_num_tests + 1):
1182            test_loop_body()
1183
1184    gp.qprint_timen("Completed all requested boot tests.")
1185
1186    boot_pass, boot_fail = boot_results.return_total_pass_fail()
1187    new_fail = boot_fail - init_boot_fail
1188    if new_fail > boot_fail_threshold:
1189        error_message = "Boot failures exceed the boot failure" +\
1190                        " threshold:\n" +\
1191                        gp.sprint_var(new_fail) +\
1192                        gp.sprint_var(boot_fail_threshold)
1193        BuiltIn().fail(gp.sprint_error(error_message))
1194