1#!/usr/bin/env python3
2
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
7import glob
8import imp
9import os
10import random
11import re
12import signal
13import time
14
15try:
16    import cPickle as pickle
17except ImportError:
18    import pickle
19
20import socket
21
22import gen_arg as ga
23import gen_cmd as gc
24import gen_misc as gm
25import gen_plug_in_utils as gpu
26import gen_print as gp
27import gen_robot_keyword as grk
28import gen_robot_plug_in as grpi
29import gen_valid as gv
30import logging_utils as log
31import pel_utils as pel
32import state as st
33import var_stack as vs
34from boot_data import *
35from robot.libraries.BuiltIn import BuiltIn
36from robot.utils import DotDict
37
38base_path = (
39    os.path.dirname(os.path.dirname(imp.find_module("gen_robot_print")[1]))
40    + os.sep
41)
42sys.path.append(base_path + "extended/")
43import run_keyword as rk  # NOQA
44
45# Setting master_pid correctly influences the behavior of plug-ins like
46# DB_Logging
47program_pid = os.getpid()
48master_pid = os.environ.get("AUTOBOOT_MASTER_PID", program_pid)
49pgm_name = re.sub("\\.py$", "", os.path.basename(__file__))
50
51# Set up boot data structures.
52os_host = BuiltIn().get_variable_value("${OS_HOST}", default="")
53
54boot_lists = read_boot_lists()
55
56# The maximum number of entries that can be in the boot_history global variable.
57max_boot_history = 10
58boot_history = []
59
60state = st.return_state_constant("default_state")
61cp_setup_called = 0
62next_boot = ""
63base_tool_dir_path = (
64    os.path.normpath(os.environ.get("AUTOBOOT_BASE_TOOL_DIR_PATH", "/tmp"))
65    + os.sep
66)
67
68ffdc_dir_path = os.path.normpath(os.environ.get("FFDC_DIR_PATH", "")) + os.sep
69boot_success = 0
70
71status_dir_path = os.environ.get(
72    "STATUS_DIR_PATH", ""
73) or BuiltIn().get_variable_value("${STATUS_DIR_PATH}", default="")
74if status_dir_path != "":
75    status_dir_path = os.path.normpath(status_dir_path) + os.sep
76    # For plugin expecting env gen_call_robot.py
77    os.environ["STATUS_DIR_PATH"] = status_dir_path
78
79redfish_support_trans_state = int(
80    os.environ.get("REDFISH_SUPPORT_TRANS_STATE", 0)
81) or int(
82    BuiltIn().get_variable_value("${REDFISH_SUPPORT_TRANS_STATE}", default=0)
83)
84redfish_supported = BuiltIn().get_variable_value(
85    "${REDFISH_SUPPORTED}", default=False
86)
87redfish_rest_supported = BuiltIn().get_variable_value(
88    "${REDFISH_REST_SUPPORTED}", default=False
89)
90redfish_delete_sessions = int(
91    BuiltIn().get_variable_value("${REDFISH_DELETE_SESSIONS}", default=1)
92)
93if redfish_supported:
94    redfish = BuiltIn().get_library_instance("redfish")
95    default_power_on = "Redfish Power On"
96    default_power_off = "Redfish Power Off"
97    if not redfish_support_trans_state:
98        delete_errlogs_cmd = "Delete Error Logs  ${quiet}=${1}"
99        delete_bmcdump_cmd = "Delete All BMC Dump"
100        default_set_power_policy = "Set BMC Power Policy  ALWAYS_POWER_OFF"
101    else:
102        delete_errlogs_cmd = "Redfish Purge Event Log"
103        delete_bmcdump_cmd = "Redfish Delete All BMC Dumps"
104        delete_sysdump_cmd = "Redfish Delete All System Dumps"
105        default_set_power_policy = (
106            "Redfish Set Power Restore Policy  AlwaysOff"
107        )
108else:
109    default_power_on = "REST Power On"
110    default_power_off = "REST Power Off"
111    delete_errlogs_cmd = "Delete Error Logs  ${quiet}=${1}"
112    delete_bmcdump_cmd = "Delete All BMC Dump"
113    default_set_power_policy = "Set BMC Power Policy  ALWAYS_POWER_OFF"
114boot_count = 0
115
116LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
117AUTOBOOT_FFDC_PREFIX = os.environ.get("AUTOBOOT_FFDC_PREFIX", "")
118ffdc_prefix = AUTOBOOT_FFDC_PREFIX
119boot_start_time = ""
120boot_end_time = ""
121save_stack = vs.var_stack("save_stack")
122main_func_parm_list = ["boot_stack", "stack_mode", "quiet"]
123
124
125def dump_ffdc_rc():
126    r"""
127    Return the constant dump ffdc test return code value.
128
129    When a plug-in call point program returns this value, it indicates that
130    this program should collect FFDC.
131    """
132
133    return 0x00000200
134
135
136def stop_test_rc():
137    r"""
138    Return the constant stop test return code value.
139
140    When a plug-in call point program returns this value, it indicates that
141    this program should stop running.
142    """
143
144    return 0x00000200
145
146
147def process_host(host, host_var_name=""):
148    r"""
149    Process a host by getting the associated host name and IP address and
150    setting them in global variables.
151
152    If the caller does not pass the host_var_name, this function will try to
153    figure out the name of the variable used by the caller for the host parm.
154    Callers are advised to explicitly specify the host_var_name when calling
155    with an exec command.  In such cases, the get_arg_name cannot figure out
156    the host variable name.
157
158    This function will then create similar global variable names by
159    removing "_host" and appending "_host_name" or "_ip" to the host variable
160    name.
161
162    Example:
163
164    If a call is made like this:
165    process_host(openbmc_host)
166
167    Global variables openbmc_host_name and openbmc_ip will be set.
168
169    Description of argument(s):
170    host           A host name or IP.  The name of the variable used should
171                   have a suffix of "_host".
172    host_var_name  The name of the variable being used as the host parm.
173    """
174
175    if host_var_name == "":
176        host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
177
178    host_name_var_name = re.sub("host", "host_name", host_var_name)
179    ip_var_name = re.sub("host", "ip", host_var_name)
180    cmd_buf = (
181        "global "
182        + host_name_var_name
183        + ", "
184        + ip_var_name
185        + " ; "
186        + host_name_var_name
187        + ", "
188        + ip_var_name
189        + " = gm.get_host_name_ip('"
190        + host
191        + "')"
192    )
193    exec(cmd_buf)
194
195
196def process_pgm_parms():
197    r"""
198    Process the program parameters by assigning them all to corresponding
199    globals.  Also, set some global values that depend on program parameters.
200    """
201
202    # Program parameter processing.
203    # Assign all program parms to python variables which are global to this
204    # module.
205
206    global parm_list
207    parm_list = BuiltIn().get_variable_value("${parm_list}")
208    # The following subset of parms should be processed as integers.
209    int_list = [
210        "max_num_tests",
211        "boot_pass",
212        "boot_fail",
213        "ffdc_only",
214        "boot_fail_threshold",
215        "delete_errlogs",
216        "call_post_stack_plug",
217        "do_pre_boot_plug_in_setup",
218        "quiet",
219        "test_mode",
220        "debug",
221    ]
222    for parm in parm_list:
223        if parm in int_list:
224            sub_cmd = (
225                'int(BuiltIn().get_variable_value("${' + parm + '}", "0"))'
226            )
227        else:
228            sub_cmd = 'BuiltIn().get_variable_value("${' + parm + '}")'
229        cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
230        gp.dpissuing(cmd_buf)
231        exec(cmd_buf)
232        if re.match(r".*_host$", parm):
233            cmd_buf = "process_host(" + parm + ", '" + parm + "')"
234            exec(cmd_buf)
235        if re.match(r".*_password$", parm):
236            # Register the value of any parm whose name ends in _password.
237            # This will cause the print functions to replace passwords with
238            # asterisks in the output.
239            cmd_buf = "gp.register_passwords(" + parm + ")"
240            exec(cmd_buf)
241
242    global ffdc_dir_path_style
243    global boot_list
244    global boot_stack
245    global boot_results_file_path
246    global boot_results
247    global boot_history
248    global ffdc_list_file_path
249    global ffdc_report_list_path
250    global ffdc_summary_list_path
251    global boot_table
252    global valid_boot_types
253
254    if ffdc_dir_path_style == "":
255        ffdc_dir_path_style = int(os.environ.get("FFDC_DIR_PATH_STYLE", "0"))
256
257    # Convert these program parms to lists for easier processing..
258    boot_list = list(filter(None, boot_list.split(":")))
259    boot_stack = list(filter(None, boot_stack.split(":")))
260
261    boot_table = create_boot_table(boot_table_path, os_host=os_host)
262    valid_boot_types = create_valid_boot_list(boot_table)
263
264    cleanup_boot_results_file()
265    boot_results_file_path = create_boot_results_file_path(
266        pgm_name, openbmc_nickname, master_pid
267    )
268
269    if os.path.isfile(boot_results_file_path):
270        # We've been called before in this run so we'll load the saved
271        # boot_results and boot_history objects.
272        boot_results, boot_history = pickle.load(
273            open(boot_results_file_path, "rb")
274        )
275    else:
276        boot_results = boot_results(boot_table, boot_pass, boot_fail)
277
278    ffdc_list_file_path = (
279        base_tool_dir_path + openbmc_nickname + "/FFDC_FILE_LIST"
280    )
281    ffdc_report_list_path = (
282        base_tool_dir_path + openbmc_nickname + "/FFDC_REPORT_FILE_LIST"
283    )
284
285    ffdc_summary_list_path = (
286        base_tool_dir_path + openbmc_nickname + "/FFDC_SUMMARY_FILE_LIST"
287    )
288
289
290def initial_plug_in_setup():
291    r"""
292    Initialize all plug-in environment variables which do not change for the
293    duration of the program.
294
295    """
296
297    global LOG_LEVEL
298    BuiltIn().set_log_level("NONE")
299
300    BuiltIn().set_global_variable("${master_pid}", master_pid)
301    BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
302    BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
303    BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
304    BuiltIn().set_global_variable(
305        "${FFDC_LIST_FILE_PATH}", ffdc_list_file_path
306    )
307    BuiltIn().set_global_variable(
308        "${FFDC_REPORT_LIST_PATH}", ffdc_report_list_path
309    )
310    BuiltIn().set_global_variable(
311        "${FFDC_SUMMARY_LIST_PATH}", ffdc_summary_list_path
312    )
313
314    BuiltIn().set_global_variable(
315        "${FFDC_DIR_PATH_STYLE}", ffdc_dir_path_style
316    )
317    BuiltIn().set_global_variable("${FFDC_CHECK}", ffdc_check)
318
319    # For each program parameter, set the corresponding AUTOBOOT_ environment
320    # variable value.  Also, set an AUTOBOOT_ environment variable for every
321    # element in additional_values.
322    additional_values = [
323        "program_pid",
324        "master_pid",
325        "ffdc_dir_path",
326        "status_dir_path",
327        "base_tool_dir_path",
328        "ffdc_list_file_path",
329        "ffdc_report_list_path",
330        "ffdc_summary_list_path",
331        "execdir",
332        "redfish_supported",
333        "redfish_rest_supported",
334        "redfish_support_trans_state",
335    ]
336
337    plug_in_vars = parm_list + additional_values
338
339    for var_name in plug_in_vars:
340        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
341        var_name = var_name.upper()
342        if var_value is None:
343            var_value = ""
344        os.environ["AUTOBOOT_" + var_name] = str(var_value)
345
346    BuiltIn().set_log_level(LOG_LEVEL)
347
348    # Make sure the ffdc list directory exists.
349    ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
350    if not os.path.exists(ffdc_list_dir_path):
351        os.makedirs(ffdc_list_dir_path)
352
353
354def plug_in_setup():
355    r"""
356    Initialize all changing plug-in environment variables for use by the
357    plug-in programs.
358    """
359
360    global LOG_LEVEL
361    global test_really_running
362
363    BuiltIn().set_log_level("NONE")
364
365    boot_pass, boot_fail = boot_results.return_total_pass_fail()
366    if boot_pass > 1:
367        test_really_running = 1
368    else:
369        test_really_running = 0
370
371    BuiltIn().set_global_variable(
372        "${test_really_running}", test_really_running
373    )
374    BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
375    BuiltIn().set_global_variable("${boot_pass}", boot_pass)
376    BuiltIn().set_global_variable("${boot_fail}", boot_fail)
377    BuiltIn().set_global_variable("${boot_success}", boot_success)
378    BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
379    BuiltIn().set_global_variable("${boot_start_time}", boot_start_time)
380    BuiltIn().set_global_variable("${boot_end_time}", boot_end_time)
381
382    # For each program parameter, set the corresponding AUTOBOOT_ environment
383    # variable value.  Also, set an AUTOBOOT_ environment variable for every
384    # element in additional_values.
385    additional_values = [
386        "boot_type_desc",
387        "boot_success",
388        "boot_pass",
389        "boot_fail",
390        "test_really_running",
391        "ffdc_prefix",
392        "boot_start_time",
393        "boot_end_time",
394    ]
395
396    plug_in_vars = additional_values
397
398    for var_name in plug_in_vars:
399        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
400        var_name = var_name.upper()
401        if var_value is None:
402            var_value = ""
403        os.environ["AUTOBOOT_" + var_name] = str(var_value)
404
405    if debug:
406        shell_rc, out_buf = gc.cmd_fnc_u(
407            "printenv | egrep AUTOBOOT_ | sort -u"
408        )
409
410    BuiltIn().set_log_level(LOG_LEVEL)
411
412
413def pre_boot_plug_in_setup():
414    # Clear the ffdc_list_file_path file.  Plug-ins may now write to it.
415    try:
416        os.remove(ffdc_list_file_path)
417    except OSError:
418        pass
419
420    # Clear the ffdc_report_list_path file.  Plug-ins may now write to it.
421    try:
422        os.remove(ffdc_report_list_path)
423    except OSError:
424        pass
425
426    # Clear the ffdc_summary_list_path file.  Plug-ins may now write to it.
427    try:
428        os.remove(ffdc_summary_list_path)
429    except OSError:
430        pass
431
432    global ffdc_prefix
433
434    seconds = time.time()
435    loc_time = time.localtime(seconds)
436    time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
437
438    ffdc_prefix = openbmc_nickname + "." + time_string
439
440
441def default_sigusr1(signal_number=0, frame=None):
442    r"""
443    Handle SIGUSR1 by doing nothing.
444
445    This function assists in debugging SIGUSR1 processing by printing messages
446    to stdout and to the log.html file.
447
448    Description of argument(s):
449    signal_number  The signal number (should always be 10 for SIGUSR1).
450    frame          The frame data.
451    """
452
453    gp.qprintn()
454    gp.qprint_executing()
455    gp.lprint_executing()
456
457
458def set_default_siguser1():
459    r"""
460    Set the default_sigusr1 function to be the SIGUSR1 handler.
461    """
462
463    gp.qprintn()
464    gp.qprint_executing()
465    gp.lprint_executing()
466    signal.signal(signal.SIGUSR1, default_sigusr1)
467
468
469def setup():
470    r"""
471    Do general program setup tasks.
472    """
473
474    global cp_setup_called
475    global transitional_boot_selected
476
477    gp.qprintn()
478
479    if redfish_supported:
480        redfish.login()
481
482    set_default_siguser1()
483    transitional_boot_selected = False
484
485    robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
486    repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
487    # If we can't find process_plug_in_packages.py, ssh_pw or
488    # validate_plug_ins.py, then we don't have our repo bin in PATH.
489    shell_rc, out_buf = gc.cmd_fnc_u(
490        "which process_plug_in_packages.py" + " ssh_pw validate_plug_ins.py",
491        quiet=1,
492        print_output=0,
493        show_err=0,
494    )
495    if shell_rc != 0:
496        os.environ["PATH"] = repo_bin_path + ":" + os.environ.get("PATH", "")
497    # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
498    if robot_pgm_dir_path not in sys.path:
499        sys.path.append(robot_pgm_dir_path)
500        PYTHONPATH = os.environ.get("PYTHONPATH", "")
501        if PYTHONPATH == "":
502            os.environ["PYTHONPATH"] = robot_pgm_dir_path
503        else:
504            os.environ["PYTHONPATH"] = robot_pgm_dir_path + ":" + PYTHONPATH
505
506    validate_parms()
507
508    gp.qprint_pgm_header()
509
510    grk.run_key_u(default_set_power_policy, ignore=1)
511
512    initial_plug_in_setup()
513
514    plug_in_setup()
515    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
516        call_point="setup"
517    )
518    if rc != 0:
519        error_message = "Plug-in setup failed.\n"
520        gp.print_error_report(error_message)
521        BuiltIn().fail(error_message)
522    # Setting cp_setup_called lets our Teardown know that it needs to call
523    # the cleanup plug-in call point.
524    cp_setup_called = 1
525
526    # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
527    BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
528    # FFDC_LOG_PATH is used by "FFDC" keyword.
529    BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
530
531    # Also printed by FFDC.
532    global host_name
533    global host_ip
534    host = socket.gethostname()
535    host_name, host_ip = gm.get_host_name_ip(host)
536
537    gp.dprint_var(boot_table)
538    gp.dprint_var(boot_lists)
539
540
541def validate_parms():
542    r"""
543    Validate all program parameters.
544    """
545
546    process_pgm_parms()
547
548    gp.qprintn()
549
550    global openbmc_model
551    if openbmc_model == "":
552        status, ret_values = grk.run_key_u("Get BMC System Model", ignore=1)
553        # Set the model to default "OPENBMC" if getting it from BMC fails.
554        if status == "FAIL":
555            openbmc_model = "OPENBMC"
556        else:
557            openbmc_model = ret_values
558        BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
559    gv.set_exit_on_error(True)
560    gv.valid_value(openbmc_host)
561    gv.valid_value(openbmc_username)
562    gv.valid_value(openbmc_password)
563    gv.valid_value(rest_username)
564    gv.valid_value(rest_password)
565    gv.valid_value(ipmi_username)
566    gv.valid_value(ipmi_password)
567    if os_host != "":
568        gv.valid_value(os_username)
569        gv.valid_value(os_password)
570    if pdu_host != "":
571        gv.valid_value(pdu_username)
572        gv.valid_value(pdu_password)
573        gv.valid_integer(pdu_slot_no)
574    if openbmc_serial_host != "":
575        gv.valid_integer(openbmc_serial_port)
576    gv.valid_value(openbmc_model)
577    gv.valid_integer(max_num_tests)
578    gv.valid_integer(boot_pass)
579    gv.valid_integer(boot_fail)
580    plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
581    BuiltIn().set_global_variable(
582        "${plug_in_packages_list}", plug_in_packages_list
583    )
584    gv.valid_value(stack_mode, valid_values=["normal", "skip"])
585    gv.set_exit_on_error(False)
586    if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
587        error_message = (
588            "You must provide either a value for either the"
589            + " boot_list or the boot_stack parm.\n"
590        )
591        BuiltIn().fail(gp.sprint_error(error_message))
592    valid_boot_list(boot_list, valid_boot_types)
593    valid_boot_list(boot_stack, valid_boot_types)
594    selected_PDU_boots = list(
595        set(boot_list + boot_stack) & set(boot_lists["PDU_reboot"])
596    )
597    if len(selected_PDU_boots) > 0 and pdu_host == "":
598        error_message = (
599            "You have selected the following boots which"
600            + " require a PDU host but no value for pdu_host:\n"
601        )
602        error_message += gp.sprint_var(selected_PDU_boots)
603        error_message += gp.sprint_var(pdu_host, fmt=gp.blank())
604        BuiltIn().fail(gp.sprint_error(error_message))
605
606    return
607
608
609def my_get_state():
610    r"""
611    Get the system state plus a little bit of wrapping.
612    """
613
614    global state
615
616    req_states = ["epoch_seconds"] + st.default_req_states
617
618    gp.qprint_timen("Getting system state.")
619    if test_mode:
620        state["epoch_seconds"] = int(time.time())
621    else:
622        state = st.get_state(req_states=req_states, quiet=quiet)
623    gp.qprint_var(state)
624
625
626def valid_state():
627    r"""
628    Verify that our state dictionary contains no blank values.  If we don't get
629    valid state data, we cannot continue to work.
630    """
631
632    if st.compare_states(state, st.invalid_state_match, "or"):
633        error_message = (
634            "The state dictionary contains blank fields which"
635            + " is illegal.\n"
636            + gp.sprint_var(state)
637        )
638        BuiltIn().fail(gp.sprint_error(error_message))
639
640
641def select_boot():
642    r"""
643    Select a boot test to be run based on our current state and return the
644    chosen boot type.
645
646    Description of arguments:
647    state  The state of the machine.
648    """
649
650    global transitional_boot_selected
651    global boot_stack
652
653    gp.qprint_timen("Selecting a boot test.")
654
655    if transitional_boot_selected and not boot_success:
656        prior_boot = next_boot
657        boot_candidate = boot_stack.pop()
658        gp.qprint_timen(
659            "The prior '"
660            + next_boot
661            + "' was chosen to"
662            + " transition to a valid state for '"
663            + boot_candidate
664            + "' which was at the top of the boot_stack.  Since"
665            + " the '"
666            + next_boot
667            + "' failed, the '"
668            + boot_candidate
669            + "' has been removed from the stack"
670            + " to avoid and endless failure loop."
671        )
672        if len(boot_stack) == 0:
673            return ""
674
675    my_get_state()
676    valid_state()
677
678    transitional_boot_selected = False
679    stack_popped = 0
680    if len(boot_stack) > 0:
681        stack_popped = 1
682        gp.qprint_dashes()
683        gp.qprint_var(boot_stack)
684        gp.qprint_dashes()
685        skip_boot_printed = 0
686        while len(boot_stack) > 0:
687            boot_candidate = boot_stack.pop()
688            if stack_mode == "normal":
689                break
690            else:
691                if st.compare_states(state, boot_table[boot_candidate]["end"]):
692                    if not skip_boot_printed:
693                        gp.qprint_var(stack_mode)
694                        gp.qprintn()
695                        gp.qprint_timen(
696                            "Skipping the following boot tests"
697                            + " which are unnecessary since their"
698                            + " required end states match the"
699                            + " current machine state:"
700                        )
701                        skip_boot_printed = 1
702                    gp.qprint_var(boot_candidate)
703                    boot_candidate = ""
704        if boot_candidate == "":
705            gp.qprint_dashes()
706            gp.qprint_var(boot_stack)
707            gp.qprint_dashes()
708            return boot_candidate
709        if st.compare_states(state, boot_table[boot_candidate]["start"]):
710            gp.qprint_timen(
711                "The machine state is valid for a '"
712                + boot_candidate
713                + "' boot test."
714            )
715            gp.qprint_dashes()
716            gp.qprint_var(boot_stack)
717            gp.qprint_dashes()
718            return boot_candidate
719        else:
720            gp.qprint_timen(
721                "The machine state does not match the required"
722                + " starting state for a '"
723                + boot_candidate
724                + "' boot test:"
725            )
726            gp.qprint_varx(
727                "boot_table_start_entry", boot_table[boot_candidate]["start"]
728            )
729            boot_stack.append(boot_candidate)
730            transitional_boot_selected = True
731            popped_boot = boot_candidate
732
733    # Loop through your list selecting a boot_candidates
734    boot_candidates = []
735    for boot_candidate in boot_list:
736        if st.compare_states(state, boot_table[boot_candidate]["start"]):
737            if stack_popped:
738                if st.compare_states(
739                    boot_table[boot_candidate]["end"],
740                    boot_table[popped_boot]["start"],
741                ):
742                    boot_candidates.append(boot_candidate)
743            else:
744                boot_candidates.append(boot_candidate)
745
746    if len(boot_candidates) == 0:
747        gp.qprint_timen(
748            "The user's boot list contained no boot tests"
749            + " which are valid for the current machine state."
750        )
751        boot_candidate = default_power_on
752        if not st.compare_states(state, boot_table[default_power_on]["start"]):
753            boot_candidate = default_power_off
754        boot_candidates.append(boot_candidate)
755        gp.qprint_timen(
756            "Using default '"
757            + boot_candidate
758            + "' boot type to transition to valid state."
759        )
760
761    gp.dprint_var(boot_candidates)
762
763    # Randomly select a boot from the candidate list.
764    boot = random.choice(boot_candidates)
765
766    return boot
767
768
769def print_defect_report(ffdc_file_list):
770    r"""
771    Print a defect report.
772
773    Description of argument(s):
774    ffdc_file_list  A list of files which were collected by our ffdc functions.
775    """
776
777    # Making deliberate choice to NOT run plug_in_setup().  We don't want
778    # ffdc_prefix updated.
779    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
780        call_point="ffdc_report", stop_on_plug_in_failure=0
781    )
782
783    # Get additional header data which may have been created by ffdc plug-ins.
784    # Also, delete the individual header files to cleanup.
785    cmd_buf = (
786        "file_list=$(cat "
787        + ffdc_report_list_path
788        + " 2>/dev/null)"
789        + ' ; [ ! -z "${file_list}" ] && cat ${file_list}'
790        + " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
791    )
792    shell_rc, more_header_info = gc.cmd_fnc_u(
793        cmd_buf, print_output=0, show_err=0
794    )
795
796    # Get additional summary data which may have been created by ffdc plug-ins.
797    # Also, delete the individual header files to cleanup.
798    cmd_buf = (
799        "file_list=$(cat "
800        + ffdc_summary_list_path
801        + " 2>/dev/null)"
802        + ' ; [ ! -z "${file_list}" ] && cat ${file_list}'
803        + " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
804    )
805    shell_rc, ffdc_summary_info = gc.cmd_fnc_u(
806        cmd_buf, print_output=0, show_err=0
807    )
808
809    # ffdc_list_file_path contains a list of any ffdc files created by plug-
810    # ins, etc.  Read that data into a list.
811    try:
812        plug_in_ffdc_list = (
813            open(ffdc_list_file_path, "r").read().rstrip("\n").split("\n")
814        )
815        plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list))
816    except IOError:
817        plug_in_ffdc_list = []
818
819    # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed
820    # in.  Eliminate duplicates and sort the list.
821    ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list))
822
823    if status_file_path != "":
824        ffdc_file_list.insert(0, status_file_path)
825
826    # Convert the list to a printable list.
827    printable_ffdc_file_list = "\n".join(ffdc_file_list)
828
829    # Open ffdc_file_list for writing.  We will write a complete list of
830    # FFDC files to it for possible use by plug-ins like cp_stop_check.
831    ffdc_list_file = open(ffdc_list_file_path, "w")
832    ffdc_list_file.write(printable_ffdc_file_list + "\n")
833    ffdc_list_file.close()
834
835    indent = 0
836    width = 90
837    linefeed = 1
838    char = "="
839
840    gp.qprintn()
841    gp.qprint_dashes(indent, width, linefeed, char)
842    gp.qprintn("Copy this data to the defect:\n")
843
844    if len(more_header_info) > 0:
845        gp.qprintn(more_header_info)
846    gp.qpvars(
847        host_name,
848        host_ip,
849        openbmc_nickname,
850        openbmc_host,
851        openbmc_host_name,
852        openbmc_ip,
853        openbmc_username,
854        openbmc_password,
855        rest_username,
856        rest_password,
857        ipmi_username,
858        ipmi_password,
859        os_host,
860        os_host_name,
861        os_ip,
862        os_username,
863        os_password,
864        pdu_host,
865        pdu_host_name,
866        pdu_ip,
867        pdu_username,
868        pdu_password,
869        pdu_slot_no,
870        openbmc_serial_host,
871        openbmc_serial_host_name,
872        openbmc_serial_ip,
873        openbmc_serial_port,
874    )
875
876    gp.qprintn()
877    print_boot_history(boot_history)
878    gp.qprintn()
879    gp.qprint_var(state)
880    gp.qprintn()
881    gp.qprintn("FFDC data files:")
882    gp.qprintn(printable_ffdc_file_list)
883    gp.qprintn()
884
885    if len(ffdc_summary_info) > 0:
886        gp.qprintn(ffdc_summary_info)
887
888    gp.qprint_dashes(indent, width, linefeed, char)
889
890
891def my_ffdc():
892    r"""
893    Collect FFDC data.
894    """
895
896    global state
897
898    plug_in_setup()
899    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
900        call_point="ffdc", stop_on_plug_in_failure=0
901    )
902
903    AUTOBOOT_FFDC_PREFIX = os.environ["AUTOBOOT_FFDC_PREFIX"]
904    status, ffdc_file_list = grk.run_key_u(
905        "FFDC  ffdc_prefix="
906        + AUTOBOOT_FFDC_PREFIX
907        + "  ffdc_function_list="
908        + ffdc_function_list,
909        ignore=1,
910    )
911    if status != "PASS":
912        gp.qprint_error("Call to ffdc failed.\n")
913        if type(ffdc_file_list) is not list:
914            ffdc_file_list = []
915        # Leave a record for caller that "soft" errors occurred.
916        soft_errors = 1
917        gpu.save_plug_in_value(soft_errors, pgm_name)
918
919    my_get_state()
920
921    print_defect_report(ffdc_file_list)
922
923
924def print_test_start_message(boot_keyword):
925    r"""
926    Print a message indicating what boot test is about to run.
927
928    Description of arguments:
929    boot_keyword  The name of the boot which is to be run
930                  (e.g. "BMC Power On").
931    """
932
933    global boot_history
934    global boot_start_time
935
936    doing_msg = gp.sprint_timen('Doing "' + boot_keyword + '".')
937
938    # Set boot_start_time for use by plug-ins.
939    boot_start_time = doing_msg[1:33]
940    gp.qprint_var(boot_start_time)
941
942    gp.qprint(doing_msg)
943
944    update_boot_history(boot_history, doing_msg, max_boot_history)
945
946
947def stop_boot_test(signal_number=0, frame=None):
948    r"""
949    Handle SIGUSR1 by aborting the boot test that is running.
950
951    Description of argument(s):
952    signal_number  The signal number (should always be 10 for SIGUSR1).
953    frame          The frame data.
954    """
955
956    gp.qprintn()
957    gp.qprint_executing()
958    gp.lprint_executing()
959
960    # Restore original sigusr1 handler.
961    set_default_siguser1()
962
963    message = "The caller has asked that the boot test be stopped and marked"
964    message += " as a failure."
965
966    function_stack = gm.get_function_stack()
967    if "wait_state" in function_stack:
968        st.set_exit_wait_early_message(message)
969    else:
970        BuiltIn().fail(gp.sprint_error(message))
971
972
973def run_boot(boot):
974    r"""
975    Run the specified boot.
976
977    Description of arguments:
978    boot  The name of the boot test to be performed.
979    """
980
981    global state
982
983    signal.signal(signal.SIGUSR1, stop_boot_test)
984    gp.qprint_timen("stop_boot_test is armed.")
985
986    print_test_start_message(boot)
987
988    plug_in_setup()
989    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
990        call_point="pre_boot"
991    )
992    if rc != 0:
993        error_message = (
994            "Plug-in failed with non-zero return code.\n"
995            + gp.sprint_var(rc, fmt=gp.hexa())
996        )
997        set_default_siguser1()
998        BuiltIn().fail(gp.sprint_error(error_message))
999
1000    if test_mode:
1001        # In test mode, we'll pretend the boot worked by assigning its
1002        # required end state to the default state value.
1003        state = st.strip_anchor_state(boot_table[boot]["end"])
1004    else:
1005        # Assertion:  We trust that the state data was made fresh by the
1006        # caller.
1007
1008        gp.qprintn()
1009
1010        if boot_table[boot]["method_type"] == "keyword":
1011            rk.my_run_keywords(
1012                boot_table[boot].get("lib_file_path", ""),
1013                boot_table[boot]["method"],
1014                quiet=quiet,
1015            )
1016
1017        if boot_table[boot]["bmc_reboot"]:
1018            st.wait_for_comm_cycle(int(state["epoch_seconds"]))
1019            plug_in_setup()
1020            rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1021                call_point="post_reboot"
1022            )
1023            if rc != 0:
1024                error_message = "Plug-in failed with non-zero return code.\n"
1025                error_message += gp.sprint_var(rc, fmt=gp.hexa())
1026                set_default_siguser1()
1027                BuiltIn().fail(gp.sprint_error(error_message))
1028        else:
1029            match_state = st.anchor_state(state)
1030            del match_state["epoch_seconds"]
1031            # Wait for the state to change in any way.
1032            st.wait_state(
1033                match_state,
1034                wait_time=state_change_timeout,
1035                interval="10 seconds",
1036                invert=1,
1037            )
1038
1039        gp.qprintn()
1040        if boot_table[boot]["end"]["chassis"] == "Off":
1041            boot_timeout = power_off_timeout
1042        else:
1043            boot_timeout = power_on_timeout
1044        st.wait_state(
1045            boot_table[boot]["end"],
1046            wait_time=boot_timeout,
1047            interval="10 seconds",
1048        )
1049
1050    plug_in_setup()
1051    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1052        call_point="post_boot"
1053    )
1054    if rc != 0:
1055        error_message = (
1056            "Plug-in failed with non-zero return code.\n"
1057            + gp.sprint_var(rc, fmt=gp.hexa())
1058        )
1059        set_default_siguser1()
1060        BuiltIn().fail(gp.sprint_error(error_message))
1061
1062    # Restore original sigusr1 handler.
1063    set_default_siguser1()
1064
1065
1066def test_loop_body():
1067    r"""
1068    The main loop body for the loop in main_py.
1069
1070    Description of arguments:
1071    boot_count  The iteration number (starts at 1).
1072    """
1073
1074    global boot_count
1075    global state
1076    global next_boot
1077    global boot_success
1078    global boot_end_time
1079
1080    gp.qprintn()
1081
1082    next_boot = select_boot()
1083    if next_boot == "":
1084        return True
1085
1086    boot_count += 1
1087    gp.qprint_timen("Starting boot " + str(boot_count) + ".")
1088
1089    pre_boot_plug_in_setup()
1090
1091    cmd_buf = ["run_boot", next_boot]
1092    boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
1093    if boot_status == "FAIL":
1094        gp.qprint(msg)
1095
1096    gp.qprintn()
1097    if boot_status == "PASS":
1098        boot_success = 1
1099        completion_msg = gp.sprint_timen(
1100            'BOOT_SUCCESS: "' + next_boot + '" succeeded.'
1101        )
1102    else:
1103        boot_success = 0
1104        completion_msg = gp.sprint_timen(
1105            'BOOT_FAILED: "' + next_boot + '" failed.'
1106        )
1107
1108    # Set boot_end_time for use by plug-ins.
1109    boot_end_time = completion_msg[1:33]
1110    gp.qprint_var(boot_end_time)
1111
1112    gp.qprint(completion_msg)
1113
1114    boot_results.update(next_boot, boot_status)
1115
1116    plug_in_setup()
1117    # NOTE: A post_test_case call point failure is NOT counted as a boot
1118    # failure.
1119    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1120        call_point="post_test_case", stop_on_plug_in_failure=0
1121    )
1122
1123    plug_in_setup()
1124    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1125        call_point="ffdc_check",
1126        shell_rc=dump_ffdc_rc(),
1127        stop_on_plug_in_failure=1,
1128        stop_on_non_zero_rc=1,
1129    )
1130    if ffdc_check == "All" or shell_rc == dump_ffdc_rc():
1131        status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
1132        if status != "PASS":
1133            gp.qprint_error("Call to my_ffdc failed.\n")
1134            # Leave a record for caller that "soft" errors occurred.
1135            soft_errors = 1
1136            gpu.save_plug_in_value(soft_errors, pgm_name)
1137
1138    if delete_errlogs:
1139        # print error logs before delete
1140        if redfish_support_trans_state:
1141            status, error_logs = grk.run_key_u("Get Redfish Event Logs")
1142            log.print_error_logs(
1143                error_logs, "AdditionalDataURI Message Severity"
1144            )
1145        else:
1146            status, error_logs = grk.run_key_u("Get Error Logs")
1147            log.print_error_logs(error_logs, "AdditionalData Message Severity")
1148        pels = pel.peltool("-l", ignore_err=1)
1149        gp.qprint_var(pels)
1150
1151        # We need to purge error logs between boots or they build up.
1152        grk.run_key(delete_errlogs_cmd, ignore=1)
1153        grk.run_key(delete_bmcdump_cmd, ignore=1)
1154        if redfish_support_trans_state:
1155            grk.run_key(delete_sysdump_cmd, ignore=1)
1156
1157    boot_results.print_report()
1158    gp.qprint_timen("Finished boot " + str(boot_count) + ".")
1159
1160    plug_in_setup()
1161    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1162        call_point="stop_check", shell_rc=stop_test_rc(), stop_on_non_zero_rc=1
1163    )
1164    if shell_rc == stop_test_rc():
1165        message = "Stopping as requested by user.\n"
1166        gp.qprint_time(message)
1167        BuiltIn().fail(message)
1168
1169    # This should help prevent ConnectionErrors.
1170    # Purge all redfish and REST connection sessions.
1171    if redfish_delete_sessions:
1172        grk.run_key_u("Close All Connections", ignore=1)
1173        grk.run_key_u("Delete All Redfish Sessions", ignore=1)
1174
1175    return True
1176
1177
1178def obmc_boot_test_teardown():
1179    r"""
1180    Clean up after the main keyword.
1181    """
1182    gp.qprint_executing()
1183
1184    if ga.psutil_imported:
1185        ga.terminate_descendants()
1186
1187    if cp_setup_called:
1188        plug_in_setup()
1189        rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1190            call_point="cleanup", stop_on_plug_in_failure=0
1191        )
1192
1193    if "boot_results_file_path" in globals():
1194        # Save boot_results and boot_history objects to a file in case they are
1195        # needed again.
1196        gp.qprint_timen("Saving boot_results to the following path.")
1197        gp.qprint_var(boot_results_file_path)
1198        pickle.dump(
1199            (boot_results, boot_history),
1200            open(boot_results_file_path, "wb"),
1201            pickle.HIGHEST_PROTOCOL,
1202        )
1203
1204    global save_stack
1205    # Restore any global values saved on the save_stack.
1206    for parm_name in main_func_parm_list:
1207        # Get the parm_value if it was saved on the stack.
1208        try:
1209            parm_value = save_stack.pop(parm_name)
1210        except BaseException:
1211            # If it was not saved, no further action is required.
1212            continue
1213
1214        # Restore the saved value.
1215        cmd_buf = (
1216            'BuiltIn().set_global_variable("${' + parm_name + '}", parm_value)'
1217        )
1218        gp.dpissuing(cmd_buf)
1219        exec(cmd_buf)
1220
1221    gp.dprintn(save_stack.sprint_obj())
1222
1223
1224def test_teardown():
1225    r"""
1226    Clean up after this test case.
1227    """
1228
1229    gp.qprintn()
1230    gp.qprint_executing()
1231
1232    if ga.psutil_imported:
1233        ga.terminate_descendants()
1234
1235    cmd_buf = [
1236        "Print Error",
1237        "A keyword timeout occurred ending this program.\n",
1238    ]
1239    BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
1240
1241    if redfish_supported:
1242        redfish.logout()
1243
1244    gp.qprint_pgm_footer()
1245
1246
1247def post_stack():
1248    r"""
1249    Process post_stack plug-in programs.
1250    """
1251
1252    if not call_post_stack_plug:
1253        # The caller does not wish to have post_stack plug-in processing done.
1254        return
1255
1256    global boot_success
1257
1258    # NOTE: A post_stack call-point failure is NOT counted as a boot failure.
1259    pre_boot_plug_in_setup()
1260    # For the purposes of the following plug-ins, mark the "boot" as a success.
1261    boot_success = 1
1262    plug_in_setup()
1263    (
1264        rc,
1265        shell_rc,
1266        failed_plug_in_name,
1267        history,
1268    ) = grpi.rprocess_plug_in_packages(
1269        call_point="post_stack", stop_on_plug_in_failure=0, return_history=True
1270    )
1271    for doing_msg in history:
1272        update_boot_history(boot_history, doing_msg, max_boot_history)
1273    if rc != 0:
1274        boot_success = 0
1275
1276    plug_in_setup()
1277    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1278        call_point="ffdc_check",
1279        shell_rc=dump_ffdc_rc(),
1280        stop_on_plug_in_failure=1,
1281        stop_on_non_zero_rc=1,
1282    )
1283    if shell_rc == dump_ffdc_rc():
1284        status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
1285        if status != "PASS":
1286            gp.qprint_error("Call to my_ffdc failed.\n")
1287            # Leave a record for caller that "soft" errors occurred.
1288            soft_errors = 1
1289            gpu.save_plug_in_value(soft_errors, pgm_name)
1290
1291    plug_in_setup()
1292    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1293        call_point="stop_check", shell_rc=stop_test_rc(), stop_on_non_zero_rc=1
1294    )
1295    if shell_rc == stop_test_rc():
1296        message = "Stopping as requested by user.\n"
1297        gp.qprint_time(message)
1298        BuiltIn().fail(message)
1299
1300
1301def obmc_boot_test_py(
1302    loc_boot_stack=None, loc_stack_mode=None, loc_quiet=None
1303):
1304    r"""
1305    Do main program processing.
1306    """
1307
1308    global save_stack
1309
1310    ga.set_term_options(
1311        term_requests={"pgm_names": ["process_plug_in_packages.py"]}
1312    )
1313
1314    gp.dprintn()
1315    # Process function parms.
1316    for parm_name in main_func_parm_list:
1317        # Get parm's value.
1318        parm_value = eval("loc_" + parm_name)
1319        gp.dpvars(parm_name, parm_value)
1320
1321        if parm_value is not None:
1322            # Save the global value on a stack.
1323            cmd_buf = (
1324                'save_stack.push(BuiltIn().get_variable_value("${'
1325                + parm_name
1326                + '}"), "'
1327                + parm_name
1328                + '")'
1329            )
1330            gp.dpissuing(cmd_buf)
1331            exec(cmd_buf)
1332
1333            # Set the global value to the passed value.
1334            cmd_buf = (
1335                'BuiltIn().set_global_variable("${'
1336                + parm_name
1337                + '}", loc_'
1338                + parm_name
1339                + ")"
1340            )
1341            gp.dpissuing(cmd_buf)
1342            exec(cmd_buf)
1343
1344    gp.dprintn(save_stack.sprint_obj())
1345
1346    setup()
1347
1348    init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail()
1349
1350    if ffdc_only:
1351        gp.qprint_timen("Caller requested ffdc_only.")
1352        if do_pre_boot_plug_in_setup:
1353            pre_boot_plug_in_setup()
1354        grk.run_key_u("my_ffdc")
1355        return
1356
1357    if delete_errlogs:
1358        # print error logs before delete
1359        if redfish_support_trans_state:
1360            status, error_logs = grk.run_key_u("Get Redfish Event Logs")
1361            log.print_error_logs(
1362                error_logs, "AdditionalDataURI Message Severity"
1363            )
1364        else:
1365            status, error_logs = grk.run_key_u("Get Error Logs")
1366            log.print_error_logs(error_logs, "AdditionalData Message Severity")
1367        pels = pel.peltool("-l", ignore_err=1)
1368        gp.qprint_var(pels)
1369
1370        # Delete errlogs prior to doing any boot tests.
1371        grk.run_key(delete_errlogs_cmd, ignore=1)
1372        grk.run_key(delete_bmcdump_cmd, ignore=1)
1373        if redfish_support_trans_state:
1374            grk.run_key(delete_sysdump_cmd, ignore=1)
1375
1376    # Process caller's boot_stack.
1377    while len(boot_stack) > 0:
1378        test_loop_body()
1379
1380    gp.qprint_timen("Finished processing stack.")
1381
1382    post_stack()
1383
1384    # Process caller's boot_list.
1385    if len(boot_list) > 0:
1386        for ix in range(1, max_num_tests + 1):
1387            test_loop_body()
1388
1389    gp.qprint_timen("Completed all requested boot tests.")
1390
1391    boot_pass, boot_fail = boot_results.return_total_pass_fail()
1392    new_fail = boot_fail - init_boot_fail
1393    if new_fail > boot_fail_threshold:
1394        error_message = (
1395            "Boot failures exceed the boot failure"
1396            + " threshold:\n"
1397            + gp.sprint_var(new_fail)
1398            + gp.sprint_var(boot_fail_threshold)
1399        )
1400        BuiltIn().fail(gp.sprint_error(error_message))
1401