1#!/usr/bin/env python3
2
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
7import os
8import imp
9import time
10import glob
11import random
12import re
13import signal
14try:
15    import cPickle as pickle
16except ImportError:
17    import pickle
18import socket
19
20from robot.utils import DotDict
21from robot.libraries.BuiltIn import BuiltIn
22
23from boot_data import *
24import gen_print as gp
25import gen_robot_plug_in as grpi
26import gen_arg as ga
27import gen_valid as gv
28import gen_misc as gm
29import gen_cmd as gc
30import gen_robot_keyword as grk
31import state as st
32import var_stack as vs
33import gen_plug_in_utils as gpu
34import pel_utils as pel
35import logging_utils as log
36
37base_path = os.path.dirname(os.path.dirname(
38                            imp.find_module("gen_robot_print")[1])) +\
39    os.sep
40sys.path.append(base_path + "extended/")
41import run_keyword as rk
42
43# Setting master_pid correctly influences the behavior of plug-ins like
44# DB_Logging
45program_pid = os.getpid()
46master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid)
47pgm_name = re.sub('\\.py$', '', os.path.basename(__file__))
48
49# Set up boot data structures.
50os_host = BuiltIn().get_variable_value("${OS_HOST}", default="")
51
52boot_lists = read_boot_lists()
53
54# The maximum number of entries that can be in the boot_history global variable.
55max_boot_history = 10
56boot_history = []
57
58state = st.return_state_constant('default_state')
59cp_setup_called = 0
60next_boot = ""
61base_tool_dir_path = os.path.normpath(os.environ.get(
62    'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep
63
64ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep
65boot_success = 0
66
67status_dir_path = os.environ.get('STATUS_DIR_PATH', "") or \
68    BuiltIn().get_variable_value("${STATUS_DIR_PATH}", default="")
69if status_dir_path != "":
70    status_dir_path = os.path.normpath(status_dir_path) + os.sep
71    # For plugin expecting env gen_call_robot.py
72    os.environ['STATUS_DIR_PATH'] = status_dir_path
73
74redfish_support_trans_state = int(os.environ.get('REDFISH_SUPPORT_TRANS_STATE', 0)) or \
75    int(BuiltIn().get_variable_value("${REDFISH_SUPPORT_TRANS_STATE}", default=0))
76redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False)
77redfish_rest_supported = BuiltIn().get_variable_value("${REDFISH_REST_SUPPORTED}", default=False)
78redfish_delete_sessions = int(BuiltIn().get_variable_value("${REDFISH_DELETE_SESSIONS}", default=1))
79if redfish_supported:
80    redfish = BuiltIn().get_library_instance('redfish')
81    default_power_on = "Redfish Power On"
82    default_power_off = "Redfish Power Off"
83    if not redfish_support_trans_state:
84        delete_errlogs_cmd = "Delete Error Logs  ${quiet}=${1}"
85        delete_bmcdump_cmd = "Delete All BMC Dump"
86        default_set_power_policy = "Set BMC Power Policy  ALWAYS_POWER_OFF"
87    else:
88        delete_errlogs_cmd = "Redfish Purge Event Log"
89        delete_bmcdump_cmd = "Redfish Delete All BMC Dumps"
90        delete_sysdump_cmd = "Redfish Delete All System Dumps"
91        default_set_power_policy = "Redfish Set Power Restore Policy  AlwaysOff"
92else:
93    default_power_on = "REST Power On"
94    default_power_off = "REST Power Off"
95    delete_errlogs_cmd = "Delete Error Logs  ${quiet}=${1}"
96    delete_bmcdump_cmd = "Delete All BMC Dump"
97    default_set_power_policy = "Set BMC Power Policy  ALWAYS_POWER_OFF"
98boot_count = 0
99
100LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
101AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '')
102ffdc_prefix = AUTOBOOT_FFDC_PREFIX
103boot_start_time = ""
104boot_end_time = ""
105save_stack = vs.var_stack('save_stack')
106main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet']
107
108
109def dump_ffdc_rc():
110    r"""
111    Return the constant dump ffdc test return code value.
112
113    When a plug-in call point program returns this value, it indicates that
114    this program should collect FFDC.
115    """
116
117    return 0x00000200
118
119
120def stop_test_rc():
121    r"""
122    Return the constant stop test return code value.
123
124    When a plug-in call point program returns this value, it indicates that
125    this program should stop running.
126    """
127
128    return 0x00000200
129
130
131def process_host(host,
132                 host_var_name=""):
133    r"""
134    Process a host by getting the associated host name and IP address and
135    setting them in global variables.
136
137    If the caller does not pass the host_var_name, this function will try to
138    figure out the name of the variable used by the caller for the host parm.
139    Callers are advised to explicitly specify the host_var_name when calling
140    with an exec command.  In such cases, the get_arg_name cannot figure out
141    the host variable name.
142
143    This function will then create similar global variable names by
144    removing "_host" and appending "_host_name" or "_ip" to the host variable
145    name.
146
147    Example:
148
149    If a call is made like this:
150    process_host(openbmc_host)
151
152    Global variables openbmc_host_name and openbmc_ip will be set.
153
154    Description of argument(s):
155    host           A host name or IP.  The name of the variable used should
156                   have a suffix of "_host".
157    host_var_name  The name of the variable being used as the host parm.
158    """
159
160    if host_var_name == "":
161        host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
162
163    host_name_var_name = re.sub("host", "host_name", host_var_name)
164    ip_var_name = re.sub("host", "ip", host_var_name)
165    cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\
166        host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\
167        host + "')"
168    exec(cmd_buf)
169
170
171def process_pgm_parms():
172    r"""
173    Process the program parameters by assigning them all to corresponding
174    globals.  Also, set some global values that depend on program parameters.
175    """
176
177    # Program parameter processing.
178    # Assign all program parms to python variables which are global to this
179    # module.
180
181    global parm_list
182    parm_list = BuiltIn().get_variable_value("${parm_list}")
183    # The following subset of parms should be processed as integers.
184    int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only',
185                'boot_fail_threshold', 'delete_errlogs',
186                'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet',
187                'test_mode', 'debug']
188    for parm in parm_list:
189        if parm in int_list:
190            sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\
191                      "}\", \"0\"))"
192        else:
193            sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")"
194        cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
195        gp.dpissuing(cmd_buf)
196        exec(cmd_buf)
197        if re.match(r".*_host$", parm):
198            cmd_buf = "process_host(" + parm + ", '" + parm + "')"
199            exec(cmd_buf)
200        if re.match(r".*_password$", parm):
201            # Register the value of any parm whose name ends in _password.
202            # This will cause the print functions to replace passwords with
203            # asterisks in the output.
204            cmd_buf = "gp.register_passwords(" + parm + ")"
205            exec(cmd_buf)
206
207    global ffdc_dir_path_style
208    global boot_list
209    global boot_stack
210    global boot_results_file_path
211    global boot_results
212    global boot_history
213    global ffdc_list_file_path
214    global ffdc_report_list_path
215    global ffdc_summary_list_path
216    global boot_table
217    global valid_boot_types
218
219    if ffdc_dir_path_style == "":
220        ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0'))
221
222    # Convert these program parms to lists for easier processing..
223    boot_list = list(filter(None, boot_list.split(":")))
224    boot_stack = list(filter(None, boot_stack.split(":")))
225
226    boot_table = create_boot_table(boot_table_path, os_host=os_host)
227    valid_boot_types = create_valid_boot_list(boot_table)
228
229    cleanup_boot_results_file()
230    boot_results_file_path = create_boot_results_file_path(pgm_name,
231                                                           openbmc_nickname,
232                                                           master_pid)
233
234    if os.path.isfile(boot_results_file_path):
235        # We've been called before in this run so we'll load the saved
236        # boot_results and boot_history objects.
237        boot_results, boot_history =\
238            pickle.load(open(boot_results_file_path, 'rb'))
239    else:
240        boot_results = boot_results(boot_table, boot_pass, boot_fail)
241
242    ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\
243        "/FFDC_FILE_LIST"
244    ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\
245        "/FFDC_REPORT_FILE_LIST"
246
247    ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\
248        "/FFDC_SUMMARY_FILE_LIST"
249
250
251def initial_plug_in_setup():
252    r"""
253    Initialize all plug-in environment variables which do not change for the
254    duration of the program.
255
256    """
257
258    global LOG_LEVEL
259    BuiltIn().set_log_level("NONE")
260
261    BuiltIn().set_global_variable("${master_pid}", master_pid)
262    BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
263    BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
264    BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
265    BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}",
266                                  ffdc_list_file_path)
267    BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}",
268                                  ffdc_report_list_path)
269    BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}",
270                                  ffdc_summary_list_path)
271
272    BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}",
273                                  ffdc_dir_path_style)
274    BuiltIn().set_global_variable("${FFDC_CHECK}",
275                                  ffdc_check)
276
277    # For each program parameter, set the corresponding AUTOBOOT_ environment
278    # variable value.  Also, set an AUTOBOOT_ environment variable for every
279    # element in additional_values.
280    additional_values = ["program_pid", "master_pid", "ffdc_dir_path",
281                         "status_dir_path", "base_tool_dir_path",
282                         "ffdc_list_file_path", "ffdc_report_list_path",
283                         "ffdc_summary_list_path", "execdir", "redfish_supported",
284                         "redfish_rest_supported", "redfish_support_trans_state"]
285
286    plug_in_vars = parm_list + additional_values
287
288    for var_name in plug_in_vars:
289        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
290        var_name = var_name.upper()
291        if var_value is None:
292            var_value = ""
293        os.environ["AUTOBOOT_" + var_name] = str(var_value)
294
295    BuiltIn().set_log_level(LOG_LEVEL)
296
297    # Make sure the ffdc list directory exists.
298    ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
299    if not os.path.exists(ffdc_list_dir_path):
300        os.makedirs(ffdc_list_dir_path)
301
302
303def plug_in_setup():
304    r"""
305    Initialize all changing plug-in environment variables for use by the
306    plug-in programs.
307    """
308
309    global LOG_LEVEL
310    global test_really_running
311
312    BuiltIn().set_log_level("NONE")
313
314    boot_pass, boot_fail = boot_results.return_total_pass_fail()
315    if boot_pass > 1:
316        test_really_running = 1
317    else:
318        test_really_running = 0
319
320    BuiltIn().set_global_variable("${test_really_running}",
321                                  test_really_running)
322    BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
323    BuiltIn().set_global_variable("${boot_pass}", boot_pass)
324    BuiltIn().set_global_variable("${boot_fail}", boot_fail)
325    BuiltIn().set_global_variable("${boot_success}", boot_success)
326    BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
327    BuiltIn().set_global_variable("${boot_start_time}", boot_start_time)
328    BuiltIn().set_global_variable("${boot_end_time}", boot_end_time)
329
330    # For each program parameter, set the corresponding AUTOBOOT_ environment
331    # variable value.  Also, set an AUTOBOOT_ environment variable for every
332    # element in additional_values.
333    additional_values = ["boot_type_desc", "boot_success", "boot_pass",
334                         "boot_fail", "test_really_running", "ffdc_prefix",
335                         "boot_start_time", "boot_end_time"]
336
337    plug_in_vars = additional_values
338
339    for var_name in plug_in_vars:
340        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
341        var_name = var_name.upper()
342        if var_value is None:
343            var_value = ""
344        os.environ["AUTOBOOT_" + var_name] = str(var_value)
345
346    if debug:
347        shell_rc, out_buf = \
348            gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u")
349
350    BuiltIn().set_log_level(LOG_LEVEL)
351
352
353def pre_boot_plug_in_setup():
354
355    # Clear the ffdc_list_file_path file.  Plug-ins may now write to it.
356    try:
357        os.remove(ffdc_list_file_path)
358    except OSError:
359        pass
360
361    # Clear the ffdc_report_list_path file.  Plug-ins may now write to it.
362    try:
363        os.remove(ffdc_report_list_path)
364    except OSError:
365        pass
366
367    # Clear the ffdc_summary_list_path file.  Plug-ins may now write to it.
368    try:
369        os.remove(ffdc_summary_list_path)
370    except OSError:
371        pass
372
373    global ffdc_prefix
374
375    seconds = time.time()
376    loc_time = time.localtime(seconds)
377    time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
378
379    ffdc_prefix = openbmc_nickname + "." + time_string
380
381
382def default_sigusr1(signal_number=0,
383                    frame=None):
384    r"""
385    Handle SIGUSR1 by doing nothing.
386
387    This function assists in debugging SIGUSR1 processing by printing messages
388    to stdout and to the log.html file.
389
390    Description of argument(s):
391    signal_number  The signal number (should always be 10 for SIGUSR1).
392    frame          The frame data.
393    """
394
395    gp.qprintn()
396    gp.qprint_executing()
397    gp.lprint_executing()
398
399
400def set_default_siguser1():
401    r"""
402    Set the default_sigusr1 function to be the SIGUSR1 handler.
403    """
404
405    gp.qprintn()
406    gp.qprint_executing()
407    gp.lprint_executing()
408    signal.signal(signal.SIGUSR1, default_sigusr1)
409
410
411def setup():
412    r"""
413    Do general program setup tasks.
414    """
415
416    global cp_setup_called
417    global transitional_boot_selected
418
419    gp.qprintn()
420
421    if redfish_supported:
422        redfish.login()
423
424    set_default_siguser1()
425    transitional_boot_selected = False
426
427    robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
428    repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
429    # If we can't find process_plug_in_packages.py, ssh_pw or
430    # validate_plug_ins.py, then we don't have our repo bin in PATH.
431    shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py"
432                                     + " ssh_pw validate_plug_ins.py", quiet=1,
433                                     print_output=0, show_err=0)
434    if shell_rc != 0:
435        os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
436    # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
437    if robot_pgm_dir_path not in sys.path:
438        sys.path.append(robot_pgm_dir_path)
439        PYTHONPATH = os.environ.get("PYTHONPATH", "")
440        if PYTHONPATH == "":
441            os.environ['PYTHONPATH'] = robot_pgm_dir_path
442        else:
443            os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
444
445    validate_parms()
446
447    gp.qprint_pgm_header()
448
449    grk.run_key_u(default_set_power_policy, ignore=1)
450
451    initial_plug_in_setup()
452
453    plug_in_setup()
454    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
455        call_point='setup')
456    if rc != 0:
457        error_message = "Plug-in setup failed.\n"
458        gp.print_error_report(error_message)
459        BuiltIn().fail(error_message)
460    # Setting cp_setup_called lets our Teardown know that it needs to call
461    # the cleanup plug-in call point.
462    cp_setup_called = 1
463
464    # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
465    BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
466    # FFDC_LOG_PATH is used by "FFDC" keyword.
467    BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
468
469    # Also printed by FFDC.
470    global host_name
471    global host_ip
472    host = socket.gethostname()
473    host_name, host_ip = gm.get_host_name_ip(host)
474
475    gp.dprint_var(boot_table)
476    gp.dprint_var(boot_lists)
477
478
479def validate_parms():
480    r"""
481    Validate all program parameters.
482    """
483
484    process_pgm_parms()
485
486    gp.qprintn()
487
488    global openbmc_model
489    if openbmc_model == "":
490        status, ret_values =\
491            grk.run_key_u("Get BMC System Model")
492        openbmc_model = ret_values
493        BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
494    gv.set_exit_on_error(True)
495    gv.valid_value(openbmc_host)
496    gv.valid_value(openbmc_username)
497    gv.valid_value(openbmc_password)
498    gv.valid_value(rest_username)
499    gv.valid_value(rest_password)
500    gv.valid_value(ipmi_username)
501    gv.valid_value(ipmi_password)
502    if os_host != "":
503        gv.valid_value(os_username)
504        gv.valid_value(os_password)
505    if pdu_host != "":
506        gv.valid_value(pdu_username)
507        gv.valid_value(pdu_password)
508        gv.valid_integer(pdu_slot_no)
509    if openbmc_serial_host != "":
510        gv.valid_integer(openbmc_serial_port)
511    gv.valid_value(openbmc_model)
512    gv.valid_integer(max_num_tests)
513    gv.valid_integer(boot_pass)
514    gv.valid_integer(boot_fail)
515    plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
516    BuiltIn().set_global_variable("${plug_in_packages_list}",
517                                  plug_in_packages_list)
518    gv.valid_value(stack_mode, valid_values=['normal', 'skip'])
519    gv.set_exit_on_error(False)
520    if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
521        error_message = "You must provide either a value for either the" +\
522            " boot_list or the boot_stack parm.\n"
523        BuiltIn().fail(gp.sprint_error(error_message))
524    valid_boot_list(boot_list, valid_boot_types)
525    valid_boot_list(boot_stack, valid_boot_types)
526    selected_PDU_boots = list(set(boot_list + boot_stack)
527                              & set(boot_lists['PDU_reboot']))
528    if len(selected_PDU_boots) > 0 and pdu_host == "":
529        error_message = "You have selected the following boots which" +\
530                        " require a PDU host but no value for pdu_host:\n"
531        error_message += gp.sprint_var(selected_PDU_boots)
532        error_message += gp.sprint_var(pdu_host, fmt=gp.blank())
533        BuiltIn().fail(gp.sprint_error(error_message))
534
535    return
536
537
538def my_get_state():
539    r"""
540    Get the system state plus a little bit of wrapping.
541    """
542
543    global state
544
545    req_states = ['epoch_seconds'] + st.default_req_states
546
547    gp.qprint_timen("Getting system state.")
548    if test_mode:
549        state['epoch_seconds'] = int(time.time())
550    else:
551        state = st.get_state(req_states=req_states, quiet=quiet)
552    gp.qprint_var(state)
553
554
555def valid_state():
556    r"""
557    Verify that our state dictionary contains no blank values.  If we don't get
558    valid state data, we cannot continue to work.
559    """
560
561    if st.compare_states(state, st.invalid_state_match, 'or'):
562        error_message = "The state dictionary contains blank fields which" +\
563            " is illegal.\n" + gp.sprint_var(state)
564        BuiltIn().fail(gp.sprint_error(error_message))
565
566
567def select_boot():
568    r"""
569    Select a boot test to be run based on our current state and return the
570    chosen boot type.
571
572    Description of arguments:
573    state  The state of the machine.
574    """
575
576    global transitional_boot_selected
577    global boot_stack
578
579    gp.qprint_timen("Selecting a boot test.")
580
581    if transitional_boot_selected and not boot_success:
582        prior_boot = next_boot
583        boot_candidate = boot_stack.pop()
584        gp.qprint_timen("The prior '" + next_boot + "' was chosen to"
585                        + " transition to a valid state for '" + boot_candidate
586                        + "' which was at the top of the boot_stack.  Since"
587                        + " the '" + next_boot + "' failed, the '"
588                        + boot_candidate + "' has been removed from the stack"
589                        + " to avoid and endless failure loop.")
590        if len(boot_stack) == 0:
591            return ""
592
593    my_get_state()
594    valid_state()
595
596    transitional_boot_selected = False
597    stack_popped = 0
598    if len(boot_stack) > 0:
599        stack_popped = 1
600        gp.qprint_dashes()
601        gp.qprint_var(boot_stack)
602        gp.qprint_dashes()
603        skip_boot_printed = 0
604        while len(boot_stack) > 0:
605            boot_candidate = boot_stack.pop()
606            if stack_mode == 'normal':
607                break
608            else:
609                if st.compare_states(state, boot_table[boot_candidate]['end']):
610                    if not skip_boot_printed:
611                        gp.qprint_var(stack_mode)
612                        gp.qprintn()
613                        gp.qprint_timen("Skipping the following boot tests"
614                                        + " which are unnecessary since their"
615                                        + " required end states match the"
616                                        + " current machine state:")
617                        skip_boot_printed = 1
618                    gp.qprint_var(boot_candidate)
619                    boot_candidate = ""
620        if boot_candidate == "":
621            gp.qprint_dashes()
622            gp.qprint_var(boot_stack)
623            gp.qprint_dashes()
624            return boot_candidate
625        if st.compare_states(state, boot_table[boot_candidate]['start']):
626            gp.qprint_timen("The machine state is valid for a '"
627                            + boot_candidate + "' boot test.")
628            gp.qprint_dashes()
629            gp.qprint_var(boot_stack)
630            gp.qprint_dashes()
631            return boot_candidate
632        else:
633            gp.qprint_timen("The machine state does not match the required"
634                            + " starting state for a '" + boot_candidate
635                            + "' boot test:")
636            gp.qprint_varx("boot_table_start_entry",
637                           boot_table[boot_candidate]['start'])
638            boot_stack.append(boot_candidate)
639            transitional_boot_selected = True
640            popped_boot = boot_candidate
641
642    # Loop through your list selecting a boot_candidates
643    boot_candidates = []
644    for boot_candidate in boot_list:
645        if st.compare_states(state, boot_table[boot_candidate]['start']):
646            if stack_popped:
647                if st.compare_states(boot_table[boot_candidate]['end'],
648                                     boot_table[popped_boot]['start']):
649                    boot_candidates.append(boot_candidate)
650            else:
651                boot_candidates.append(boot_candidate)
652
653    if len(boot_candidates) == 0:
654        gp.qprint_timen("The user's boot list contained no boot tests"
655                        + " which are valid for the current machine state.")
656        boot_candidate = default_power_on
657        if not st.compare_states(state, boot_table[default_power_on]['start']):
658            boot_candidate = default_power_off
659        boot_candidates.append(boot_candidate)
660        gp.qprint_timen("Using default '" + boot_candidate
661                        + "' boot type to transition to valid state.")
662
663    gp.dprint_var(boot_candidates)
664
665    # Randomly select a boot from the candidate list.
666    boot = random.choice(boot_candidates)
667
668    return boot
669
670
671def print_defect_report(ffdc_file_list):
672    r"""
673    Print a defect report.
674
675    Description of argument(s):
676    ffdc_file_list  A list of files which were collected by our ffdc functions.
677    """
678
679    # Making deliberate choice to NOT run plug_in_setup().  We don't want
680    # ffdc_prefix updated.
681    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
682        call_point='ffdc_report', stop_on_plug_in_failure=0)
683
684    # Get additional header data which may have been created by ffdc plug-ins.
685    # Also, delete the individual header files to cleanup.
686    cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\
687              " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
688              " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
689    shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
690                                              show_err=0)
691
692    # Get additional summary data which may have been created by ffdc plug-ins.
693    # Also, delete the individual header files to cleanup.
694    cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\
695              " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
696              " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
697    shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
698                                               show_err=0)
699
700    # ffdc_list_file_path contains a list of any ffdc files created by plug-
701    # ins, etc.  Read that data into a list.
702    try:
703        plug_in_ffdc_list = \
704            open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n")
705        plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list))
706    except IOError:
707        plug_in_ffdc_list = []
708
709    # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed
710    # in.  Eliminate duplicates and sort the list.
711    ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list))
712
713    if status_file_path != "":
714        ffdc_file_list.insert(0, status_file_path)
715
716    # Convert the list to a printable list.
717    printable_ffdc_file_list = "\n".join(ffdc_file_list)
718
719    # Open ffdc_file_list for writing.  We will write a complete list of
720    # FFDC files to it for possible use by plug-ins like cp_stop_check.
721    ffdc_list_file = open(ffdc_list_file_path, 'w')
722    ffdc_list_file.write(printable_ffdc_file_list + "\n")
723    ffdc_list_file.close()
724
725    indent = 0
726    width = 90
727    linefeed = 1
728    char = "="
729
730    gp.qprintn()
731    gp.qprint_dashes(indent, width, linefeed, char)
732    gp.qprintn("Copy this data to the defect:\n")
733
734    if len(more_header_info) > 0:
735        gp.qprintn(more_header_info)
736    gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host,
737              openbmc_host_name, openbmc_ip, openbmc_username,
738              openbmc_password, rest_username, rest_password, ipmi_username,
739              ipmi_password, os_host, os_host_name, os_ip, os_username,
740              os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username,
741              pdu_password, pdu_slot_no, openbmc_serial_host,
742              openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port)
743
744    gp.qprintn()
745    print_boot_history(boot_history)
746    gp.qprintn()
747    gp.qprint_var(state)
748    gp.qprintn()
749    gp.qprintn("FFDC data files:")
750    gp.qprintn(printable_ffdc_file_list)
751    gp.qprintn()
752
753    if len(ffdc_summary_info) > 0:
754        gp.qprintn(ffdc_summary_info)
755
756    gp.qprint_dashes(indent, width, linefeed, char)
757
758
759def my_ffdc():
760    r"""
761    Collect FFDC data.
762    """
763
764    global state
765
766    plug_in_setup()
767    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
768        call_point='ffdc', stop_on_plug_in_failure=0)
769
770    AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX']
771    status, ffdc_file_list = grk.run_key_u("FFDC  ffdc_prefix="
772                                           + AUTOBOOT_FFDC_PREFIX
773                                           + "  ffdc_function_list="
774                                           + ffdc_function_list, ignore=1)
775    if status != 'PASS':
776        gp.qprint_error("Call to ffdc failed.\n")
777        if type(ffdc_file_list) is not list:
778            ffdc_file_list = []
779        # Leave a record for caller that "soft" errors occurred.
780        soft_errors = 1
781        gpu.save_plug_in_value(soft_errors, pgm_name)
782
783    my_get_state()
784
785    print_defect_report(ffdc_file_list)
786
787
788def print_test_start_message(boot_keyword):
789    r"""
790    Print a message indicating what boot test is about to run.
791
792    Description of arguments:
793    boot_keyword  The name of the boot which is to be run
794                  (e.g. "BMC Power On").
795    """
796
797    global boot_history
798    global boot_start_time
799
800    doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".")
801
802    # Set boot_start_time for use by plug-ins.
803    boot_start_time = doing_msg[1:33]
804    gp.qprint_var(boot_start_time)
805
806    gp.qprint(doing_msg)
807
808    update_boot_history(boot_history, doing_msg, max_boot_history)
809
810
811def stop_boot_test(signal_number=0,
812                   frame=None):
813    r"""
814    Handle SIGUSR1 by aborting the boot test that is running.
815
816    Description of argument(s):
817    signal_number  The signal number (should always be 10 for SIGUSR1).
818    frame          The frame data.
819    """
820
821    gp.qprintn()
822    gp.qprint_executing()
823    gp.lprint_executing()
824
825    # Restore original sigusr1 handler.
826    set_default_siguser1()
827
828    message = "The caller has asked that the boot test be stopped and marked"
829    message += " as a failure."
830
831    function_stack = gm.get_function_stack()
832    if "wait_state" in function_stack:
833        st.set_exit_wait_early_message(message)
834    else:
835        BuiltIn().fail(gp.sprint_error(message))
836
837
838def run_boot(boot):
839    r"""
840    Run the specified boot.
841
842    Description of arguments:
843    boot  The name of the boot test to be performed.
844    """
845
846    global state
847
848    signal.signal(signal.SIGUSR1, stop_boot_test)
849    gp.qprint_timen("stop_boot_test is armed.")
850
851    print_test_start_message(boot)
852
853    plug_in_setup()
854    rc, shell_rc, failed_plug_in_name = \
855        grpi.rprocess_plug_in_packages(call_point="pre_boot")
856    if rc != 0:
857        error_message = "Plug-in failed with non-zero return code.\n" +\
858            gp.sprint_var(rc, fmt=gp.hexa())
859        set_default_siguser1()
860        BuiltIn().fail(gp.sprint_error(error_message))
861
862    if test_mode:
863        # In test mode, we'll pretend the boot worked by assigning its
864        # required end state to the default state value.
865        state = st.strip_anchor_state(boot_table[boot]['end'])
866    else:
867        # Assertion:  We trust that the state data was made fresh by the
868        # caller.
869
870        gp.qprintn()
871
872        if boot_table[boot]['method_type'] == "keyword":
873            rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''),
874                               boot_table[boot]['method'],
875                               quiet=quiet)
876
877        if boot_table[boot]['bmc_reboot']:
878            st.wait_for_comm_cycle(int(state['epoch_seconds']))
879            plug_in_setup()
880            rc, shell_rc, failed_plug_in_name = \
881                grpi.rprocess_plug_in_packages(call_point="post_reboot")
882            if rc != 0:
883                error_message = "Plug-in failed with non-zero return code.\n"
884                error_message += gp.sprint_var(rc, fmt=gp.hexa())
885                set_default_siguser1()
886                BuiltIn().fail(gp.sprint_error(error_message))
887        else:
888            match_state = st.anchor_state(state)
889            del match_state['epoch_seconds']
890            # Wait for the state to change in any way.
891            st.wait_state(match_state, wait_time=state_change_timeout,
892                          interval="10 seconds", invert=1)
893
894        gp.qprintn()
895        if boot_table[boot]['end']['chassis'] == "Off":
896            boot_timeout = power_off_timeout
897        else:
898            boot_timeout = power_on_timeout
899        st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout,
900                      interval="10 seconds")
901
902    plug_in_setup()
903    rc, shell_rc, failed_plug_in_name = \
904        grpi.rprocess_plug_in_packages(call_point="post_boot")
905    if rc != 0:
906        error_message = "Plug-in failed with non-zero return code.\n" +\
907            gp.sprint_var(rc, fmt=gp.hexa())
908        set_default_siguser1()
909        BuiltIn().fail(gp.sprint_error(error_message))
910
911    # Restore original sigusr1 handler.
912    set_default_siguser1()
913
914
915def test_loop_body():
916    r"""
917    The main loop body for the loop in main_py.
918
919    Description of arguments:
920    boot_count  The iteration number (starts at 1).
921    """
922
923    global boot_count
924    global state
925    global next_boot
926    global boot_success
927    global boot_end_time
928
929    gp.qprintn()
930
931    next_boot = select_boot()
932    if next_boot == "":
933        return True
934
935    boot_count += 1
936    gp.qprint_timen("Starting boot " + str(boot_count) + ".")
937
938    pre_boot_plug_in_setup()
939
940    cmd_buf = ["run_boot", next_boot]
941    boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
942    if boot_status == "FAIL":
943        gp.qprint(msg)
944
945    gp.qprintn()
946    if boot_status == "PASS":
947        boot_success = 1
948        completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot
949                                         + "\" succeeded.")
950    else:
951        boot_success = 0
952        completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot
953                                         + "\" failed.")
954
955    # Set boot_end_time for use by plug-ins.
956    boot_end_time = completion_msg[1:33]
957    gp.qprint_var(boot_end_time)
958
959    gp.qprint(completion_msg)
960
961    boot_results.update(next_boot, boot_status)
962
963    plug_in_setup()
964    # NOTE: A post_test_case call point failure is NOT counted as a boot
965    # failure.
966    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
967        call_point='post_test_case', stop_on_plug_in_failure=0)
968
969    plug_in_setup()
970    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
971        call_point='ffdc_check', shell_rc=dump_ffdc_rc(),
972        stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
973    if ffdc_check == "All" or\
974       shell_rc == dump_ffdc_rc():
975        status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
976        if status != 'PASS':
977            gp.qprint_error("Call to my_ffdc failed.\n")
978            # Leave a record for caller that "soft" errors occurred.
979            soft_errors = 1
980            gpu.save_plug_in_value(soft_errors, pgm_name)
981
982    if delete_errlogs:
983        # print error logs before delete
984        if redfish_support_trans_state:
985            status, error_logs = grk.run_key_u("Get Redfish Event Logs")
986            log.print_error_logs(error_logs, "AdditionalDataURI Message Severity")
987        else:
988            status, error_logs = grk.run_key_u("Get Error Logs")
989            log.print_error_logs(error_logs, "AdditionalData Message Severity")
990        pels = pel.peltool("-l", ignore_err=1)
991        gp.qprint_var(pels)
992
993        # We need to purge error logs between boots or they build up.
994        grk.run_key(delete_errlogs_cmd, ignore=1)
995        grk.run_key(delete_bmcdump_cmd, ignore=1)
996        if redfish_support_trans_state:
997            grk.run_key(delete_sysdump_cmd, ignore=1)
998
999    boot_results.print_report()
1000    gp.qprint_timen("Finished boot " + str(boot_count) + ".")
1001
1002    plug_in_setup()
1003    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1004        call_point='stop_check', shell_rc=stop_test_rc(),
1005        stop_on_non_zero_rc=1)
1006    if shell_rc == stop_test_rc():
1007        message = "Stopping as requested by user.\n"
1008        gp.qprint_time(message)
1009        BuiltIn().fail(message)
1010
1011    # This should help prevent ConnectionErrors.
1012    # Purge all redfish and REST connection sessions.
1013    if redfish_delete_sessions:
1014        grk.run_key_u("Close All Connections", ignore=1)
1015        grk.run_key_u("Delete All Redfish Sessions", ignore=1)
1016
1017    return True
1018
1019
1020def obmc_boot_test_teardown():
1021    r"""
1022    Clean up after the main keyword.
1023    """
1024    gp.qprint_executing()
1025
1026    if ga.psutil_imported:
1027        ga.terminate_descendants()
1028
1029    if cp_setup_called:
1030        plug_in_setup()
1031        rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1032            call_point='cleanup', stop_on_plug_in_failure=0)
1033
1034    if 'boot_results_file_path' in globals():
1035        # Save boot_results and boot_history objects to a file in case they are
1036        # needed again.
1037        gp.qprint_timen("Saving boot_results to the following path.")
1038        gp.qprint_var(boot_results_file_path)
1039        pickle.dump((boot_results, boot_history),
1040                    open(boot_results_file_path, 'wb'),
1041                    pickle.HIGHEST_PROTOCOL)
1042
1043    global save_stack
1044    # Restore any global values saved on the save_stack.
1045    for parm_name in main_func_parm_list:
1046        # Get the parm_value if it was saved on the stack.
1047        try:
1048            parm_value = save_stack.pop(parm_name)
1049        except BaseException:
1050            # If it was not saved, no further action is required.
1051            continue
1052
1053        # Restore the saved value.
1054        cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1055            "}\", parm_value)"
1056        gp.dpissuing(cmd_buf)
1057        exec(cmd_buf)
1058
1059    gp.dprintn(save_stack.sprint_obj())
1060
1061
1062def test_teardown():
1063    r"""
1064    Clean up after this test case.
1065    """
1066
1067    gp.qprintn()
1068    gp.qprint_executing()
1069
1070    if ga.psutil_imported:
1071        ga.terminate_descendants()
1072
1073    cmd_buf = ["Print Error",
1074               "A keyword timeout occurred ending this program.\n"]
1075    BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
1076
1077    if redfish_supported:
1078        redfish.logout()
1079
1080    gp.qprint_pgm_footer()
1081
1082
1083def post_stack():
1084    r"""
1085    Process post_stack plug-in programs.
1086    """
1087
1088    if not call_post_stack_plug:
1089        # The caller does not wish to have post_stack plug-in processing done.
1090        return
1091
1092    global boot_success
1093
1094    # NOTE: A post_stack call-point failure is NOT counted as a boot failure.
1095    pre_boot_plug_in_setup()
1096    # For the purposes of the following plug-ins, mark the "boot" as a success.
1097    boot_success = 1
1098    plug_in_setup()
1099    rc, shell_rc, failed_plug_in_name, history =\
1100        grpi.rprocess_plug_in_packages(call_point='post_stack',
1101                                       stop_on_plug_in_failure=0,
1102                                       return_history=True)
1103    for doing_msg in history:
1104        update_boot_history(boot_history, doing_msg, max_boot_history)
1105    if rc != 0:
1106        boot_success = 0
1107
1108    plug_in_setup()
1109    rc, shell_rc, failed_plug_in_name =\
1110        grpi.rprocess_plug_in_packages(call_point='ffdc_check',
1111                                       shell_rc=dump_ffdc_rc(),
1112                                       stop_on_plug_in_failure=1,
1113                                       stop_on_non_zero_rc=1)
1114    if shell_rc == dump_ffdc_rc():
1115        status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
1116        if status != 'PASS':
1117            gp.qprint_error("Call to my_ffdc failed.\n")
1118            # Leave a record for caller that "soft" errors occurred.
1119            soft_errors = 1
1120            gpu.save_plug_in_value(soft_errors, pgm_name)
1121
1122    plug_in_setup()
1123    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1124        call_point='stop_check', shell_rc=stop_test_rc(),
1125        stop_on_non_zero_rc=1)
1126    if shell_rc == stop_test_rc():
1127        message = "Stopping as requested by user.\n"
1128        gp.qprint_time(message)
1129        BuiltIn().fail(message)
1130
1131
1132def obmc_boot_test_py(loc_boot_stack=None,
1133                      loc_stack_mode=None,
1134                      loc_quiet=None):
1135    r"""
1136    Do main program processing.
1137    """
1138
1139    global save_stack
1140
1141    ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']})
1142
1143    gp.dprintn()
1144    # Process function parms.
1145    for parm_name in main_func_parm_list:
1146        # Get parm's value.
1147        parm_value = eval("loc_" + parm_name)
1148        gp.dpvars(parm_name, parm_value)
1149
1150        if parm_value is not None:
1151            # Save the global value on a stack.
1152            cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\
1153                parm_name + "}\"), \"" + parm_name + "\")"
1154            gp.dpissuing(cmd_buf)
1155            exec(cmd_buf)
1156
1157            # Set the global value to the passed value.
1158            cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1159                "}\", loc_" + parm_name + ")"
1160            gp.dpissuing(cmd_buf)
1161            exec(cmd_buf)
1162
1163    gp.dprintn(save_stack.sprint_obj())
1164
1165    setup()
1166
1167    init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail()
1168
1169    if ffdc_only:
1170        gp.qprint_timen("Caller requested ffdc_only.")
1171        if do_pre_boot_plug_in_setup:
1172            pre_boot_plug_in_setup()
1173        grk.run_key_u("my_ffdc")
1174        return
1175
1176    if delete_errlogs:
1177        # print error logs before delete
1178        if redfish_support_trans_state:
1179            status, error_logs = grk.run_key_u("Get Redfish Event Logs")
1180            log.print_error_logs(error_logs, "AdditionalDataURI Message Severity")
1181        else:
1182            status, error_logs = grk.run_key_u("Get Error Logs")
1183            log.print_error_logs(error_logs, "AdditionalData Message Severity")
1184        pels = pel.peltool("-l", ignore_err=1)
1185        gp.qprint_var(pels)
1186
1187        # Delete errlogs prior to doing any boot tests.
1188        grk.run_key(delete_errlogs_cmd, ignore=1)
1189        grk.run_key(delete_bmcdump_cmd, ignore=1)
1190        if redfish_support_trans_state:
1191            grk.run_key(delete_sysdump_cmd, ignore=1)
1192
1193    # Process caller's boot_stack.
1194    while (len(boot_stack) > 0):
1195        test_loop_body()
1196
1197    gp.qprint_timen("Finished processing stack.")
1198
1199    post_stack()
1200
1201    # Process caller's boot_list.
1202    if len(boot_list) > 0:
1203        for ix in range(1, max_num_tests + 1):
1204            test_loop_body()
1205
1206    gp.qprint_timen("Completed all requested boot tests.")
1207
1208    boot_pass, boot_fail = boot_results.return_total_pass_fail()
1209    new_fail = boot_fail - init_boot_fail
1210    if new_fail > boot_fail_threshold:
1211        error_message = "Boot failures exceed the boot failure" +\
1212                        " threshold:\n" +\
1213                        gp.sprint_var(new_fail) +\
1214                        gp.sprint_var(boot_fail_threshold)
1215        BuiltIn().fail(gp.sprint_error(error_message))
1216