1#!/usr/bin/env python3
2
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
7import os
8import imp
9import time
10import glob
11import random
12import re
13import signal
14try:
15    import cPickle as pickle
16except ImportError:
17    import pickle
18import socket
19
20from robot.utils import DotDict
21from robot.libraries.BuiltIn import BuiltIn
22
23from boot_data import *
24import gen_print as gp
25import gen_robot_plug_in as grpi
26import gen_arg as ga
27import gen_valid as gv
28import gen_misc as gm
29import gen_cmd as gc
30import gen_robot_keyword as grk
31import state as st
32import var_stack as vs
33import gen_plug_in_utils as gpu
34import pel_utils as pel
35import logging_utils as log
36
37base_path = os.path.dirname(os.path.dirname(
38                            imp.find_module("gen_robot_print")[1])) +\
39    os.sep
40sys.path.append(base_path + "extended/")
41import run_keyword as rk
42
43# Setting master_pid correctly influences the behavior of plug-ins like
44# DB_Logging
45program_pid = os.getpid()
46master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid)
47pgm_name = re.sub('\\.py$', '', os.path.basename(__file__))
48
49# Set up boot data structures.
50os_host = BuiltIn().get_variable_value("${OS_HOST}", default="")
51
52boot_lists = read_boot_lists()
53
54# The maximum number of entries that can be in the boot_history global variable.
55max_boot_history = 10
56boot_history = []
57
58state = st.return_state_constant('default_state')
59cp_setup_called = 0
60next_boot = ""
61base_tool_dir_path = os.path.normpath(os.environ.get(
62    'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep
63
64ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep
65boot_success = 0
66
67status_dir_path = os.environ.get('STATUS_DIR_PATH', "") or \
68    BuiltIn().get_variable_value("${STATUS_DIR_PATH}", default="")
69if status_dir_path != "":
70    status_dir_path = os.path.normpath(status_dir_path) + os.sep
71    # For plugin expecting env gen_call_robot.py
72    os.environ['STATUS_DIR_PATH'] = status_dir_path
73
74redfish_support_trans_state = int(os.environ.get('REDFISH_SUPPORT_TRANS_STATE', 0)) or \
75    int(BuiltIn().get_variable_value("${REDFISH_SUPPORT_TRANS_STATE}", default=0))
76redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False)
77redfish_rest_supported = BuiltIn().get_variable_value("${REDFISH_REST_SUPPORTED}", default=False)
78redfish_delete_sessions = int(BuiltIn().get_variable_value("${REDFISH_DELETE_SESSIONS}", default=1))
79if redfish_supported:
80    redfish = BuiltIn().get_library_instance('redfish')
81    default_power_on = "Redfish Power On"
82    default_power_off = "Redfish Power Off"
83    if not redfish_support_trans_state:
84        delete_errlogs_cmd = "Delete Error Logs  ${quiet}=${1}"
85        delete_bmcdump_cmd = "Delete All BMC Dump"
86        default_set_power_policy = "Set BMC Power Policy  ALWAYS_POWER_OFF"
87    else:
88        delete_errlogs_cmd = "Redfish Purge Event Log"
89        delete_bmcdump_cmd = "Redfish Delete All BMC Dumps"
90        delete_sysdump_cmd = "Redfish Delete All System Dumps"
91        default_set_power_policy = "Redfish Set Power Restore Policy  AlwaysOff"
92else:
93    default_power_on = "REST Power On"
94    default_power_off = "REST Power Off"
95    delete_errlogs_cmd = "Delete Error Logs  ${quiet}=${1}"
96    delete_bmcdump_cmd = "Delete All BMC Dump"
97    default_set_power_policy = "Set BMC Power Policy  ALWAYS_POWER_OFF"
98boot_count = 0
99
100LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
101AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '')
102ffdc_prefix = AUTOBOOT_FFDC_PREFIX
103boot_start_time = ""
104boot_end_time = ""
105save_stack = vs.var_stack('save_stack')
106main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet']
107
108
109def dump_ffdc_rc():
110    r"""
111    Return the constant dump ffdc test return code value.
112
113    When a plug-in call point program returns this value, it indicates that
114    this program should collect FFDC.
115    """
116
117    return 0x00000200
118
119
120def stop_test_rc():
121    r"""
122    Return the constant stop test return code value.
123
124    When a plug-in call point program returns this value, it indicates that
125    this program should stop running.
126    """
127
128    return 0x00000200
129
130
131def process_host(host,
132                 host_var_name=""):
133    r"""
134    Process a host by getting the associated host name and IP address and
135    setting them in global variables.
136
137    If the caller does not pass the host_var_name, this function will try to
138    figure out the name of the variable used by the caller for the host parm.
139    Callers are advised to explicitly specify the host_var_name when calling
140    with an exec command.  In such cases, the get_arg_name cannot figure out
141    the host variable name.
142
143    This function will then create similar global variable names by
144    removing "_host" and appending "_host_name" or "_ip" to the host variable
145    name.
146
147    Example:
148
149    If a call is made like this:
150    process_host(openbmc_host)
151
152    Global variables openbmc_host_name and openbmc_ip will be set.
153
154    Description of argument(s):
155    host           A host name or IP.  The name of the variable used should
156                   have a suffix of "_host".
157    host_var_name  The name of the variable being used as the host parm.
158    """
159
160    if host_var_name == "":
161        host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
162
163    host_name_var_name = re.sub("host", "host_name", host_var_name)
164    ip_var_name = re.sub("host", "ip", host_var_name)
165    cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\
166        host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\
167        host + "')"
168    exec(cmd_buf)
169
170
171def process_pgm_parms():
172    r"""
173    Process the program parameters by assigning them all to corresponding
174    globals.  Also, set some global values that depend on program parameters.
175    """
176
177    # Program parameter processing.
178    # Assign all program parms to python variables which are global to this
179    # module.
180
181    global parm_list
182    parm_list = BuiltIn().get_variable_value("${parm_list}")
183    # The following subset of parms should be processed as integers.
184    int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only',
185                'boot_fail_threshold', 'delete_errlogs',
186                'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet',
187                'test_mode', 'debug']
188    for parm in parm_list:
189        if parm in int_list:
190            sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\
191                      "}\", \"0\"))"
192        else:
193            sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")"
194        cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
195        gp.dpissuing(cmd_buf)
196        exec(cmd_buf)
197        if re.match(r".*_host$", parm):
198            cmd_buf = "process_host(" + parm + ", '" + parm + "')"
199            exec(cmd_buf)
200        if re.match(r".*_password$", parm):
201            # Register the value of any parm whose name ends in _password.
202            # This will cause the print functions to replace passwords with
203            # asterisks in the output.
204            cmd_buf = "gp.register_passwords(" + parm + ")"
205            exec(cmd_buf)
206
207    global ffdc_dir_path_style
208    global boot_list
209    global boot_stack
210    global boot_results_file_path
211    global boot_results
212    global boot_history
213    global ffdc_list_file_path
214    global ffdc_report_list_path
215    global ffdc_summary_list_path
216    global boot_table
217    global valid_boot_types
218
219    if ffdc_dir_path_style == "":
220        ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0'))
221
222    # Convert these program parms to lists for easier processing..
223    boot_list = list(filter(None, boot_list.split(":")))
224    boot_stack = list(filter(None, boot_stack.split(":")))
225
226    boot_table = create_boot_table(boot_table_path, os_host=os_host)
227    valid_boot_types = create_valid_boot_list(boot_table)
228
229    cleanup_boot_results_file()
230    boot_results_file_path = create_boot_results_file_path(pgm_name,
231                                                           openbmc_nickname,
232                                                           master_pid)
233
234    if os.path.isfile(boot_results_file_path):
235        # We've been called before in this run so we'll load the saved
236        # boot_results and boot_history objects.
237        boot_results, boot_history =\
238            pickle.load(open(boot_results_file_path, 'rb'))
239    else:
240        boot_results = boot_results(boot_table, boot_pass, boot_fail)
241
242    ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\
243        "/FFDC_FILE_LIST"
244    ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\
245        "/FFDC_REPORT_FILE_LIST"
246
247    ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\
248        "/FFDC_SUMMARY_FILE_LIST"
249
250
251def initial_plug_in_setup():
252    r"""
253    Initialize all plug-in environment variables which do not change for the
254    duration of the program.
255
256    """
257
258    global LOG_LEVEL
259    BuiltIn().set_log_level("NONE")
260
261    BuiltIn().set_global_variable("${master_pid}", master_pid)
262    BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
263    BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
264    BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
265    BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}",
266                                  ffdc_list_file_path)
267    BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}",
268                                  ffdc_report_list_path)
269    BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}",
270                                  ffdc_summary_list_path)
271
272    BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}",
273                                  ffdc_dir_path_style)
274    BuiltIn().set_global_variable("${FFDC_CHECK}",
275                                  ffdc_check)
276
277    # For each program parameter, set the corresponding AUTOBOOT_ environment
278    # variable value.  Also, set an AUTOBOOT_ environment variable for every
279    # element in additional_values.
280    additional_values = ["program_pid", "master_pid", "ffdc_dir_path",
281                         "status_dir_path", "base_tool_dir_path",
282                         "ffdc_list_file_path", "ffdc_report_list_path",
283                         "ffdc_summary_list_path", "execdir", "redfish_supported",
284                         "redfish_rest_supported", "redfish_support_trans_state"]
285
286    plug_in_vars = parm_list + additional_values
287
288    for var_name in plug_in_vars:
289        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
290        var_name = var_name.upper()
291        if var_value is None:
292            var_value = ""
293        os.environ["AUTOBOOT_" + var_name] = str(var_value)
294
295    BuiltIn().set_log_level(LOG_LEVEL)
296
297    # Make sure the ffdc list directory exists.
298    ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
299    if not os.path.exists(ffdc_list_dir_path):
300        os.makedirs(ffdc_list_dir_path)
301
302
303def plug_in_setup():
304    r"""
305    Initialize all changing plug-in environment variables for use by the
306    plug-in programs.
307    """
308
309    global LOG_LEVEL
310    global test_really_running
311
312    BuiltIn().set_log_level("NONE")
313
314    boot_pass, boot_fail = boot_results.return_total_pass_fail()
315    if boot_pass > 1:
316        test_really_running = 1
317    else:
318        test_really_running = 0
319
320    BuiltIn().set_global_variable("${test_really_running}",
321                                  test_really_running)
322    BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
323    BuiltIn().set_global_variable("${boot_pass}", boot_pass)
324    BuiltIn().set_global_variable("${boot_fail}", boot_fail)
325    BuiltIn().set_global_variable("${boot_success}", boot_success)
326    BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
327    BuiltIn().set_global_variable("${boot_start_time}", boot_start_time)
328    BuiltIn().set_global_variable("${boot_end_time}", boot_end_time)
329
330    # For each program parameter, set the corresponding AUTOBOOT_ environment
331    # variable value.  Also, set an AUTOBOOT_ environment variable for every
332    # element in additional_values.
333    additional_values = ["boot_type_desc", "boot_success", "boot_pass",
334                         "boot_fail", "test_really_running", "ffdc_prefix",
335                         "boot_start_time", "boot_end_time"]
336
337    plug_in_vars = additional_values
338
339    for var_name in plug_in_vars:
340        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
341        var_name = var_name.upper()
342        if var_value is None:
343            var_value = ""
344        os.environ["AUTOBOOT_" + var_name] = str(var_value)
345
346    if debug:
347        shell_rc, out_buf = \
348            gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u")
349
350    BuiltIn().set_log_level(LOG_LEVEL)
351
352
353def pre_boot_plug_in_setup():
354
355    # Clear the ffdc_list_file_path file.  Plug-ins may now write to it.
356    try:
357        os.remove(ffdc_list_file_path)
358    except OSError:
359        pass
360
361    # Clear the ffdc_report_list_path file.  Plug-ins may now write to it.
362    try:
363        os.remove(ffdc_report_list_path)
364    except OSError:
365        pass
366
367    # Clear the ffdc_summary_list_path file.  Plug-ins may now write to it.
368    try:
369        os.remove(ffdc_summary_list_path)
370    except OSError:
371        pass
372
373    global ffdc_prefix
374
375    seconds = time.time()
376    loc_time = time.localtime(seconds)
377    time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
378
379    ffdc_prefix = openbmc_nickname + "." + time_string
380
381
382def default_sigusr1(signal_number=0,
383                    frame=None):
384    r"""
385    Handle SIGUSR1 by doing nothing.
386
387    This function assists in debugging SIGUSR1 processing by printing messages
388    to stdout and to the log.html file.
389
390    Description of argument(s):
391    signal_number  The signal number (should always be 10 for SIGUSR1).
392    frame          The frame data.
393    """
394
395    gp.qprintn()
396    gp.qprint_executing()
397    gp.lprint_executing()
398
399
400def set_default_siguser1():
401    r"""
402    Set the default_sigusr1 function to be the SIGUSR1 handler.
403    """
404
405    gp.qprintn()
406    gp.qprint_executing()
407    gp.lprint_executing()
408    signal.signal(signal.SIGUSR1, default_sigusr1)
409
410
411def setup():
412    r"""
413    Do general program setup tasks.
414    """
415
416    global cp_setup_called
417    global transitional_boot_selected
418
419    gp.qprintn()
420
421    if redfish_supported:
422        redfish.login()
423
424    set_default_siguser1()
425    transitional_boot_selected = False
426
427    robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
428    repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
429    # If we can't find process_plug_in_packages.py, ssh_pw or
430    # validate_plug_ins.py, then we don't have our repo bin in PATH.
431    shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py"
432                                     + " ssh_pw validate_plug_ins.py", quiet=1,
433                                     print_output=0, show_err=0)
434    if shell_rc != 0:
435        os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
436    # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
437    if robot_pgm_dir_path not in sys.path:
438        sys.path.append(robot_pgm_dir_path)
439        PYTHONPATH = os.environ.get("PYTHONPATH", "")
440        if PYTHONPATH == "":
441            os.environ['PYTHONPATH'] = robot_pgm_dir_path
442        else:
443            os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
444
445    validate_parms()
446
447    gp.qprint_pgm_header()
448
449    grk.run_key_u(default_set_power_policy, ignore=1)
450
451    initial_plug_in_setup()
452
453    plug_in_setup()
454    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
455        call_point='setup')
456    if rc != 0:
457        error_message = "Plug-in setup failed.\n"
458        gp.print_error_report(error_message)
459        BuiltIn().fail(error_message)
460    # Setting cp_setup_called lets our Teardown know that it needs to call
461    # the cleanup plug-in call point.
462    cp_setup_called = 1
463
464    # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
465    BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
466    # FFDC_LOG_PATH is used by "FFDC" keyword.
467    BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
468
469    # Also printed by FFDC.
470    global host_name
471    global host_ip
472    host = socket.gethostname()
473    host_name, host_ip = gm.get_host_name_ip(host)
474
475    gp.dprint_var(boot_table)
476    gp.dprint_var(boot_lists)
477
478
479def validate_parms():
480    r"""
481    Validate all program parameters.
482    """
483
484    process_pgm_parms()
485
486    gp.qprintn()
487
488    global openbmc_model
489    if openbmc_model == "":
490        status, ret_values =\
491            grk.run_key_u("Get BMC System Model", ignore=1)
492        # Set the model to default "OPENBMC" if getting it from BMC fails.
493        if status == 'FAIL':
494            openbmc_model = 'OPENBMC'
495        else:
496            openbmc_model = ret_values
497        BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
498    gv.set_exit_on_error(True)
499    gv.valid_value(openbmc_host)
500    gv.valid_value(openbmc_username)
501    gv.valid_value(openbmc_password)
502    gv.valid_value(rest_username)
503    gv.valid_value(rest_password)
504    gv.valid_value(ipmi_username)
505    gv.valid_value(ipmi_password)
506    if os_host != "":
507        gv.valid_value(os_username)
508        gv.valid_value(os_password)
509    if pdu_host != "":
510        gv.valid_value(pdu_username)
511        gv.valid_value(pdu_password)
512        gv.valid_integer(pdu_slot_no)
513    if openbmc_serial_host != "":
514        gv.valid_integer(openbmc_serial_port)
515    gv.valid_value(openbmc_model)
516    gv.valid_integer(max_num_tests)
517    gv.valid_integer(boot_pass)
518    gv.valid_integer(boot_fail)
519    plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
520    BuiltIn().set_global_variable("${plug_in_packages_list}",
521                                  plug_in_packages_list)
522    gv.valid_value(stack_mode, valid_values=['normal', 'skip'])
523    gv.set_exit_on_error(False)
524    if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
525        error_message = "You must provide either a value for either the" +\
526            " boot_list or the boot_stack parm.\n"
527        BuiltIn().fail(gp.sprint_error(error_message))
528    valid_boot_list(boot_list, valid_boot_types)
529    valid_boot_list(boot_stack, valid_boot_types)
530    selected_PDU_boots = list(set(boot_list + boot_stack)
531                              & set(boot_lists['PDU_reboot']))
532    if len(selected_PDU_boots) > 0 and pdu_host == "":
533        error_message = "You have selected the following boots which" +\
534                        " require a PDU host but no value for pdu_host:\n"
535        error_message += gp.sprint_var(selected_PDU_boots)
536        error_message += gp.sprint_var(pdu_host, fmt=gp.blank())
537        BuiltIn().fail(gp.sprint_error(error_message))
538
539    return
540
541
542def my_get_state():
543    r"""
544    Get the system state plus a little bit of wrapping.
545    """
546
547    global state
548
549    req_states = ['epoch_seconds'] + st.default_req_states
550
551    gp.qprint_timen("Getting system state.")
552    if test_mode:
553        state['epoch_seconds'] = int(time.time())
554    else:
555        state = st.get_state(req_states=req_states, quiet=quiet)
556    gp.qprint_var(state)
557
558
559def valid_state():
560    r"""
561    Verify that our state dictionary contains no blank values.  If we don't get
562    valid state data, we cannot continue to work.
563    """
564
565    if st.compare_states(state, st.invalid_state_match, 'or'):
566        error_message = "The state dictionary contains blank fields which" +\
567            " is illegal.\n" + gp.sprint_var(state)
568        BuiltIn().fail(gp.sprint_error(error_message))
569
570
571def select_boot():
572    r"""
573    Select a boot test to be run based on our current state and return the
574    chosen boot type.
575
576    Description of arguments:
577    state  The state of the machine.
578    """
579
580    global transitional_boot_selected
581    global boot_stack
582
583    gp.qprint_timen("Selecting a boot test.")
584
585    if transitional_boot_selected and not boot_success:
586        prior_boot = next_boot
587        boot_candidate = boot_stack.pop()
588        gp.qprint_timen("The prior '" + next_boot + "' was chosen to"
589                        + " transition to a valid state for '" + boot_candidate
590                        + "' which was at the top of the boot_stack.  Since"
591                        + " the '" + next_boot + "' failed, the '"
592                        + boot_candidate + "' has been removed from the stack"
593                        + " to avoid and endless failure loop.")
594        if len(boot_stack) == 0:
595            return ""
596
597    my_get_state()
598    valid_state()
599
600    transitional_boot_selected = False
601    stack_popped = 0
602    if len(boot_stack) > 0:
603        stack_popped = 1
604        gp.qprint_dashes()
605        gp.qprint_var(boot_stack)
606        gp.qprint_dashes()
607        skip_boot_printed = 0
608        while len(boot_stack) > 0:
609            boot_candidate = boot_stack.pop()
610            if stack_mode == 'normal':
611                break
612            else:
613                if st.compare_states(state, boot_table[boot_candidate]['end']):
614                    if not skip_boot_printed:
615                        gp.qprint_var(stack_mode)
616                        gp.qprintn()
617                        gp.qprint_timen("Skipping the following boot tests"
618                                        + " which are unnecessary since their"
619                                        + " required end states match the"
620                                        + " current machine state:")
621                        skip_boot_printed = 1
622                    gp.qprint_var(boot_candidate)
623                    boot_candidate = ""
624        if boot_candidate == "":
625            gp.qprint_dashes()
626            gp.qprint_var(boot_stack)
627            gp.qprint_dashes()
628            return boot_candidate
629        if st.compare_states(state, boot_table[boot_candidate]['start']):
630            gp.qprint_timen("The machine state is valid for a '"
631                            + boot_candidate + "' boot test.")
632            gp.qprint_dashes()
633            gp.qprint_var(boot_stack)
634            gp.qprint_dashes()
635            return boot_candidate
636        else:
637            gp.qprint_timen("The machine state does not match the required"
638                            + " starting state for a '" + boot_candidate
639                            + "' boot test:")
640            gp.qprint_varx("boot_table_start_entry",
641                           boot_table[boot_candidate]['start'])
642            boot_stack.append(boot_candidate)
643            transitional_boot_selected = True
644            popped_boot = boot_candidate
645
646    # Loop through your list selecting a boot_candidates
647    boot_candidates = []
648    for boot_candidate in boot_list:
649        if st.compare_states(state, boot_table[boot_candidate]['start']):
650            if stack_popped:
651                if st.compare_states(boot_table[boot_candidate]['end'],
652                                     boot_table[popped_boot]['start']):
653                    boot_candidates.append(boot_candidate)
654            else:
655                boot_candidates.append(boot_candidate)
656
657    if len(boot_candidates) == 0:
658        gp.qprint_timen("The user's boot list contained no boot tests"
659                        + " which are valid for the current machine state.")
660        boot_candidate = default_power_on
661        if not st.compare_states(state, boot_table[default_power_on]['start']):
662            boot_candidate = default_power_off
663        boot_candidates.append(boot_candidate)
664        gp.qprint_timen("Using default '" + boot_candidate
665                        + "' boot type to transition to valid state.")
666
667    gp.dprint_var(boot_candidates)
668
669    # Randomly select a boot from the candidate list.
670    boot = random.choice(boot_candidates)
671
672    return boot
673
674
675def print_defect_report(ffdc_file_list):
676    r"""
677    Print a defect report.
678
679    Description of argument(s):
680    ffdc_file_list  A list of files which were collected by our ffdc functions.
681    """
682
683    # Making deliberate choice to NOT run plug_in_setup().  We don't want
684    # ffdc_prefix updated.
685    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
686        call_point='ffdc_report', stop_on_plug_in_failure=0)
687
688    # Get additional header data which may have been created by ffdc plug-ins.
689    # Also, delete the individual header files to cleanup.
690    cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\
691              " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
692              " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
693    shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
694                                              show_err=0)
695
696    # Get additional summary data which may have been created by ffdc plug-ins.
697    # Also, delete the individual header files to cleanup.
698    cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\
699              " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
700              " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
701    shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
702                                               show_err=0)
703
704    # ffdc_list_file_path contains a list of any ffdc files created by plug-
705    # ins, etc.  Read that data into a list.
706    try:
707        plug_in_ffdc_list = \
708            open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n")
709        plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list))
710    except IOError:
711        plug_in_ffdc_list = []
712
713    # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed
714    # in.  Eliminate duplicates and sort the list.
715    ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list))
716
717    if status_file_path != "":
718        ffdc_file_list.insert(0, status_file_path)
719
720    # Convert the list to a printable list.
721    printable_ffdc_file_list = "\n".join(ffdc_file_list)
722
723    # Open ffdc_file_list for writing.  We will write a complete list of
724    # FFDC files to it for possible use by plug-ins like cp_stop_check.
725    ffdc_list_file = open(ffdc_list_file_path, 'w')
726    ffdc_list_file.write(printable_ffdc_file_list + "\n")
727    ffdc_list_file.close()
728
729    indent = 0
730    width = 90
731    linefeed = 1
732    char = "="
733
734    gp.qprintn()
735    gp.qprint_dashes(indent, width, linefeed, char)
736    gp.qprintn("Copy this data to the defect:\n")
737
738    if len(more_header_info) > 0:
739        gp.qprintn(more_header_info)
740    gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host,
741              openbmc_host_name, openbmc_ip, openbmc_username,
742              openbmc_password, rest_username, rest_password, ipmi_username,
743              ipmi_password, os_host, os_host_name, os_ip, os_username,
744              os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username,
745              pdu_password, pdu_slot_no, openbmc_serial_host,
746              openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port)
747
748    gp.qprintn()
749    print_boot_history(boot_history)
750    gp.qprintn()
751    gp.qprint_var(state)
752    gp.qprintn()
753    gp.qprintn("FFDC data files:")
754    gp.qprintn(printable_ffdc_file_list)
755    gp.qprintn()
756
757    if len(ffdc_summary_info) > 0:
758        gp.qprintn(ffdc_summary_info)
759
760    gp.qprint_dashes(indent, width, linefeed, char)
761
762
763def my_ffdc():
764    r"""
765    Collect FFDC data.
766    """
767
768    global state
769
770    plug_in_setup()
771    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
772        call_point='ffdc', stop_on_plug_in_failure=0)
773
774    AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX']
775    status, ffdc_file_list = grk.run_key_u("FFDC  ffdc_prefix="
776                                           + AUTOBOOT_FFDC_PREFIX
777                                           + "  ffdc_function_list="
778                                           + ffdc_function_list, ignore=1)
779    if status != 'PASS':
780        gp.qprint_error("Call to ffdc failed.\n")
781        if type(ffdc_file_list) is not list:
782            ffdc_file_list = []
783        # Leave a record for caller that "soft" errors occurred.
784        soft_errors = 1
785        gpu.save_plug_in_value(soft_errors, pgm_name)
786
787    my_get_state()
788
789    print_defect_report(ffdc_file_list)
790
791
792def print_test_start_message(boot_keyword):
793    r"""
794    Print a message indicating what boot test is about to run.
795
796    Description of arguments:
797    boot_keyword  The name of the boot which is to be run
798                  (e.g. "BMC Power On").
799    """
800
801    global boot_history
802    global boot_start_time
803
804    doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".")
805
806    # Set boot_start_time for use by plug-ins.
807    boot_start_time = doing_msg[1:33]
808    gp.qprint_var(boot_start_time)
809
810    gp.qprint(doing_msg)
811
812    update_boot_history(boot_history, doing_msg, max_boot_history)
813
814
815def stop_boot_test(signal_number=0,
816                   frame=None):
817    r"""
818    Handle SIGUSR1 by aborting the boot test that is running.
819
820    Description of argument(s):
821    signal_number  The signal number (should always be 10 for SIGUSR1).
822    frame          The frame data.
823    """
824
825    gp.qprintn()
826    gp.qprint_executing()
827    gp.lprint_executing()
828
829    # Restore original sigusr1 handler.
830    set_default_siguser1()
831
832    message = "The caller has asked that the boot test be stopped and marked"
833    message += " as a failure."
834
835    function_stack = gm.get_function_stack()
836    if "wait_state" in function_stack:
837        st.set_exit_wait_early_message(message)
838    else:
839        BuiltIn().fail(gp.sprint_error(message))
840
841
842def run_boot(boot):
843    r"""
844    Run the specified boot.
845
846    Description of arguments:
847    boot  The name of the boot test to be performed.
848    """
849
850    global state
851
852    signal.signal(signal.SIGUSR1, stop_boot_test)
853    gp.qprint_timen("stop_boot_test is armed.")
854
855    print_test_start_message(boot)
856
857    plug_in_setup()
858    rc, shell_rc, failed_plug_in_name = \
859        grpi.rprocess_plug_in_packages(call_point="pre_boot")
860    if rc != 0:
861        error_message = "Plug-in failed with non-zero return code.\n" +\
862            gp.sprint_var(rc, fmt=gp.hexa())
863        set_default_siguser1()
864        BuiltIn().fail(gp.sprint_error(error_message))
865
866    if test_mode:
867        # In test mode, we'll pretend the boot worked by assigning its
868        # required end state to the default state value.
869        state = st.strip_anchor_state(boot_table[boot]['end'])
870    else:
871        # Assertion:  We trust that the state data was made fresh by the
872        # caller.
873
874        gp.qprintn()
875
876        if boot_table[boot]['method_type'] == "keyword":
877            rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''),
878                               boot_table[boot]['method'],
879                               quiet=quiet)
880
881        if boot_table[boot]['bmc_reboot']:
882            st.wait_for_comm_cycle(int(state['epoch_seconds']))
883            plug_in_setup()
884            rc, shell_rc, failed_plug_in_name = \
885                grpi.rprocess_plug_in_packages(call_point="post_reboot")
886            if rc != 0:
887                error_message = "Plug-in failed with non-zero return code.\n"
888                error_message += gp.sprint_var(rc, fmt=gp.hexa())
889                set_default_siguser1()
890                BuiltIn().fail(gp.sprint_error(error_message))
891        else:
892            match_state = st.anchor_state(state)
893            del match_state['epoch_seconds']
894            # Wait for the state to change in any way.
895            st.wait_state(match_state, wait_time=state_change_timeout,
896                          interval="10 seconds", invert=1)
897
898        gp.qprintn()
899        if boot_table[boot]['end']['chassis'] == "Off":
900            boot_timeout = power_off_timeout
901        else:
902            boot_timeout = power_on_timeout
903        st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout,
904                      interval="10 seconds")
905
906    plug_in_setup()
907    rc, shell_rc, failed_plug_in_name = \
908        grpi.rprocess_plug_in_packages(call_point="post_boot")
909    if rc != 0:
910        error_message = "Plug-in failed with non-zero return code.\n" +\
911            gp.sprint_var(rc, fmt=gp.hexa())
912        set_default_siguser1()
913        BuiltIn().fail(gp.sprint_error(error_message))
914
915    # Restore original sigusr1 handler.
916    set_default_siguser1()
917
918
919def test_loop_body():
920    r"""
921    The main loop body for the loop in main_py.
922
923    Description of arguments:
924    boot_count  The iteration number (starts at 1).
925    """
926
927    global boot_count
928    global state
929    global next_boot
930    global boot_success
931    global boot_end_time
932
933    gp.qprintn()
934
935    next_boot = select_boot()
936    if next_boot == "":
937        return True
938
939    boot_count += 1
940    gp.qprint_timen("Starting boot " + str(boot_count) + ".")
941
942    pre_boot_plug_in_setup()
943
944    cmd_buf = ["run_boot", next_boot]
945    boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
946    if boot_status == "FAIL":
947        gp.qprint(msg)
948
949    gp.qprintn()
950    if boot_status == "PASS":
951        boot_success = 1
952        completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot
953                                         + "\" succeeded.")
954    else:
955        boot_success = 0
956        completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot
957                                         + "\" failed.")
958
959    # Set boot_end_time for use by plug-ins.
960    boot_end_time = completion_msg[1:33]
961    gp.qprint_var(boot_end_time)
962
963    gp.qprint(completion_msg)
964
965    boot_results.update(next_boot, boot_status)
966
967    plug_in_setup()
968    # NOTE: A post_test_case call point failure is NOT counted as a boot
969    # failure.
970    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
971        call_point='post_test_case', stop_on_plug_in_failure=0)
972
973    plug_in_setup()
974    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
975        call_point='ffdc_check', shell_rc=dump_ffdc_rc(),
976        stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
977    if ffdc_check == "All" or\
978       shell_rc == dump_ffdc_rc():
979        status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
980        if status != 'PASS':
981            gp.qprint_error("Call to my_ffdc failed.\n")
982            # Leave a record for caller that "soft" errors occurred.
983            soft_errors = 1
984            gpu.save_plug_in_value(soft_errors, pgm_name)
985
986    if delete_errlogs:
987        # print error logs before delete
988        if redfish_support_trans_state:
989            status, error_logs = grk.run_key_u("Get Redfish Event Logs")
990            log.print_error_logs(error_logs, "AdditionalDataURI Message Severity")
991        else:
992            status, error_logs = grk.run_key_u("Get Error Logs")
993            log.print_error_logs(error_logs, "AdditionalData Message Severity")
994        pels = pel.peltool("-l", ignore_err=1)
995        gp.qprint_var(pels)
996
997        # We need to purge error logs between boots or they build up.
998        grk.run_key(delete_errlogs_cmd, ignore=1)
999        grk.run_key(delete_bmcdump_cmd, ignore=1)
1000        if redfish_support_trans_state:
1001            grk.run_key(delete_sysdump_cmd, ignore=1)
1002
1003    boot_results.print_report()
1004    gp.qprint_timen("Finished boot " + str(boot_count) + ".")
1005
1006    plug_in_setup()
1007    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1008        call_point='stop_check', shell_rc=stop_test_rc(),
1009        stop_on_non_zero_rc=1)
1010    if shell_rc == stop_test_rc():
1011        message = "Stopping as requested by user.\n"
1012        gp.qprint_time(message)
1013        BuiltIn().fail(message)
1014
1015    # This should help prevent ConnectionErrors.
1016    # Purge all redfish and REST connection sessions.
1017    if redfish_delete_sessions:
1018        grk.run_key_u("Close All Connections", ignore=1)
1019        grk.run_key_u("Delete All Redfish Sessions", ignore=1)
1020
1021    return True
1022
1023
1024def obmc_boot_test_teardown():
1025    r"""
1026    Clean up after the main keyword.
1027    """
1028    gp.qprint_executing()
1029
1030    if ga.psutil_imported:
1031        ga.terminate_descendants()
1032
1033    if cp_setup_called:
1034        plug_in_setup()
1035        rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1036            call_point='cleanup', stop_on_plug_in_failure=0)
1037
1038    if 'boot_results_file_path' in globals():
1039        # Save boot_results and boot_history objects to a file in case they are
1040        # needed again.
1041        gp.qprint_timen("Saving boot_results to the following path.")
1042        gp.qprint_var(boot_results_file_path)
1043        pickle.dump((boot_results, boot_history),
1044                    open(boot_results_file_path, 'wb'),
1045                    pickle.HIGHEST_PROTOCOL)
1046
1047    global save_stack
1048    # Restore any global values saved on the save_stack.
1049    for parm_name in main_func_parm_list:
1050        # Get the parm_value if it was saved on the stack.
1051        try:
1052            parm_value = save_stack.pop(parm_name)
1053        except BaseException:
1054            # If it was not saved, no further action is required.
1055            continue
1056
1057        # Restore the saved value.
1058        cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1059            "}\", parm_value)"
1060        gp.dpissuing(cmd_buf)
1061        exec(cmd_buf)
1062
1063    gp.dprintn(save_stack.sprint_obj())
1064
1065
1066def test_teardown():
1067    r"""
1068    Clean up after this test case.
1069    """
1070
1071    gp.qprintn()
1072    gp.qprint_executing()
1073
1074    if ga.psutil_imported:
1075        ga.terminate_descendants()
1076
1077    cmd_buf = ["Print Error",
1078               "A keyword timeout occurred ending this program.\n"]
1079    BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
1080
1081    if redfish_supported:
1082        redfish.logout()
1083
1084    gp.qprint_pgm_footer()
1085
1086
1087def post_stack():
1088    r"""
1089    Process post_stack plug-in programs.
1090    """
1091
1092    if not call_post_stack_plug:
1093        # The caller does not wish to have post_stack plug-in processing done.
1094        return
1095
1096    global boot_success
1097
1098    # NOTE: A post_stack call-point failure is NOT counted as a boot failure.
1099    pre_boot_plug_in_setup()
1100    # For the purposes of the following plug-ins, mark the "boot" as a success.
1101    boot_success = 1
1102    plug_in_setup()
1103    rc, shell_rc, failed_plug_in_name, history =\
1104        grpi.rprocess_plug_in_packages(call_point='post_stack',
1105                                       stop_on_plug_in_failure=0,
1106                                       return_history=True)
1107    for doing_msg in history:
1108        update_boot_history(boot_history, doing_msg, max_boot_history)
1109    if rc != 0:
1110        boot_success = 0
1111
1112    plug_in_setup()
1113    rc, shell_rc, failed_plug_in_name =\
1114        grpi.rprocess_plug_in_packages(call_point='ffdc_check',
1115                                       shell_rc=dump_ffdc_rc(),
1116                                       stop_on_plug_in_failure=1,
1117                                       stop_on_non_zero_rc=1)
1118    if shell_rc == dump_ffdc_rc():
1119        status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
1120        if status != 'PASS':
1121            gp.qprint_error("Call to my_ffdc failed.\n")
1122            # Leave a record for caller that "soft" errors occurred.
1123            soft_errors = 1
1124            gpu.save_plug_in_value(soft_errors, pgm_name)
1125
1126    plug_in_setup()
1127    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1128        call_point='stop_check', shell_rc=stop_test_rc(),
1129        stop_on_non_zero_rc=1)
1130    if shell_rc == stop_test_rc():
1131        message = "Stopping as requested by user.\n"
1132        gp.qprint_time(message)
1133        BuiltIn().fail(message)
1134
1135
1136def obmc_boot_test_py(loc_boot_stack=None,
1137                      loc_stack_mode=None,
1138                      loc_quiet=None):
1139    r"""
1140    Do main program processing.
1141    """
1142
1143    global save_stack
1144
1145    ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']})
1146
1147    gp.dprintn()
1148    # Process function parms.
1149    for parm_name in main_func_parm_list:
1150        # Get parm's value.
1151        parm_value = eval("loc_" + parm_name)
1152        gp.dpvars(parm_name, parm_value)
1153
1154        if parm_value is not None:
1155            # Save the global value on a stack.
1156            cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\
1157                parm_name + "}\"), \"" + parm_name + "\")"
1158            gp.dpissuing(cmd_buf)
1159            exec(cmd_buf)
1160
1161            # Set the global value to the passed value.
1162            cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1163                "}\", loc_" + parm_name + ")"
1164            gp.dpissuing(cmd_buf)
1165            exec(cmd_buf)
1166
1167    gp.dprintn(save_stack.sprint_obj())
1168
1169    setup()
1170
1171    init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail()
1172
1173    if ffdc_only:
1174        gp.qprint_timen("Caller requested ffdc_only.")
1175        if do_pre_boot_plug_in_setup:
1176            pre_boot_plug_in_setup()
1177        grk.run_key_u("my_ffdc")
1178        return
1179
1180    if delete_errlogs:
1181        # print error logs before delete
1182        if redfish_support_trans_state:
1183            status, error_logs = grk.run_key_u("Get Redfish Event Logs")
1184            log.print_error_logs(error_logs, "AdditionalDataURI Message Severity")
1185        else:
1186            status, error_logs = grk.run_key_u("Get Error Logs")
1187            log.print_error_logs(error_logs, "AdditionalData Message Severity")
1188        pels = pel.peltool("-l", ignore_err=1)
1189        gp.qprint_var(pels)
1190
1191        # Delete errlogs prior to doing any boot tests.
1192        grk.run_key(delete_errlogs_cmd, ignore=1)
1193        grk.run_key(delete_bmcdump_cmd, ignore=1)
1194        if redfish_support_trans_state:
1195            grk.run_key(delete_sysdump_cmd, ignore=1)
1196
1197    # Process caller's boot_stack.
1198    while (len(boot_stack) > 0):
1199        test_loop_body()
1200
1201    gp.qprint_timen("Finished processing stack.")
1202
1203    post_stack()
1204
1205    # Process caller's boot_list.
1206    if len(boot_list) > 0:
1207        for ix in range(1, max_num_tests + 1):
1208            test_loop_body()
1209
1210    gp.qprint_timen("Completed all requested boot tests.")
1211
1212    boot_pass, boot_fail = boot_results.return_total_pass_fail()
1213    new_fail = boot_fail - init_boot_fail
1214    if new_fail > boot_fail_threshold:
1215        error_message = "Boot failures exceed the boot failure" +\
1216                        " threshold:\n" +\
1217                        gp.sprint_var(new_fail) +\
1218                        gp.sprint_var(boot_fail_threshold)
1219        BuiltIn().fail(gp.sprint_error(error_message))
1220