1#!/usr/bin/env python
2
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
7import os
8import imp
9import time
10import glob
11import random
12import re
13import signal
14try:
15    import cPickle as pickle
16except ImportError:
17    import pickle
18import socket
19
20from robot.utils import DotDict
21from robot.libraries.BuiltIn import BuiltIn
22
23from boot_data import *
24import gen_print as gp
25import gen_robot_plug_in as grpi
26import gen_valid as gv
27import gen_misc as gm
28import gen_cmd as gc
29import gen_robot_keyword as grk
30import state as st
31import var_stack as vs
32import gen_plug_in_utils as gpu
33
34base_path = os.path.dirname(os.path.dirname(
35                            imp.find_module("gen_robot_print")[1])) +\
36    os.sep
37sys.path.append(base_path + "extended/")
38import run_keyword as rk
39
40# Setting master_pid correctly influences the behavior of plug-ins like
41# DB_Logging
42program_pid = os.getpid()
43master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid)
44pgm_name = re.sub('\\.py$', '', os.path.basename(__file__))
45
46# Set up boot data structures.
47os_host = BuiltIn().get_variable_value("${OS_HOST}", default="")
48boot_table = create_boot_table(os_host=os_host)
49valid_boot_types = create_valid_boot_list(boot_table)
50
51boot_lists = read_boot_lists()
52
53# The maximum number of entries that can be in the boot_history global variable.
54max_boot_history = 10
55boot_history = []
56
57state = st.return_state_constant('default_state')
58cp_setup_called = 0
59next_boot = ""
60base_tool_dir_path = os.path.normpath(os.environ.get(
61    'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep
62
63ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep
64boot_success = 0
65status_dir_path = os.environ.get('STATUS_DIR_PATH', "")
66if status_dir_path != "":
67    status_dir_path = os.path.normpath(status_dir_path) + os.sep
68redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False)
69if redfish_supported:
70    default_power_on = "Redfish Power On"
71    default_power_off = "Redfish Power Off"
72else:
73    default_power_on = "REST Power On"
74    default_power_off = "REST Power Off"
75boot_count = 0
76
77LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
78AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '')
79ffdc_prefix = AUTOBOOT_FFDC_PREFIX
80boot_start_time = ""
81boot_end_time = ""
82save_stack = vs.var_stack('save_stack')
83main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet']
84
85
86def dump_ffdc_rc():
87    r"""
88    Return the constant dump ffdc test return code value.
89
90    When a plug-in call point program returns this value, it indicates that
91    this program should collect FFDC.
92    """
93
94    return 0x00000200
95
96
97def stop_test_rc():
98    r"""
99    Return the constant stop test return code value.
100
101    When a plug-in call point program returns this value, it indicates that
102    this program should stop running.
103    """
104
105    return 0x00000200
106
107
108def process_host(host,
109                 host_var_name=""):
110    r"""
111    Process a host by getting the associated host name and IP address and
112    setting them in global variables.
113
114    If the caller does not pass the host_var_name, this function will try to
115    figure out the name of the variable used by the caller for the host parm.
116    Callers are advised to explicitly specify the host_var_name when calling
117    with an exec command.  In such cases, the get_arg_name cannot figure out
118    the host variable name.
119
120    This function will then create similar global variable names by
121    removing "_host" and appending "_host_name" or "_ip" to the host variable
122    name.
123
124    Example:
125
126    If a call is made like this:
127    process_host(openbmc_host)
128
129    Global variables openbmc_host_name and openbmc_ip will be set.
130
131    Description of argument(s):
132    host           A host name or IP.  The name of the variable used should
133                   have a suffix of "_host".
134    host_var_name  The name of the variable being used as the host parm.
135    """
136
137    if host_var_name == "":
138        host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
139
140    host_name_var_name = re.sub("host", "host_name", host_var_name)
141    ip_var_name = re.sub("host", "ip", host_var_name)
142    cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\
143        host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\
144        host + "')"
145    exec(cmd_buf)
146
147
148def process_pgm_parms():
149    r"""
150    Process the program parameters by assigning them all to corresponding
151    globals.  Also, set some global values that depend on program parameters.
152    """
153
154    # Program parameter processing.
155    # Assign all program parms to python variables which are global to this
156    # module.
157
158    global parm_list
159    parm_list = BuiltIn().get_variable_value("${parm_list}")
160    # The following subset of parms should be processed as integers.
161    int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only',
162                'boot_fail_threshold', 'delete_errlogs',
163                'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet',
164                'test_mode', 'debug']
165    for parm in parm_list:
166        if parm in int_list:
167            sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\
168                      "}\", \"0\"))"
169        else:
170            sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")"
171        cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
172        gp.dpissuing(cmd_buf)
173        exec(cmd_buf)
174        if re.match(r".*_host$", parm):
175            cmd_buf = "process_host(" + parm + ", '" + parm + "')"
176            exec(cmd_buf)
177        if re.match(r".*_password$", parm):
178            # Register the value of any parm whose name ends in _password.
179            # This will cause the print functions to replace passwords with
180            # asterisks in the output.
181            cmd_buf = "gp.register_passwords(" + parm + ")"
182            exec(cmd_buf)
183
184    global ffdc_dir_path_style
185    global boot_list
186    global boot_stack
187    global boot_results_file_path
188    global boot_results
189    global boot_history
190    global ffdc_list_file_path
191    global ffdc_report_list_path
192    global ffdc_summary_list_path
193
194    if ffdc_dir_path_style == "":
195        ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0'))
196
197    # Convert these program parms to lists for easier processing..
198    boot_list = list(filter(None, boot_list.split(":")))
199    boot_stack = list(filter(None, boot_stack.split(":")))
200
201    cleanup_boot_results_file()
202    boot_results_file_path = create_boot_results_file_path(pgm_name,
203                                                           openbmc_nickname,
204                                                           master_pid)
205
206    if os.path.isfile(boot_results_file_path):
207        # We've been called before in this run so we'll load the saved
208        # boot_results and boot_history objects.
209        boot_results, boot_history =\
210            pickle.load(open(boot_results_file_path, 'rb'))
211    else:
212        boot_results = boot_results(boot_table, boot_pass, boot_fail)
213
214    ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\
215        "/FFDC_FILE_LIST"
216    ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\
217        "/FFDC_REPORT_FILE_LIST"
218
219    ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\
220        "/FFDC_SUMMARY_FILE_LIST"
221
222
223def initial_plug_in_setup():
224    r"""
225    Initialize all plug-in environment variables which do not change for the
226    duration of the program.
227
228    """
229
230    global LOG_LEVEL
231    BuiltIn().set_log_level("NONE")
232
233    BuiltIn().set_global_variable("${master_pid}", master_pid)
234    BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
235    BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
236    BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
237    BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}",
238                                  ffdc_list_file_path)
239    BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}",
240                                  ffdc_report_list_path)
241    BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}",
242                                  ffdc_summary_list_path)
243
244    BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}",
245                                  ffdc_dir_path_style)
246    BuiltIn().set_global_variable("${FFDC_CHECK}",
247                                  ffdc_check)
248
249    # For each program parameter, set the corresponding AUTOBOOT_ environment
250    # variable value.  Also, set an AUTOBOOT_ environment variable for every
251    # element in additional_values.
252    additional_values = ["program_pid", "master_pid", "ffdc_dir_path",
253                         "status_dir_path", "base_tool_dir_path",
254                         "ffdc_list_file_path", "ffdc_report_list_path",
255                         "ffdc_summary_list_path", "execdir"]
256
257    plug_in_vars = parm_list + additional_values
258
259    for var_name in plug_in_vars:
260        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
261        var_name = var_name.upper()
262        if var_value is None:
263            var_value = ""
264        os.environ["AUTOBOOT_" + var_name] = str(var_value)
265
266    BuiltIn().set_log_level(LOG_LEVEL)
267
268    # Make sure the ffdc list directory exists.
269    ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
270    if not os.path.exists(ffdc_list_dir_path):
271        os.makedirs(ffdc_list_dir_path)
272
273
274def plug_in_setup():
275    r"""
276    Initialize all changing plug-in environment variables for use by the
277    plug-in programs.
278    """
279
280    global LOG_LEVEL
281    global test_really_running
282
283    BuiltIn().set_log_level("NONE")
284
285    boot_pass, boot_fail = boot_results.return_total_pass_fail()
286    if boot_pass > 1:
287        test_really_running = 1
288    else:
289        test_really_running = 0
290
291    BuiltIn().set_global_variable("${test_really_running}",
292                                  test_really_running)
293    BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
294    BuiltIn().set_global_variable("${boot_pass}", boot_pass)
295    BuiltIn().set_global_variable("${boot_fail}", boot_fail)
296    BuiltIn().set_global_variable("${boot_success}", boot_success)
297    BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
298    BuiltIn().set_global_variable("${boot_start_time}", boot_start_time)
299    BuiltIn().set_global_variable("${boot_end_time}", boot_end_time)
300
301    # For each program parameter, set the corresponding AUTOBOOT_ environment
302    # variable value.  Also, set an AUTOBOOT_ environment variable for every
303    # element in additional_values.
304    additional_values = ["boot_type_desc", "boot_success", "boot_pass",
305                         "boot_fail", "test_really_running", "ffdc_prefix",
306                         "boot_start_time", "boot_end_time"]
307
308    plug_in_vars = additional_values
309
310    for var_name in plug_in_vars:
311        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
312        var_name = var_name.upper()
313        if var_value is None:
314            var_value = ""
315        os.environ["AUTOBOOT_" + var_name] = str(var_value)
316
317    if debug:
318        shell_rc, out_buf = \
319            gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u")
320
321    BuiltIn().set_log_level(LOG_LEVEL)
322
323
324def pre_boot_plug_in_setup():
325
326    # Clear the ffdc_list_file_path file.  Plug-ins may now write to it.
327    try:
328        os.remove(ffdc_list_file_path)
329    except OSError:
330        pass
331
332    # Clear the ffdc_report_list_path file.  Plug-ins may now write to it.
333    try:
334        os.remove(ffdc_report_list_path)
335    except OSError:
336        pass
337
338    # Clear the ffdc_summary_list_path file.  Plug-ins may now write to it.
339    try:
340        os.remove(ffdc_summary_list_path)
341    except OSError:
342        pass
343
344    global ffdc_prefix
345
346    seconds = time.time()
347    loc_time = time.localtime(seconds)
348    time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
349
350    ffdc_prefix = openbmc_nickname + "." + time_string
351
352
353def default_sigusr1(signal_number=0,
354                    frame=None):
355    r"""
356    Handle SIGUSR1 by doing nothing.
357
358    This function assists in debugging SIGUSR1 processing by printing messages
359    to stdout and to the log.html file.
360
361    Description of argument(s):
362    signal_number  The signal number (should always be 10 for SIGUSR1).
363    frame          The frame data.
364    """
365
366    gp.printn()
367    gp.print_executing()
368    gp.lprint_executing()
369
370
371def set_default_siguser1():
372    r"""
373    Set the default_sigusr1 function to be the SIGUSR1 handler.
374    """
375
376    gp.printn()
377    gp.print_executing()
378    gp.lprint_executing()
379    signal.signal(signal.SIGUSR1, default_sigusr1)
380
381
382def setup():
383    r"""
384    Do general program setup tasks.
385    """
386
387    global cp_setup_called
388    global transitional_boot_selected
389
390    gp.qprintn()
391
392    set_default_siguser1()
393    transitional_boot_selected = False
394
395    robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
396    repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
397    # If we can't find process_plug_in_packages.py, ssh_pw or
398    # validate_plug_ins.py, then we don't have our repo bin in PATH.
399    shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py"
400                                     + " ssh_pw validate_plug_ins.py", quiet=1,
401                                     print_output=0, show_err=0)
402    if shell_rc != 0:
403        os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
404    # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
405    if robot_pgm_dir_path not in sys.path:
406        sys.path.append(robot_pgm_dir_path)
407        PYTHONPATH = os.environ.get("PYTHONPATH", "")
408        if PYTHONPATH == "":
409            os.environ['PYTHONPATH'] = robot_pgm_dir_path
410        else:
411            os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
412
413    validate_parms()
414
415    gp.qprint_pgm_header()
416
417    grk.run_key("Set BMC Power Policy  ALWAYS_POWER_OFF")
418
419    initial_plug_in_setup()
420
421    plug_in_setup()
422    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
423        call_point='setup')
424    if rc != 0:
425        error_message = "Plug-in setup failed.\n"
426        gp.print_error_report(error_message)
427        BuiltIn().fail(error_message)
428    # Setting cp_setup_called lets our Teardown know that it needs to call
429    # the cleanup plug-in call point.
430    cp_setup_called = 1
431
432    # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
433    BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
434    # FFDC_LOG_PATH is used by "FFDC" keyword.
435    BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
436
437    # Also printed by FFDC.
438    global host_name
439    global host_ip
440    host = socket.gethostname()
441    host_name, host_ip = gm.get_host_name_ip(host)
442
443    gp.dprint_var(boot_table)
444    gp.dprint_var(boot_lists)
445
446
447def validate_parms():
448    r"""
449    Validate all program parameters.
450    """
451
452    process_pgm_parms()
453
454    gp.qprintn()
455
456    global openbmc_model
457    gv.valid_value(openbmc_host)
458    gv.valid_value(openbmc_username)
459    gv.valid_value(openbmc_password)
460    gv.valid_value(rest_username)
461    gv.valid_value(rest_password)
462    gv.valid_value(ipmi_username)
463    gv.valid_value(ipmi_password)
464    if os_host != "":
465        gv.valid_value(os_username)
466        gv.valid_value(os_password)
467
468    if pdu_host != "":
469        gv.valid_value(pdu_username)
470        gv.valid_value(pdu_password)
471        gv.valid_integer(pdu_slot_no)
472    if openbmc_serial_host != "":
473        gv.valid_integer(openbmc_serial_port)
474    if openbmc_model == "":
475        status, ret_values =\
476            grk.run_key_u("Get BMC System Model")
477        openbmc_model = ret_values
478        BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
479    gv.valid_value(openbmc_model)
480    gv.valid_integer(max_num_tests)
481    gv.valid_integer(boot_pass)
482    gv.valid_integer(boot_fail)
483
484    plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
485    BuiltIn().set_global_variable("${plug_in_packages_list}",
486                                  plug_in_packages_list)
487
488    gv.valid_value(stack_mode, valid_values=['normal', 'skip'])
489    if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
490        error_message = "You must provide either a value for either the" +\
491            " boot_list or the boot_stack parm.\n"
492        BuiltIn().fail(gp.sprint_error(error_message))
493
494    valid_boot_list(boot_list, valid_boot_types)
495    valid_boot_list(boot_stack, valid_boot_types)
496
497    selected_PDU_boots = list(set(boot_list + boot_stack)
498                              & set(boot_lists['PDU_reboot']))
499
500    if len(selected_PDU_boots) > 0 and pdu_host == "":
501        error_message = "You have selected the following boots which" +\
502                        " require a PDU host but no value for pdu_host:\n"
503        error_message += gp.sprint_var(selected_PDU_boots)
504        error_message += gp.sprint_var(pdu_host, fmt=gp.blank())
505        BuiltIn().fail(gp.sprint_error(error_message))
506
507    return
508
509
510def my_get_state():
511    r"""
512    Get the system state plus a little bit of wrapping.
513    """
514
515    global state
516
517    req_states = ['epoch_seconds'] + st.default_req_states
518
519    gp.qprint_timen("Getting system state.")
520    if test_mode:
521        state['epoch_seconds'] = int(time.time())
522    else:
523        state = st.get_state(req_states=req_states, quiet=quiet)
524    gp.qprint_var(state)
525
526
527def valid_state():
528    r"""
529    Verify that our state dictionary contains no blank values.  If we don't get
530    valid state data, we cannot continue to work.
531    """
532
533    if st.compare_states(state, st.invalid_state_match, 'or'):
534        error_message = "The state dictionary contains blank fields which" +\
535            " is illegal.\n" + gp.sprint_var(state)
536        BuiltIn().fail(gp.sprint_error(error_message))
537
538
539def select_boot():
540    r"""
541    Select a boot test to be run based on our current state and return the
542    chosen boot type.
543
544    Description of arguments:
545    state  The state of the machine.
546    """
547
548    global transitional_boot_selected
549    global boot_stack
550
551    gp.qprint_timen("Selecting a boot test.")
552
553    if transitional_boot_selected and not boot_success:
554        prior_boot = next_boot
555        boot_candidate = boot_stack.pop()
556        gp.qprint_timen("The prior '" + next_boot + "' was chosen to"
557                        + " transition to a valid state for '" + boot_candidate
558                        + "' which was at the top of the boot_stack.  Since"
559                        + " the '" + next_boot + "' failed, the '"
560                        + boot_candidate + "' has been removed from the stack"
561                        + " to avoid and endless failure loop.")
562        if len(boot_stack) == 0:
563            return ""
564
565    my_get_state()
566    valid_state()
567
568    transitional_boot_selected = False
569    stack_popped = 0
570    if len(boot_stack) > 0:
571        stack_popped = 1
572        gp.qprint_dashes()
573        gp.qprint_var(boot_stack)
574        gp.qprint_dashes()
575        skip_boot_printed = 0
576        while len(boot_stack) > 0:
577            boot_candidate = boot_stack.pop()
578            if stack_mode == 'normal':
579                break
580            else:
581                if st.compare_states(state, boot_table[boot_candidate]['end']):
582                    if not skip_boot_printed:
583                        gp.qprint_var(stack_mode)
584                        gp.qprintn()
585                        gp.qprint_timen("Skipping the following boot tests"
586                                        + " which are unnecessary since their"
587                                        + " required end states match the"
588                                        + " current machine state:")
589                        skip_boot_printed = 1
590                    gp.qprint_var(boot_candidate)
591                    boot_candidate = ""
592        if boot_candidate == "":
593            gp.qprint_dashes()
594            gp.qprint_var(boot_stack)
595            gp.qprint_dashes()
596            return boot_candidate
597        if st.compare_states(state, boot_table[boot_candidate]['start']):
598            gp.qprint_timen("The machine state is valid for a '"
599                            + boot_candidate + "' boot test.")
600            gp.qprint_dashes()
601            gp.qprint_var(boot_stack)
602            gp.qprint_dashes()
603            return boot_candidate
604        else:
605            gp.qprint_timen("The machine state does not match the required"
606                            + " starting state for a '" + boot_candidate
607                            + "' boot test:")
608            gp.qprint_varx("boot_table_start_entry",
609                           boot_table[boot_candidate]['start'])
610            boot_stack.append(boot_candidate)
611            transitional_boot_selected = True
612            popped_boot = boot_candidate
613
614    # Loop through your list selecting a boot_candidates
615    boot_candidates = []
616    for boot_candidate in boot_list:
617        if st.compare_states(state, boot_table[boot_candidate]['start']):
618            if stack_popped:
619                if st.compare_states(boot_table[boot_candidate]['end'],
620                                     boot_table[popped_boot]['start']):
621                    boot_candidates.append(boot_candidate)
622            else:
623                boot_candidates.append(boot_candidate)
624
625    if len(boot_candidates) == 0:
626        gp.qprint_timen("The user's boot list contained no boot tests"
627                        + " which are valid for the current machine state.")
628        boot_candidate = default_power_on
629        if not st.compare_states(state, boot_table[default_power_on]['start']):
630            boot_candidate = default_power_off
631        boot_candidates.append(boot_candidate)
632        gp.qprint_timen("Using default '" + boot_candidate
633                        + "' boot type to transition to valid state.")
634
635    gp.dprint_var(boot_candidates)
636
637    # Randomly select a boot from the candidate list.
638    boot = random.choice(boot_candidates)
639
640    return boot
641
642
643def print_defect_report(ffdc_file_list):
644    r"""
645    Print a defect report.
646
647    Description of argument(s):
648    ffdc_file_list  A list of files which were collected by our ffdc functions.
649    """
650
651    # Making deliberate choice to NOT run plug_in_setup().  We don't want
652    # ffdc_prefix updated.
653    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
654        call_point='ffdc_report', stop_on_plug_in_failure=0)
655
656    # Get additional header data which may have been created by ffdc plug-ins.
657    # Also, delete the individual header files to cleanup.
658    cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\
659              " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
660              " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
661    shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
662                                              show_err=0)
663
664    # Get additional summary data which may have been created by ffdc plug-ins.
665    # Also, delete the individual header files to cleanup.
666    cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\
667              " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
668              " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
669    shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
670                                               show_err=0)
671
672    # ffdc_list_file_path contains a list of any ffdc files created by plug-
673    # ins, etc.  Read that data into a list.
674    try:
675        plug_in_ffdc_list = \
676            open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n")
677        plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list))
678    except IOError:
679        plug_in_ffdc_list = []
680
681    # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed
682    # in.  Eliminate duplicates and sort the list.
683    ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list))
684
685    if status_file_path != "":
686        ffdc_file_list.insert(0, status_file_path)
687
688    # Convert the list to a printable list.
689    printable_ffdc_file_list = "\n".join(ffdc_file_list)
690
691    # Open ffdc_file_list for writing.  We will write a complete list of
692    # FFDC files to it for possible use by plug-ins like cp_stop_check.
693    ffdc_list_file = open(ffdc_list_file_path, 'w')
694    ffdc_list_file.write(printable_ffdc_file_list + "\n")
695    ffdc_list_file.close()
696
697    indent = 0
698    width = 90
699    linefeed = 1
700    char = "="
701
702    gp.qprintn()
703    gp.qprint_dashes(indent, width, linefeed, char)
704    gp.qprintn("Copy this data to the defect:\n")
705
706    if len(more_header_info) > 0:
707        gp.qprintn(more_header_info)
708    gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host,
709              openbmc_host_name, openbmc_ip, openbmc_username,
710              openbmc_password, rest_username, rest_password, ipmi_username,
711              ipmi_password, os_host, os_host_name, os_ip, os_username,
712              os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username,
713              pdu_password, pdu_slot_no, openbmc_serial_host,
714              openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port)
715
716    gp.qprintn()
717    print_boot_history(boot_history)
718    gp.qprintn()
719    gp.qprint_var(state)
720    gp.qprintn()
721    gp.qprintn("FFDC data files:")
722    gp.qprintn(printable_ffdc_file_list)
723    gp.qprintn()
724
725    if len(ffdc_summary_info) > 0:
726        gp.qprintn(ffdc_summary_info)
727
728    gp.qprint_dashes(indent, width, linefeed, char)
729
730
731def my_ffdc():
732    r"""
733    Collect FFDC data.
734    """
735
736    global state
737
738    plug_in_setup()
739    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
740        call_point='ffdc', stop_on_plug_in_failure=0)
741
742    AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX']
743    status, ffdc_file_list = grk.run_key_u("FFDC  ffdc_prefix="
744                                           + AUTOBOOT_FFDC_PREFIX
745                                           + "  ffdc_function_list="
746                                           + ffdc_function_list, ignore=1)
747    if status != 'PASS':
748        gp.qprint_error("Call to ffdc failed.\n")
749        if type(ffdc_file_list) is not list:
750            ffdc_file_list = []
751        # Leave a record for caller that "soft" errors occurred.
752        soft_errors = 1
753        gpu.save_plug_in_value(soft_errors, pgm_name)
754
755    my_get_state()
756
757    print_defect_report(ffdc_file_list)
758
759
760def print_test_start_message(boot_keyword):
761    r"""
762    Print a message indicating what boot test is about to run.
763
764    Description of arguments:
765    boot_keyword  The name of the boot which is to be run
766                  (e.g. "BMC Power On").
767    """
768
769    global boot_history
770    global boot_start_time
771
772    doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".")
773
774    # Set boot_start_time for use by plug-ins.
775    boot_start_time = doing_msg[1:33]
776    gp.qprint_var(boot_start_time)
777
778    gp.qprint(doing_msg)
779
780    update_boot_history(boot_history, doing_msg, max_boot_history)
781
782
783def stop_boot_test(signal_number=0,
784                   frame=None):
785    r"""
786    Handle SIGUSR1 by aborting the boot test that is running.
787
788    Description of argument(s):
789    signal_number  The signal number (should always be 10 for SIGUSR1).
790    frame          The frame data.
791    """
792
793    gp.printn()
794    gp.print_executing()
795    gp.lprint_executing()
796
797    # Restore original sigusr1 handler.
798    set_default_siguser1()
799
800    message = "The caller has asked that the boot test be stopped and marked"
801    message += " as a failure."
802
803    function_stack = gm.get_function_stack()
804    if "wait_state" in function_stack:
805        st.set_exit_wait_early_message(message)
806    else:
807        BuiltIn().fail(gp.sprint_error(message))
808
809
810def run_boot(boot):
811    r"""
812    Run the specified boot.
813
814    Description of arguments:
815    boot  The name of the boot test to be performed.
816    """
817
818    global state
819
820    signal.signal(signal.SIGUSR1, stop_boot_test)
821    gp.qprint_timen("stop_boot_test is armed.")
822
823    print_test_start_message(boot)
824
825    plug_in_setup()
826    rc, shell_rc, failed_plug_in_name = \
827        grpi.rprocess_plug_in_packages(call_point="pre_boot")
828    if rc != 0:
829        error_message = "Plug-in failed with non-zero return code.\n" +\
830            gp.sprint_var(rc, fmt=gp.hexa())
831        set_default_siguser1()
832        BuiltIn().fail(gp.sprint_error(error_message))
833
834    if test_mode:
835        # In test mode, we'll pretend the boot worked by assigning its
836        # required end state to the default state value.
837        state = st.strip_anchor_state(boot_table[boot]['end'])
838    else:
839        # Assertion:  We trust that the state data was made fresh by the
840        # caller.
841
842        gp.qprintn()
843
844        if boot_table[boot]['method_type'] == "keyword":
845            rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''),
846                               boot_table[boot]['method'],
847                               quiet=quiet)
848
849        if boot_table[boot]['bmc_reboot']:
850            st.wait_for_comm_cycle(int(state['epoch_seconds']))
851            plug_in_setup()
852            rc, shell_rc, failed_plug_in_name = \
853                grpi.rprocess_plug_in_packages(call_point="post_reboot")
854            if rc != 0:
855                error_message = "Plug-in failed with non-zero return code.\n"
856                error_message += gp.sprint_var(rc, fmt=gp.hexa())
857                set_default_siguser1()
858                BuiltIn().fail(gp.sprint_error(error_message))
859        else:
860            match_state = st.anchor_state(state)
861            del match_state['epoch_seconds']
862            # Wait for the state to change in any way.
863            st.wait_state(match_state, wait_time=state_change_timeout,
864                          interval="10 seconds", invert=1)
865
866        gp.qprintn()
867        if boot_table[boot]['end']['chassis'] == "Off":
868            boot_timeout = power_off_timeout
869        else:
870            boot_timeout = power_on_timeout
871        st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout,
872                      interval="10 seconds")
873
874    plug_in_setup()
875    rc, shell_rc, failed_plug_in_name = \
876        grpi.rprocess_plug_in_packages(call_point="post_boot")
877    if rc != 0:
878        error_message = "Plug-in failed with non-zero return code.\n" +\
879            gp.sprint_var(rc, fmt=gp.hexa())
880        set_default_siguser1()
881        BuiltIn().fail(gp.sprint_error(error_message))
882
883    # Restore original sigusr1 handler.
884    set_default_siguser1()
885
886
887def test_loop_body():
888    r"""
889    The main loop body for the loop in main_py.
890
891    Description of arguments:
892    boot_count  The iteration number (starts at 1).
893    """
894
895    global boot_count
896    global state
897    global next_boot
898    global boot_success
899    global boot_end_time
900
901    gp.qprintn()
902
903    next_boot = select_boot()
904    if next_boot == "":
905        return True
906
907    boot_count += 1
908    gp.qprint_timen("Starting boot " + str(boot_count) + ".")
909
910    pre_boot_plug_in_setup()
911
912    cmd_buf = ["run_boot", next_boot]
913    boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
914    if boot_status == "FAIL":
915        gp.qprint(msg)
916
917    gp.qprintn()
918    if boot_status == "PASS":
919        boot_success = 1
920        completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot
921                                         + "\" succeeded.")
922    else:
923        boot_success = 0
924        completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot
925                                         + "\" failed.")
926
927    # Set boot_end_time for use by plug-ins.
928    boot_end_time = completion_msg[1:33]
929    gp.qprint_var(boot_end_time)
930
931    gp.qprint(completion_msg)
932
933    boot_results.update(next_boot, boot_status)
934
935    plug_in_setup()
936    # NOTE: A post_test_case call point failure is NOT counted as a boot
937    # failure.
938    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
939        call_point='post_test_case', stop_on_plug_in_failure=0)
940
941    plug_in_setup()
942    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
943        call_point='ffdc_check', shell_rc=dump_ffdc_rc(),
944        stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
945    if ffdc_check == "All" or\
946       shell_rc == dump_ffdc_rc():
947        status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
948        if status != 'PASS':
949            gp.qprint_error("Call to my_ffdc failed.\n")
950            # Leave a record for caller that "soft" errors occurred.
951            soft_errors = 1
952            gpu.save_plug_in_value(soft_errors, pgm_name)
953
954    if delete_errlogs:
955        # We need to purge error logs between boots or they build up.
956        grk.run_key("Delete Error logs", ignore=1)
957
958    boot_results.print_report()
959    gp.qprint_timen("Finished boot " + str(boot_count) + ".")
960
961    plug_in_setup()
962    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
963        call_point='stop_check', shell_rc=stop_test_rc(),
964        stop_on_non_zero_rc=1)
965    if shell_rc == stop_test_rc():
966        message = "Stopping as requested by user.\n"
967        gp.print_time(message)
968        BuiltIn().fail(message)
969
970    # This should help prevent ConnectionErrors.
971    grk.run_key_u("Close All Connections")
972
973    return True
974
975
976def obmc_boot_test_teardown():
977    r"""
978    Clean up after the Main keyword.
979    """
980
981    if cp_setup_called:
982        plug_in_setup()
983        rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
984            call_point='cleanup', stop_on_plug_in_failure=0)
985
986    if 'boot_results_file_path' in globals():
987        # Save boot_results and boot_history objects to a file in case they are
988        # needed again.
989        gp.qprint_timen("Saving boot_results to the following path.")
990        gp.qprint_var(boot_results_file_path)
991        pickle.dump((boot_results, boot_history),
992                    open(boot_results_file_path, 'wb'),
993                    pickle.HIGHEST_PROTOCOL)
994
995    global save_stack
996    # Restore any global values saved on the save_stack.
997    for parm_name in main_func_parm_list:
998        # Get the parm_value if it was saved on the stack.
999        try:
1000            parm_value = save_stack.pop(parm_name)
1001        except BaseException:
1002            # If it was not saved, no further action is required.
1003            continue
1004
1005        # Restore the saved value.
1006        cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1007            "}\", parm_value)"
1008        gp.dpissuing(cmd_buf)
1009        exec(cmd_buf)
1010
1011    gp.dprintn(save_stack.sprint_obj())
1012
1013
1014def test_teardown():
1015    r"""
1016    Clean up after this test case.
1017    """
1018
1019    gp.qprintn()
1020    cmd_buf = ["Print Error",
1021               "A keyword timeout occurred ending this program.\n"]
1022    BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
1023
1024    gp.qprint_pgm_footer()
1025
1026
1027def post_stack():
1028    r"""
1029    Process post_stack plug-in programs.
1030    """
1031
1032    if not call_post_stack_plug:
1033        # The caller does not wish to have post_stack plug-in processing done.
1034        return
1035
1036    global boot_success
1037
1038    # NOTE: A post_stack call-point failure is NOT counted as a boot failure.
1039    pre_boot_plug_in_setup()
1040    # For the purposes of the following plug-ins, mark the "boot" as a success.
1041    boot_success = 1
1042    plug_in_setup()
1043    rc, shell_rc, failed_plug_in_name, history =\
1044        grpi.rprocess_plug_in_packages(call_point='post_stack',
1045                                       stop_on_plug_in_failure=0,
1046                                       return_history=True)
1047    for doing_msg in history:
1048        update_boot_history(boot_history, doing_msg, max_boot_history)
1049    if rc != 0:
1050        boot_success = 0
1051
1052    plug_in_setup()
1053    rc, shell_rc, failed_plug_in_name =\
1054        grpi.rprocess_plug_in_packages(call_point='ffdc_check',
1055                                       shell_rc=dump_ffdc_rc(),
1056                                       stop_on_plug_in_failure=1,
1057                                       stop_on_non_zero_rc=1)
1058    if shell_rc == dump_ffdc_rc():
1059        status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
1060        if status != 'PASS':
1061            gp.qprint_error("Call to my_ffdc failed.\n")
1062            # Leave a record for caller that "soft" errors occurred.
1063            soft_errors = 1
1064            gpu.save_plug_in_value(soft_errors, pgm_name)
1065
1066    plug_in_setup()
1067    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1068        call_point='stop_check', shell_rc=stop_test_rc(),
1069        stop_on_non_zero_rc=1)
1070    if shell_rc == stop_test_rc():
1071        message = "Stopping as requested by user.\n"
1072        gp.print_time(message)
1073        BuiltIn().fail(message)
1074
1075
1076def obmc_boot_test_py(loc_boot_stack=None,
1077                      loc_stack_mode=None,
1078                      loc_quiet=None):
1079    r"""
1080    Do main program processing.
1081    """
1082
1083    global save_stack
1084
1085    gp.dprintn()
1086    # Process function parms.
1087    for parm_name in main_func_parm_list:
1088        # Get parm's value.
1089        parm_value = eval("loc_" + parm_name)
1090        gp.dpvars(parm_name, parm_value)
1091
1092        if parm_value is not None:
1093            # Save the global value on a stack.
1094            cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\
1095                parm_name + "}\"), \"" + parm_name + "\")"
1096            gp.dpissuing(cmd_buf)
1097            exec(cmd_buf)
1098
1099            # Set the global value to the passed value.
1100            cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1101                "}\", loc_" + parm_name + ")"
1102            gp.dpissuing(cmd_buf)
1103            exec(cmd_buf)
1104
1105    gp.dprintn(save_stack.sprint_obj())
1106
1107    setup()
1108
1109    init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail()
1110
1111    if ffdc_only:
1112        gp.qprint_timen("Caller requested ffdc_only.")
1113        if do_pre_boot_plug_in_setup:
1114            pre_boot_plug_in_setup()
1115        grk.run_key_u("my_ffdc")
1116        return
1117
1118    # Process caller's boot_stack.
1119    while (len(boot_stack) > 0):
1120        test_loop_body()
1121
1122    gp.qprint_timen("Finished processing stack.")
1123
1124    post_stack()
1125
1126    # Process caller's boot_list.
1127    if len(boot_list) > 0:
1128        for ix in range(1, max_num_tests + 1):
1129            test_loop_body()
1130
1131    gp.qprint_timen("Completed all requested boot tests.")
1132
1133    boot_pass, boot_fail = boot_results.return_total_pass_fail()
1134    new_fail = boot_fail - init_boot_fail
1135    if new_fail > boot_fail_threshold:
1136        error_message = "Boot failures exceed the boot failure" +\
1137                        " threshold:\n" +\
1138                        gp.sprint_var(new_fail) +\
1139                        gp.sprint_var(boot_fail_threshold)
1140        BuiltIn().fail(gp.sprint_error(error_message))
1141