xref: /openbmc/openbmc-test-automation/lib/obmc_boot_test.py (revision 4e8c09b1a6aace6f6bd1dfae1b999ba871f5c9b5)
1#!/usr/bin/env python
2
3r"""
4This module is the python counterpart to obmc_boot_test.
5"""
6
7import os
8import imp
9import time
10import glob
11import random
12import re
13import cPickle as pickle
14import socket
15
16from robot.utils import DotDict
17from robot.libraries.BuiltIn import BuiltIn
18
19from boot_data import *
20import gen_print as gp
21import gen_robot_print as grp
22import gen_robot_plug_in as grpi
23import gen_robot_valid as grv
24import gen_misc as gm
25import gen_cmd as gc
26import gen_robot_keyword as grk
27import state as st
28import var_stack as vs
29
30base_path = os.path.dirname(os.path.dirname(
31                            imp.find_module("gen_robot_print")[1])) +\
32    os.sep
33sys.path.append(base_path + "extended/")
34import run_keyword as rk
35
36# Setting master_pid correctly influences the behavior of plug-ins like
37# DB_Logging
38program_pid = os.getpid()
39master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid)
40pgm_name = re.sub('\.py$', '', os.path.basename(__file__))
41
42# Set up boot data structures.
43boot_table = create_boot_table()
44valid_boot_types = create_valid_boot_list(boot_table)
45
46boot_lists = read_boot_lists()
47last_ten = []
48
49state = st.return_default_state()
50cp_setup_called = 0
51next_boot = ""
52base_tool_dir_path = os.path.normpath(os.environ.get(
53    'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep
54
55ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep
56boot_success = 0
57status_dir_path = os.environ.get('STATUS_DIR_PATH', "")
58if status_dir_path != "":
59    status_dir_path = os.path.normpath(status_dir_path) + os.sep
60default_power_on = "REST Power On"
61default_power_off = "REST Power Off"
62boot_count = 0
63
64LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
65ffdc_prefix = ""
66boot_start_time = ""
67boot_end_time = ""
68save_stack = vs.var_stack('save_stack')
69main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet']
70
71
72###############################################################################
73def process_host(host,
74                 host_var_name=""):
75
76    r"""
77    Process a host by getting the associated host name and IP address and
78    setting them in global variables.
79
80    If the caller does not pass the host_var_name, this function will try to
81    figure out the name of the variable used by the caller for the host parm.
82    Callers are advised to explicitly specify the host_var_name when calling
83    with an exec command.  In such cases, the get_arg_name cannot figure out
84    the host variable name.
85
86    This function will then create similar global variable names by
87    removing "_host" and appending "_host_name" or "_ip" to the host variable
88    name.
89
90    Example:
91
92    If a call is made like this:
93    process_host(openbmc_host)
94
95    Global variables openbmc_host_name and openbmc_ip will be set.
96
97    Description of argument(s):
98    host           A host name or IP.  The name of the variable used should
99                   have a suffix of "_host".
100    host_var_name  The name of the variable being used as the host parm.
101    """
102
103    if host_var_name == "":
104        host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
105
106    host_name_var_name = re.sub("host", "host_name", host_var_name)
107    ip_var_name = re.sub("host", "ip", host_var_name)
108    cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\
109        host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\
110        host + "')"
111    exec(cmd_buf)
112
113###############################################################################
114
115
116###############################################################################
117def process_pgm_parms():
118
119    r"""
120    Process the program parameters by assigning them all to corresponding
121    globals.  Also, set some global values that depend on program parameters.
122    """
123
124    # Program parameter processing.
125    # Assign all program parms to python variables which are global to this
126    # module.
127
128    global parm_list
129    parm_list = BuiltIn().get_variable_value("${parm_list}")
130    # The following subset of parms should be processed as integers.
131    int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only',
132                'boot_fail_threshold', 'delete_errlogs', 'quiet', 'test_mode',
133                'debug']
134    for parm in parm_list:
135        if parm in int_list:
136            sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\
137                      "}\", \"0\"))"
138        else:
139            sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")"
140        cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
141        gp.dpissuing(cmd_buf)
142        exec(cmd_buf)
143        if re.match(r".*_host$", parm):
144            cmd_buf = "process_host(" + parm + ", '" + parm + "')"
145            exec(cmd_buf)
146        if re.match(r".*_password$", parm):
147            # Register the value of any parm whose name ends in _password.
148            # This will cause the print functions to replace passwords with
149            # asterisks in the output.
150            cmd_buf = "gp.register_passwords(" + parm + ")"
151            exec(cmd_buf)
152
153    global ffdc_dir_path_style
154    global boot_list
155    global boot_stack
156    global boot_results_file_path
157    global boot_results
158    global ffdc_list_file_path
159    global ffdc_report_list_path
160    global ffdc_summary_list_path
161
162    if ffdc_dir_path_style == "":
163        ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0'))
164
165    # Convert these program parms to lists for easier processing..
166    boot_list = filter(None, boot_list.split(":"))
167    boot_stack = filter(None, boot_stack.split(":"))
168
169    cleanup_boot_results_file()
170    boot_results_file_path = create_boot_results_file_path(pgm_name,
171                                                           openbmc_nickname,
172                                                           master_pid)
173
174    if os.path.isfile(boot_results_file_path):
175        # We've been called before in this run so we'll load the saved
176        # boot_results object.
177        boot_results = pickle.load(open(boot_results_file_path, 'rb'))
178    else:
179        boot_results = boot_results(boot_table, boot_pass, boot_fail)
180
181    ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\
182        "/FFDC_FILE_LIST"
183    ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\
184        "/FFDC_REPORT_FILE_LIST"
185
186    ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\
187        "/FFDC_SUMMARY_FILE_LIST"
188
189###############################################################################
190
191
192###############################################################################
193def initial_plug_in_setup():
194
195    r"""
196    Initialize all plug-in environment variables which do not change for the
197    duration of the program.
198
199    """
200
201    global LOG_LEVEL
202    BuiltIn().set_log_level("NONE")
203
204    BuiltIn().set_global_variable("${master_pid}", master_pid)
205    BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
206    BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
207    BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
208    BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}",
209                                  ffdc_list_file_path)
210    BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}",
211                                  ffdc_report_list_path)
212    BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}",
213                                  ffdc_summary_list_path)
214
215    BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}",
216                                  ffdc_dir_path_style)
217    BuiltIn().set_global_variable("${FFDC_CHECK}",
218                                  ffdc_check)
219
220    # For each program parameter, set the corresponding AUTOBOOT_ environment
221    # variable value.  Also, set an AUTOBOOT_ environment variable for every
222    # element in additional_values.
223    additional_values = ["program_pid", "master_pid", "ffdc_dir_path",
224                         "status_dir_path", "base_tool_dir_path",
225                         "ffdc_list_file_path", "ffdc_report_list_path",
226                         "ffdc_summary_list_path"]
227
228    plug_in_vars = parm_list + additional_values
229
230    for var_name in plug_in_vars:
231        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
232        var_name = var_name.upper()
233        if var_value is None:
234            var_value = ""
235        os.environ["AUTOBOOT_" + var_name] = str(var_value)
236
237    BuiltIn().set_log_level(LOG_LEVEL)
238
239    # Make sure the ffdc list directory exists.
240    ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
241    if not os.path.exists(ffdc_list_dir_path):
242        os.makedirs(ffdc_list_dir_path)
243
244###############################################################################
245
246
247###############################################################################
248def plug_in_setup():
249
250    r"""
251    Initialize all changing plug-in environment variables for use by the
252    plug-in programs.
253    """
254
255    global LOG_LEVEL
256    global test_really_running
257
258    BuiltIn().set_log_level("NONE")
259
260    boot_pass, boot_fail = boot_results.return_total_pass_fail()
261    if boot_pass > 1:
262        test_really_running = 1
263    else:
264        test_really_running = 0
265
266    BuiltIn().set_global_variable("${test_really_running}",
267                                  test_really_running)
268    BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
269    BuiltIn().set_global_variable("${boot_pass}", boot_pass)
270    BuiltIn().set_global_variable("${boot_fail}", boot_fail)
271    BuiltIn().set_global_variable("${boot_success}", boot_success)
272    BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
273    BuiltIn().set_global_variable("${boot_start_time}", boot_start_time)
274    BuiltIn().set_global_variable("${boot_end_time}", boot_end_time)
275
276    # For each program parameter, set the corresponding AUTOBOOT_ environment
277    # variable value.  Also, set an AUTOBOOT_ environment variable for every
278    # element in additional_values.
279    additional_values = ["boot_type_desc", "boot_success", "boot_pass",
280                         "boot_fail", "test_really_running", "ffdc_prefix",
281                         "boot_start_time", "boot_end_time"]
282
283    plug_in_vars = additional_values
284
285    for var_name in plug_in_vars:
286        var_value = BuiltIn().get_variable_value("${" + var_name + "}")
287        var_name = var_name.upper()
288        if var_value is None:
289            var_value = ""
290        os.environ["AUTOBOOT_" + var_name] = str(var_value)
291
292    if debug:
293        shell_rc, out_buf = \
294            gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u")
295
296    BuiltIn().set_log_level(LOG_LEVEL)
297
298###############################################################################
299
300
301###############################################################################
302def pre_boot_plug_in_setup():
303
304    # Clear the ffdc_list_file_path file.  Plug-ins may now write to it.
305    try:
306        os.remove(ffdc_list_file_path)
307    except OSError:
308        pass
309
310    # Clear the ffdc_report_list_path file.  Plug-ins may now write to it.
311    try:
312        os.remove(ffdc_report_list_path)
313    except OSError:
314        pass
315
316    # Clear the ffdc_summary_list_path file.  Plug-ins may now write to it.
317    try:
318        os.remove(ffdc_summary_list_path)
319    except OSError:
320        pass
321
322    global ffdc_prefix
323
324    seconds = time.time()
325    loc_time = time.localtime(seconds)
326    time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
327
328    ffdc_prefix = openbmc_nickname + "." + time_string
329
330###############################################################################
331
332
333###############################################################################
334def setup():
335
336    r"""
337    Do general program setup tasks.
338    """
339
340    global cp_setup_called
341    global transitional_boot_selected
342
343    gp.qprintn()
344
345    transitional_boot_selected = False
346
347    robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
348    repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
349    # If we can't find process_plug_in_packages.py, ssh_pw or
350    # validate_plug_ins.py, then we don't have our repo bin in PATH.
351    shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py" +
352                                     " ssh_pw validate_plug_ins.py", quiet=1,
353                                     print_output=0, show_err=0)
354    if shell_rc != 0:
355        os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
356    # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
357    if robot_pgm_dir_path not in sys.path:
358        sys.path.append(robot_pgm_dir_path)
359        PYTHONPATH = os.environ.get("PYTHONPATH", "")
360        if PYTHONPATH == "":
361            os.environ['PYTHONPATH'] = robot_pgm_dir_path
362        else:
363            os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
364
365    validate_parms()
366
367    grp.rqprint_pgm_header()
368
369    grk.run_key("Set BMC Power Policy  RESTORE_LAST_STATE")
370
371    initial_plug_in_setup()
372
373    plug_in_setup()
374    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
375        call_point='setup')
376    if rc != 0:
377        error_message = "Plug-in setup failed.\n"
378        grp.rprint_error_report(error_message)
379        BuiltIn().fail(error_message)
380    # Setting cp_setup_called lets our Teardown know that it needs to call
381    # the cleanup plug-in call point.
382    cp_setup_called = 1
383
384    # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
385    BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
386    # FFDC_LOG_PATH is used by "FFDC" keyword.
387    BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
388
389    # Also printed by FFDC.
390    global host_name
391    global host_ip
392    host = socket.gethostname()
393    host_name, host_ip = gm.get_host_name_ip(host)
394
395    gp.dprint_var(boot_table, 1)
396    gp.dprint_var(boot_lists)
397
398###############################################################################
399
400
401###############################################################################
402def validate_parms():
403
404    r"""
405    Validate all program parameters.
406    """
407
408    process_pgm_parms()
409
410    gp.qprintn()
411
412    global openbmc_model
413    grv.rvalid_value("openbmc_host")
414    grv.rvalid_value("openbmc_username")
415    grv.rvalid_value("openbmc_password")
416    if os_host != "":
417        grv.rvalid_value("os_username")
418        grv.rvalid_value("os_password")
419
420    if pdu_host != "":
421        grv.rvalid_value("pdu_username")
422        grv.rvalid_value("pdu_password")
423        grv.rvalid_integer("pdu_slot_no")
424    if openbmc_serial_host != "":
425        grv.rvalid_integer("openbmc_serial_port")
426    if openbmc_model == "":
427        status, ret_values =\
428            grk.run_key_u("Get BMC System Model")
429        openbmc_model = ret_values
430        BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
431    grv.rvalid_value("openbmc_model")
432    grv.rvalid_integer("max_num_tests")
433    grv.rvalid_integer("boot_pass")
434    grv.rvalid_integer("boot_fail")
435
436    plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
437    BuiltIn().set_global_variable("${plug_in_packages_list}",
438                                  plug_in_packages_list)
439
440    grv.rvalid_value("stack_mode", valid_values=['normal', 'skip'])
441    if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
442        error_message = "You must provide either a value for either the" +\
443            " boot_list or the boot_stack parm.\n"
444        BuiltIn().fail(gp.sprint_error(error_message))
445
446    valid_boot_list(boot_list, valid_boot_types)
447    valid_boot_list(boot_stack, valid_boot_types)
448
449    selected_PDU_boots = list(set(boot_list + boot_stack) &
450                              set(boot_lists['PDU_reboot']))
451
452    if len(selected_PDU_boots) > 0 and pdu_host == "":
453        error_message = "You have selected the following boots which" +\
454                        " require a PDU host but no value for pdu_host:\n"
455        error_message += gp.sprint_var(selected_PDU_boots)
456        error_message += gp.sprint_var(pdu_host, 2)
457        BuiltIn().fail(gp.sprint_error(error_message))
458
459    return
460
461###############################################################################
462
463
464###############################################################################
465def my_get_state():
466
467    r"""
468    Get the system state plus a little bit of wrapping.
469    """
470
471    global state
472
473    req_states = ['epoch_seconds'] + st.default_req_states
474
475    gp.qprint_timen("Getting system state.")
476    if test_mode:
477        state['epoch_seconds'] = int(time.time())
478    else:
479        state = st.get_state(req_states=req_states, quiet=quiet)
480    gp.qprint_var(state)
481
482###############################################################################
483
484
485###############################################################################
486def valid_state():
487
488    r"""
489    Verify that our state dictionary contains no blank values.  If we don't get
490    valid state data, we cannot continue to work.
491    """
492
493    if st.compare_states(state, st.invalid_state_match, 'or'):
494        error_message = "The state dictionary contains blank fields which" +\
495            " is illegal.\n" + gp.sprint_var(state)
496        BuiltIn().fail(gp.sprint_error(error_message))
497
498###############################################################################
499
500
501###############################################################################
502def select_boot():
503
504    r"""
505    Select a boot test to be run based on our current state and return the
506    chosen boot type.
507
508    Description of arguments:
509    state  The state of the machine.
510    """
511
512    global transitional_boot_selected
513    global boot_stack
514
515    gp.qprint_timen("Selecting a boot test.")
516
517    if transitional_boot_selected and not boot_success:
518        prior_boot = next_boot
519        boot_candidate = boot_stack.pop()
520        gp.qprint_timen("The prior '" + next_boot + "' was chosen to" +
521                        " transition to a valid state for '" + boot_candidate +
522                        "' which was at the top of the boot_stack.  Since" +
523                        " the '" + next_boot + "' failed, the '" +
524                        boot_candidate + "' has been removed from the stack" +
525                        " to avoid and endless failure loop.")
526        if len(boot_stack) == 0:
527            return ""
528
529    my_get_state()
530    valid_state()
531
532    transitional_boot_selected = False
533    stack_popped = 0
534    if len(boot_stack) > 0:
535        stack_popped = 1
536        gp.qprint_dashes()
537        gp.qprint_var(boot_stack)
538        gp.qprint_dashes()
539        skip_boot_printed = 0
540        while len(boot_stack) > 0:
541            boot_candidate = boot_stack.pop()
542            if stack_mode == 'normal':
543                break
544            else:
545                if st.compare_states(state, boot_table[boot_candidate]['end']):
546                    if not skip_boot_printed:
547                        gp.qprint_var(stack_mode)
548                        gp.qprintn()
549                        gp.qprint_timen("Skipping the following boot tests" +
550                                        " which are unnecessary since their" +
551                                        " required end states match the" +
552                                        " current machine state:")
553                        skip_boot_printed = 1
554                    gp.qprint_var(boot_candidate)
555                    boot_candidate = ""
556        if boot_candidate == "":
557            gp.qprint_dashes()
558            gp.qprint_var(boot_stack)
559            gp.qprint_dashes()
560            return boot_candidate
561        if st.compare_states(state, boot_table[boot_candidate]['start']):
562            gp.qprint_timen("The machine state is valid for a '" +
563                            boot_candidate + "' boot test.")
564            gp.qprint_dashes()
565            gp.qprint_var(boot_stack)
566            gp.qprint_dashes()
567            return boot_candidate
568        else:
569            gp.qprint_timen("The machine state does not match the required" +
570                            " starting state for a '" + boot_candidate +
571                            "' boot test:")
572            gp.qprint_varx("boot_table[" + boot_candidate + "][start]",
573                           boot_table[boot_candidate]['start'], 1)
574            boot_stack.append(boot_candidate)
575            transitional_boot_selected = True
576            popped_boot = boot_candidate
577
578    # Loop through your list selecting a boot_candidates
579    boot_candidates = []
580    for boot_candidate in boot_list:
581        if st.compare_states(state, boot_table[boot_candidate]['start']):
582            if stack_popped:
583                if st.compare_states(boot_table[boot_candidate]['end'],
584                   boot_table[popped_boot]['start']):
585                    boot_candidates.append(boot_candidate)
586            else:
587                boot_candidates.append(boot_candidate)
588
589    if len(boot_candidates) == 0:
590        gp.qprint_timen("The user's boot list contained no boot tests" +
591                        " which are valid for the current machine state.")
592        boot_candidate = default_power_on
593        if not st.compare_states(state, boot_table[default_power_on]['start']):
594            boot_candidate = default_power_off
595        boot_candidates.append(boot_candidate)
596        gp.qprint_timen("Using default '" + boot_candidate +
597                        "' boot type to transition to valid state.")
598
599    gp.dprint_var(boot_candidates)
600
601    # Randomly select a boot from the candidate list.
602    boot = random.choice(boot_candidates)
603
604    return boot
605
606###############################################################################
607
608
609###############################################################################
610def print_last_boots():
611
612    r"""
613    Print the last ten boots done with their time stamps.
614    """
615
616    # indent 0, 90 chars wide, linefeed, char is "="
617    gp.qprint_dashes(0, 90)
618    gp.qprintn("Last 10 boots:\n")
619
620    for boot_entry in last_ten:
621        grp.rqprint(boot_entry)
622    gp.qprint_dashes(0, 90)
623
624###############################################################################
625
626
627###############################################################################
628def print_defect_report(ffdc_file_list):
629
630    r"""
631    Print a defect report.
632
633    Description of argument(s):
634    ffdc_file_list  A list of files which were collected by our ffdc functions.
635    """
636
637    # Making deliberate choice to NOT run plug_in_setup().  We don't want
638    # ffdc_prefix updated.
639    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
640        call_point='ffdc_report', stop_on_plug_in_failure=0)
641
642    # Get additional header data which may have been created by ffdc plug-ins.
643    # Also, delete the individual header files to cleanup.
644    cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\
645              " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
646              " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
647    shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
648                                              show_err=0)
649
650    # Get additional summary data which may have been created by ffdc plug-ins.
651    # Also, delete the individual header files to cleanup.
652    cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\
653              " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
654              " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
655    shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
656                                               show_err=0)
657
658    # ffdc_list_file_path contains a list of any ffdc files created by plug-
659    # ins, etc.  Read that data into a list.
660    try:
661        plug_in_ffdc_list = \
662            open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n")
663        plug_in_ffdc_list = filter(None, plug_in_ffdc_list)
664    except IOError:
665        plug_in_ffdc_list = []
666
667    # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed
668    # in.  Eliminate duplicates and sort the list.
669    ffdc_file_list = list(set(ffdc_file_list + plug_in_ffdc_list))
670    ffdc_file_list.sort()
671
672    if status_file_path != "":
673        ffdc_file_list.insert(0, status_file_path)
674
675    # Convert the list to a printable list.
676    printable_ffdc_file_list = "\n".join(ffdc_file_list)
677
678    # Open ffdc_file_list for writing.  We will write a complete list of
679    # FFDC files to it for possible use by plug-ins like cp_stop_check.
680    ffdc_list_file = open(ffdc_list_file_path, 'w')
681    ffdc_list_file.write(printable_ffdc_file_list + "\n")
682    ffdc_list_file.close()
683
684    indent = 0
685    width = 90
686    linefeed = 1
687    char = "="
688
689    gp.qprintn()
690    gp.qprint_dashes(indent, width, linefeed, char)
691    gp.qprintn("Copy this data to the defect:\n")
692
693    if len(more_header_info) > 0:
694        gp.qprintn(more_header_info)
695    gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host,
696              openbmc_host_name, openbmc_ip, openbmc_username,
697              openbmc_password, os_host, os_host_name, os_ip, os_username,
698              os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username,
699              pdu_password, pdu_slot_no, openbmc_serial_host,
700              openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port)
701
702    gp.qprintn()
703    print_last_boots()
704    gp.qprintn()
705    gp.qprint_var(state)
706    gp.qprintn()
707    gp.qprintn("FFDC data files:")
708    gp.qprintn(printable_ffdc_file_list)
709    gp.qprintn()
710
711    if len(ffdc_summary_info) > 0:
712        gp.qprintn(ffdc_summary_info)
713
714    gp.qprint_dashes(indent, width, linefeed, char)
715
716###############################################################################
717
718
719###############################################################################
720def my_ffdc():
721
722    r"""
723    Collect FFDC data.
724    """
725
726    global state
727
728    plug_in_setup()
729    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
730        call_point='ffdc', stop_on_plug_in_failure=0)
731
732    AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX']
733    status, ffdc_file_list = grk.run_key_u("FFDC  ffdc_prefix=" +
734                                           AUTOBOOT_FFDC_PREFIX +
735                                           "  ffdc_function_list=" +
736                                           ffdc_function_list, ignore=1)
737    if status != 'PASS':
738        gp.qprint_error("Call to ffdc failed.\n")
739
740    my_get_state()
741
742    print_defect_report(ffdc_file_list)
743
744###############################################################################
745
746
747###############################################################################
748def print_test_start_message(boot_keyword):
749
750    r"""
751    Print a message indicating what boot test is about to run.
752
753    Description of arguments:
754    boot_keyword  The name of the boot which is to be run
755                  (e.g. "BMC Power On").
756    """
757
758    global last_ten
759    global boot_start_time
760
761    doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".")
762
763    # Set boot_start_time for use by plug-ins.
764    boot_start_time = doing_msg[1:33]
765    gp.qprint_var(boot_start_time)
766
767    gp.qprint(doing_msg)
768
769    last_ten.append(doing_msg)
770
771    if len(last_ten) > 10:
772        del last_ten[0]
773
774###############################################################################
775
776
777###############################################################################
778def run_boot(boot):
779
780    r"""
781    Run the specified boot.
782
783    Description of arguments:
784    boot  The name of the boot test to be performed.
785    """
786
787    global state
788
789    print_test_start_message(boot)
790
791    plug_in_setup()
792    rc, shell_rc, failed_plug_in_name = \
793        grpi.rprocess_plug_in_packages(call_point="pre_boot")
794    if rc != 0:
795        error_message = "Plug-in failed with non-zero return code.\n" +\
796            gp.sprint_var(rc, 1)
797        BuiltIn().fail(gp.sprint_error(error_message))
798
799    if test_mode:
800        # In test mode, we'll pretend the boot worked by assigning its
801        # required end state to the default state value.
802        state = st.strip_anchor_state(boot_table[boot]['end'])
803    else:
804        # Assertion:  We trust that the state data was made fresh by the
805        # caller.
806
807        gp.qprintn()
808
809        if boot_table[boot]['method_type'] == "keyword":
810            rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''),
811                               boot_table[boot]['method'],
812                               quiet=quiet)
813
814        if boot_table[boot]['bmc_reboot']:
815            st.wait_for_comm_cycle(int(state['epoch_seconds']))
816            plug_in_setup()
817            rc, shell_rc, failed_plug_in_name = \
818                grpi.rprocess_plug_in_packages(call_point="post_reboot")
819            if rc != 0:
820                error_message = "Plug-in failed with non-zero return code.\n"
821                error_message += gp.sprint_var(rc, 1)
822                BuiltIn().fail(gp.sprint_error(error_message))
823        else:
824            match_state = st.anchor_state(state)
825            del match_state['epoch_seconds']
826            # Wait for the state to change in any way.
827            st.wait_state(match_state, wait_time=state_change_timeout,
828                          interval="10 seconds", invert=1)
829
830        gp.qprintn()
831        if boot_table[boot]['end']['chassis'] == "Off":
832            boot_timeout = power_off_timeout
833        else:
834            boot_timeout = power_on_timeout
835        st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout,
836                      interval="10 seconds")
837
838    plug_in_setup()
839    rc, shell_rc, failed_plug_in_name = \
840        grpi.rprocess_plug_in_packages(call_point="post_boot")
841    if rc != 0:
842        error_message = "Plug-in failed with non-zero return code.\n" +\
843            gp.sprint_var(rc, 1)
844        BuiltIn().fail(gp.sprint_error(error_message))
845
846###############################################################################
847
848
849###############################################################################
850def test_loop_body():
851
852    r"""
853    The main loop body for the loop in main_py.
854
855    Description of arguments:
856    boot_count  The iteration number (starts at 1).
857    """
858
859    global boot_count
860    global state
861    global next_boot
862    global boot_success
863    global boot_end_time
864
865    gp.qprintn()
866
867    next_boot = select_boot()
868    if next_boot == "":
869        return True
870
871    boot_count += 1
872    gp.qprint_timen("Starting boot " + str(boot_count) + ".")
873
874    pre_boot_plug_in_setup()
875
876    cmd_buf = ["run_boot", next_boot]
877    boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
878    if boot_status == "FAIL":
879        gp.qprint(msg)
880
881    gp.qprintn()
882    if boot_status == "PASS":
883        boot_success = 1
884        completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot +
885                                         "\" succeeded.")
886    else:
887        boot_success = 0
888        completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot +
889                                         "\" failed.")
890
891    # Set boot_end_time for use by plug-ins.
892    boot_end_time = completion_msg[1:33]
893    gp.qprint_var(boot_end_time)
894
895    gp.qprint(completion_msg)
896
897    boot_results.update(next_boot, boot_status)
898
899    plug_in_setup()
900    # NOTE: A post_test_case call point failure is NOT counted as a boot
901    # failure.
902    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
903        call_point='post_test_case', stop_on_plug_in_failure=0)
904
905    plug_in_setup()
906    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
907        call_point='ffdc_check', shell_rc=0x00000200,
908        stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
909    if boot_status != "PASS" or ffdc_check == "All" or shell_rc == 0x00000200:
910        status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
911        if status != 'PASS':
912            gp.qprint_error("Call to my_ffdc failed.\n")
913
914    if delete_errlogs:
915        # We need to purge error logs between boots or they build up.
916        grk.run_key("Delete Error logs", ignore=1)
917
918    boot_results.print_report()
919    gp.qprint_timen("Finished boot " + str(boot_count) + ".")
920
921    plug_in_setup()
922    rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
923        call_point='stop_check')
924    if rc != 0:
925        error_message = "Stopping as requested by user.\n"
926        grp.rprint_error_report(error_message)
927        BuiltIn().fail(error_message)
928
929    # This should help prevent ConnectionErrors.
930    grk.run_key_u("Close All Connections")
931
932    return True
933
934###############################################################################
935
936
937###############################################################################
938def obmc_boot_test_teardown():
939
940    r"""
941    Clean up after the Main keyword.
942    """
943
944    if cp_setup_called:
945        plug_in_setup()
946        rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
947            call_point='cleanup', stop_on_plug_in_failure=0)
948
949    if 'boot_results_file_path' in globals():
950        # Save boot_results object to a file in case it is needed again.
951        gp.qprint_timen("Saving boot_results to the following path.")
952        gp.qprint_var(boot_results_file_path)
953        pickle.dump(boot_results, open(boot_results_file_path, 'wb'),
954                    pickle.HIGHEST_PROTOCOL)
955
956    global save_stack
957    # Restore any global values saved on the save_stack.
958    for parm_name in main_func_parm_list:
959        # Get the parm_value if it was saved on the stack.
960        try:
961            parm_value = save_stack.pop(parm_name)
962        except:
963            # If it was not saved, no further action is required.
964            continue
965
966        # Restore the saved value.
967        cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
968            "}\", parm_value)"
969        gp.dpissuing(cmd_buf)
970        exec(cmd_buf)
971
972    gp.dprintn(save_stack.sprint_obj())
973
974###############################################################################
975
976
977###############################################################################
978def test_teardown():
979
980    r"""
981    Clean up after this test case.
982    """
983
984    gp.qprintn()
985    cmd_buf = ["Print Error",
986               "A keyword timeout occurred ending this program.\n"]
987    BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
988
989    grp.rqprint_pgm_footer()
990
991###############################################################################
992
993
994###############################################################################
995def obmc_boot_test_py(loc_boot_stack=None,
996                      loc_stack_mode=None,
997                      loc_quiet=None):
998
999    r"""
1000    Do main program processing.
1001    """
1002
1003    global save_stack
1004
1005    # Process function parms.
1006    for parm_name in main_func_parm_list:
1007        # Get parm's value.
1008        cmd_buf = "parm_value = loc_" + parm_name
1009        exec(cmd_buf)
1010        gp.dpvar(parm_name)
1011        gp.dpvar(parm_value)
1012
1013        if parm_value is None:
1014            # Parm was not specified by the calling function so set it to its
1015            # corresponding global value.
1016            cmd_buf = "loc_" + parm_name + " = BuiltIn().get_variable_value" +\
1017                "(\"${" + parm_name + "}\")"
1018            gp.dpissuing(cmd_buf)
1019            exec(cmd_buf)
1020        else:
1021            # Save the global value on a stack.
1022            cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\
1023                parm_name + "}\"), \"" + parm_name + "\")"
1024            gp.dpissuing(cmd_buf)
1025            exec(cmd_buf)
1026
1027            # Set the global value to the passed value.
1028            cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1029                "}\", loc_" + parm_name + ")"
1030            gp.dpissuing(cmd_buf)
1031            exec(cmd_buf)
1032
1033    gp.dprintn(save_stack.sprint_obj())
1034
1035    setup()
1036
1037    init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail()
1038
1039    if ffdc_only:
1040        gp.qprint_timen("Caller requested ffdc_only.")
1041        pre_boot_plug_in_setup()
1042        grk.run_key_u("my_ffdc")
1043        return
1044
1045    # Process caller's boot_stack.
1046    while (len(boot_stack) > 0):
1047        test_loop_body()
1048
1049    gp.qprint_timen("Finished processing stack.")
1050
1051    # Process caller's boot_list.
1052    if len(boot_list) > 0:
1053        for ix in range(1, max_num_tests + 1):
1054            test_loop_body()
1055
1056    gp.qprint_timen("Completed all requested boot tests.")
1057
1058    boot_pass, boot_fail = boot_results.return_total_pass_fail()
1059    new_fail = boot_fail - init_boot_fail
1060    if new_fail > boot_fail_threshold:
1061        error_message = "Boot failures exceed the boot failure" +\
1062                        " threshold:\n" +\
1063                        gp.sprint_var(new_fail) +\
1064                        gp.sprint_var(boot_fail_threshold)
1065        BuiltIn().fail(gp.sprint_error(error_message))
1066
1067###############################################################################
1068