xref: /openbmc/openbmc-test-automation/lib/obmc_boot_test.py (revision 438fd3b6c69c11da78a5b01123a0b4ce50e7face)
1 #!/usr/bin/env python
2 
3 r"""
4 This module is the python counterpart to obmc_boot_test.
5 """
6 
7 import os
8 import imp
9 import time
10 import glob
11 import random
12 import re
13 import signal
14 try:
15     import cPickle as pickle
16 except ImportError:
17     import pickle
18 import socket
19 
20 from robot.utils import DotDict
21 from robot.libraries.BuiltIn import BuiltIn
22 
23 from boot_data import *
24 import gen_print as gp
25 import gen_robot_plug_in as grpi
26 import gen_arg as ga
27 import gen_valid as gv
28 import gen_misc as gm
29 import gen_cmd as gc
30 import gen_robot_keyword as grk
31 import state as st
32 import var_stack as vs
33 import gen_plug_in_utils as gpu
34 import pel_utils as pel
35 import logging_utils as log
36 
37 base_path = os.path.dirname(os.path.dirname(
38                             imp.find_module("gen_robot_print")[1])) +\
39     os.sep
40 sys.path.append(base_path + "extended/")
41 import run_keyword as rk
42 
43 # Setting master_pid correctly influences the behavior of plug-ins like
44 # DB_Logging
45 program_pid = os.getpid()
46 master_pid = os.environ.get('AUTOBOOT_MASTER_PID', program_pid)
47 pgm_name = re.sub('\\.py$', '', os.path.basename(__file__))
48 
49 # Set up boot data structures.
50 os_host = BuiltIn().get_variable_value("${OS_HOST}", default="")
51 
52 boot_lists = read_boot_lists()
53 
54 # The maximum number of entries that can be in the boot_history global variable.
55 max_boot_history = 10
56 boot_history = []
57 
58 state = st.return_state_constant('default_state')
59 cp_setup_called = 0
60 next_boot = ""
61 base_tool_dir_path = os.path.normpath(os.environ.get(
62     'AUTOBOOT_BASE_TOOL_DIR_PATH', "/tmp")) + os.sep
63 
64 ffdc_dir_path = os.path.normpath(os.environ.get('FFDC_DIR_PATH', '')) + os.sep
65 boot_success = 0
66 status_dir_path = os.environ.get('STATUS_DIR_PATH', "")
67 if status_dir_path != "":
68     status_dir_path = os.path.normpath(status_dir_path) + os.sep
69 redfish_support_trans_state = int(os.environ.get('REDFISH_SUPPORT_TRANS_STATE', 0)) or \
70     int(BuiltIn().get_variable_value("${REDFISH_SUPPORT_TRANS_STATE}", default=0))
71 redfish_supported = BuiltIn().get_variable_value("${REDFISH_SUPPORTED}", default=False)
72 redfish_rest_supported = BuiltIn().get_variable_value("${REDFISH_REST_SUPPORTED}", default=False)
73 redfish_delete_sessions = int(BuiltIn().get_variable_value("${REDFISH_DELETE_SESSIONS}", default=1))
74 if redfish_supported:
75     redfish = BuiltIn().get_library_instance('redfish')
76     default_power_on = "Redfish Power On"
77     default_power_off = "Redfish Power Off"
78     if not redfish_support_trans_state:
79         delete_errlogs_cmd = "Delete Error Logs  ${quiet}=${1}"
80         delete_bmcdump_cmd = "Delete All BMC Dump"
81         default_set_power_policy = "Set BMC Power Policy  ALWAYS_POWER_OFF"
82     else:
83         delete_errlogs_cmd = "Redfish Purge Event Log"
84         delete_bmcdump_cmd = "Redfish Delete All BMC Dumps"
85         delete_sysdump_cmd = "Redfish Delete All System Dumps"
86         default_set_power_policy = "Redfish Set Power Restore Policy  AlwaysOff"
87 else:
88     default_power_on = "REST Power On"
89     default_power_off = "REST Power Off"
90     delete_errlogs_cmd = "Delete Error Logs  ${quiet}=${1}"
91     delete_bmcdump_cmd = "Delete All BMC Dump"
92     default_set_power_policy = "Set BMC Power Policy  ALWAYS_POWER_OFF"
93 boot_count = 0
94 
95 LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
96 AUTOBOOT_FFDC_PREFIX = os.environ.get('AUTOBOOT_FFDC_PREFIX', '')
97 ffdc_prefix = AUTOBOOT_FFDC_PREFIX
98 boot_start_time = ""
99 boot_end_time = ""
100 save_stack = vs.var_stack('save_stack')
101 main_func_parm_list = ['boot_stack', 'stack_mode', 'quiet']
102 
103 
104 def dump_ffdc_rc():
105     r"""
106     Return the constant dump ffdc test return code value.
107 
108     When a plug-in call point program returns this value, it indicates that
109     this program should collect FFDC.
110     """
111 
112     return 0x00000200
113 
114 
115 def stop_test_rc():
116     r"""
117     Return the constant stop test return code value.
118 
119     When a plug-in call point program returns this value, it indicates that
120     this program should stop running.
121     """
122 
123     return 0x00000200
124 
125 
126 def process_host(host,
127                  host_var_name=""):
128     r"""
129     Process a host by getting the associated host name and IP address and
130     setting them in global variables.
131 
132     If the caller does not pass the host_var_name, this function will try to
133     figure out the name of the variable used by the caller for the host parm.
134     Callers are advised to explicitly specify the host_var_name when calling
135     with an exec command.  In such cases, the get_arg_name cannot figure out
136     the host variable name.
137 
138     This function will then create similar global variable names by
139     removing "_host" and appending "_host_name" or "_ip" to the host variable
140     name.
141 
142     Example:
143 
144     If a call is made like this:
145     process_host(openbmc_host)
146 
147     Global variables openbmc_host_name and openbmc_ip will be set.
148 
149     Description of argument(s):
150     host           A host name or IP.  The name of the variable used should
151                    have a suffix of "_host".
152     host_var_name  The name of the variable being used as the host parm.
153     """
154 
155     if host_var_name == "":
156         host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
157 
158     host_name_var_name = re.sub("host", "host_name", host_var_name)
159     ip_var_name = re.sub("host", "ip", host_var_name)
160     cmd_buf = "global " + host_name_var_name + ", " + ip_var_name + " ; " +\
161         host_name_var_name + ", " + ip_var_name + " = gm.get_host_name_ip('" +\
162         host + "')"
163     exec(cmd_buf)
164 
165 
166 def process_pgm_parms():
167     r"""
168     Process the program parameters by assigning them all to corresponding
169     globals.  Also, set some global values that depend on program parameters.
170     """
171 
172     # Program parameter processing.
173     # Assign all program parms to python variables which are global to this
174     # module.
175 
176     global parm_list
177     parm_list = BuiltIn().get_variable_value("${parm_list}")
178     # The following subset of parms should be processed as integers.
179     int_list = ['max_num_tests', 'boot_pass', 'boot_fail', 'ffdc_only',
180                 'boot_fail_threshold', 'delete_errlogs',
181                 'call_post_stack_plug', 'do_pre_boot_plug_in_setup', 'quiet',
182                 'test_mode', 'debug']
183     for parm in parm_list:
184         if parm in int_list:
185             sub_cmd = "int(BuiltIn().get_variable_value(\"${" + parm +\
186                       "}\", \"0\"))"
187         else:
188             sub_cmd = "BuiltIn().get_variable_value(\"${" + parm + "}\")"
189         cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
190         gp.dpissuing(cmd_buf)
191         exec(cmd_buf)
192         if re.match(r".*_host$", parm):
193             cmd_buf = "process_host(" + parm + ", '" + parm + "')"
194             exec(cmd_buf)
195         if re.match(r".*_password$", parm):
196             # Register the value of any parm whose name ends in _password.
197             # This will cause the print functions to replace passwords with
198             # asterisks in the output.
199             cmd_buf = "gp.register_passwords(" + parm + ")"
200             exec(cmd_buf)
201 
202     global ffdc_dir_path_style
203     global boot_list
204     global boot_stack
205     global boot_results_file_path
206     global boot_results
207     global boot_history
208     global ffdc_list_file_path
209     global ffdc_report_list_path
210     global ffdc_summary_list_path
211     global boot_table
212     global valid_boot_types
213 
214     if ffdc_dir_path_style == "":
215         ffdc_dir_path_style = int(os.environ.get('FFDC_DIR_PATH_STYLE', '0'))
216 
217     # Convert these program parms to lists for easier processing..
218     boot_list = list(filter(None, boot_list.split(":")))
219     boot_stack = list(filter(None, boot_stack.split(":")))
220 
221     boot_table = create_boot_table(boot_table_path, os_host=os_host)
222     valid_boot_types = create_valid_boot_list(boot_table)
223 
224     cleanup_boot_results_file()
225     boot_results_file_path = create_boot_results_file_path(pgm_name,
226                                                            openbmc_nickname,
227                                                            master_pid)
228 
229     if os.path.isfile(boot_results_file_path):
230         # We've been called before in this run so we'll load the saved
231         # boot_results and boot_history objects.
232         boot_results, boot_history =\
233             pickle.load(open(boot_results_file_path, 'rb'))
234     else:
235         boot_results = boot_results(boot_table, boot_pass, boot_fail)
236 
237     ffdc_list_file_path = base_tool_dir_path + openbmc_nickname +\
238         "/FFDC_FILE_LIST"
239     ffdc_report_list_path = base_tool_dir_path + openbmc_nickname +\
240         "/FFDC_REPORT_FILE_LIST"
241 
242     ffdc_summary_list_path = base_tool_dir_path + openbmc_nickname +\
243         "/FFDC_SUMMARY_FILE_LIST"
244 
245 
246 def initial_plug_in_setup():
247     r"""
248     Initialize all plug-in environment variables which do not change for the
249     duration of the program.
250 
251     """
252 
253     global LOG_LEVEL
254     BuiltIn().set_log_level("NONE")
255 
256     BuiltIn().set_global_variable("${master_pid}", master_pid)
257     BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
258     BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
259     BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
260     BuiltIn().set_global_variable("${FFDC_LIST_FILE_PATH}",
261                                   ffdc_list_file_path)
262     BuiltIn().set_global_variable("${FFDC_REPORT_LIST_PATH}",
263                                   ffdc_report_list_path)
264     BuiltIn().set_global_variable("${FFDC_SUMMARY_LIST_PATH}",
265                                   ffdc_summary_list_path)
266 
267     BuiltIn().set_global_variable("${FFDC_DIR_PATH_STYLE}",
268                                   ffdc_dir_path_style)
269     BuiltIn().set_global_variable("${FFDC_CHECK}",
270                                   ffdc_check)
271 
272     # For each program parameter, set the corresponding AUTOBOOT_ environment
273     # variable value.  Also, set an AUTOBOOT_ environment variable for every
274     # element in additional_values.
275     additional_values = ["program_pid", "master_pid", "ffdc_dir_path",
276                          "status_dir_path", "base_tool_dir_path",
277                          "ffdc_list_file_path", "ffdc_report_list_path",
278                          "ffdc_summary_list_path", "execdir", "redfish_supported",
279                          "redfish_rest_supported", "redfish_support_trans_state"]
280 
281     plug_in_vars = parm_list + additional_values
282 
283     for var_name in plug_in_vars:
284         var_value = BuiltIn().get_variable_value("${" + var_name + "}")
285         var_name = var_name.upper()
286         if var_value is None:
287             var_value = ""
288         os.environ["AUTOBOOT_" + var_name] = str(var_value)
289 
290     BuiltIn().set_log_level(LOG_LEVEL)
291 
292     # Make sure the ffdc list directory exists.
293     ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
294     if not os.path.exists(ffdc_list_dir_path):
295         os.makedirs(ffdc_list_dir_path)
296 
297 
298 def plug_in_setup():
299     r"""
300     Initialize all changing plug-in environment variables for use by the
301     plug-in programs.
302     """
303 
304     global LOG_LEVEL
305     global test_really_running
306 
307     BuiltIn().set_log_level("NONE")
308 
309     boot_pass, boot_fail = boot_results.return_total_pass_fail()
310     if boot_pass > 1:
311         test_really_running = 1
312     else:
313         test_really_running = 0
314 
315     BuiltIn().set_global_variable("${test_really_running}",
316                                   test_really_running)
317     BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
318     BuiltIn().set_global_variable("${boot_pass}", boot_pass)
319     BuiltIn().set_global_variable("${boot_fail}", boot_fail)
320     BuiltIn().set_global_variable("${boot_success}", boot_success)
321     BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
322     BuiltIn().set_global_variable("${boot_start_time}", boot_start_time)
323     BuiltIn().set_global_variable("${boot_end_time}", boot_end_time)
324 
325     # For each program parameter, set the corresponding AUTOBOOT_ environment
326     # variable value.  Also, set an AUTOBOOT_ environment variable for every
327     # element in additional_values.
328     additional_values = ["boot_type_desc", "boot_success", "boot_pass",
329                          "boot_fail", "test_really_running", "ffdc_prefix",
330                          "boot_start_time", "boot_end_time"]
331 
332     plug_in_vars = additional_values
333 
334     for var_name in plug_in_vars:
335         var_value = BuiltIn().get_variable_value("${" + var_name + "}")
336         var_name = var_name.upper()
337         if var_value is None:
338             var_value = ""
339         os.environ["AUTOBOOT_" + var_name] = str(var_value)
340 
341     if debug:
342         shell_rc, out_buf = \
343             gc.cmd_fnc_u("printenv | egrep AUTOBOOT_ | sort -u")
344 
345     BuiltIn().set_log_level(LOG_LEVEL)
346 
347 
348 def pre_boot_plug_in_setup():
349 
350     # Clear the ffdc_list_file_path file.  Plug-ins may now write to it.
351     try:
352         os.remove(ffdc_list_file_path)
353     except OSError:
354         pass
355 
356     # Clear the ffdc_report_list_path file.  Plug-ins may now write to it.
357     try:
358         os.remove(ffdc_report_list_path)
359     except OSError:
360         pass
361 
362     # Clear the ffdc_summary_list_path file.  Plug-ins may now write to it.
363     try:
364         os.remove(ffdc_summary_list_path)
365     except OSError:
366         pass
367 
368     global ffdc_prefix
369 
370     seconds = time.time()
371     loc_time = time.localtime(seconds)
372     time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
373 
374     ffdc_prefix = openbmc_nickname + "." + time_string
375 
376 
377 def default_sigusr1(signal_number=0,
378                     frame=None):
379     r"""
380     Handle SIGUSR1 by doing nothing.
381 
382     This function assists in debugging SIGUSR1 processing by printing messages
383     to stdout and to the log.html file.
384 
385     Description of argument(s):
386     signal_number  The signal number (should always be 10 for SIGUSR1).
387     frame          The frame data.
388     """
389 
390     gp.qprintn()
391     gp.qprint_executing()
392     gp.lprint_executing()
393 
394 
395 def set_default_siguser1():
396     r"""
397     Set the default_sigusr1 function to be the SIGUSR1 handler.
398     """
399 
400     gp.qprintn()
401     gp.qprint_executing()
402     gp.lprint_executing()
403     signal.signal(signal.SIGUSR1, default_sigusr1)
404 
405 
406 def setup():
407     r"""
408     Do general program setup tasks.
409     """
410 
411     global cp_setup_called
412     global transitional_boot_selected
413 
414     gp.qprintn()
415 
416     if redfish_supported:
417         redfish.login()
418 
419     set_default_siguser1()
420     transitional_boot_selected = False
421 
422     robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
423     repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
424     # If we can't find process_plug_in_packages.py, ssh_pw or
425     # validate_plug_ins.py, then we don't have our repo bin in PATH.
426     shell_rc, out_buf = gc.cmd_fnc_u("which process_plug_in_packages.py"
427                                      + " ssh_pw validate_plug_ins.py", quiet=1,
428                                      print_output=0, show_err=0)
429     if shell_rc != 0:
430         os.environ['PATH'] = repo_bin_path + ":" + os.environ.get('PATH', "")
431     # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
432     if robot_pgm_dir_path not in sys.path:
433         sys.path.append(robot_pgm_dir_path)
434         PYTHONPATH = os.environ.get("PYTHONPATH", "")
435         if PYTHONPATH == "":
436             os.environ['PYTHONPATH'] = robot_pgm_dir_path
437         else:
438             os.environ['PYTHONPATH'] = robot_pgm_dir_path + ":" + PYTHONPATH
439 
440     validate_parms()
441 
442     gp.qprint_pgm_header()
443 
444     grk.run_key_u(default_set_power_policy)
445 
446     initial_plug_in_setup()
447 
448     plug_in_setup()
449     rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
450         call_point='setup')
451     if rc != 0:
452         error_message = "Plug-in setup failed.\n"
453         gp.print_error_report(error_message)
454         BuiltIn().fail(error_message)
455     # Setting cp_setup_called lets our Teardown know that it needs to call
456     # the cleanup plug-in call point.
457     cp_setup_called = 1
458 
459     # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
460     BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
461     # FFDC_LOG_PATH is used by "FFDC" keyword.
462     BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
463 
464     # Also printed by FFDC.
465     global host_name
466     global host_ip
467     host = socket.gethostname()
468     host_name, host_ip = gm.get_host_name_ip(host)
469 
470     gp.dprint_var(boot_table)
471     gp.dprint_var(boot_lists)
472 
473 
474 def validate_parms():
475     r"""
476     Validate all program parameters.
477     """
478 
479     process_pgm_parms()
480 
481     gp.qprintn()
482 
483     global openbmc_model
484     if openbmc_model == "":
485         status, ret_values =\
486             grk.run_key_u("Get BMC System Model")
487         openbmc_model = ret_values
488         BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
489     gv.set_exit_on_error(True)
490     gv.valid_value(openbmc_host)
491     gv.valid_value(openbmc_username)
492     gv.valid_value(openbmc_password)
493     gv.valid_value(rest_username)
494     gv.valid_value(rest_password)
495     gv.valid_value(ipmi_username)
496     gv.valid_value(ipmi_password)
497     if os_host != "":
498         gv.valid_value(os_username)
499         gv.valid_value(os_password)
500     if pdu_host != "":
501         gv.valid_value(pdu_username)
502         gv.valid_value(pdu_password)
503         gv.valid_integer(pdu_slot_no)
504     if openbmc_serial_host != "":
505         gv.valid_integer(openbmc_serial_port)
506     gv.valid_value(openbmc_model)
507     gv.valid_integer(max_num_tests)
508     gv.valid_integer(boot_pass)
509     gv.valid_integer(boot_fail)
510     plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
511     BuiltIn().set_global_variable("${plug_in_packages_list}",
512                                   plug_in_packages_list)
513     gv.valid_value(stack_mode, valid_values=['normal', 'skip'])
514     gv.set_exit_on_error(False)
515     if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
516         error_message = "You must provide either a value for either the" +\
517             " boot_list or the boot_stack parm.\n"
518         BuiltIn().fail(gp.sprint_error(error_message))
519     valid_boot_list(boot_list, valid_boot_types)
520     valid_boot_list(boot_stack, valid_boot_types)
521     selected_PDU_boots = list(set(boot_list + boot_stack)
522                               & set(boot_lists['PDU_reboot']))
523     if len(selected_PDU_boots) > 0 and pdu_host == "":
524         error_message = "You have selected the following boots which" +\
525                         " require a PDU host but no value for pdu_host:\n"
526         error_message += gp.sprint_var(selected_PDU_boots)
527         error_message += gp.sprint_var(pdu_host, fmt=gp.blank())
528         BuiltIn().fail(gp.sprint_error(error_message))
529 
530     return
531 
532 
533 def my_get_state():
534     r"""
535     Get the system state plus a little bit of wrapping.
536     """
537 
538     global state
539 
540     req_states = ['epoch_seconds'] + st.default_req_states
541 
542     gp.qprint_timen("Getting system state.")
543     if test_mode:
544         state['epoch_seconds'] = int(time.time())
545     else:
546         state = st.get_state(req_states=req_states, quiet=quiet)
547     gp.qprint_var(state)
548 
549 
550 def valid_state():
551     r"""
552     Verify that our state dictionary contains no blank values.  If we don't get
553     valid state data, we cannot continue to work.
554     """
555 
556     if st.compare_states(state, st.invalid_state_match, 'or'):
557         error_message = "The state dictionary contains blank fields which" +\
558             " is illegal.\n" + gp.sprint_var(state)
559         BuiltIn().fail(gp.sprint_error(error_message))
560 
561 
562 def select_boot():
563     r"""
564     Select a boot test to be run based on our current state and return the
565     chosen boot type.
566 
567     Description of arguments:
568     state  The state of the machine.
569     """
570 
571     global transitional_boot_selected
572     global boot_stack
573 
574     gp.qprint_timen("Selecting a boot test.")
575 
576     if transitional_boot_selected and not boot_success:
577         prior_boot = next_boot
578         boot_candidate = boot_stack.pop()
579         gp.qprint_timen("The prior '" + next_boot + "' was chosen to"
580                         + " transition to a valid state for '" + boot_candidate
581                         + "' which was at the top of the boot_stack.  Since"
582                         + " the '" + next_boot + "' failed, the '"
583                         + boot_candidate + "' has been removed from the stack"
584                         + " to avoid and endless failure loop.")
585         if len(boot_stack) == 0:
586             return ""
587 
588     my_get_state()
589     valid_state()
590 
591     transitional_boot_selected = False
592     stack_popped = 0
593     if len(boot_stack) > 0:
594         stack_popped = 1
595         gp.qprint_dashes()
596         gp.qprint_var(boot_stack)
597         gp.qprint_dashes()
598         skip_boot_printed = 0
599         while len(boot_stack) > 0:
600             boot_candidate = boot_stack.pop()
601             if stack_mode == 'normal':
602                 break
603             else:
604                 if st.compare_states(state, boot_table[boot_candidate]['end']):
605                     if not skip_boot_printed:
606                         gp.qprint_var(stack_mode)
607                         gp.qprintn()
608                         gp.qprint_timen("Skipping the following boot tests"
609                                         + " which are unnecessary since their"
610                                         + " required end states match the"
611                                         + " current machine state:")
612                         skip_boot_printed = 1
613                     gp.qprint_var(boot_candidate)
614                     boot_candidate = ""
615         if boot_candidate == "":
616             gp.qprint_dashes()
617             gp.qprint_var(boot_stack)
618             gp.qprint_dashes()
619             return boot_candidate
620         if st.compare_states(state, boot_table[boot_candidate]['start']):
621             gp.qprint_timen("The machine state is valid for a '"
622                             + boot_candidate + "' boot test.")
623             gp.qprint_dashes()
624             gp.qprint_var(boot_stack)
625             gp.qprint_dashes()
626             return boot_candidate
627         else:
628             gp.qprint_timen("The machine state does not match the required"
629                             + " starting state for a '" + boot_candidate
630                             + "' boot test:")
631             gp.qprint_varx("boot_table_start_entry",
632                            boot_table[boot_candidate]['start'])
633             boot_stack.append(boot_candidate)
634             transitional_boot_selected = True
635             popped_boot = boot_candidate
636 
637     # Loop through your list selecting a boot_candidates
638     boot_candidates = []
639     for boot_candidate in boot_list:
640         if st.compare_states(state, boot_table[boot_candidate]['start']):
641             if stack_popped:
642                 if st.compare_states(boot_table[boot_candidate]['end'],
643                                      boot_table[popped_boot]['start']):
644                     boot_candidates.append(boot_candidate)
645             else:
646                 boot_candidates.append(boot_candidate)
647 
648     if len(boot_candidates) == 0:
649         gp.qprint_timen("The user's boot list contained no boot tests"
650                         + " which are valid for the current machine state.")
651         boot_candidate = default_power_on
652         if not st.compare_states(state, boot_table[default_power_on]['start']):
653             boot_candidate = default_power_off
654         boot_candidates.append(boot_candidate)
655         gp.qprint_timen("Using default '" + boot_candidate
656                         + "' boot type to transition to valid state.")
657 
658     gp.dprint_var(boot_candidates)
659 
660     # Randomly select a boot from the candidate list.
661     boot = random.choice(boot_candidates)
662 
663     return boot
664 
665 
666 def print_defect_report(ffdc_file_list):
667     r"""
668     Print a defect report.
669 
670     Description of argument(s):
671     ffdc_file_list  A list of files which were collected by our ffdc functions.
672     """
673 
674     # Making deliberate choice to NOT run plug_in_setup().  We don't want
675     # ffdc_prefix updated.
676     rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
677         call_point='ffdc_report', stop_on_plug_in_failure=0)
678 
679     # Get additional header data which may have been created by ffdc plug-ins.
680     # Also, delete the individual header files to cleanup.
681     cmd_buf = "file_list=$(cat " + ffdc_report_list_path + " 2>/dev/null)" +\
682               " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
683               " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
684     shell_rc, more_header_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
685                                               show_err=0)
686 
687     # Get additional summary data which may have been created by ffdc plug-ins.
688     # Also, delete the individual header files to cleanup.
689     cmd_buf = "file_list=$(cat " + ffdc_summary_list_path + " 2>/dev/null)" +\
690               " ; [ ! -z \"${file_list}\" ] && cat ${file_list}" +\
691               " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
692     shell_rc, ffdc_summary_info = gc.cmd_fnc_u(cmd_buf, print_output=0,
693                                                show_err=0)
694 
695     # ffdc_list_file_path contains a list of any ffdc files created by plug-
696     # ins, etc.  Read that data into a list.
697     try:
698         plug_in_ffdc_list = \
699             open(ffdc_list_file_path, 'r').read().rstrip("\n").split("\n")
700         plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list))
701     except IOError:
702         plug_in_ffdc_list = []
703 
704     # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed
705     # in.  Eliminate duplicates and sort the list.
706     ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list))
707 
708     if status_file_path != "":
709         ffdc_file_list.insert(0, status_file_path)
710 
711     # Convert the list to a printable list.
712     printable_ffdc_file_list = "\n".join(ffdc_file_list)
713 
714     # Open ffdc_file_list for writing.  We will write a complete list of
715     # FFDC files to it for possible use by plug-ins like cp_stop_check.
716     ffdc_list_file = open(ffdc_list_file_path, 'w')
717     ffdc_list_file.write(printable_ffdc_file_list + "\n")
718     ffdc_list_file.close()
719 
720     indent = 0
721     width = 90
722     linefeed = 1
723     char = "="
724 
725     gp.qprintn()
726     gp.qprint_dashes(indent, width, linefeed, char)
727     gp.qprintn("Copy this data to the defect:\n")
728 
729     if len(more_header_info) > 0:
730         gp.qprintn(more_header_info)
731     gp.qpvars(host_name, host_ip, openbmc_nickname, openbmc_host,
732               openbmc_host_name, openbmc_ip, openbmc_username,
733               openbmc_password, rest_username, rest_password, ipmi_username,
734               ipmi_password, os_host, os_host_name, os_ip, os_username,
735               os_password, pdu_host, pdu_host_name, pdu_ip, pdu_username,
736               pdu_password, pdu_slot_no, openbmc_serial_host,
737               openbmc_serial_host_name, openbmc_serial_ip, openbmc_serial_port)
738 
739     gp.qprintn()
740     print_boot_history(boot_history)
741     gp.qprintn()
742     gp.qprint_var(state)
743     gp.qprintn()
744     gp.qprintn("FFDC data files:")
745     gp.qprintn(printable_ffdc_file_list)
746     gp.qprintn()
747 
748     if len(ffdc_summary_info) > 0:
749         gp.qprintn(ffdc_summary_info)
750 
751     gp.qprint_dashes(indent, width, linefeed, char)
752 
753 
754 def my_ffdc():
755     r"""
756     Collect FFDC data.
757     """
758 
759     global state
760 
761     plug_in_setup()
762     rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
763         call_point='ffdc', stop_on_plug_in_failure=0)
764 
765     AUTOBOOT_FFDC_PREFIX = os.environ['AUTOBOOT_FFDC_PREFIX']
766     status, ffdc_file_list = grk.run_key_u("FFDC  ffdc_prefix="
767                                            + AUTOBOOT_FFDC_PREFIX
768                                            + "  ffdc_function_list="
769                                            + ffdc_function_list, ignore=1)
770     if status != 'PASS':
771         gp.qprint_error("Call to ffdc failed.\n")
772         if type(ffdc_file_list) is not list:
773             ffdc_file_list = []
774         # Leave a record for caller that "soft" errors occurred.
775         soft_errors = 1
776         gpu.save_plug_in_value(soft_errors, pgm_name)
777 
778     my_get_state()
779 
780     print_defect_report(ffdc_file_list)
781 
782 
783 def print_test_start_message(boot_keyword):
784     r"""
785     Print a message indicating what boot test is about to run.
786 
787     Description of arguments:
788     boot_keyword  The name of the boot which is to be run
789                   (e.g. "BMC Power On").
790     """
791 
792     global boot_history
793     global boot_start_time
794 
795     doing_msg = gp.sprint_timen("Doing \"" + boot_keyword + "\".")
796 
797     # Set boot_start_time for use by plug-ins.
798     boot_start_time = doing_msg[1:33]
799     gp.qprint_var(boot_start_time)
800 
801     gp.qprint(doing_msg)
802 
803     update_boot_history(boot_history, doing_msg, max_boot_history)
804 
805 
806 def stop_boot_test(signal_number=0,
807                    frame=None):
808     r"""
809     Handle SIGUSR1 by aborting the boot test that is running.
810 
811     Description of argument(s):
812     signal_number  The signal number (should always be 10 for SIGUSR1).
813     frame          The frame data.
814     """
815 
816     gp.qprintn()
817     gp.qprint_executing()
818     gp.lprint_executing()
819 
820     # Restore original sigusr1 handler.
821     set_default_siguser1()
822 
823     message = "The caller has asked that the boot test be stopped and marked"
824     message += " as a failure."
825 
826     function_stack = gm.get_function_stack()
827     if "wait_state" in function_stack:
828         st.set_exit_wait_early_message(message)
829     else:
830         BuiltIn().fail(gp.sprint_error(message))
831 
832 
833 def run_boot(boot):
834     r"""
835     Run the specified boot.
836 
837     Description of arguments:
838     boot  The name of the boot test to be performed.
839     """
840 
841     global state
842 
843     signal.signal(signal.SIGUSR1, stop_boot_test)
844     gp.qprint_timen("stop_boot_test is armed.")
845 
846     print_test_start_message(boot)
847 
848     plug_in_setup()
849     rc, shell_rc, failed_plug_in_name = \
850         grpi.rprocess_plug_in_packages(call_point="pre_boot")
851     if rc != 0:
852         error_message = "Plug-in failed with non-zero return code.\n" +\
853             gp.sprint_var(rc, fmt=gp.hexa())
854         set_default_siguser1()
855         BuiltIn().fail(gp.sprint_error(error_message))
856 
857     if test_mode:
858         # In test mode, we'll pretend the boot worked by assigning its
859         # required end state to the default state value.
860         state = st.strip_anchor_state(boot_table[boot]['end'])
861     else:
862         # Assertion:  We trust that the state data was made fresh by the
863         # caller.
864 
865         gp.qprintn()
866 
867         if boot_table[boot]['method_type'] == "keyword":
868             rk.my_run_keywords(boot_table[boot].get('lib_file_path', ''),
869                                boot_table[boot]['method'],
870                                quiet=quiet)
871 
872         if boot_table[boot]['bmc_reboot']:
873             st.wait_for_comm_cycle(int(state['epoch_seconds']))
874             plug_in_setup()
875             rc, shell_rc, failed_plug_in_name = \
876                 grpi.rprocess_plug_in_packages(call_point="post_reboot")
877             if rc != 0:
878                 error_message = "Plug-in failed with non-zero return code.\n"
879                 error_message += gp.sprint_var(rc, fmt=gp.hexa())
880                 set_default_siguser1()
881                 BuiltIn().fail(gp.sprint_error(error_message))
882         else:
883             match_state = st.anchor_state(state)
884             del match_state['epoch_seconds']
885             # Wait for the state to change in any way.
886             st.wait_state(match_state, wait_time=state_change_timeout,
887                           interval="10 seconds", invert=1)
888 
889         gp.qprintn()
890         if boot_table[boot]['end']['chassis'] == "Off":
891             boot_timeout = power_off_timeout
892         else:
893             boot_timeout = power_on_timeout
894         st.wait_state(boot_table[boot]['end'], wait_time=boot_timeout,
895                       interval="10 seconds")
896 
897     plug_in_setup()
898     rc, shell_rc, failed_plug_in_name = \
899         grpi.rprocess_plug_in_packages(call_point="post_boot")
900     if rc != 0:
901         error_message = "Plug-in failed with non-zero return code.\n" +\
902             gp.sprint_var(rc, fmt=gp.hexa())
903         set_default_siguser1()
904         BuiltIn().fail(gp.sprint_error(error_message))
905 
906     # Restore original sigusr1 handler.
907     set_default_siguser1()
908 
909 
910 def test_loop_body():
911     r"""
912     The main loop body for the loop in main_py.
913 
914     Description of arguments:
915     boot_count  The iteration number (starts at 1).
916     """
917 
918     global boot_count
919     global state
920     global next_boot
921     global boot_success
922     global boot_end_time
923 
924     gp.qprintn()
925 
926     next_boot = select_boot()
927     if next_boot == "":
928         return True
929 
930     boot_count += 1
931     gp.qprint_timen("Starting boot " + str(boot_count) + ".")
932 
933     pre_boot_plug_in_setup()
934 
935     cmd_buf = ["run_boot", next_boot]
936     boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
937     if boot_status == "FAIL":
938         gp.qprint(msg)
939 
940     gp.qprintn()
941     if boot_status == "PASS":
942         boot_success = 1
943         completion_msg = gp.sprint_timen("BOOT_SUCCESS: \"" + next_boot
944                                          + "\" succeeded.")
945     else:
946         boot_success = 0
947         completion_msg = gp.sprint_timen("BOOT_FAILED: \"" + next_boot
948                                          + "\" failed.")
949 
950     # Set boot_end_time for use by plug-ins.
951     boot_end_time = completion_msg[1:33]
952     gp.qprint_var(boot_end_time)
953 
954     gp.qprint(completion_msg)
955 
956     boot_results.update(next_boot, boot_status)
957 
958     plug_in_setup()
959     # NOTE: A post_test_case call point failure is NOT counted as a boot
960     # failure.
961     rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
962         call_point='post_test_case', stop_on_plug_in_failure=0)
963 
964     plug_in_setup()
965     rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
966         call_point='ffdc_check', shell_rc=dump_ffdc_rc(),
967         stop_on_plug_in_failure=1, stop_on_non_zero_rc=1)
968     if ffdc_check == "All" or\
969        shell_rc == dump_ffdc_rc():
970         status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
971         if status != 'PASS':
972             gp.qprint_error("Call to my_ffdc failed.\n")
973             # Leave a record for caller that "soft" errors occurred.
974             soft_errors = 1
975             gpu.save_plug_in_value(soft_errors, pgm_name)
976 
977     if delete_errlogs:
978         # print error logs before delete
979         if redfish_support_trans_state:
980             status, error_logs = grk.run_key_u("Get Redfish Event Logs")
981             log.print_error_logs(error_logs, "AdditionalDataURI Message Severity")
982         else:
983             status, error_logs = grk.run_key_u("Get Error Logs")
984             log.print_error_logs(error_logs, "AdditionalData Message Severity")
985         pels = pel.peltool("-l", ignore_err=1)
986         gp.qprint_var(pels)
987 
988         # We need to purge error logs between boots or they build up.
989         grk.run_key(delete_errlogs_cmd, ignore=1)
990         grk.run_key(delete_bmcdump_cmd, ignore=1)
991         if redfish_support_trans_state:
992             grk.run_key(delete_sysdump_cmd, ignore=1)
993 
994     boot_results.print_report()
995     gp.qprint_timen("Finished boot " + str(boot_count) + ".")
996 
997     plug_in_setup()
998     rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
999         call_point='stop_check', shell_rc=stop_test_rc(),
1000         stop_on_non_zero_rc=1)
1001     if shell_rc == stop_test_rc():
1002         message = "Stopping as requested by user.\n"
1003         gp.qprint_time(message)
1004         BuiltIn().fail(message)
1005 
1006     # This should help prevent ConnectionErrors.
1007     # Purge all redfish and REST connection sessions.
1008     if redfish_delete_sessions:
1009         grk.run_key_u("Close All Connections", ignore=1)
1010         grk.run_key_u("Delete All Redfish Sessions", ignore=1)
1011 
1012     return True
1013 
1014 
1015 def obmc_boot_test_teardown():
1016     r"""
1017     Clean up after the main keyword.
1018     """
1019     gp.qprint_executing()
1020 
1021     if ga.psutil_imported:
1022         ga.terminate_descendants()
1023 
1024     if cp_setup_called:
1025         plug_in_setup()
1026         rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1027             call_point='cleanup', stop_on_plug_in_failure=0)
1028 
1029     if 'boot_results_file_path' in globals():
1030         # Save boot_results and boot_history objects to a file in case they are
1031         # needed again.
1032         gp.qprint_timen("Saving boot_results to the following path.")
1033         gp.qprint_var(boot_results_file_path)
1034         pickle.dump((boot_results, boot_history),
1035                     open(boot_results_file_path, 'wb'),
1036                     pickle.HIGHEST_PROTOCOL)
1037 
1038     global save_stack
1039     # Restore any global values saved on the save_stack.
1040     for parm_name in main_func_parm_list:
1041         # Get the parm_value if it was saved on the stack.
1042         try:
1043             parm_value = save_stack.pop(parm_name)
1044         except BaseException:
1045             # If it was not saved, no further action is required.
1046             continue
1047 
1048         # Restore the saved value.
1049         cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1050             "}\", parm_value)"
1051         gp.dpissuing(cmd_buf)
1052         exec(cmd_buf)
1053 
1054     gp.dprintn(save_stack.sprint_obj())
1055 
1056 
1057 def test_teardown():
1058     r"""
1059     Clean up after this test case.
1060     """
1061 
1062     gp.qprintn()
1063     gp.qprint_executing()
1064 
1065     if ga.psutil_imported:
1066         ga.terminate_descendants()
1067 
1068     cmd_buf = ["Print Error",
1069                "A keyword timeout occurred ending this program.\n"]
1070     BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
1071 
1072     if redfish_supported:
1073         redfish.logout()
1074 
1075     gp.qprint_pgm_footer()
1076 
1077 
1078 def post_stack():
1079     r"""
1080     Process post_stack plug-in programs.
1081     """
1082 
1083     if not call_post_stack_plug:
1084         # The caller does not wish to have post_stack plug-in processing done.
1085         return
1086 
1087     global boot_success
1088 
1089     # NOTE: A post_stack call-point failure is NOT counted as a boot failure.
1090     pre_boot_plug_in_setup()
1091     # For the purposes of the following plug-ins, mark the "boot" as a success.
1092     boot_success = 1
1093     plug_in_setup()
1094     rc, shell_rc, failed_plug_in_name, history =\
1095         grpi.rprocess_plug_in_packages(call_point='post_stack',
1096                                        stop_on_plug_in_failure=0,
1097                                        return_history=True)
1098     for doing_msg in history:
1099         update_boot_history(boot_history, doing_msg, max_boot_history)
1100     if rc != 0:
1101         boot_success = 0
1102 
1103     plug_in_setup()
1104     rc, shell_rc, failed_plug_in_name =\
1105         grpi.rprocess_plug_in_packages(call_point='ffdc_check',
1106                                        shell_rc=dump_ffdc_rc(),
1107                                        stop_on_plug_in_failure=1,
1108                                        stop_on_non_zero_rc=1)
1109     if shell_rc == dump_ffdc_rc():
1110         status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
1111         if status != 'PASS':
1112             gp.qprint_error("Call to my_ffdc failed.\n")
1113             # Leave a record for caller that "soft" errors occurred.
1114             soft_errors = 1
1115             gpu.save_plug_in_value(soft_errors, pgm_name)
1116 
1117     plug_in_setup()
1118     rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1119         call_point='stop_check', shell_rc=stop_test_rc(),
1120         stop_on_non_zero_rc=1)
1121     if shell_rc == stop_test_rc():
1122         message = "Stopping as requested by user.\n"
1123         gp.qprint_time(message)
1124         BuiltIn().fail(message)
1125 
1126 
1127 def obmc_boot_test_py(loc_boot_stack=None,
1128                       loc_stack_mode=None,
1129                       loc_quiet=None):
1130     r"""
1131     Do main program processing.
1132     """
1133 
1134     global save_stack
1135 
1136     ga.set_term_options(term_requests={'pgm_names': ['process_plug_in_packages.py']})
1137 
1138     gp.dprintn()
1139     # Process function parms.
1140     for parm_name in main_func_parm_list:
1141         # Get parm's value.
1142         parm_value = eval("loc_" + parm_name)
1143         gp.dpvars(parm_name, parm_value)
1144 
1145         if parm_value is not None:
1146             # Save the global value on a stack.
1147             cmd_buf = "save_stack.push(BuiltIn().get_variable_value(\"${" +\
1148                 parm_name + "}\"), \"" + parm_name + "\")"
1149             gp.dpissuing(cmd_buf)
1150             exec(cmd_buf)
1151 
1152             # Set the global value to the passed value.
1153             cmd_buf = "BuiltIn().set_global_variable(\"${" + parm_name +\
1154                 "}\", loc_" + parm_name + ")"
1155             gp.dpissuing(cmd_buf)
1156             exec(cmd_buf)
1157 
1158     gp.dprintn(save_stack.sprint_obj())
1159 
1160     setup()
1161 
1162     init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail()
1163 
1164     if ffdc_only:
1165         gp.qprint_timen("Caller requested ffdc_only.")
1166         if do_pre_boot_plug_in_setup:
1167             pre_boot_plug_in_setup()
1168         grk.run_key_u("my_ffdc")
1169         return
1170 
1171     if delete_errlogs:
1172         # print error logs before delete
1173         if redfish_support_trans_state:
1174             status, error_logs = grk.run_key_u("Get Redfish Event Logs")
1175             log.print_error_logs(error_logs, "AdditionalDataURI Message Severity")
1176         else:
1177             status, error_logs = grk.run_key_u("Get Error Logs")
1178             log.print_error_logs(error_logs, "AdditionalData Message Severity")
1179         pels = pel.peltool("-l", ignore_err=1)
1180         gp.qprint_var(pels)
1181 
1182         # Delete errlogs prior to doing any boot tests.
1183         grk.run_key(delete_errlogs_cmd, ignore=1)
1184         grk.run_key(delete_bmcdump_cmd, ignore=1)
1185         if redfish_support_trans_state:
1186             grk.run_key(delete_sysdump_cmd, ignore=1)
1187 
1188     # Process caller's boot_stack.
1189     while (len(boot_stack) > 0):
1190         test_loop_body()
1191 
1192     gp.qprint_timen("Finished processing stack.")
1193 
1194     post_stack()
1195 
1196     # Process caller's boot_list.
1197     if len(boot_list) > 0:
1198         for ix in range(1, max_num_tests + 1):
1199             test_loop_body()
1200 
1201     gp.qprint_timen("Completed all requested boot tests.")
1202 
1203     boot_pass, boot_fail = boot_results.return_total_pass_fail()
1204     new_fail = boot_fail - init_boot_fail
1205     if new_fail > boot_fail_threshold:
1206         error_message = "Boot failures exceed the boot failure" +\
1207                         " threshold:\n" +\
1208                         gp.sprint_var(new_fail) +\
1209                         gp.sprint_var(boot_fail_threshold)
1210         BuiltIn().fail(gp.sprint_error(error_message))
1211