xref: /openbmc/openbmc-test-automation/lib/obmc_boot_test.py (revision 139f1da69baea53bab830baafcbc75d59f51459b)
1 #!/usr/bin/env python3
2 
3 r"""
4 This module is the python counterpart to obmc_boot_test.
5 """
6 
7 import glob
8 import importlib.util
9 import os
10 import random
11 import re
12 import signal
13 import time
14 
15 try:
16     import cPickle as pickle
17 except ImportError:
18     import pickle
19 
20 import socket
21 
22 import gen_arg as ga
23 import gen_cmd as gc
24 import gen_misc as gm
25 import gen_plug_in_utils as gpu
26 import gen_print as gp
27 import gen_robot_keyword as grk
28 import gen_robot_plug_in as grpi
29 import gen_valid as gv
30 import logging_utils as log
31 import pel_utils as pel
32 import state as st
33 import var_stack as vs
34 from boot_data import *
35 from robot.libraries.BuiltIn import BuiltIn
36 from robot.utils import DotDict
37 
38 base_path = (
39     os.path.dirname(
40         os.path.dirname(importlib.util.find_spec("gen_robot_print").origin)
41     )
42     + os.sep
43 )
44 sys.path.append(base_path + "extended/")
45 import run_keyword as rk  # NOQA
46 
47 # Setting master_pid correctly influences the behavior of plug-ins like
48 # DB_Logging
49 program_pid = os.getpid()
50 master_pid = os.environ.get("AUTOBOOT_MASTER_PID", program_pid)
51 pgm_name = re.sub("\\.py$", "", os.path.basename(__file__))
52 
53 # Set up boot data structures.
54 os_host = BuiltIn().get_variable_value("${OS_HOST}", default="")
55 
56 boot_lists = read_boot_lists()
57 
58 # The maximum number of entries that can be in the boot_history global variable.
59 max_boot_history = 10
60 boot_history = []
61 
62 state = st.return_state_constant("default_state")
63 cp_setup_called = 0
64 next_boot = ""
65 base_tool_dir_path = (
66     os.path.normpath(os.environ.get("AUTOBOOT_BASE_TOOL_DIR_PATH", "/tmp"))
67     + os.sep
68 )
69 
70 ffdc_dir_path = os.path.normpath(os.environ.get("FFDC_DIR_PATH", "")) + os.sep
71 boot_success = 0
72 
73 status_dir_path = os.environ.get(
74     "STATUS_DIR_PATH", ""
75 ) or BuiltIn().get_variable_value("${STATUS_DIR_PATH}", default="")
76 if status_dir_path != "":
77     status_dir_path = os.path.normpath(status_dir_path) + os.sep
78     # For plugin expecting env gen_call_robot.py
79     os.environ["STATUS_DIR_PATH"] = status_dir_path
80 
81 redfish_support_trans_state = int(
82     os.environ.get("REDFISH_SUPPORT_TRANS_STATE", 0)
83 ) or int(
84     BuiltIn().get_variable_value("${REDFISH_SUPPORT_TRANS_STATE}", default=0)
85 )
86 redfish_supported = BuiltIn().get_variable_value(
87     "${REDFISH_SUPPORTED}", default=False
88 )
89 redfish_rest_supported = BuiltIn().get_variable_value(
90     "${REDFISH_REST_SUPPORTED}", default=False
91 )
92 redfish_delete_sessions = int(
93     BuiltIn().get_variable_value("${REDFISH_DELETE_SESSIONS}", default=1)
94 )
95 if redfish_supported:
96     redfish = BuiltIn().get_library_instance("redfish")
97     default_power_on = "Redfish Power On"
98     default_power_off = "Redfish Power Off"
99     if not redfish_support_trans_state:
100         delete_errlogs_cmd = "Delete Error Logs  ${quiet}=${1}"
101         delete_bmcdump_cmd = "Delete All BMC Dump"
102         default_set_power_policy = "Set BMC Power Policy  ALWAYS_POWER_OFF"
103     else:
104         delete_errlogs_cmd = "Redfish Purge Event Log"
105         delete_bmcdump_cmd = "Redfish Delete All BMC Dumps"
106         delete_sysdump_cmd = "Redfish Delete All System Dumps"
107         default_set_power_policy = (
108             "Redfish Set Power Restore Policy  AlwaysOff"
109         )
110 else:
111     default_power_on = "REST Power On"
112     default_power_off = "REST Power Off"
113     delete_errlogs_cmd = "Delete Error Logs  ${quiet}=${1}"
114     delete_bmcdump_cmd = "Delete All BMC Dump"
115     default_set_power_policy = "Set BMC Power Policy  ALWAYS_POWER_OFF"
116 boot_count = 0
117 
118 LOG_LEVEL = BuiltIn().get_variable_value("${LOG_LEVEL}")
119 AUTOBOOT_FFDC_PREFIX = os.environ.get("AUTOBOOT_FFDC_PREFIX", "")
120 ffdc_prefix = AUTOBOOT_FFDC_PREFIX
121 boot_start_time = ""
122 boot_end_time = ""
123 save_stack = vs.var_stack("save_stack")
124 main_func_parm_list = ["boot_stack", "stack_mode", "quiet"]
125 
126 
127 def dump_ffdc_rc():
128     r"""
129     Return the constant dump ffdc test return code value.
130 
131     When a plug-in call point program returns this value, it indicates that
132     this program should collect FFDC.
133     """
134 
135     return 0x00000200
136 
137 
138 def stop_test_rc():
139     r"""
140     Return the constant stop test return code value.
141 
142     When a plug-in call point program returns this value, it indicates that
143     this program should stop running.
144     """
145 
146     return 0x00000200
147 
148 
149 def process_host(host, host_var_name=""):
150     r"""
151     Process a host by getting the associated host name and IP address and
152     setting them in global variables.
153 
154     If the caller does not pass the host_var_name, this function will try to
155     figure out the name of the variable used by the caller for the host parm.
156     Callers are advised to explicitly specify the host_var_name when calling
157     with an exec command.  In such cases, the get_arg_name cannot figure out
158     the host variable name.
159 
160     This function will then create similar global variable names by
161     removing "_host" and appending "_host_name" or "_ip" to the host variable
162     name.
163 
164     Example:
165 
166     If a call is made like this:
167     process_host(openbmc_host)
168 
169     Global variables openbmc_host_name and openbmc_ip will be set.
170 
171     Description of argument(s):
172     host           A host name or IP.  The name of the variable used should
173                    have a suffix of "_host".
174     host_var_name  The name of the variable being used as the host parm.
175     """
176 
177     if host_var_name == "":
178         host_var_name = gp.get_arg_name(0, 1, stack_frame_ix=2)
179 
180     host_name_var_name = re.sub("host", "host_name", host_var_name)
181     ip_var_name = re.sub("host", "ip", host_var_name)
182     cmd_buf = (
183         "global "
184         + host_name_var_name
185         + ", "
186         + ip_var_name
187         + " ; "
188         + host_name_var_name
189         + ", "
190         + ip_var_name
191         + " = gm.get_host_name_ip('"
192         + host
193         + "')"
194     )
195     exec(cmd_buf)
196 
197 
198 def process_pgm_parms():
199     r"""
200     Process the program parameters by assigning them all to corresponding
201     globals.  Also, set some global values that depend on program parameters.
202     """
203 
204     # Program parameter processing.
205     # Assign all program parms to python variables which are global to this
206     # module.
207 
208     global parm_list
209     parm_list = BuiltIn().get_variable_value("${parm_list}")
210     # The following subset of parms should be processed as integers.
211     int_list = [
212         "max_num_tests",
213         "boot_pass",
214         "boot_fail",
215         "ffdc_only",
216         "boot_fail_threshold",
217         "delete_errlogs",
218         "call_post_stack_plug",
219         "do_pre_boot_plug_in_setup",
220         "quiet",
221         "test_mode",
222         "debug",
223     ]
224     for parm in parm_list:
225         if parm in int_list:
226             sub_cmd = (
227                 'int(BuiltIn().get_variable_value("${' + parm + '}", "0"))'
228             )
229         else:
230             sub_cmd = 'BuiltIn().get_variable_value("${' + parm + '}")'
231         cmd_buf = "global " + parm + " ; " + parm + " = " + sub_cmd
232         gp.dpissuing(cmd_buf)
233         exec(cmd_buf)
234         if re.match(r".*_host$", parm):
235             cmd_buf = "process_host(" + parm + ", '" + parm + "')"
236             exec(cmd_buf)
237         if re.match(r".*_password$", parm):
238             # Register the value of any parm whose name ends in _password.
239             # This will cause the print functions to replace passwords with
240             # asterisks in the output.
241             cmd_buf = "gp.register_passwords(" + parm + ")"
242             exec(cmd_buf)
243 
244     global ffdc_dir_path_style
245     global boot_list
246     global boot_stack
247     global boot_results_file_path
248     global boot_results
249     global boot_history
250     global ffdc_list_file_path
251     global ffdc_report_list_path
252     global ffdc_summary_list_path
253     global boot_table
254     global valid_boot_types
255 
256     if ffdc_dir_path_style == "":
257         ffdc_dir_path_style = int(os.environ.get("FFDC_DIR_PATH_STYLE", "0"))
258 
259     # Convert these program parms to lists for easier processing..
260     boot_list = list(filter(None, boot_list.split(":")))
261     boot_stack = list(filter(None, boot_stack.split(":")))
262 
263     boot_table = create_boot_table(boot_table_path, os_host=os_host)
264     valid_boot_types = create_valid_boot_list(boot_table)
265 
266     cleanup_boot_results_file()
267     boot_results_file_path = create_boot_results_file_path(
268         pgm_name, openbmc_nickname, master_pid
269     )
270 
271     if os.path.isfile(boot_results_file_path):
272         # We've been called before in this run so we'll load the saved
273         # boot_results and boot_history objects.
274         boot_results, boot_history = pickle.load(
275             open(boot_results_file_path, "rb")
276         )
277     else:
278         boot_results = boot_results(boot_table, boot_pass, boot_fail)
279 
280     ffdc_list_file_path = (
281         base_tool_dir_path + openbmc_nickname + "/FFDC_FILE_LIST"
282     )
283     ffdc_report_list_path = (
284         base_tool_dir_path + openbmc_nickname + "/FFDC_REPORT_FILE_LIST"
285     )
286 
287     ffdc_summary_list_path = (
288         base_tool_dir_path + openbmc_nickname + "/FFDC_SUMMARY_FILE_LIST"
289     )
290 
291 
292 def initial_plug_in_setup():
293     r"""
294     Initialize all plug-in environment variables which do not change for the
295     duration of the program.
296 
297     """
298 
299     global LOG_LEVEL
300     BuiltIn().set_log_level("NONE")
301 
302     BuiltIn().set_global_variable("${master_pid}", master_pid)
303     BuiltIn().set_global_variable("${FFDC_DIR_PATH}", ffdc_dir_path)
304     BuiltIn().set_global_variable("${STATUS_DIR_PATH}", status_dir_path)
305     BuiltIn().set_global_variable("${BASE_TOOL_DIR_PATH}", base_tool_dir_path)
306     BuiltIn().set_global_variable(
307         "${FFDC_LIST_FILE_PATH}", ffdc_list_file_path
308     )
309     BuiltIn().set_global_variable(
310         "${FFDC_REPORT_LIST_PATH}", ffdc_report_list_path
311     )
312     BuiltIn().set_global_variable(
313         "${FFDC_SUMMARY_LIST_PATH}", ffdc_summary_list_path
314     )
315 
316     BuiltIn().set_global_variable(
317         "${FFDC_DIR_PATH_STYLE}", ffdc_dir_path_style
318     )
319     BuiltIn().set_global_variable("${FFDC_CHECK}", ffdc_check)
320 
321     # For each program parameter, set the corresponding AUTOBOOT_ environment
322     # variable value.  Also, set an AUTOBOOT_ environment variable for every
323     # element in additional_values.
324     additional_values = [
325         "program_pid",
326         "master_pid",
327         "ffdc_dir_path",
328         "status_dir_path",
329         "base_tool_dir_path",
330         "ffdc_list_file_path",
331         "ffdc_report_list_path",
332         "ffdc_summary_list_path",
333         "execdir",
334         "redfish_supported",
335         "redfish_rest_supported",
336         "redfish_support_trans_state",
337     ]
338 
339     plug_in_vars = parm_list + additional_values
340 
341     for var_name in plug_in_vars:
342         var_value = BuiltIn().get_variable_value("${" + var_name + "}")
343         var_name = var_name.upper()
344         if var_value is None:
345             var_value = ""
346         os.environ["AUTOBOOT_" + var_name] = str(var_value)
347 
348     BuiltIn().set_log_level(LOG_LEVEL)
349 
350     # Make sure the ffdc list directory exists.
351     ffdc_list_dir_path = os.path.dirname(ffdc_list_file_path) + os.sep
352     if not os.path.exists(ffdc_list_dir_path):
353         os.makedirs(ffdc_list_dir_path)
354 
355 
356 def plug_in_setup():
357     r"""
358     Initialize all changing plug-in environment variables for use by the
359     plug-in programs.
360     """
361 
362     global LOG_LEVEL
363     global test_really_running
364 
365     BuiltIn().set_log_level("NONE")
366 
367     boot_pass, boot_fail = boot_results.return_total_pass_fail()
368     if boot_pass > 1:
369         test_really_running = 1
370     else:
371         test_really_running = 0
372 
373     BuiltIn().set_global_variable(
374         "${test_really_running}", test_really_running
375     )
376     BuiltIn().set_global_variable("${boot_type_desc}", next_boot)
377     BuiltIn().set_global_variable("${boot_pass}", boot_pass)
378     BuiltIn().set_global_variable("${boot_fail}", boot_fail)
379     BuiltIn().set_global_variable("${boot_success}", boot_success)
380     BuiltIn().set_global_variable("${ffdc_prefix}", ffdc_prefix)
381     BuiltIn().set_global_variable("${boot_start_time}", boot_start_time)
382     BuiltIn().set_global_variable("${boot_end_time}", boot_end_time)
383 
384     # For each program parameter, set the corresponding AUTOBOOT_ environment
385     # variable value.  Also, set an AUTOBOOT_ environment variable for every
386     # element in additional_values.
387     additional_values = [
388         "boot_type_desc",
389         "boot_success",
390         "boot_pass",
391         "boot_fail",
392         "test_really_running",
393         "ffdc_prefix",
394         "boot_start_time",
395         "boot_end_time",
396     ]
397 
398     plug_in_vars = additional_values
399 
400     for var_name in plug_in_vars:
401         var_value = BuiltIn().get_variable_value("${" + var_name + "}")
402         var_name = var_name.upper()
403         if var_value is None:
404             var_value = ""
405         os.environ["AUTOBOOT_" + var_name] = str(var_value)
406 
407     if debug:
408         shell_rc, out_buf = gc.cmd_fnc_u(
409             "printenv | egrep AUTOBOOT_ | sort -u"
410         )
411 
412     BuiltIn().set_log_level(LOG_LEVEL)
413 
414 
415 def pre_boot_plug_in_setup():
416     # Clear the ffdc_list_file_path file.  Plug-ins may now write to it.
417     try:
418         os.remove(ffdc_list_file_path)
419     except OSError:
420         pass
421 
422     # Clear the ffdc_report_list_path file.  Plug-ins may now write to it.
423     try:
424         os.remove(ffdc_report_list_path)
425     except OSError:
426         pass
427 
428     # Clear the ffdc_summary_list_path file.  Plug-ins may now write to it.
429     try:
430         os.remove(ffdc_summary_list_path)
431     except OSError:
432         pass
433 
434     global ffdc_prefix
435 
436     seconds = time.time()
437     loc_time = time.localtime(seconds)
438     time_string = time.strftime("%y%m%d.%H%M%S.", loc_time)
439 
440     ffdc_prefix = openbmc_nickname + "." + time_string
441 
442 
443 def default_sigusr1(signal_number=0, frame=None):
444     r"""
445     Handle SIGUSR1 by doing nothing.
446 
447     This function assists in debugging SIGUSR1 processing by printing messages
448     to stdout and to the log.html file.
449 
450     Description of argument(s):
451     signal_number  The signal number (should always be 10 for SIGUSR1).
452     frame          The frame data.
453     """
454 
455     gp.qprintn()
456     gp.qprint_executing()
457     gp.lprint_executing()
458 
459 
460 def set_default_siguser1():
461     r"""
462     Set the default_sigusr1 function to be the SIGUSR1 handler.
463     """
464 
465     gp.qprintn()
466     gp.qprint_executing()
467     gp.lprint_executing()
468     signal.signal(signal.SIGUSR1, default_sigusr1)
469 
470 
471 def setup():
472     r"""
473     Do general program setup tasks.
474     """
475 
476     global cp_setup_called
477     global transitional_boot_selected
478 
479     gp.qprintn()
480 
481     if redfish_supported:
482         redfish.login()
483 
484     set_default_siguser1()
485     transitional_boot_selected = False
486 
487     robot_pgm_dir_path = os.path.dirname(__file__) + os.sep
488     repo_bin_path = robot_pgm_dir_path.replace("/lib/", "/bin/")
489     # If we can't find process_plug_in_packages.py, ssh_pw or
490     # validate_plug_ins.py, then we don't have our repo bin in PATH.
491     shell_rc, out_buf = gc.cmd_fnc_u(
492         "which process_plug_in_packages.py" + " ssh_pw validate_plug_ins.py",
493         quiet=1,
494         print_output=0,
495         show_err=0,
496     )
497     if shell_rc != 0:
498         os.environ["PATH"] = repo_bin_path + ":" + os.environ.get("PATH", "")
499     # Likewise, our repo lib subdir needs to be in sys.path and PYTHONPATH.
500     if robot_pgm_dir_path not in sys.path:
501         sys.path.append(robot_pgm_dir_path)
502         PYTHONPATH = os.environ.get("PYTHONPATH", "")
503         if PYTHONPATH == "":
504             os.environ["PYTHONPATH"] = robot_pgm_dir_path
505         else:
506             os.environ["PYTHONPATH"] = robot_pgm_dir_path + ":" + PYTHONPATH
507 
508     validate_parms()
509 
510     gp.qprint_pgm_header()
511 
512     grk.run_key_u(default_set_power_policy, ignore=1)
513 
514     initial_plug_in_setup()
515 
516     plug_in_setup()
517     rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
518         call_point="setup"
519     )
520     if rc != 0:
521         error_message = "Plug-in setup failed.\n"
522         gp.print_error_report(error_message)
523         BuiltIn().fail(error_message)
524     # Setting cp_setup_called lets our Teardown know that it needs to call
525     # the cleanup plug-in call point.
526     cp_setup_called = 1
527 
528     # Keyword "FFDC" will fail if TEST_MESSAGE is not set.
529     BuiltIn().set_global_variable("${TEST_MESSAGE}", "${EMPTY}")
530     # FFDC_LOG_PATH is used by "FFDC" keyword.
531     BuiltIn().set_global_variable("${FFDC_LOG_PATH}", ffdc_dir_path)
532 
533     # Also printed by FFDC.
534     global host_name
535     global host_ip
536     host = socket.gethostname()
537     host_name, host_ip = gm.get_host_name_ip(host)
538 
539     gp.dprint_var(boot_table)
540     gp.dprint_var(boot_lists)
541 
542 
543 def validate_parms():
544     r"""
545     Validate all program parameters.
546     """
547 
548     process_pgm_parms()
549 
550     gp.qprintn()
551 
552     global openbmc_model
553     if openbmc_model == "":
554         status, ret_values = grk.run_key_u("Get BMC System Model", ignore=1)
555         # Set the model to default "OPENBMC" if getting it from BMC fails.
556         if status == "FAIL":
557             openbmc_model = "OPENBMC"
558         else:
559             openbmc_model = ret_values
560         BuiltIn().set_global_variable("${openbmc_model}", openbmc_model)
561     gv.set_exit_on_error(True)
562     gv.valid_value(openbmc_host)
563     gv.valid_value(openbmc_username)
564     gv.valid_value(openbmc_password)
565     gv.valid_value(rest_username)
566     gv.valid_value(rest_password)
567     gv.valid_value(ipmi_username)
568     gv.valid_value(ipmi_password)
569     if os_host != "":
570         gv.valid_value(os_username)
571         gv.valid_value(os_password)
572     if pdu_host != "":
573         gv.valid_value(pdu_username)
574         gv.valid_value(pdu_password)
575         gv.valid_integer(pdu_slot_no)
576     if openbmc_serial_host != "":
577         gv.valid_integer(openbmc_serial_port)
578     gv.valid_value(openbmc_model)
579     gv.valid_integer(max_num_tests)
580     gv.valid_integer(boot_pass)
581     gv.valid_integer(boot_fail)
582     plug_in_packages_list = grpi.rvalidate_plug_ins(plug_in_dir_paths)
583     BuiltIn().set_global_variable(
584         "${plug_in_packages_list}", plug_in_packages_list
585     )
586     gv.valid_value(stack_mode, valid_values=["normal", "skip"])
587     gv.set_exit_on_error(False)
588     if len(boot_list) == 0 and len(boot_stack) == 0 and not ffdc_only:
589         error_message = (
590             "You must provide either a value for either the"
591             + " boot_list or the boot_stack parm.\n"
592         )
593         BuiltIn().fail(gp.sprint_error(error_message))
594     valid_boot_list(boot_list, valid_boot_types)
595     valid_boot_list(boot_stack, valid_boot_types)
596     selected_PDU_boots = list(
597         set(boot_list + boot_stack) & set(boot_lists["PDU_reboot"])
598     )
599     if len(selected_PDU_boots) > 0 and pdu_host == "":
600         error_message = (
601             "You have selected the following boots which"
602             + " require a PDU host but no value for pdu_host:\n"
603         )
604         error_message += gp.sprint_var(selected_PDU_boots)
605         error_message += gp.sprint_var(pdu_host, fmt=gp.blank())
606         BuiltIn().fail(gp.sprint_error(error_message))
607 
608     return
609 
610 
611 def my_get_state():
612     r"""
613     Get the system state plus a little bit of wrapping.
614     """
615 
616     global state
617 
618     req_states = ["epoch_seconds"] + st.default_req_states
619 
620     gp.qprint_timen("Getting system state.")
621     if test_mode:
622         state["epoch_seconds"] = int(time.time())
623     else:
624         state = st.get_state(req_states=req_states, quiet=quiet)
625     gp.qprint_var(state)
626 
627 
628 def valid_state():
629     r"""
630     Verify that our state dictionary contains no blank values.  If we don't get
631     valid state data, we cannot continue to work.
632     """
633 
634     if st.compare_states(state, st.invalid_state_match, "or"):
635         error_message = (
636             "The state dictionary contains blank fields which"
637             + " is illegal.\n"
638             + gp.sprint_var(state)
639         )
640         BuiltIn().fail(gp.sprint_error(error_message))
641 
642 
643 def select_boot():
644     r"""
645     Select a boot test to be run based on our current state and return the
646     chosen boot type.
647 
648     Description of arguments:
649     state  The state of the machine.
650     """
651 
652     global transitional_boot_selected
653     global boot_stack
654 
655     gp.qprint_timen("Selecting a boot test.")
656 
657     if transitional_boot_selected and not boot_success:
658         prior_boot = next_boot
659         boot_candidate = boot_stack.pop()
660         gp.qprint_timen(
661             "The prior '"
662             + next_boot
663             + "' was chosen to"
664             + " transition to a valid state for '"
665             + boot_candidate
666             + "' which was at the top of the boot_stack.  Since"
667             + " the '"
668             + next_boot
669             + "' failed, the '"
670             + boot_candidate
671             + "' has been removed from the stack"
672             + " to avoid and endless failure loop."
673         )
674         if len(boot_stack) == 0:
675             return ""
676 
677     my_get_state()
678     valid_state()
679 
680     transitional_boot_selected = False
681     stack_popped = 0
682     if len(boot_stack) > 0:
683         stack_popped = 1
684         gp.qprint_dashes()
685         gp.qprint_var(boot_stack)
686         gp.qprint_dashes()
687         skip_boot_printed = 0
688         while len(boot_stack) > 0:
689             boot_candidate = boot_stack.pop()
690             if stack_mode == "normal":
691                 break
692             else:
693                 if st.compare_states(state, boot_table[boot_candidate]["end"]):
694                     if not skip_boot_printed:
695                         gp.qprint_var(stack_mode)
696                         gp.qprintn()
697                         gp.qprint_timen(
698                             "Skipping the following boot tests"
699                             + " which are unnecessary since their"
700                             + " required end states match the"
701                             + " current machine state:"
702                         )
703                         skip_boot_printed = 1
704                     gp.qprint_var(boot_candidate)
705                     boot_candidate = ""
706         if boot_candidate == "":
707             gp.qprint_dashes()
708             gp.qprint_var(boot_stack)
709             gp.qprint_dashes()
710             return boot_candidate
711         if st.compare_states(state, boot_table[boot_candidate]["start"]):
712             gp.qprint_timen(
713                 "The machine state is valid for a '"
714                 + boot_candidate
715                 + "' boot test."
716             )
717             gp.qprint_dashes()
718             gp.qprint_var(boot_stack)
719             gp.qprint_dashes()
720             return boot_candidate
721         else:
722             gp.qprint_timen(
723                 "The machine state does not match the required"
724                 + " starting state for a '"
725                 + boot_candidate
726                 + "' boot test:"
727             )
728             gp.qprint_varx(
729                 "boot_table_start_entry", boot_table[boot_candidate]["start"]
730             )
731             boot_stack.append(boot_candidate)
732             transitional_boot_selected = True
733             popped_boot = boot_candidate
734 
735     # Loop through your list selecting a boot_candidates
736     boot_candidates = []
737     for boot_candidate in boot_list:
738         if st.compare_states(state, boot_table[boot_candidate]["start"]):
739             if stack_popped:
740                 if st.compare_states(
741                     boot_table[boot_candidate]["end"],
742                     boot_table[popped_boot]["start"],
743                 ):
744                     boot_candidates.append(boot_candidate)
745             else:
746                 boot_candidates.append(boot_candidate)
747 
748     if len(boot_candidates) == 0:
749         gp.qprint_timen(
750             "The user's boot list contained no boot tests"
751             + " which are valid for the current machine state."
752         )
753         boot_candidate = default_power_on
754         if not st.compare_states(state, boot_table[default_power_on]["start"]):
755             boot_candidate = default_power_off
756         boot_candidates.append(boot_candidate)
757         gp.qprint_timen(
758             "Using default '"
759             + boot_candidate
760             + "' boot type to transition to valid state."
761         )
762 
763     gp.dprint_var(boot_candidates)
764 
765     # Randomly select a boot from the candidate list.
766     boot = random.choice(boot_candidates)
767 
768     return boot
769 
770 
771 def print_defect_report(ffdc_file_list):
772     r"""
773     Print a defect report.
774 
775     Description of argument(s):
776     ffdc_file_list  A list of files which were collected by our ffdc functions.
777     """
778 
779     # Making deliberate choice to NOT run plug_in_setup().  We don't want
780     # ffdc_prefix updated.
781     rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
782         call_point="ffdc_report", stop_on_plug_in_failure=0
783     )
784 
785     # Get additional header data which may have been created by ffdc plug-ins.
786     # Also, delete the individual header files to cleanup.
787     cmd_buf = (
788         "file_list=$(cat "
789         + ffdc_report_list_path
790         + " 2>/dev/null)"
791         + ' ; [ ! -z "${file_list}" ] && cat ${file_list}'
792         + " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
793     )
794     shell_rc, more_header_info = gc.cmd_fnc_u(
795         cmd_buf, print_output=0, show_err=0
796     )
797 
798     # Get additional summary data which may have been created by ffdc plug-ins.
799     # Also, delete the individual header files to cleanup.
800     cmd_buf = (
801         "file_list=$(cat "
802         + ffdc_summary_list_path
803         + " 2>/dev/null)"
804         + ' ; [ ! -z "${file_list}" ] && cat ${file_list}'
805         + " 2>/dev/null ; rm -rf ${file_list} 2>/dev/null || :"
806     )
807     shell_rc, ffdc_summary_info = gc.cmd_fnc_u(
808         cmd_buf, print_output=0, show_err=0
809     )
810 
811     # ffdc_list_file_path contains a list of any ffdc files created by plug-
812     # ins, etc.  Read that data into a list.
813     try:
814         plug_in_ffdc_list = (
815             open(ffdc_list_file_path, "r").read().rstrip("\n").split("\n")
816         )
817         plug_in_ffdc_list = list(filter(None, plug_in_ffdc_list))
818     except IOError:
819         plug_in_ffdc_list = []
820 
821     # Combine the files from plug_in_ffdc_list with the ffdc_file_list passed
822     # in.  Eliminate duplicates and sort the list.
823     ffdc_file_list = sorted(set(ffdc_file_list + plug_in_ffdc_list))
824 
825     if status_file_path != "":
826         ffdc_file_list.insert(0, status_file_path)
827 
828     # Convert the list to a printable list.
829     printable_ffdc_file_list = "\n".join(ffdc_file_list)
830 
831     # Open ffdc_file_list for writing.  We will write a complete list of
832     # FFDC files to it for possible use by plug-ins like cp_stop_check.
833     ffdc_list_file = open(ffdc_list_file_path, "w")
834     ffdc_list_file.write(printable_ffdc_file_list + "\n")
835     ffdc_list_file.close()
836 
837     indent = 0
838     width = 90
839     linefeed = 1
840     char = "="
841 
842     gp.qprintn()
843     gp.qprint_dashes(indent, width, linefeed, char)
844     gp.qprintn("Copy this data to the defect:\n")
845 
846     if len(more_header_info) > 0:
847         gp.qprintn(more_header_info)
848     gp.qpvars(
849         host_name,
850         host_ip,
851         openbmc_nickname,
852         openbmc_host,
853         openbmc_host_name,
854         openbmc_ip,
855         openbmc_username,
856         openbmc_password,
857         rest_username,
858         rest_password,
859         ipmi_username,
860         ipmi_password,
861         os_host,
862         os_host_name,
863         os_ip,
864         os_username,
865         os_password,
866         pdu_host,
867         pdu_host_name,
868         pdu_ip,
869         pdu_username,
870         pdu_password,
871         pdu_slot_no,
872         openbmc_serial_host,
873         openbmc_serial_host_name,
874         openbmc_serial_ip,
875         openbmc_serial_port,
876     )
877 
878     gp.qprintn()
879     print_boot_history(boot_history)
880     gp.qprintn()
881     gp.qprint_var(state)
882     gp.qprintn()
883     gp.qprintn("FFDC data files:")
884     gp.qprintn(printable_ffdc_file_list)
885     gp.qprintn()
886 
887     if len(ffdc_summary_info) > 0:
888         gp.qprintn(ffdc_summary_info)
889 
890     gp.qprint_dashes(indent, width, linefeed, char)
891 
892 
893 def my_ffdc():
894     r"""
895     Collect FFDC data.
896     """
897 
898     global state
899 
900     plug_in_setup()
901     rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
902         call_point="ffdc", stop_on_plug_in_failure=0
903     )
904 
905     AUTOBOOT_FFDC_PREFIX = os.environ["AUTOBOOT_FFDC_PREFIX"]
906     status, ffdc_file_list = grk.run_key_u(
907         "FFDC  ffdc_prefix="
908         + AUTOBOOT_FFDC_PREFIX
909         + "  ffdc_function_list="
910         + ffdc_function_list,
911         ignore=1,
912     )
913     if status != "PASS":
914         gp.qprint_error("Call to ffdc failed.\n")
915         if type(ffdc_file_list) is not list:
916             ffdc_file_list = []
917         # Leave a record for caller that "soft" errors occurred.
918         soft_errors = 1
919         gpu.save_plug_in_value(soft_errors, pgm_name)
920 
921     my_get_state()
922 
923     print_defect_report(ffdc_file_list)
924 
925 
926 def print_test_start_message(boot_keyword):
927     r"""
928     Print a message indicating what boot test is about to run.
929 
930     Description of arguments:
931     boot_keyword  The name of the boot which is to be run
932                   (e.g. "BMC Power On").
933     """
934 
935     global boot_history
936     global boot_start_time
937 
938     doing_msg = gp.sprint_timen('Doing "' + boot_keyword + '".')
939 
940     # Set boot_start_time for use by plug-ins.
941     boot_start_time = doing_msg[1:33]
942     gp.qprint_var(boot_start_time)
943 
944     gp.qprint(doing_msg)
945 
946     update_boot_history(boot_history, doing_msg, max_boot_history)
947 
948 
949 def stop_boot_test(signal_number=0, frame=None):
950     r"""
951     Handle SIGUSR1 by aborting the boot test that is running.
952 
953     Description of argument(s):
954     signal_number  The signal number (should always be 10 for SIGUSR1).
955     frame          The frame data.
956     """
957 
958     gp.qprintn()
959     gp.qprint_executing()
960     gp.lprint_executing()
961 
962     # Restore original sigusr1 handler.
963     set_default_siguser1()
964 
965     message = "The caller has asked that the boot test be stopped and marked"
966     message += " as a failure."
967 
968     function_stack = gm.get_function_stack()
969     if "wait_state" in function_stack:
970         st.set_exit_wait_early_message(message)
971     else:
972         BuiltIn().fail(gp.sprint_error(message))
973 
974 
975 def run_boot(boot):
976     r"""
977     Run the specified boot.
978 
979     Description of arguments:
980     boot  The name of the boot test to be performed.
981     """
982 
983     global state
984 
985     signal.signal(signal.SIGUSR1, stop_boot_test)
986     gp.qprint_timen("stop_boot_test is armed.")
987 
988     print_test_start_message(boot)
989 
990     plug_in_setup()
991     rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
992         call_point="pre_boot"
993     )
994     if rc != 0:
995         error_message = (
996             "Plug-in failed with non-zero return code.\n"
997             + gp.sprint_var(rc, fmt=gp.hexa())
998         )
999         set_default_siguser1()
1000         BuiltIn().fail(gp.sprint_error(error_message))
1001 
1002     if test_mode:
1003         # In test mode, we'll pretend the boot worked by assigning its
1004         # required end state to the default state value.
1005         state = st.strip_anchor_state(boot_table[boot]["end"])
1006     else:
1007         # Assertion:  We trust that the state data was made fresh by the
1008         # caller.
1009 
1010         gp.qprintn()
1011 
1012         if boot_table[boot]["method_type"] == "keyword":
1013             rk.my_run_keywords(
1014                 boot_table[boot].get("lib_file_path", ""),
1015                 boot_table[boot]["method"],
1016                 quiet=quiet,
1017             )
1018 
1019         if boot_table[boot]["bmc_reboot"]:
1020             st.wait_for_comm_cycle(int(state["epoch_seconds"]))
1021             plug_in_setup()
1022             rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1023                 call_point="post_reboot"
1024             )
1025             if rc != 0:
1026                 error_message = "Plug-in failed with non-zero return code.\n"
1027                 error_message += gp.sprint_var(rc, fmt=gp.hexa())
1028                 set_default_siguser1()
1029                 BuiltIn().fail(gp.sprint_error(error_message))
1030         else:
1031             match_state = st.anchor_state(state)
1032             del match_state["epoch_seconds"]
1033             # Wait for the state to change in any way.
1034             st.wait_state(
1035                 match_state,
1036                 wait_time=state_change_timeout,
1037                 interval="10 seconds",
1038                 invert=1,
1039             )
1040 
1041         gp.qprintn()
1042         if boot_table[boot]["end"]["chassis"] == "Off":
1043             boot_timeout = power_off_timeout
1044         else:
1045             boot_timeout = power_on_timeout
1046         st.wait_state(
1047             boot_table[boot]["end"],
1048             wait_time=boot_timeout,
1049             interval="10 seconds",
1050         )
1051 
1052     plug_in_setup()
1053     rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1054         call_point="post_boot"
1055     )
1056     if rc != 0:
1057         error_message = (
1058             "Plug-in failed with non-zero return code.\n"
1059             + gp.sprint_var(rc, fmt=gp.hexa())
1060         )
1061         set_default_siguser1()
1062         BuiltIn().fail(gp.sprint_error(error_message))
1063 
1064     # Restore original sigusr1 handler.
1065     set_default_siguser1()
1066 
1067 
1068 def test_loop_body():
1069     r"""
1070     The main loop body for the loop in main_py.
1071 
1072     Description of arguments:
1073     boot_count  The iteration number (starts at 1).
1074     """
1075 
1076     global boot_count
1077     global state
1078     global next_boot
1079     global boot_success
1080     global boot_end_time
1081 
1082     # The flag can be enabled or disabled on the go
1083     redfish_delete_sessions = int(
1084         BuiltIn().get_variable_value("${REDFISH_DELETE_SESSIONS}", default=1)
1085     )
1086 
1087     gp.qprintn()
1088 
1089     next_boot = select_boot()
1090     if next_boot == "":
1091         return True
1092 
1093     boot_count += 1
1094     gp.qprint_timen("Starting boot " + str(boot_count) + ".")
1095 
1096     pre_boot_plug_in_setup()
1097 
1098     cmd_buf = ["run_boot", next_boot]
1099     boot_status, msg = BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
1100     if boot_status == "FAIL":
1101         gp.qprint(msg)
1102 
1103     gp.qprintn()
1104     if boot_status == "PASS":
1105         boot_success = 1
1106         completion_msg = gp.sprint_timen(
1107             'BOOT_SUCCESS: "' + next_boot + '" succeeded.'
1108         )
1109     else:
1110         boot_success = 0
1111         completion_msg = gp.sprint_timen(
1112             'BOOT_FAILED: "' + next_boot + '" failed.'
1113         )
1114 
1115     # Set boot_end_time for use by plug-ins.
1116     boot_end_time = completion_msg[1:33]
1117     gp.qprint_var(boot_end_time)
1118 
1119     gp.qprint(completion_msg)
1120 
1121     boot_results.update(next_boot, boot_status)
1122 
1123     plug_in_setup()
1124     # NOTE: A post_test_case call point failure is NOT counted as a boot
1125     # failure.
1126     rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1127         call_point="post_test_case", stop_on_plug_in_failure=0
1128     )
1129 
1130     plug_in_setup()
1131     rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1132         call_point="ffdc_check",
1133         shell_rc=dump_ffdc_rc(),
1134         stop_on_plug_in_failure=1,
1135         stop_on_non_zero_rc=1,
1136     )
1137     if ffdc_check == "All" or shell_rc == dump_ffdc_rc():
1138         status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
1139         if status != "PASS":
1140             gp.qprint_error("Call to my_ffdc failed.\n")
1141             # Leave a record for caller that "soft" errors occurred.
1142             soft_errors = 1
1143             gpu.save_plug_in_value(soft_errors, pgm_name)
1144 
1145     if delete_errlogs:
1146         # print error logs before delete
1147         if redfish_support_trans_state:
1148             status, error_logs = grk.run_key_u("Get Redfish Event Logs")
1149             log.print_error_logs(
1150                 error_logs, "AdditionalDataURI Message Severity"
1151             )
1152         else:
1153             status, error_logs = grk.run_key_u("Get Error Logs")
1154             log.print_error_logs(error_logs, "AdditionalData Message Severity")
1155         pels = pel.peltool("-l", ignore_err=1)
1156         gp.qprint_var(pels)
1157 
1158         # We need to purge error logs between boots or they build up.
1159         grk.run_key(delete_errlogs_cmd, ignore=1)
1160         grk.run_key(delete_bmcdump_cmd, ignore=1)
1161         if redfish_support_trans_state:
1162             grk.run_key(delete_sysdump_cmd, ignore=1)
1163 
1164     boot_results.print_report()
1165     gp.qprint_timen("Finished boot " + str(boot_count) + ".")
1166 
1167     plug_in_setup()
1168     rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1169         call_point="stop_check", shell_rc=stop_test_rc(), stop_on_non_zero_rc=1
1170     )
1171     if shell_rc == stop_test_rc():
1172         message = "Stopping as requested by user.\n"
1173         gp.qprint_time(message)
1174         BuiltIn().fail(message)
1175 
1176     # This should help prevent ConnectionErrors.
1177     # Purge all redfish and REST connection sessions.
1178     if redfish_delete_sessions:
1179         grk.run_key_u("Close All Connections", ignore=1)
1180         grk.run_key_u("Delete All Redfish Sessions", ignore=1)
1181 
1182     return True
1183 
1184 
1185 def obmc_boot_test_teardown():
1186     r"""
1187     Clean up after the main keyword.
1188     """
1189     gp.qprint_executing()
1190 
1191     if ga.psutil_imported:
1192         ga.terminate_descendants()
1193 
1194     if cp_setup_called:
1195         plug_in_setup()
1196         rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1197             call_point="cleanup", stop_on_plug_in_failure=0
1198         )
1199 
1200     if "boot_results_file_path" in globals():
1201         # Save boot_results and boot_history objects to a file in case they are
1202         # needed again.
1203         gp.qprint_timen("Saving boot_results to the following path.")
1204         gp.qprint_var(boot_results_file_path)
1205         pickle.dump(
1206             (boot_results, boot_history),
1207             open(boot_results_file_path, "wb"),
1208             pickle.HIGHEST_PROTOCOL,
1209         )
1210 
1211     global save_stack
1212     # Restore any global values saved on the save_stack.
1213     for parm_name in main_func_parm_list:
1214         # Get the parm_value if it was saved on the stack.
1215         try:
1216             parm_value = save_stack.pop(parm_name)
1217         except BaseException:
1218             # If it was not saved, no further action is required.
1219             continue
1220 
1221         # Restore the saved value.
1222         cmd_buf = (
1223             'BuiltIn().set_global_variable("${' + parm_name + '}", parm_value)'
1224         )
1225         gp.dpissuing(cmd_buf)
1226         exec(cmd_buf)
1227 
1228     gp.dprintn(save_stack.sprint_obj())
1229 
1230 
1231 def test_teardown():
1232     r"""
1233     Clean up after this test case.
1234     """
1235 
1236     gp.qprintn()
1237     gp.qprint_executing()
1238 
1239     if ga.psutil_imported:
1240         ga.terminate_descendants()
1241 
1242     cmd_buf = [
1243         "Print Error",
1244         "A keyword timeout occurred ending this program.\n",
1245     ]
1246     BuiltIn().run_keyword_if_timeout_occurred(*cmd_buf)
1247 
1248     if redfish_supported:
1249         redfish.logout()
1250 
1251     gp.qprint_pgm_footer()
1252 
1253 
1254 def post_stack():
1255     r"""
1256     Process post_stack plug-in programs.
1257     """
1258 
1259     if not call_post_stack_plug:
1260         # The caller does not wish to have post_stack plug-in processing done.
1261         return
1262 
1263     global boot_success
1264 
1265     # NOTE: A post_stack call-point failure is NOT counted as a boot failure.
1266     pre_boot_plug_in_setup()
1267     # For the purposes of the following plug-ins, mark the "boot" as a success.
1268     boot_success = 1
1269     plug_in_setup()
1270     (
1271         rc,
1272         shell_rc,
1273         failed_plug_in_name,
1274         history,
1275     ) = grpi.rprocess_plug_in_packages(
1276         call_point="post_stack", stop_on_plug_in_failure=0, return_history=True
1277     )
1278     for doing_msg in history:
1279         update_boot_history(boot_history, doing_msg, max_boot_history)
1280     if rc != 0:
1281         boot_success = 0
1282 
1283     plug_in_setup()
1284     rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1285         call_point="ffdc_check",
1286         shell_rc=dump_ffdc_rc(),
1287         stop_on_plug_in_failure=1,
1288         stop_on_non_zero_rc=1,
1289     )
1290     if shell_rc == dump_ffdc_rc():
1291         status, ret_values = grk.run_key_u("my_ffdc", ignore=1)
1292         if status != "PASS":
1293             gp.qprint_error("Call to my_ffdc failed.\n")
1294             # Leave a record for caller that "soft" errors occurred.
1295             soft_errors = 1
1296             gpu.save_plug_in_value(soft_errors, pgm_name)
1297 
1298     plug_in_setup()
1299     rc, shell_rc, failed_plug_in_name = grpi.rprocess_plug_in_packages(
1300         call_point="stop_check", shell_rc=stop_test_rc(), stop_on_non_zero_rc=1
1301     )
1302     if shell_rc == stop_test_rc():
1303         message = "Stopping as requested by user.\n"
1304         gp.qprint_time(message)
1305         BuiltIn().fail(message)
1306 
1307 
1308 def obmc_boot_test_py(
1309     loc_boot_stack=None, loc_stack_mode=None, loc_quiet=None
1310 ):
1311     r"""
1312     Do main program processing.
1313     """
1314 
1315     global save_stack
1316 
1317     ga.set_term_options(
1318         term_requests={"pgm_names": ["process_plug_in_packages.py"]}
1319     )
1320 
1321     gp.dprintn()
1322     # Process function parms.
1323     for parm_name in main_func_parm_list:
1324         # Get parm's value.
1325         parm_value = eval("loc_" + parm_name)
1326         gp.dpvars(parm_name, parm_value)
1327 
1328         if parm_value is not None:
1329             # Save the global value on a stack.
1330             cmd_buf = (
1331                 'save_stack.push(BuiltIn().get_variable_value("${'
1332                 + parm_name
1333                 + '}"), "'
1334                 + parm_name
1335                 + '")'
1336             )
1337             gp.dpissuing(cmd_buf)
1338             exec(cmd_buf)
1339 
1340             # Set the global value to the passed value.
1341             cmd_buf = (
1342                 'BuiltIn().set_global_variable("${'
1343                 + parm_name
1344                 + '}", loc_'
1345                 + parm_name
1346                 + ")"
1347             )
1348             gp.dpissuing(cmd_buf)
1349             exec(cmd_buf)
1350 
1351     gp.dprintn(save_stack.sprint_obj())
1352 
1353     setup()
1354 
1355     init_boot_pass, init_boot_fail = boot_results.return_total_pass_fail()
1356 
1357     if ffdc_only:
1358         gp.qprint_timen("Caller requested ffdc_only.")
1359         if do_pre_boot_plug_in_setup:
1360             pre_boot_plug_in_setup()
1361         grk.run_key_u("my_ffdc")
1362         return
1363 
1364     if delete_errlogs:
1365         # print error logs before delete
1366         if redfish_support_trans_state:
1367             status, error_logs = grk.run_key_u("Get Redfish Event Logs")
1368             log.print_error_logs(
1369                 error_logs, "AdditionalDataURI Message Severity"
1370             )
1371         else:
1372             status, error_logs = grk.run_key_u("Get Error Logs")
1373             log.print_error_logs(error_logs, "AdditionalData Message Severity")
1374         pels = pel.peltool("-l", ignore_err=1)
1375         gp.qprint_var(pels)
1376 
1377         # Delete errlogs prior to doing any boot tests.
1378         grk.run_key(delete_errlogs_cmd, ignore=1)
1379         grk.run_key(delete_bmcdump_cmd, ignore=1)
1380         if redfish_support_trans_state:
1381             grk.run_key(delete_sysdump_cmd, ignore=1)
1382 
1383     # Process caller's boot_stack.
1384     while len(boot_stack) > 0:
1385         test_loop_body()
1386 
1387     gp.qprint_timen("Finished processing stack.")
1388 
1389     post_stack()
1390 
1391     # Process caller's boot_list.
1392     if len(boot_list) > 0:
1393         for ix in range(1, max_num_tests + 1):
1394             test_loop_body()
1395 
1396     gp.qprint_timen("Completed all requested boot tests.")
1397 
1398     boot_pass, boot_fail = boot_results.return_total_pass_fail()
1399     new_fail = boot_fail - init_boot_fail
1400     if new_fail > boot_fail_threshold:
1401         error_message = (
1402             "Boot failures exceed the boot failure"
1403             + " threshold:\n"
1404             + gp.sprint_var(new_fail)
1405             + gp.sprint_var(boot_fail_threshold)
1406         )
1407         BuiltIn().fail(gp.sprint_error(error_message))
1408