xref: /openbmc/openbmc-test-automation/lib/state.py (revision 89c983777d108d72b4cac0bd7adcf999bf6cea32)
1 #!/usr/bin/env python
2 
3 r"""
4 This module contains functions having to do with machine state: get_state,
5 check_state, wait_state, etc.
6 
7 The 'State' is a composite of many pieces of data.  Therefore, the functions
8 in this module define state as an ordered dictionary.  Here is an example of
9 some test output showing machine state:
10 
11 default_state:
12   default_state[chassis]:                         On
13   default_state[boot_progress]:                   OSStart
14   default_state[operating_system]:                BootComplete
15   default_state[host]:                            Running
16   default_state[os_ping]:                         1
17   default_state[os_login]:                        1
18   default_state[os_run_cmd]:                      1
19 
20 Different users may very well have different needs when inquiring about
21 state.  Support for new pieces of state information may be added to this
22 module as needed.
23 
24 By using the wait_state function, a caller can start a boot and then wait for
25 a precisely defined state to indicate that the boot has succeeded.  If
26 the boot fails, they can see exactly why by looking at the current state as
27 compared with the expected state.
28 """
29 
30 import gen_print as gp
31 import gen_valid as gv
32 import gen_robot_utils as gru
33 import gen_cmd as gc
34 import bmc_ssh_utils as bsu
35 
36 from robot.libraries.BuiltIn import BuiltIn
37 from robot.utils import DotDict
38 
39 import re
40 import os
41 import sys
42 import imp
43 
44 
45 # NOTE: Avoid importing utils.robot because utils.robot imports state.py
46 # (indirectly) which will cause failures.
47 gru.my_import_resource("rest_client.robot")
48 
49 base_path = os.path.dirname(os.path.dirname(
50                             imp.find_module("gen_robot_print")[1])) + os.sep
51 sys.path.append(base_path + "data/")
52 
53 # Previously, I had this coded:
54 # import variables as var
55 # However, we ran into a problem where a robot program did this...
56 # Variables           ../../lib/ras/variables.py
57 # Prior to doing this...
58 # Library            ../lib/state.py
59 
60 # This caused the wrong variables.py file to be selected.  Attempts to fix this
61 # have failed so far.  For the moment, we will hard-code the value we need from
62 # the file.
63 
64 SYSTEM_STATE_URI = "/xyz/openbmc_project/state/"
65 
66 # The BMC code has recently been changed as far as what states are defined and
67 # what the state values can be.  This module now has a means of processing both
68 # the old style state (i.e. OBMC_STATES_VERSION = 0) and the new style (i.e.
69 # OBMC_STATES_VERSION = 1).
70 # The caller can set environment variable OBMC_STATES_VERSION to dictate
71 # whether we're processing old or new style states.  If OBMC_STATES_VERSION is
72 # not set it will default to 1.
73 
74 # As of the present moment, OBMC_STATES_VERSION of 0 is for cold that is so old
75 # that it is no longer worthwhile to maintain.  The OBMC_STATES_VERSION 0 code
76 # is being removed but the OBMC_STATES_VERSION value will stay for now in the
77 # event that it is needed in the future.
78 
79 OBMC_STATES_VERSION = int(os.environ.get('OBMC_STATES_VERSION', 1))
80 
81 redfish_support_trans_state = int(os.environ.get('REDFISH_SUPPORT_TRANS_STATE', 0)) or \
82     int(BuiltIn().get_variable_value("${REDFISH_SUPPORT_TRANS_STATE}", default=0))
83 
84 platform_arch_type = os.environ.get('PLATFORM_ARCH_TYPE', '') or \
85     BuiltIn().get_variable_value("${PLATFORM_ARCH_TYPE}", default="power")
86 
87 # valid_os_req_states and default_os_req_states are used by the os_get_state
88 # function.
89 # valid_os_req_states is a list of state information supported by the
90 # get_os_state function.
91 valid_os_req_states = ['os_ping',
92                        'os_login',
93                        'os_run_cmd']
94 
95 # When a user calls get_os_state w/o specifying req_states,
96 # default_os_req_states is used as its value.
97 default_os_req_states = ['os_ping',
98                          'os_login',
99                          'os_run_cmd']
100 
101 # Presently, some BMCs appear to not keep time very well.  This environment
102 # variable directs the get_state function to use either the BMC's epoch time
103 # or the local epoch time.
104 USE_BMC_EPOCH_TIME = int(os.environ.get('USE_BMC_EPOCH_TIME', 0))
105 
106 # Useful state constant definition(s).
107 if not redfish_support_trans_state:
108     # When a user calls get_state w/o specifying req_states, default_req_states
109     # is used as its value.
110     default_req_states = ['rest',
111                           'chassis',
112                           'bmc',
113                           'boot_progress',
114                           'operating_system',
115                           'host',
116                           'os_ping',
117                           'os_login',
118                           'os_run_cmd']
119 
120     # valid_req_states is a list of sub states supported by the get_state function.
121     # valid_req_states, default_req_states and master_os_up_match are used by the
122     # get_state function.
123 
124     valid_req_states = ['ping',
125                         'packet_loss',
126                         'uptime',
127                         'epoch_seconds',
128                         'elapsed_boot_time',
129                         'rest',
130                         'chassis',
131                         'requested_chassis',
132                         'bmc',
133                         'requested_bmc',
134                         'boot_progress',
135                         'operating_system',
136                         'host',
137                         'requested_host',
138                         'attempts_left',
139                         'os_ping',
140                         'os_login',
141                         'os_run_cmd']
142 
143     # default_state is an initial value which may be of use to callers.
144     default_state = DotDict([('rest', '1'),
145                              ('chassis', 'On'),
146                              ('bmc', 'Ready'),
147                              ('boot_progress', 'OSStart'),
148                              ('operating_system', 'BootComplete'),
149                              ('host', 'Running'),
150                              ('os_ping', '1'),
151                              ('os_login', '1'),
152                              ('os_run_cmd', '1')])
153 
154     # A match state for checking that the system is at "standby".
155     standby_match_state = DotDict([('rest', '^1$'),
156                                    ('chassis', '^Off$'),
157                                    ('bmc', '^Ready$'),
158                                    ('boot_progress', '^Off|Unspecified$'),
159                                    ('operating_system', '^Inactive$'),
160                                    ('host', '^Off$')])
161 
162     # A match state for checking that the system is at "os running".
163     os_running_match_state = DotDict([('chassis', '^On$'),
164                                       ('bmc', '^Ready$'),
165                                       ('boot_progress',
166                                        'FW Progress, Starting OS|OSStart'),
167                                       ('operating_system', 'BootComplete'),
168                                       ('host', '^Running$'),
169                                       ('os_ping', '^1$'),
170                                       ('os_login', '^1$'),
171                                       ('os_run_cmd', '^1$')])
172 
173     # A master dictionary to determine whether the os may be up.
174     master_os_up_match = DotDict([('chassis', '^On$'),
175                                   ('bmc', '^Ready$'),
176                                   ('boot_progress',
177                                    'FW Progress, Starting OS|OSStart'),
178                                   ('operating_system', 'BootComplete'),
179                                   ('host', '^Running|Quiesced$')])
180 
181     invalid_state_match = DotDict([('rest', '^$'),
182                                    ('chassis', '^$'),
183                                    ('bmc', '^$'),
184                                    ('boot_progress', '^$'),
185                                    ('operating_system', '^$'),
186                                    ('host', '^$')])
187 else:
188     # When a user calls get_state w/o specifying req_states, default_req_states
189     # is used as its value.
190     default_req_states = ['redfish',
191                           'chassis',
192                           'bmc',
193                           'boot_progress',
194                           'host',
195                           'os_ping',
196                           'os_login',
197                           'os_run_cmd']
198 
199     # valid_req_states is a list of sub states supported by the get_state function.
200     # valid_req_states, default_req_states and master_os_up_match are used by the
201     # get_state function.
202 
203     valid_req_states = ['ping',
204                         'packet_loss',
205                         'uptime',
206                         'epoch_seconds',
207                         'elapsed_boot_time',
208                         'redfish',
209                         'chassis',
210                         'requested_chassis',
211                         'bmc',
212                         'requested_bmc',
213                         'boot_progress',
214                         'host',
215                         'requested_host',
216                         'attempts_left',
217                         'os_ping',
218                         'os_login',
219                         'os_run_cmd']
220 
221     # default_state is an initial value which may be of use to callers.
222     default_state = DotDict([('redfish', '1'),
223                              ('chassis', 'On'),
224                              ('bmc', 'Enabled'),
225                              ('boot_progress',
226                               'SystemHardwareInitializationComplete|OSBootStarted|OSRunning'),
227                              ('host', 'Enabled'),
228                              ('os_ping', '1'),
229                              ('os_login', '1'),
230                              ('os_run_cmd', '1')])
231 
232     # A match state for checking that the system is at "standby".
233     standby_match_state = DotDict([('redfish', '^1$'),
234                                    ('chassis', '^Off$'),
235                                    ('bmc', '^Enabled$'),
236                                    ('boot_progress', '^None$'),
237                                    ('host', '^Disabled$')])
238 
239     # A match state for checking that the system is at "os running".
240     os_running_match_state = DotDict([('chassis', '^On$'),
241                                       ('bmc', '^Enabled$'),
242                                       ('boot_progress',
243                                        'SystemHardwareInitializationComplete|OSBootStarted|OSRunning'),
244                                       ('host', '^Enabled$'),
245                                       ('os_ping', '^1$'),
246                                       ('os_login', '^1$'),
247                                       ('os_run_cmd', '^1$')])
248 
249     # A master dictionary to determine whether the os may be up.
250     master_os_up_match = DotDict([('chassis', '^On$'),
251                                   ('bmc', '^Enabled$'),
252                                   ('boot_progress',
253                                    'SystemHardwareInitializationComplete|OSBootStarted|OSRunning'),
254                                   ('host', '^Enabled$')])
255 
256     invalid_state_match = DotDict([('redfish', '^$'),
257                                    ('chassis', '^$'),
258                                    ('bmc', '^$'),
259                                    ('boot_progress', '^$'),
260                                    ('host', '^$')])
261 
262 # Filter the states based on platform type.
263 if platform_arch_type == "x86":
264     default_req_states.remove("operating_system")
265     default_req_states.remove("boot_progress")
266     valid_req_states.remove("operating_system")
267     valid_req_states.remove("boot_progress")
268     del default_state["operating_system"]
269     del default_state["boot_progress"]
270     del standby_match_state["operating_system"]
271     del standby_match_state["boot_progress"]
272     del os_running_match_state["operating_system"]
273     del os_running_match_state["boot_progress"]
274     del master_os_up_match["operating_system"]
275     del master_os_up_match["boot_progress"]
276     del invalid_state_match["operating_system"]
277     del invalid_state_match["boot_progress"]
278 
279 
280 def return_state_constant(state_name='default_state'):
281     r"""
282     Return the named state dictionary constant.
283     """
284 
285     return eval(state_name)
286 
287 
288 def anchor_state(state):
289     r"""
290     Add regular expression anchors ("^" and "$") to the beginning and end of
291     each item in the state dictionary passed in.  Return the resulting
292     dictionary.
293 
294     Description of argument(s):
295     state    A dictionary such as the one returned by the get_state()
296              function.
297     """
298 
299     anchored_state = state.copy()
300     for key in anchored_state.keys():
301         anchored_state[key] = "^" + str(anchored_state[key]) + "$"
302 
303     return anchored_state
304 
305 
306 def strip_anchor_state(state):
307     r"""
308     Strip regular expression anchors ("^" and "$") from the beginning and end
309     of each item in the state dictionary passed in.  Return the resulting
310     dictionary.
311 
312     Description of argument(s):
313     state    A dictionary such as the one returned by the get_state()
314              function.
315     """
316 
317     stripped_state = state.copy()
318     for key in stripped_state.keys():
319         stripped_state[key] = stripped_state[key].strip("^$")
320 
321     return stripped_state
322 
323 
324 def expressions_key():
325     r"""
326     Return expressions key constant.
327     """
328     return '<expressions>'
329 
330 
331 def compare_states(state,
332                    match_state,
333                    match_type='and'):
334     r"""
335     Compare 2 state dictionaries.  Return True if they match and False if they
336     don't.  Note that the match_state dictionary does not need to have an entry
337     corresponding to each entry in the state dictionary.  But for each entry
338     that it does have, the corresponding state entry will be checked for a
339     match.
340 
341     Description of argument(s):
342     state           A state dictionary such as the one returned by the
343                     get_state function.
344     match_state     A dictionary whose key/value pairs are "state field"/
345                     "state value".  The state value is interpreted as a
346                     regular expression.  Every value in this dictionary is
347                     considered.  When match_type is 'and', if each and every
348                     comparison matches, the two dictionaries are considered to
349                     be matching.  If match_type is 'or', if any two of the
350                     elements compared match, the two dictionaries are
351                     considered to be matching.
352 
353                     This value may also be any string accepted by
354                     return_state_constant (e.g. "standby_match_state").  In
355                     such a case this function will call return_state_constant
356                     to convert it to a proper dictionary as described above.
357 
358                     Finally, one special value is accepted for the key field:
359                     expression_key().  If such an entry exists, its value is
360                     taken to be a list of expressions to be evaluated.  These
361                     expressions may reference state dictionary entries by
362                     simply coding them in standard python syntax (e.g.
363                     state['key1']).  What follows is an example expression:
364 
365                     "int(float(state['uptime'])) < int(state['elapsed_boot_time'])"
366 
367                     In this example, if the state dictionary's 'uptime' entry
368                     is less than its 'elapsed_boot_time' entry, it would
369                     qualify as a match.
370     match_type      This may be 'and' or 'or'.
371     """
372 
373     error_message = gv.valid_value(match_type, valid_values=['and', 'or'])
374     if error_message != "":
375         BuiltIn().fail(gp.sprint_error(error_message))
376 
377     try:
378         match_state = return_state_constant(match_state)
379     except TypeError:
380         pass
381 
382     default_match = (match_type == 'and')
383     for key, match_state_value in match_state.items():
384         # Blank match_state_value means "don't care".
385         if match_state_value == "":
386             continue
387         if key == expressions_key():
388             for expr in match_state_value:
389                 # Use python interpreter to evaluate the expression.
390                 match = eval(expr)
391                 if match != default_match:
392                     return match
393         else:
394             try:
395                 match = (re.match(match_state_value, str(state[key])) is not None)
396             except KeyError:
397                 match = False
398             if match != default_match:
399                 return match
400 
401     return default_match
402 
403 
404 def get_os_state(os_host="",
405                  os_username="",
406                  os_password="",
407                  req_states=default_os_req_states,
408                  os_up=True,
409                  quiet=None):
410     r"""
411     Get component states for the operating system such as ping, login,
412     etc, put them into a dictionary and return them to the caller.
413 
414     Note that all substate values are strings.
415 
416     Description of argument(s):
417     os_host      The DNS name or IP address of the operating system.
418                  This defaults to global ${OS_HOST}.
419     os_username  The username to be used to login to the OS.
420                  This defaults to global ${OS_USERNAME}.
421     os_password  The password to be used to login to the OS.
422                  This defaults to global ${OS_PASSWORD}.
423     req_states   This is a list of states whose values are being requested by
424                  the caller.
425     os_up        If the caller knows that the os can't possibly be up, it can
426                  improve performance by passing os_up=False.  This function
427                  will then simply return default values for all requested os
428                  sub states.
429     quiet        Indicates whether status details (e.g. curl commands) should
430                  be written to the console.
431                  Defaults to either global value of ${QUIET} or to 1.
432     """
433 
434     quiet = int(gp.get_var_value(quiet, 0))
435 
436     # Set parm defaults where necessary and validate all parms.
437     if os_host == "":
438         os_host = BuiltIn().get_variable_value("${OS_HOST}")
439     error_message = gv.valid_value(os_host, invalid_values=[None, ""])
440     if error_message != "":
441         BuiltIn().fail(gp.sprint_error(error_message))
442 
443     if os_username == "":
444         os_username = BuiltIn().get_variable_value("${OS_USERNAME}")
445     error_message = gv.valid_value(os_username, invalid_values=[None, ""])
446     if error_message != "":
447         BuiltIn().fail(gp.sprint_error(error_message))
448 
449     if os_password == "":
450         os_password = BuiltIn().get_variable_value("${OS_PASSWORD}")
451     error_message = gv.valid_value(os_password, invalid_values=[None, ""])
452     if error_message != "":
453         BuiltIn().fail(gp.sprint_error(error_message))
454 
455     invalid_req_states = [sub_state for sub_state in req_states
456                           if sub_state not in valid_os_req_states]
457     if len(invalid_req_states) > 0:
458         error_message = "The following req_states are not supported:\n" +\
459             gp.sprint_var(invalid_req_states)
460         BuiltIn().fail(gp.sprint_error(error_message))
461 
462     # Initialize all substate values supported by this function.
463     os_ping = 0
464     os_login = 0
465     os_run_cmd = 0
466 
467     if os_up:
468         if 'os_ping' in req_states:
469             # See if the OS pings.
470             rc, out_buf = gc.shell_cmd("ping -c 1 -w 2 " + os_host,
471                                        print_output=0, show_err=0,
472                                        ignore_err=1)
473             if rc == 0:
474                 os_ping = 1
475 
476         # Programming note: All attributes which do not require an ssh login
477         # should have been processed by this point.
478         master_req_login = ['os_login', 'os_run_cmd']
479         req_login = [sub_state for sub_state in req_states if sub_state in
480                      master_req_login]
481         must_login = (len(req_login) > 0)
482 
483         if must_login:
484             output, stderr, rc = bsu.os_execute_command("uptime", quiet=quiet,
485                                                         ignore_err=1,
486                                                         time_out=20,
487                                                         os_host=os_host,
488                                                         os_username=os_username,
489                                                         os_password=os_password)
490             if rc == 0:
491                 os_login = 1
492                 os_run_cmd = 1
493             else:
494                 gp.dprint_vars(output, stderr)
495                 gp.dprint_vars(rc, 1)
496 
497     os_state = DotDict()
498     for sub_state in req_states:
499         cmd_buf = "os_state['" + sub_state + "'] = str(" + sub_state + ")"
500         exec(cmd_buf)
501 
502     return os_state
503 
504 
505 def get_state(openbmc_host="",
506               openbmc_username="",
507               openbmc_password="",
508               os_host="",
509               os_username="",
510               os_password="",
511               req_states=default_req_states,
512               quiet=None):
513     r"""
514     Get component states such as chassis state, bmc state, etc, put them into a
515     dictionary and return them to the caller.
516 
517     Note that all substate values are strings.
518 
519     Note: If elapsed_boot_time is included in req_states, it is the caller's
520     duty to call set_start_boot_seconds() in order to set global
521     start_boot_seconds.  elapsed_boot_time is the current time minus
522     start_boot_seconds.
523 
524     Description of argument(s):
525     openbmc_host      The DNS name or IP address of the BMC.
526                       This defaults to global ${OPENBMC_HOST}.
527     openbmc_username  The username to be used to login to the BMC.
528                       This defaults to global ${OPENBMC_USERNAME}.
529     openbmc_password  The password to be used to login to the BMC.
530                       This defaults to global ${OPENBMC_PASSWORD}.
531     os_host           The DNS name or IP address of the operating system.
532                       This defaults to global ${OS_HOST}.
533     os_username       The username to be used to login to the OS.
534                       This defaults to global ${OS_USERNAME}.
535     os_password       The password to be used to login to the OS.
536                       This defaults to global ${OS_PASSWORD}.
537     req_states        This is a list of states whose values are being requested
538                       by the caller.
539     quiet             Indicates whether status details (e.g. curl commands)
540                       should be written to the console.
541                       Defaults to either global value of ${QUIET} or to 1.
542     """
543 
544     quiet = int(gp.get_var_value(quiet, 0))
545 
546     # Set parm defaults where necessary and validate all parms.
547     if openbmc_host == "":
548         openbmc_host = BuiltIn().get_variable_value("${OPENBMC_HOST}")
549     error_message = gv.valid_value(openbmc_host, invalid_values=[None, ""])
550     if error_message != "":
551         BuiltIn().fail(gp.sprint_error(error_message))
552 
553     if openbmc_username == "":
554         openbmc_username = BuiltIn().get_variable_value("${OPENBMC_USERNAME}")
555     error_message = gv.valid_value(openbmc_username, invalid_values=[None, ""])
556     if error_message != "":
557         BuiltIn().fail(gp.sprint_error(error_message))
558 
559     if openbmc_password == "":
560         openbmc_password = BuiltIn().get_variable_value("${OPENBMC_PASSWORD}")
561     error_message = gv.valid_value(openbmc_password, invalid_values=[None, ""])
562     if error_message != "":
563         BuiltIn().fail(gp.sprint_error(error_message))
564 
565     # NOTE: OS parms are optional.
566     if os_host == "":
567         os_host = BuiltIn().get_variable_value("${OS_HOST}")
568         if os_host is None:
569             os_host = ""
570 
571     if os_username is "":
572         os_username = BuiltIn().get_variable_value("${OS_USERNAME}")
573         if os_username is None:
574             os_username = ""
575 
576     if os_password is "":
577         os_password = BuiltIn().get_variable_value("${OS_PASSWORD}")
578         if os_password is None:
579             os_password = ""
580 
581     invalid_req_states = [sub_state for sub_state in req_states
582                           if sub_state not in valid_req_states]
583     if len(invalid_req_states) > 0:
584         error_message = "The following req_states are not supported:\n" +\
585             gp.sprint_var(invalid_req_states)
586         BuiltIn().fail(gp.sprint_error(error_message))
587 
588     # Initialize all substate values supported by this function.
589     ping = 0
590     packet_loss = ''
591     uptime = ''
592     epoch_seconds = ''
593     elapsed_boot_time = ''
594     rest = ''
595     redfish = ''
596     chassis = ''
597     requested_chassis = ''
598     bmc = ''
599     requested_bmc = ''
600     # BootProgress state will get populated when state logic enumerates the
601     # state URI. This is to prevent state dictionary  boot_progress value
602     # getting empty when the BootProgress is NOT found, making it optional.
603     boot_progress = 'NA'
604     operating_system = ''
605     host = ''
606     requested_host = ''
607     attempts_left = ''
608 
609     # Get the component states.
610     if 'ping' in req_states:
611         # See if the OS pings.
612         rc, out_buf = gc.shell_cmd("ping -c 1 -w 2 " + openbmc_host,
613                                    print_output=0, show_err=0,
614                                    ignore_err=1)
615         if rc == 0:
616             ping = 1
617 
618     if 'packet_loss' in req_states:
619         # See if the OS pings.
620         cmd_buf = "ping -c 5 -w 5 " + openbmc_host +\
621             " | egrep 'packet loss' | sed -re 's/.* ([0-9]+)%.*/\\1/g'"
622         rc, out_buf = gc.shell_cmd(cmd_buf,
623                                    print_output=0, show_err=0,
624                                    ignore_err=1)
625         if rc == 0:
626             packet_loss = out_buf.rstrip("\n")
627 
628     if 'uptime' in req_states:
629         # Sometimes reading uptime results in a blank value. Call with
630         # wait_until_keyword_succeeds to ensure a non-blank value is obtained.
631         remote_cmd_buf = "read uptime filler 2>/dev/null < /proc/uptime" +\
632             " && [ ! -z \"${uptime}\" ] && echo ${uptime}"
633         cmd_buf = ["BMC Execute Command",
634                    re.sub('\\$', '\\$', remote_cmd_buf), 'quiet=1',
635                    'test_mode=0', 'time_out=5']
636         gp.qprint_issuing(cmd_buf, 0)
637         gp.qprint_issuing(remote_cmd_buf, 0)
638         try:
639             stdout, stderr, rc =\
640                 BuiltIn().wait_until_keyword_succeeds("10 sec", "5 sec",
641                                                       *cmd_buf)
642             if rc == 0 and stderr == "":
643                 uptime = stdout
644         except AssertionError as my_assertion_error:
645             pass
646 
647     if 'epoch_seconds' in req_states or 'elapsed_boot_time' in req_states:
648         date_cmd_buf = "date -u +%s"
649         if USE_BMC_EPOCH_TIME:
650             cmd_buf = ["BMC Execute Command", date_cmd_buf, 'quiet=${1}']
651             if not quiet:
652                 gp.print_issuing(cmd_buf)
653             status, ret_values = \
654                 BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
655             if status == "PASS":
656                 stdout, stderr, rc = ret_values
657                 if rc == 0 and stderr == "":
658                     epoch_seconds = stdout.rstrip("\n")
659         else:
660             shell_rc, out_buf = gc.cmd_fnc_u(date_cmd_buf,
661                                              quiet=quiet,
662                                              print_output=0)
663             if shell_rc == 0:
664                 epoch_seconds = out_buf.rstrip("\n")
665 
666     if 'elapsed_boot_time' in req_states:
667         global start_boot_seconds
668         elapsed_boot_time = int(epoch_seconds) - start_boot_seconds
669 
670     if not redfish_support_trans_state:
671         master_req_rest = ['rest', 'host', 'requested_host', 'operating_system',
672                            'attempts_left', 'boot_progress', 'chassis',
673                            'requested_chassis' 'bmc' 'requested_bmc']
674 
675         req_rest = [sub_state for sub_state in req_states if sub_state in
676                     master_req_rest]
677         need_rest = (len(req_rest) > 0)
678         state = DotDict()
679         if need_rest:
680             cmd_buf = ["Read Properties", SYSTEM_STATE_URI + "enumerate",
681                        "quiet=${" + str(quiet) + "}", "timeout=30"]
682             gp.dprint_issuing(cmd_buf)
683             status, ret_values = \
684                 BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
685             if status == "PASS":
686                 state['rest'] = '1'
687             else:
688                 state['rest'] = '0'
689 
690             if int(state['rest']):
691                 for url_path in ret_values:
692                     # Skip conflicting "CurrentHostState" URL from the enum
693                     # /xyz/openbmc_project/state/hypervisor0
694                     if "hypervisor0" in url_path:
695                         continue
696 
697                     if platform_arch_type == "x86":
698                         # Skip conflicting "CurrentPowerState" URL from the enum
699                         # /xyz/openbmc_project/state/chassis_system0
700                         if "chassis_system0" in url_path:
701                             continue
702 
703                     for attr_name in ret_values[url_path]:
704                         # Create a state key value based on the attr_name.
705                         try:
706                             ret_values[url_path][attr_name] = \
707                                 re.sub(r'.*\.', "",
708                                        ret_values[url_path][attr_name])
709                         except TypeError:
710                             pass
711                         # Do some key name manipulations.
712                         new_attr_name = re.sub(r'^Current|(State|Transition)$',
713                                                "", attr_name)
714                         new_attr_name = re.sub(r'BMC', r'Bmc', new_attr_name)
715                         new_attr_name = re.sub(r'([A-Z][a-z])', r'_\1',
716                                                new_attr_name)
717                         new_attr_name = new_attr_name.lower().lstrip("_")
718                         new_attr_name = re.sub(r'power', r'chassis', new_attr_name)
719                         if new_attr_name in req_states:
720                             state[new_attr_name] = ret_values[url_path][attr_name]
721     else:
722         master_req_rf = ['redfish', 'host', 'requested_host',
723                          'attempts_left', 'boot_progress', 'chassis',
724                          'requested_chassis' 'bmc' 'requested_bmc']
725 
726         req_rf = [sub_state for sub_state in req_states if sub_state in
727                   master_req_rf]
728         need_rf = (len(req_rf) > 0)
729         state = DotDict()
730         if need_rf:
731             cmd_buf = ["Redfish Get States"]
732             gp.dprint_issuing(cmd_buf)
733             try:
734                 status, ret_values = \
735                     BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
736             except Exception as ex:
737                 # Robot raised UserKeywordExecutionFailed error exception.
738                 gp.dprint_issuing("Retrying Redfish Get States")
739                 status, ret_values = \
740                     BuiltIn().run_keyword_and_ignore_error(*cmd_buf)
741 
742             gp.dprint_vars(status, ret_values)
743             if status == "PASS":
744                 state['redfish'] = '1'
745             else:
746                 state['redfish'] = '0'
747 
748             if int(state['redfish']):
749                 state['chassis'] = ret_values['chassis']
750                 state['boot_progress'] = ret_values['boot_progress']
751                 state['host'] = ret_values['host']
752                 state['bmc'] = ret_values['bmc']
753 
754     for sub_state in req_states:
755         if sub_state in state:
756             continue
757         if sub_state.startswith("os_"):
758             # We pass "os_" requests on to get_os_state.
759             continue
760         cmd_buf = "state['" + sub_state + "'] = str(" + sub_state + ")"
761         exec(cmd_buf)
762 
763     if os_host == "":
764         # The caller has not specified an os_host so as far as we're concerned,
765         # it doesn't exist.
766         return state
767 
768     os_req_states = [sub_state for sub_state in req_states
769                      if sub_state.startswith('os_')]
770 
771     if len(os_req_states) > 0:
772         # The caller has specified an os_host and they have requested
773         # information on os substates.
774 
775         # Based on the information gathered on bmc, we'll try to make a
776         # determination of whether the os is even up.  We'll pass the result
777         # of that assessment to get_os_state to enhance performance.
778         os_up_match = DotDict()
779         for sub_state in master_os_up_match:
780             if sub_state in req_states:
781                 os_up_match[sub_state] = master_os_up_match[sub_state]
782         os_up = compare_states(state, os_up_match)
783         os_state = get_os_state(os_host=os_host,
784                                 os_username=os_username,
785                                 os_password=os_password,
786                                 req_states=os_req_states,
787                                 os_up=os_up,
788                                 quiet=quiet)
789         # Append os_state dictionary to ours.
790         state.update(os_state)
791 
792     return state
793 
794 
795 exit_wait_early_message = ""
796 
797 
798 def set_exit_wait_early_message(value):
799     r"""
800     Set global exit_wait_early_message to the indicated value.
801 
802     This is a mechanism by which the programmer can do an early exit from
803     wait_until_keyword_succeeds() based on some special condition.
804 
805     Description of argument(s):
806     value                           The value to assign to the global
807                                     exit_wait_early_message.
808     """
809 
810     global exit_wait_early_message
811     exit_wait_early_message = value
812 
813 
814 def check_state(match_state,
815                 invert=0,
816                 print_string="",
817                 openbmc_host="",
818                 openbmc_username="",
819                 openbmc_password="",
820                 os_host="",
821                 os_username="",
822                 os_password="",
823                 quiet=None):
824     r"""
825     Check that the Open BMC machine's composite state matches the specified
826     state.  On success, this keyword returns the machine's composite state as a
827     dictionary.
828 
829     Description of argument(s):
830     match_state       A dictionary whose key/value pairs are "state field"/
831                       "state value".  The state value is interpreted as a
832                       regular expression.  Example call from robot:
833                       ${match_state}=  Create Dictionary  chassis=^On$
834                       ...  bmc=^Ready$
835                       ...  boot_progress=^OSStart$
836                       ${state}=  Check State  &{match_state}
837     invert            If this flag is set, this function will succeed if the
838                       states do NOT match.
839     print_string      This function will print this string to the console prior
840                       to getting the state.
841     openbmc_host      The DNS name or IP address of the BMC.
842                       This defaults to global ${OPENBMC_HOST}.
843     openbmc_username  The username to be used to login to the BMC.
844                       This defaults to global ${OPENBMC_USERNAME}.
845     openbmc_password  The password to be used to login to the BMC.
846                       This defaults to global ${OPENBMC_PASSWORD}.
847     os_host           The DNS name or IP address of the operating system.
848                       This defaults to global ${OS_HOST}.
849     os_username       The username to be used to login to the OS.
850                       This defaults to global ${OS_USERNAME}.
851     os_password       The password to be used to login to the OS.
852                       This defaults to global ${OS_PASSWORD}.
853     quiet             Indicates whether status details should be written to the
854                       console.  Defaults to either global value of ${QUIET} or
855                       to 1.
856     """
857 
858     quiet = int(gp.get_var_value(quiet, 0))
859 
860     gp.gp_print(print_string)
861 
862     try:
863         match_state = return_state_constant(match_state)
864     except TypeError:
865         pass
866 
867     req_states = list(match_state.keys())
868     # Remove special-case match key from req_states.
869     if expressions_key() in req_states:
870         req_states.remove(expressions_key())
871     # Initialize state.
872     state = get_state(openbmc_host=openbmc_host,
873                       openbmc_username=openbmc_username,
874                       openbmc_password=openbmc_password,
875                       os_host=os_host,
876                       os_username=os_username,
877                       os_password=os_password,
878                       req_states=req_states,
879                       quiet=quiet)
880     if not quiet:
881         gp.print_var(state)
882 
883     if exit_wait_early_message != "":
884         # The exit_wait_early_message has been set by a signal handler so we
885         # will exit "successfully".  It is incumbent upon the calling function
886         # (e.g. wait_state) to check/clear this variable and to fail
887         # appropriately.
888         return state
889 
890     match = compare_states(state, match_state)
891 
892     if invert and match:
893         fail_msg = "The current state of the machine matches the match" +\
894                    " state:\n" + gp.sprint_varx("state", state)
895         BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
896     elif not invert and not match:
897         fail_msg = "The current state of the machine does NOT match the" +\
898                    " match state:\n" +\
899                    gp.sprint_varx("state", state)
900         BuiltIn().fail("\n" + gp.sprint_error(fail_msg))
901 
902     return state
903 
904 
905 def wait_state(match_state=(),
906                wait_time="1 min",
907                interval="1 second",
908                invert=0,
909                openbmc_host="",
910                openbmc_username="",
911                openbmc_password="",
912                os_host="",
913                os_username="",
914                os_password="",
915                quiet=None):
916     r"""
917     Wait for the Open BMC machine's composite state to match the specified
918     state.  On success, this keyword returns the machine's composite state as
919     a dictionary.
920 
921     Description of argument(s):
922     match_state       A dictionary whose key/value pairs are "state field"/
923                       "state value".  See check_state (above) for details.
924                       This value may also be any string accepted by
925                       return_state_constant (e.g. "standby_match_state").
926                       In such a case this function will call
927                       return_state_constant to convert it to a proper
928                       dictionary as described above.
929     wait_time         The total amount of time to wait for the desired state.
930                       This value may be expressed in Robot Framework's time
931                       format (e.g. 1 minute, 2 min 3 s, 4.5).
932     interval          The amount of time between state checks.
933                       This value may be expressed in Robot Framework's time
934                       format (e.g. 1 minute, 2 min 3 s, 4.5).
935     invert            If this flag is set, this function will for the state of
936                       the machine to cease to match the match state.
937     openbmc_host      The DNS name or IP address of the BMC.
938                       This defaults to global ${OPENBMC_HOST}.
939     openbmc_username  The username to be used to login to the BMC.
940                       This defaults to global ${OPENBMC_USERNAME}.
941     openbmc_password  The password to be used to login to the BMC.
942                       This defaults to global ${OPENBMC_PASSWORD}.
943     os_host           The DNS name or IP address of the operating system.
944                       This defaults to global ${OS_HOST}.
945     os_username       The username to be used to login to the OS.
946                       This defaults to global ${OS_USERNAME}.
947     os_password       The password to be used to login to the OS.
948                       This defaults to global ${OS_PASSWORD}.
949     quiet             Indicates whether status details should be written to the
950                       console.  Defaults to either global value of ${QUIET} or
951                       to 1.
952     """
953 
954     quiet = int(gp.get_var_value(quiet, 0))
955 
956     try:
957         match_state = return_state_constant(match_state)
958     except TypeError:
959         pass
960 
961     if not quiet:
962         if invert:
963             alt_text = "cease to "
964         else:
965             alt_text = ""
966         gp.print_timen("Checking every " + str(interval) + " for up to "
967                        + str(wait_time) + " for the state of the machine to "
968                        + alt_text + "match the state shown below.")
969         gp.print_var(match_state)
970 
971     if quiet:
972         print_string = ""
973     else:
974         print_string = "#"
975 
976     debug = int(BuiltIn().get_variable_value("${debug}", "0"))
977     if debug:
978         # In debug we print state so no need to print the "#".
979         print_string = ""
980     check_state_quiet = 1 - debug
981     cmd_buf = ["Check State", match_state, "invert=${" + str(invert) + "}",
982                "print_string=" + print_string, "openbmc_host=" + openbmc_host,
983                "openbmc_username=" + openbmc_username,
984                "openbmc_password=" + openbmc_password, "os_host=" + os_host,
985                "os_username=" + os_username, "os_password=" + os_password,
986                "quiet=${" + str(check_state_quiet) + "}"]
987     gp.dprint_issuing(cmd_buf)
988     try:
989         state = BuiltIn().wait_until_keyword_succeeds(wait_time, interval,
990                                                       *cmd_buf)
991     except AssertionError as my_assertion_error:
992         gp.printn()
993         message = my_assertion_error.args[0]
994         BuiltIn().fail(message)
995 
996     if exit_wait_early_message:
997         # The global exit_wait_early_message was set by a signal handler
998         # indicating that we should fail.
999         message = exit_wait_early_message
1000         # Clear the exit_wait_early_message variable for future use.
1001         set_exit_wait_early_message("")
1002         BuiltIn().fail(gp.sprint_error(message))
1003 
1004     if not quiet:
1005         gp.printn()
1006         if invert:
1007             gp.print_timen("The states no longer match:")
1008         else:
1009             gp.print_timen("The states match:")
1010         gp.print_var(state)
1011 
1012     return state
1013 
1014 
1015 def set_start_boot_seconds(value=0):
1016     global start_boot_seconds
1017     start_boot_seconds = int(value)
1018 
1019 
1020 set_start_boot_seconds(0)
1021 
1022 
1023 def wait_for_comm_cycle(start_boot_seconds,
1024                         quiet=None):
1025     r"""
1026     Wait for the BMC uptime to be less than elapsed_boot_time.
1027 
1028     This function will tolerate an expected loss of communication to the BMC.
1029     This function is useful when some kind of reboot has been initiated by the
1030     caller.
1031 
1032     Description of argument(s):
1033     start_boot_seconds  The time that the boot test started.  The format is the
1034                         epoch time in seconds, i.e. the number of seconds since
1035                         1970-01-01 00:00:00 UTC.  This value should be obtained
1036                         from the BMC so that it is not dependent on any kind of
1037                         synchronization between this machine and the target BMC
1038                         This will allow this program to work correctly even in
1039                         a simulated environment.  This value should be obtained
1040                         by the caller prior to initiating a reboot.  It can be
1041                         obtained as follows:
1042                         state = st.get_state(req_states=['epoch_seconds'])
1043     """
1044 
1045     quiet = int(gp.get_var_value(quiet, 0))
1046 
1047     # Validate parms.
1048     error_message = gv.valid_integer(start_boot_seconds)
1049     if error_message:
1050         BuiltIn().fail(gp.sprint_error(error_message))
1051 
1052     # Wait for uptime to be less than elapsed_boot_time.
1053     set_start_boot_seconds(start_boot_seconds)
1054     expr = 'int(float(state[\'uptime\'])) < int(state[\'elapsed_boot_time\'])'
1055     match_state = DotDict([('uptime', '^[0-9\\.]+$'),
1056                            ('elapsed_boot_time', '^[0-9]+$'),
1057                            (expressions_key(), [expr])])
1058     wait_state(match_state, wait_time="12 mins", interval="5 seconds")
1059 
1060     gp.qprint_timen("Verifying that REST/Redfish API interface is working.")
1061     if not redfish_support_trans_state:
1062         match_state = DotDict([('rest', '^1$')])
1063     else:
1064         match_state = DotDict([('redfish', '^1$')])
1065     state = wait_state(match_state, wait_time="5 mins", interval="2 seconds")
1066