1*** Settings ***
2
3Documentation  Utilities for Robot keywords that do not use REST.
4
5Resource                ../lib/resource.robot
6Resource                ../lib/connection_client.robot
7Resource                ../lib/boot_utils.robot
8Library                 String
9Library                 DateTime
10Library                 Process
11Library                 OperatingSystem
12Library                 gen_print.py
13Library                 gen_robot_print.py
14Library                 gen_cmd.py
15Library                 gen_robot_keyword.py
16Library                 bmc_ssh_utils.py
17Library                 utils.py
18Library                 var_funcs.py
19Library                 SCPLibrary  WITH NAME  scp
20
21*** Variables ***
22
23${pflash_cmd}             /usr/sbin/pflash -r /dev/stdout -P VERSION
24
25${dbuscmdBase}
26...  dbus-send --system --print-reply --dest=${OPENBMC_BASE_DBUS}.settings.Host
27${dbuscmdGet}
28...  ${SETTINGS_URI}host0  org.freedesktop.DBus.Properties.Get
29${dbuscmdString}=  string:"xyz.openbmc_project.settings.Host" string:
30
31# Assign default value to QUIET for programs which may not define it.
32${QUIET}  ${0}
33
34${bmc_mem_free_cmd}=   free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f4
35${bmc_mem_total_cmd}=   free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f2
36${bmc_cpu_usage_cmd}=   top -n 1  | grep CPU: | cut -c 7-9
37${HOST_SETTING}    ${SETTINGS_URI}host0
38
39# /run/initramfs/ro associate filesystem  should be 100% full always
40${bmc_file_system_usage_cmd}=  df -h | cut -c 52-54 | grep 100 | wc -l
41${total_pnor_ro_file_system_cmd}=  df -h | grep /media/pnor-ro | wc -l
42${total_bmc_ro_file_system_cmd}=  df -h | grep /media/rofs | wc -l
43
44${BOOT_TIME}     ${0}
45${BOOT_COUNT}    ${0}
46${count}  ${0}
47${devicetree_base}  /sys/firmware/devicetree/base/model
48
49# Initialize default debug value to 0.
50${DEBUG}         ${0}
51
52${probe_cpu_tool_path}     ${EXECDIR}/tools/ras/probe_cpus.sh
53${scom_addrs_tool_path}    ${EXECDIR}/tools/ras/scom_addr_p9.sh
54${target_file_path}        /root/
55
56${default_tarball}  ${EXECDIR}/obmc-phosphor-debug-tarball-witherspoon.tar.xz
57
58# These variables are used to straddle between new and old methods of setting
59# values.
60${bmc_power_policy_method}        ${EMPTY}
61@{valid_power_policy_vars}        RESTORE_LAST_STATE  ALWAYS_POWER_ON
62...                               ALWAYS_POWER_OFF
63
64
65*** Keywords ***
66
67Check BMC Performance
68    [Documentation]  Check BMC basic CPU Mem File system performance.
69
70    Check BMC CPU Performance
71    Check BMC Mem Performance
72    Check BMC File System Performance
73
74
75Verify PNOR Update
76    [Documentation]  Verify that the PNOR is not corrupted.
77    # Example:
78    # FFS: Flash header not found. Code: 100
79    # Error 100 opening ffs !
80
81    ${stdout}  ${stderr}  ${rc}=
82    ...  BMC Execute Command  /usr/sbin/pflash -h | egrep -q skip
83    ...  ignore_err=${1}
84    ${pflash_cmd}=  Set Variable If  ${rc} == ${0}  ${pflash_cmd} --skip=4096
85    ...  ${pflash_cmd}
86    ${pnor_info}=  BMC Execute Command  ${pflash_cmd}
87    Should Not Contain Any  ${pnor_info}  Flash header not found  Error
88
89
90Get BMC System Model
91    [Documentation]  Get the BMC model from the device tree and return it.
92
93    ${bmc_model}  ${stderr}  ${rc}=  BMC Execute Command
94    ...  cat ${devicetree_base} | cut -d " " -f 1  return_stderr=True
95    ...  test_mode=0
96    Should Be Empty  ${stderr}
97    Should Not Be Empty  ${bmc_model}  msg=BMC model is empty.
98    [Return]  ${bmc_model}
99
100
101Verify BMC System Model
102    [Documentation]  Verify the BMC model with ${OPENBMC_MODEL}.
103    [Arguments]  ${bmc_model}
104
105    # Description of argument(s):
106    # bmc_model System model (e.g. "witherspoon").
107
108    ${tmp_bmc_model}=  Fetch From Right  ${OPENBMC_MODEL}  /
109    ${tmp_bmc_model}=  Fetch From Left  ${tmp_bmc_model}  .
110    ${ret}=  Run Keyword And Return Status  Should Contain  ${bmc_model}
111    ...  ${tmp_bmc_model}  ignore_case=True
112    [Return]  ${ret}
113
114
115Wait For Host To Ping
116    [Documentation]  Wait for the given host to ping.
117    [Arguments]  ${host}  ${timeout}=${OPENBMC_REBOOT_TIMEOUT}min
118    ...          ${interval}=5 sec
119
120    # Description of argument(s):
121    # host      The host name or IP of the host to ping.
122    # timeout   The amount of time after which ping attempts cease.
123    #           This should be expressed in Robot Framework's time format
124    #           (e.g. "10 seconds").
125    # interval  The amount of time in between attempts to ping.
126    #           This should be expressed in Robot Framework's time format
127    #           (e.g. "5 seconds").
128
129    Wait Until Keyword Succeeds  ${timeout}  ${interval}  Ping Host  ${host}
130
131
132Ping Host
133    [Documentation]  Ping the given host.
134    [Arguments]     ${host}
135
136    # Description of argument(s):
137    # host      The host name or IP of the host to ping.
138
139    Should Not Be Empty    ${host}   msg=No host provided
140    ${RC}   ${output}=     Run and return RC and Output    ping -c 4 ${host}
141    Log     RC: ${RC}\nOutput:\n${output}
142    Should be equal     ${RC}   ${0}
143
144
145Check OS
146    [Documentation]  Attempts to ping the host OS and then checks that the host
147    ...              OS is up by running an SSH command.
148
149    [Arguments]  ${os_host}=${OS_HOST}  ${os_username}=${OS_USERNAME}
150    ...          ${os_password}=${OS_PASSWORD}  ${quiet}=${QUIET}
151    ...          ${print_string}=${EMPTY}
152    [Teardown]  SSHLibrary.Close Connection
153
154    # Description of argument(s):
155    # os_host           The DNS name/IP of the OS host associated with our BMC.
156    # os_username       The username to be used to sign on to the OS host.
157    # os_password       The password to be used to sign on to the OS host.
158    # quiet             Indicates whether this keyword should write to console.
159    # print_string      A string to be printed before checking the OS.
160
161    Log To Console  ${print_string}  no_newline=True
162
163    # Attempt to ping the OS. Store the return code to check later.
164    ${ping_rc}=  Run Keyword and Return Status  Ping Host  ${os_host}
165
166    SSHLibrary.Open connection  ${os_host}
167
168    ${status}  ${msg}=  Run Keyword And Ignore Error  SSHLibrary.Login  ${os_username}
169    ...  ${os_password}
170    ${err_msg1}=  Sprint Error  ${msg}
171    ${err_msg}=  Catenate  SEPARATOR=  \n  ${err_msg1}
172    Run Keyword If  '${status}' == 'FAIL'  Fail  msg=${err_msg}
173    ${output}  ${stderr}  ${rc}=  Execute Command  uptime  return_stderr=True
174    ...        return_rc=True
175
176    ${temp_msg}=  Catenate  Could not execute a command on the operating
177    ...  system.\n
178    ${err_msg1}=  Sprint Error  ${temp_msg}
179    ${err_msg}=  Catenate  SEPARATOR=  \n  ${err_msg1}
180
181    # If the return code returned by "Execute Command" is non-zero, this
182    # keyword will fail.
183    Should Be Equal  ${rc}  ${0}  msg=${err_msg}
184    # We will likewise fail if there is any stderr data.
185    Should Be Empty  ${stderr}
186
187    ${temp_msg}=  Set Variable  Could not ping the operating system.\n
188    ${err_msg1}=  Sprint Error  ${temp_msg}
189    ${err_msg}=  Catenate  SEPARATOR=  \n  ${err_msg1}
190    # We will likewise fail if the OS did not ping, as we could SSH but not
191    # ping
192    Should Be Equal As Strings  ${ping_rc}  ${TRUE}  msg=${err_msg}
193
194
195Wait for OS
196    [Documentation]  Waits for the host OS to come up via calls to "Check OS".
197    [Arguments]  ${os_host}=${OS_HOST}  ${os_username}=${OS_USERNAME}
198    ...          ${os_password}=${OS_PASSWORD}  ${timeout}=${OS_WAIT_TIMEOUT}
199    ...          ${quiet}=${0}
200    [Teardown]  Printn
201
202    # Description of argument(s):
203    # os_host           The DNS name or IP of the OS host associated with our
204    #                   BMC.
205    # os_username       The username to be used to sign on to the OS host.
206    # os_password       The password to be used to sign on to the OS host.
207    # timeout           The timeout in seconds indicating how long you're
208    #                   willing to wait for the OS to respond.
209    # quiet             Indicates whether this keyword should write to console.
210
211    # The interval to be used between calls to "Check OS".
212    ${interval}=  Set Variable  5
213
214    ${message}=  Catenate  Checking every ${interval} seconds for up to
215    ...  ${timeout} seconds for the operating system to communicate.
216    Qprint Timen  ${message}
217
218    Wait Until Keyword Succeeds  ${timeout} sec  ${interval}  Check OS
219    ...                          ${os_host}  ${os_username}  ${os_password}
220    ...                          print_string=\#
221
222    Qprintn
223
224    Qprint Timen  The operating system is now communicating.
225
226
227Copy PNOR to BMC
228    [Documentation]  Copy the PNOR image to the BMC.
229    Import Library      SCPLibrary      WITH NAME       scp
230    Open Connection for SCP
231    Log    Copying ${PNOR_IMAGE_PATH} to /tmp
232    scp.Put File    ${PNOR_IMAGE_PATH}   /tmp
233
234
235Is OS Starting
236    [Documentation]  Check if boot progress is OS starting.
237    ${boot_progress}=  Get Boot Progress
238    Should Be Equal  ${boot_progress}  OSStart
239
240
241Is OS Off
242    [Documentation]  Check if boot progress is "Off".
243    ${boot_progress}=  Get Boot Progress
244    Should Be Equal  ${boot_progress}  Off
245
246
247Get Boot Progress To OS Starting State
248    [Documentation]  Get the system to a boot progress state of 'FW Progress,
249    ...  Starting OS'.
250
251    ${boot_progress}=  Get Boot Progress
252    Run Keyword If  '${boot_progress}' == 'OSStart'
253    ...  Log  Host is already in OS starting state
254    ...  ELSE
255    ...  Run Keywords  Initiate Host PowerOff  AND  Initiate Host Boot
256    ...  AND  Wait Until Keyword Succeeds  10 min  10 sec  Is OS Starting
257
258
259Check If warmReset is Initiated
260    [Documentation]  Ping would be still alive, so try SSH to connect
261    ...              if fails the ports are down indicating reboot
262    ...              is in progress
263
264    # Warm reset adds 3 seconds delay before forcing reboot
265    # To minimize race conditions, we wait for 7 seconds
266    Sleep  7s
267    ${alive}=   Run Keyword and Return Status
268    ...    Open Connection And Log In
269    Return From Keyword If   '${alive}' == '${False}'    ${False}
270    [Return]    ${True}
271
272
273Initialize DBUS cmd
274    [Documentation]  Initialize dbus string with property string to extract
275    [Arguments]   ${boot_property}
276
277    # Description of argument(s):
278    # boot_property   Property string.
279
280    ${cmd}=     Catenate  ${dbuscmdBase} ${dbuscmdGet} ${dbuscmdString}
281    ${cmd}=     Catenate  ${cmd}${boot_property}
282    Set Global Variable   ${dbuscmd}     ${cmd}
283
284
285Create OS Console Command String
286    [Documentation]  Return a command string to start OS console logging.
287
288    # First make sure that the ssh_pw program is available.
289    ${cmd}=  Catenate  which ssh_pw 2>/dev/null || find
290    ...  ${EXECDIR} -name 'ssh_pw'
291
292    Dprint Issuing  ${cmd}
293    ${rc}  ${output}=  Run And Return Rc And Output  ${cmd}
294    Rdpvars  rc  output
295
296    Should Be Equal As Integers  0  ${rc}  msg=Could not find ssh_pw.
297
298    ${ssh_pw_file_path}=  Set Variable  ${output}
299
300    ${cmd}=  Catenate  ${ssh_pw_file_path} ${OPENBMC_PASSWORD} -p 2200
301    ...  -o "StrictHostKeyChecking no" ${OPENBMC_USERNAME}@${OPENBMC_HOST}
302
303    [Return]  ${cmd}
304
305
306Get SOL Console Pid
307    [Documentation]  Get the pid of the active SOL console job.
308    [Arguments]  ${expect_running}=${0}
309
310    # Description of argument(s):
311    # expect_running  If set and if no SOL console job is found, print debug
312    #                 info and fail.
313
314    # Find the pid of the active system console logging session (if any).
315    ${search_string}=  Create OS Console Command String
316    # At least in some cases, ps output does not show double quotes so we must
317    # replace them in our search string with the regexes to indicate that they
318    # are optional.
319    ${search_string}=  Replace String  ${search_string}  "  ["]?
320    ${ps_cmd}=  Catenate  ps axwwo user,pid,cmd
321    ${cmd_buf}=  Catenate  echo $(${ps_cmd} | egrep '${search_string}' |
322    ...  egrep -v grep | cut -c10-14)
323    Dprint Issuing  ${cmd_buf}
324    ${rc}  ${os_con_pid}=  Run And Return Rc And Output  ${cmd_buf}
325    Rdpvars  os_con_pid
326    # If rc is not zero it just means that there is no OS Console process
327    # running.
328
329    Return From Keyword If  '${os_con_pid}' != '${EMPTY}'  ${os_con_pid}
330    Return From Keyword If  '${expect_running}' == '${0}'  ${os_con_pid}
331
332    Cmd Fnc  cat ${log_file_path} ; echo ; ${ps_cmd}  quiet=${0}
333    ...  print_output=${1}  show_err=${1}
334
335    Should Not Be Empty  ${os_con_pid}
336
337
338Stop SOL Console Logging
339    [Documentation]  Stop system console logging and return log output.
340    [Arguments]  ${log_file_path}=${EMPTY}
341    ...          ${targ_file_path}=${EXECDIR}${/}logs${/}
342    ...          ${return_data}=${1}
343
344    # If there are muliple system console processes, they will all be stopped.
345    # If there is no existing log file this keyword will return an error
346    # message to that effect (and write that message to targ_file_path, if
347    # specified).
348    # NOTE: This keyword will not fail if there is no running system console
349    # process.
350
351    # Description of arguments:
352    # log_file_path   The file path that was used to call "Start SOL
353    #                 Console Logging".  See that keyword (above) for details.
354    # targ_file_path  If specified, the file path to which the source
355    #                 file path (i.e. "log_file_path") should be copied.
356    # return_data     If this is set to ${1}, this keyword will return the SOL
357    #                 data to the caller as a unicode string.
358
359    ${log_file_path}=  Create OS Console File Path  ${log_file_path}
360
361    ${os_con_pid}=  Get SOL Console Pid
362
363    ${cmd_buf}=  Catenate  kill -9 ${os_con_pid}
364    Run Keyword If  '${os_con_pid}' != '${EMPTY}'  Dprint Issuing  ${cmd_buf}
365    ${rc}  ${output}=  Run Keyword If  '${os_con_pid}' != '${EMPTY}'
366    ...  Run And Return Rc And Output  ${cmd_buf}
367    Run Keyword If  '${os_con_pid}' != '${EMPTY}'  Rdpvars  rc  output
368
369    Run Keyword If  '${targ_file_path}' != '${EMPTY}'
370    ...  Run Keyword And Ignore Error
371    ...  Copy File  ${log_file_path}  ${targ_file_path}
372
373    ${output}=  Set Variable  ${EMPTY}
374    ${loc_quiet}=  Evaluate  ${debug}^1
375    ${rc}  ${output}=  Run Keyword If  '${return_data}' == '${1}'
376    ...  Cmd Fnc  cat ${log_file_path} 2>/dev/null  quiet=${loc_quiet}
377    ...  print_output=${0}  show_err=${0}
378
379    [Return]  ${output}
380
381
382Start SOL Console Logging
383    [Documentation]  Start system console log to file.
384    [Arguments]  ${log_file_path}=${EMPTY}  ${return_data}=${1}
385
386    # This keyword will first call "Stop SOL Console Logging".  Only then will
387    # it start SOL console logging.  The data returned by "Stop SOL Console
388    # Logging" will in turn be returned by this keyword.
389
390    # Description of arguments:
391    # log_file_path   The file path to which system console log data should be
392    #                 written.  Note that this path is taken to be a location
393    #                 on the machine where this program is running rather than
394    #                 on the Open BMC system.
395    # return_data     If this is set to ${1}, this keyword will return any SOL
396    #                 data to the caller as a unicode string.
397
398    ${log_file_path}=  Create OS Console File Path  ${log_file_path}
399
400    ${log_output}=  Stop SOL Console Logging  ${log_file_path}
401    ...  return_data=${return_data}
402
403    # Validate by making sure we can create the file.  Problems creating the
404    # file would not be noticed by the subsequent ssh command because we fork
405    # the command.
406    Create File  ${log_file_path}
407    ${sub_cmd_buf}=  Create OS Console Command String
408    # Routing stderr to stdout so that any startup error text will go to the
409    # output file.
410    # TODO: Doesn't work with tox so reverting temporarily.
411    # nohup detaches the process completely from our pty.
412    #${cmd_buf}=  Catenate  nohup ${sub_cmd_buf} &> ${log_file_path} &
413    ${cmd_buf}=  Catenate  ${sub_cmd_buf} > ${log_file_path} 2>&1 &
414    Dprint Issuing  ${cmd_buf}
415    ${rc}  ${output}=  Run And Return Rc And Output  ${cmd_buf}
416    # Because we are forking this command, we essentially will never get a
417    # non-zero return code or any output.
418    Should Be Equal  ${rc}  ${0}
419
420    Wait Until Keyword Succeeds  10 seconds  0 seconds
421    ...   Get SOL Console Pid  ${1}
422
423    [Return]  ${log_output}
424
425
426Get Time Stamp
427    [Documentation]     Get the current time stamp data
428    ${cur_time}=    Get Current Date   result_format=%Y%m%d%H%M%S%f
429    [Return]   ${cur_time}
430
431
432Start Journal Log
433    [Documentation]   Start capturing journal log to a file in /tmp using
434    ...               journalctl command. By default journal log is collected
435    ...               at /tmp/journal_log else user input location.
436    ...               The File is appended with datetime.
437    [Arguments]       ${file_path}=/tmp/journal_log  ${filter}=${EMPTY}
438
439    # Description of arguments:
440    # file_path   The file path of the journal file.
441
442    ${cur_time}=    Get Time Stamp
443    Set Global Variable   ${LOG_TIME}   ${cur_time}
444    Open Connection And Log In
445    Start Command
446    ...  journalctl -f ${filter} > ${file_path}-${LOG_TIME}
447    Log    Journal Log Started: ${file_path}-${LOG_TIME}
448
449
450Stop Journal Log
451    [Documentation]   Stop journalctl process if its running.
452    ...               By default return log from /tmp/journal_log else
453    ...               user input location.
454    [Arguments]       ${file_path}=/tmp/journal_log
455
456    # Description of arguments:
457    # file_path   The file path of the journal file.
458
459    Open Connection And Log In
460
461    ${rc}=
462    ...  Execute Command
463    ...  ps | grep journalctl | grep -v grep
464    ...  return_stdout=False  return_rc=True
465
466    Return From Keyword If   '${rc}' == '${1}'
467    ...   No journal log process running
468
469    ${output}  ${stderr}=
470    ...  Execute Command   killall journalctl
471    ...  return_stderr=True
472    Should Be Empty     ${stderr}
473
474    ${journal_log}  ${stderr}=
475    ...  Execute Command
476    ...  cat ${file_path}-${LOG_TIME}
477    ...  return_stderr=True
478    Should Be Empty     ${stderr}
479
480    Log    ${journal_log}
481
482    Execute Command    rm ${file_path}-${LOG_TIME}
483
484    [Return]    ${journal_log}
485
486
487Mac Address To Hex String
488    [Documentation]   Converts MAC address into hex format.
489    ...               Example
490    ...               Given the following MAC: 00:01:6C:80:02:78
491    ...               This keyword will return: 0x00 0x01 0x6C 0x80 0x02 0x78
492    ...               Description of arguments:
493    ...               i_macaddress  MAC address in the following format
494    ...               00:01:6C:80:02:78
495    [Arguments]    ${i_macaddress}
496
497    # Description of arguments:
498    # i_macaddress   The MAC address.
499
500    ${mac_hex}=  Catenate  0x${i_macaddress.replace(':', ' 0x')}
501    [Return]    ${mac_hex}
502
503
504IP Address To Hex String
505    [Documentation]   Converts IP address into hex format.
506    ...               Example:
507    ...               Given the following IP: 10.3.164.100
508    ...               This keyword will return: 0xa 0x3 0xa4 0xa0
509    [Arguments]    ${i_ipaddress}
510
511    # Description of arguments:
512    # i_macaddress   The IP address in the format 10.10.10.10.
513
514    @{ip}=  Split String  ${i_ipaddress}    .
515    ${index}=  Set Variable  ${0}
516
517    :FOR    ${item}     IN      @{ip}
518    \   ${hex}=  Convert To Hex    ${item}    prefix=0x    lowercase=yes
519    \   Set List Value    ${ip}    ${index}    ${hex}
520    \   ${index}=  Set Variable    ${index + 1}
521    ${ip_hex}=  Catenate    @{ip}
522
523    [Return]    ${ip_hex}
524
525
526BMC CPU Performance Check
527   [Documentation]   Minimal 10% of proc should be free in this instance
528
529    ${bmc_cpu_usage_output}  ${stderr}  ${rc}=  BMC Execute Command
530    ...  ${bmc_cpu_usage_cmd}
531    ${bmc_cpu_usage_output}  ${stderr}  ${rc}=  BMC Execute Command
532    ...  ${bmc_cpu_usage_cmd}
533    ${bmc_cpu_percentage}=  Fetch From Left  ${bmc_cpu_usage_output}  %
534    Rprint Vars  bmc_cpu_percentage
535    Should be true  ${bmc_cpu_percentage} < 90
536
537
538BMC Mem Performance Check
539    [Documentation]   Minimal 10% of memory should be free in this instance
540
541    ${bmc_mem_free_output}  ${stderr}  ${rc}=   BMC Execute Command
542    ...  ${bmc_mem_free_cmd}
543
544    ${bmc_mem_total_output}  ${stderr}  ${rc}=  BMC Execute Command
545    ...  ${bmc_mem_total_cmd}
546    ${bmc_mem_free_output}  ${stderr}  ${rc}=   BMC Execute Command
547    ...  ${bmc_mem_free_cmd}
548
549    ${bmc_mem_total_output}  ${stderr}  ${rc}=  BMC Execute Command
550    ...  ${bmc_mem_total_cmd}
551
552    ${bmc_mem_percentage}=  Evaluate  ${bmc_mem_free_output}*100
553    ${bmc_mem_percentage}=  Evaluate
554    ...   ${bmc_mem_percentage}/${bmc_mem_total_output}
555    Rprint Vars  bmc_mem_percentage
556    Should be true  ${bmc_mem_percentage} > 10
557
558
559BMC File System Usage Check
560    [Documentation]   Check the file system space. 4 file system should be
561    ...  100% full which is expected
562    # Filesystem            Size    Used Available Use% Mounted on
563    # /dev/root            14.4M     14.4M       0 100% /
564    # /dev/ubiblock0_0     14.4M     14.4M       0 100% /media/rofs-c9249b0e
565    # /dev/ubiblock8_0     19.6M     19.6M       0 100% /media/pnor-ro-8764baa3
566    # /dev/ubiblock4_0     14.4M     14.4M       0 100% /media/rofs-407816c
567    # /dev/ubiblock8_4     21.1M     21.1M       0 100% /media/pnor-ro-cecc64c4
568    ${bmc_fs_usage_output}  ${stderr}  ${rc}=  BMC Execute Command
569    ...  ${bmc_file_system_usage_cmd}
570    ${bmc_pnor_fs_usage_output}  ${stderr}  ${rc}=  BMC Execute Command
571    ...  ${total_pnor_ro_file_system_cmd}
572    ${bmc_bmc_fs_usage_output}  ${stderr}  ${rc}=  BMC Execute Command
573    ...  ${total_bmc_ro_file_system_cmd}
574    ${total_bmc_pnor_image}=  Evaluate
575    ...  ${bmc_pnor_fs_usage_output}+${bmc_bmc_fs_usage_output}
576    # Considering /dev/root also in total 100% used file system
577    ${total_full_fs}=  Evaluate  ${total_bmc_pnor_image}+1
578    Rprint Vars  bmc_fs_usage_output  bmc_pnor_fs_usage_output  bmc_bmc_fs_usage_output
579    ...  total_full_fs
580    Should Be True  ${bmc_fs_usage_output}==${total_full_fs}
581
582
583Check BMC CPU Performance
584    [Documentation]   Minimal 10% of proc should be free in 3 sample
585    :FOR  ${var}  IN RANGE  1  4
586    \     BMC CPU Performance check
587
588
589Check BMC Mem Performance
590    [Documentation]   Minimal 10% of memory should be free
591
592    :FOR  ${var}  IN RANGE  1  4
593    \     BMC Mem Performance check
594
595
596Check BMC File System Performance
597    [Documentation]  Check for file system usage for 4 times
598
599    :FOR  ${var}  IN RANGE  1  4
600    \     BMC File System Usage check
601
602
603Get URL List
604    [Documentation]  Return list of URLs under given URL.
605    [Arguments]  ${openbmc_url}
606
607    # Description of argument(s):
608    # openbmc_url  URL for list operation (e.g.
609    #              /xyz/openbmc_project/inventory).
610
611    ${url_list}=  Read Properties  ${openbmc_url}list  quiet=${1}
612    Sort List  ${url_list}
613
614    [Return]  ${url_list}
615
616
617Check Zombie Process
618    [Documentation]    Check if any defunct process exist or not on BMC
619    ${count}  ${stderr}  ${rc}=  Execute Command  ps -o stat | grep Z | wc -l
620    ...    return_stderr=True  return_rc=True
621    Should Be True    ${count}==0
622    Should Be Empty    ${stderr}
623
624
625Prune Journal Log
626    [Documentation]   Prune archived journal logs.
627    [Arguments]   ${vacuum_size}=1M
628
629    # This keyword can be used to prevent the journal
630    # log from filling up the /run filesystem.
631    # This command will retain only the latest logs
632    # of the user specified size.
633
634    # Description of argument(s):
635    # vacuum_size    Size of journal.
636
637    Open Connection And Log In
638    ${output}  ${stderr}  ${rc}=
639    ...  Execute Command
640    ...  journalctl --vacuum-size=${vacuum_size}
641    ...  return_stderr=True  return_rc=True
642
643    Should Be Equal  ${rc}  ${0}  msg=${stderr}
644
645
646Get System Power Policy
647    [Documentation]  Returns the BMC power policy.
648
649    # Set the bmc_power_policy_method to either 'Old' or 'New'.
650    Set Power Policy Method
651    ${cmd_buf}=  Create List  ${bmc_power_policy_method} Get Power Policy
652    # Run the appropriate keyword.
653    ${currentPolicy}=  Run Keyword  @{cmd_buf}
654
655    [Return]  ${currentPolicy}
656
657
658Set BMC Reset Reference Time
659    [Documentation]  Set current boot time as a reference and increment
660    ...              boot count.
661
662    ${cur_btime}=  Get BMC Boot Time
663    Run Keyword If  ${BOOT_TIME} == ${0} and ${BOOT_COUNT} == ${0}
664    ...  Set Global Variable  ${BOOT_TIME}  ${cur_btime}
665    ...  ELSE IF  ${cur_btime} > ${BOOT_TIME}
666    ...  Run Keywords  Set Global Variable  ${BOOT_TIME}  ${cur_btime}
667    ...  AND
668    ...  Set Global Variable  ${BOOT_COUNT}  ${BOOT_COUNT + 1}
669
670
671Get BMC Boot Time
672    [Documentation]  Returns boot time from /proc/stat.
673
674    Open Connection And Log In
675    ${output}  ${stderr}=
676    ...  Execute Command  egrep '^btime ' /proc/stat | cut -f 2 -d ' '
677    ...  return_stderr=True
678    Should Be Empty  ${stderr}
679    ${btime}=  Convert To Integer  ${output}
680    [Return]  ${btime}
681
682
683Enable Core Dump On BMC
684    [Documentation]  Enable core dump collection.
685    ${core_pattern}  ${stderr}  ${rc}=  BMC Execute Command
686    ...  echo '/tmp/core_%e.%p' | tee /proc/sys/kernel/core_pattern
687    Should Be Equal As Strings  ${core_pattern}  /tmp/core_%e.%p
688
689
690Get Number Of BMC Core Dump Files
691    [Documentation]  Returns number of core dump files on BMC.
692    Open Connection And Log In
693    ${num_of_core_dump}=  Execute Command
694    ...  ls /tmp/core* 2>/dev/null | wc -l
695    [Return]  ${num_of_core_dump}
696
697
698Set Core Dump File Size Unlimited
699    [Documentation]  Set core dump file size to unlimited.
700    BMC Execute Command  ulimit -c unlimited
701
702
703Check For Core Dumps
704    [Documentation]  Check for any core dumps exist.
705    ${output}=  Get Number Of BMC Core Dump Files
706    Run Keyword If  ${output} > 0
707    ...  Log  **Warning** BMC core dump files exist  level=WARN
708
709
710Configure Initial Settings
711    [Documentation]  Restore old IP and route.
712    ...  This keyword requires initial settings viz IP address,
713    ...  Network Mask, default gatway and serial console IP and port
714    ...  information which should be provided in command line.
715
716    [Arguments]  ${host}=${OPENBMC_HOST}  ${mask}=${NET_MASK}
717    ...          ${gw_ip}=${GW_IP}
718
719    # Description of arguments:
720    # host  IP address of the OS Host.
721    # mask  Network mask.
722    # gu_ip  Gateway IP address or hostname.
723
724    # Open telnet connection and ignore the error, in case telnet session is
725    # already opened by the program calling this keyword.
726    Run Keyword And Ignore Error  Open Telnet Connection to BMC Serial Console
727    Telnet.write  ifconfig eth0 ${host} netmask ${mask}
728    Telnet.write  route add default gw ${gw_ip}
729
730
731Install Debug Tarball On BMC
732    [Documentation]  Copy the debug tar file to BMC and install.
733    [Arguments]  ${tarball_file_path}=${default_tarball}
734    ...  ${targ_tarball_dir_path}=/tmp/tarball/
735
736    # Description of arguments:
737    # tarball_file_path      Path of the debug tarball file.
738    #                        The tar file is downloaded from the build page
739    #                        https://openpower.xyz/job/openbmc-build/
740    #                        obmc-phosphor-debug-tarball-witherspoon.tar.xz
741    #
742    # targ_tarball_dir_path  The directory path where the tarball is to be
743    #                        installed.
744
745    OperatingSystem.File Should Exist  ${tarball_file_path}
746    ...  msg=${tarball_file_path} doesn't exist.
747
748    # Upload the file to BMC.
749    Import Library  SCPLibrary  WITH NAME  scp
750    Open Connection for SCP
751    scp.Put File  ${tarball_file_path}  /tmp/debug-tarball.tar.xz
752
753    # Create tarball directory and install.
754    BMC Execute Command  mkdir -p ${targ_tarball_dir_path}
755    BMC Execute Command
756    ...  tar -xf /tmp/debug-tarball.tar.xz -C ${targ_tarball_dir_path}
757
758    # Remove the tarball file from BMC.
759    BMC Execute Command  rm -f /tmp/debug-tarball.tar.xz
760
761
762Get BMC Boot Count
763    [Documentation]  Returns BMC boot count based on boot time.
764    ${cur_btime}=  Get BMC Boot Time
765
766    # Set global variable BOOT_TIME to current boot time if current boot time
767    # is changed. Also increase value of global variable BOOT_COUNT by 1.
768    Run Keyword If  ${cur_btime} > ${BOOT_TIME}
769    ...  Run Keywords  Set Global Variable  ${BOOT_TIME}  ${cur_btime}
770    ...  AND
771    ...  Set Global Variable  ${BOOT_COUNT}  ${BOOT_COUNT + 1}
772
773    [Return]  ${BOOT_COUNT}
774
775
776Set BMC Boot Count
777    [Documentation]  Set BMC boot count to given value.
778    [Arguments]  ${count}
779
780    # Description of arguments:
781    # count  boot count value.
782    ${cur_btime}=  Get BMC Boot Time
783
784    # Set global variable BOOT_COUNT to given value.
785    Set Global Variable  ${BOOT_COUNT}  ${count}
786
787    # Set BOOT_TIME variable to current boot time.
788    Set Global Variable  ${BOOT_COUNT}  ${count}
789
790
791Delete Error Log Entry
792    [Documentation]  Delete error log entry.
793    [Arguments]  ${entry_path}
794
795    # Description of argument(s):
796    # entry_path  Delete an error log entry.
797    #             Ex. /xyz/openbmc_project/logging/entry/1
798
799    # Skip delete if entry URI is a callout.
800    # Examples:
801    # /xyz/openbmc_project/logging/entry/1/callout
802    # /xyz/openbmc_project/logging/entry/1/callouts/0
803    ${callout_entry}=  Run Keyword And Return Status
804    ...  Should Match Regexp  ${entry_path}  /callout[s]?(/|$)
805    Return From Keyword If  ${callout_entry}
806
807    ${data}=  Create Dictionary  data=@{EMPTY}
808    ${resp}=  Openbmc Delete Request  ${entry_path}  data=${data}
809    Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
810
811
812Get BMC Version
813    [Documentation]  Returns BMC version from /etc/os-release.
814    ...              e.g. "v1.99.6-141-ge662190"
815
816    ${cmd}=  Set Variable  grep ^VERSION_ID= /etc/os-release | cut -f 2 -d '='
817    ${output}  ${stderr}  ${rc}=  BMC Execute Command  ${cmd}
818    [Return]  ${output}
819
820
821Get PNOR Version
822    [Documentation]  Returns the PNOR version from the BMC.
823
824    ${pnor_attrs}=  Get PNOR Attributes
825    [Return]  ${pnor_attrs['version']}
826
827
828Get PNOR Attributes
829    [Documentation]  Return PNOR software attributes as a dictionary.
830
831    # This keyword parses /var/lib/phosphor-software-manager/pnor/ro/pnor.toc
832    # into key/value pairs.
833
834    ${outbuf}  ${stderr}  ${rc}=  BMC Execute Command
835    ...  cat /var/lib/phosphor-software-manager/pnor/ro/pnor.toc
836    ${pnor_attrs}=  Key Value Outbuf To Dict  ${outbuf}  delim==
837
838    [Return]  ${pnor_attrs}
839
840
841GET BMC PNOR Version
842    [Documentation]  Return BMC & PNOR version from openbmc shell.
843
844    ${bmc_version}=  GET BMC Version
845    ${pnor_version}=  GET PNOR Version
846    Log  ${bmc_version}
847    Rprint Vars  bmc_version
848    Log  ${pnor_version}
849    Rprint Vars  pnor_version
850
851    [Return]  ${bmc_version}  ${pnor_version}
852
853
854Copy Address Translation Utils To HOST OS
855    [Documentation]  Copy address translation utils to host OS.
856
857    OperatingSystem.File Should Exist  ${probe_cpu_tool_path}
858    ...  msg=${probe_cpu_tool_path} doesn't exist.
859    OperatingSystem.File Should Exist  ${probe_cpu_tool_path}
860    ...  msg=${probe_cpu_tool_path} doesn't exist.
861
862    scp.Open connection  ${OS_HOST}  username=${OS_USERNAME}
863    ...  password=${OS_PASSWORD}
864    scp.Put File  ${probe_cpu_tool_path}  ${target_file_path}
865    scp.Put File  ${scom_addrs_tool_path}  ${target_file_path}
866
867
868Verify BMC RTC And UTC Time Drift
869    [Documentation]  Verify that the RTC and UTC time difference is less than
870    ...              the given time_drift_max.
871    [Arguments]  ${time_diff_max}=${10}
872
873    # Description of argument(s):
874    # time_diff_max   The max allowable RTC and UTC time difference in seconds.
875
876    # Example:
877    # time_dict:
878    #   [local_time]:               Fri 2017-11-03 152756 UTC
879    #   [local_time_seconds]:       1509740876
880    #   [universal_time]:           Fri 2017-11-03 152756 UTC
881    #   [universal_time_seconds]:   1509740876
882    #   [rtc_time]:                 Fri 2016-05-20 163403
883    #   [rtc_time_seconds]:         1463780043
884    #   [time_zone]:                n/a (UTC, +0000)
885    #   [network_time_on]:          yes
886    #   [ntp_synchronized]:         no
887    #   [rtc_in_local_tz]:          no
888
889    ${time}=  Get BMC Date Time
890    ${time_diff}=  Evaluate
891    ...  ${time['universal_time_seconds']} - ${time['rtc_time_seconds']}
892    Should Be True  ${time_diff} < ${time_diff_max}
893
894
895Validate IP On BMC
896    [Documentation]  Validate IP address is present in set of IP addresses.
897    [Arguments]  ${ip_address}  ${ip_data}
898
899    # Description of argument(s):
900    # ip_address  IP address to check (e.g. xx.xx.xx.xx).
901    # ip_data     Set of the IP addresses present.
902
903    Should Contain Match  ${ip_data}  ${ip_address}/*
904    ...  msg=${ip_address} not found in the list provided.
905
906
907Remove Journald Logs
908    [Documentation]  Remove all journald logs and restart service.
909
910    ${cmd}=  Catenate  systemctl stop systemd-journald.service &&
911    ...  rm -rf /var/log/journal && systemctl start systemd-journald.service
912
913    BMC Execute Command  ${cmd}
914
915
916Check For Regex In Journald
917    [Documentation]  Parse the journal log and check for regex string.
918    [Arguments]  ${regex}=${ERROR_REGEX}  ${error_check}=${0}  ${boot}=${EMPTY}
919
920    # Description of argument(s):
921    # regex            Strings to be filter.
922    # error_check      Check for errors.
923    # boot             Argument to check current or persistent full boot log
924    #                  (e.g. "-b").
925
926    ${journal_log}  ${stderr}  ${rc}=  BMC Execute Command
927    ...  journalctl --no-pager ${boot} | egrep '${regex}'  ignore_err=1
928
929    Run Keyword If  ${error_check} == ${0}
930    ...    Should Be Empty  ${journal_log}
931    ...  ELSE
932    ...    Should Not Be Empty  ${journal_log}
933
934
935Get Service Attribute
936    [Documentation]  Get service attribute policy output.
937    [Arguments]  ${option}  ${servicename}
938
939    # Description of argument(s):
940    # option       systemctl supported options
941    # servicename  Qualified service name
942    ${cmd}=  Set Variable
943    ...  systemctl -p ${option} show ${servicename} | cut -d = -f2
944    ${attr}  ${stderr}  ${rc}=  BMC Execute Command  ${cmd}
945    [Return]  ${attr}
946
947
948Verify Watchdog Enabled
949    [Documentation]  Check that watchdog settings are enabled.
950
951    ${properties}=  Read Properties  /xyz/openbmc_project/watchdog/host0
952    Should Be Equal As Strings  ${properties["Enabled"]}  ${True}
953    Should Not Be Equal As Strings  ${properties["TimeRemaining"]}  0
954