1*** Settings ***
2Resource                ../lib/resource.txt
3Resource                ../lib/rest_client.robot
4Resource                ../lib/connection_client.robot
5Library                 DateTime
6Library                 Process
7Library                 OperatingSystem
8Library                 gen_print.py
9Library                 gen_robot_print.py
10
11*** Variables ***
12${SYSTEM_SHUTDOWN_TIME}       ${5}
13${dbuscmdBase}
14...  dbus-send --system --print-reply --dest=${OPENBMC_BASE_DBUS}.settings.Host
15${dbuscmdGet}
16...  ${SETTINGS_URI}host0  org.freedesktop.DBus.Properties.Get
17# Enable when ready with openbmc/openbmc-test-automation#203
18#${dbuscmdString}=  string:"xyz.openbmc_project.settings.Host" string:
19${dbuscmdString}=   string:"org.openbmc.settings.Host" string:
20
21# Assign default value to QUIET for programs which may not define it.
22${QUIET}  ${0}
23${bmc_mem_free_cmd}=   free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f4
24${bmc_mem_total_cmd}=   free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f2
25${bmc_cpu_usage_cmd}=   top -n 1  | grep CPU: | cut -c 7-9
26${HOST_SETTING}    ${SETTINGS_URI}host0
27# /run/initramfs/ro associate filesystem  should be 100% full always
28${bmc_file_system_usage_cmd}=
29...  df -h | grep -v /run/initramfs/ro | cut -c 52-54 | grep 100 | wc -l
30
31${BOOT_TIME}     ${0}
32${BOOT_COUNT}    ${0}
33${devicetree_base}  /sys/firmware/devicetree/base/model
34
35*** Keywords ***
36
37Get BMC System Model
38    [Documentation]  Get the BMC model from the device tree.
39
40    ${bmc_model}  ${stderr}=  Execute Command  cat ${devicetree_base} | cut -f 1
41    ...  -d ${SPACE}  return_stderr=True
42    Should Be Empty  ${stderr}
43    Should Not Be Empty  ${bmc_model}
44    [Return]  ${bmc_model}
45
46Verify BMC System Model
47    [Documentation]  Verify the BMC model with ${OPENBMC_MODEL}.
48    [Arguments]  ${bmc_model}
49
50    ${tmp_bmc_model}=  Fetch From Right  ${OPENBMC_MODEL}  /
51    ${tmp_bmc_model}=  Fetch From Left  ${tmp_bmc_model}  .
52    ${ret}=  Run Keyword And Return Status  Should Contain  ${bmc_model}
53    ...  ${tmp_bmc_model}  ignore_case=True
54    [Return]  ${ret}
55
56Wait For Host To Ping
57    [Arguments]  ${host}  ${timeout}=${OPENBMC_REBOOT_TIMEOUT}min
58    ...          ${interval}=5 sec
59
60    # host      The DNS name or IP of the host to ping.
61    # timeout   The amount of time after which attempts to ping cease.
62    # interval  The amount of time in between attempts to ping.
63
64    Wait Until Keyword Succeeds  ${timeout}  ${interval}  Ping Host  ${host}
65
66Ping Host
67    [Arguments]     ${host}
68    Should Not Be Empty    ${host}   msg=No host provided
69    ${RC}   ${output}=     Run and return RC and Output    ping -c 4 ${host}
70    Log     RC: ${RC}\nOutput:\n${output}
71    Should be equal     ${RC}   ${0}
72
73Get Boot Progress
74    [Arguments]  ${quiet}=${QUIET}
75
76    ${state}=  Read Attribute  ${OPENBMC_BASE_URI}sensors/host/BootProgress
77    ...  value  quiet=${quiet}
78    [Return]  ${state}
79
80Is Power On
81    ${state}=  Get Power State
82    Should be equal  ${state}  ${1}
83
84Is Power Off
85    ${state}=  Get Power State
86    Should be equal  ${state}  ${0}
87
88Initiate Power On
89    [Documentation]  Initiates the power on and waits until the Is Power On
90    ...  keyword returns that the power state has switched to on.
91    [Arguments]  ${wait}=${1}
92
93    @{arglist}=   Create List
94    ${args}=     Create Dictionary    data=@{arglist}
95    ${resp}=  Call Method  ${OPENBMC_BASE_URI}control/chassis0/  powerOn
96    ...  data=${args}
97    should be equal as strings      ${resp.status_code}     ${HTTP_OK}
98
99    # Does caller want to wait for power on status?
100    Run Keyword If  '${wait}' == '${0}'  Return From Keyword
101    Wait Until Keyword Succeeds  3 min  10 sec  Is Power On
102
103Initiate Power Off
104    [Documentation]  Initiates the power off and waits until the Is Power Off
105    ...  keyword returns that the power state has switched to off.
106    @{arglist}=   Create List
107    ${args}=     Create Dictionary    data=@{arglist}
108    ${resp}=  Call Method  ${OPENBMC_BASE_URI}control/chassis0/  powerOff
109    ...  data=${args}
110    should be equal as strings      ${resp.status_code}     ${HTTP_OK}
111    Wait Until Keyword Succeeds  1 min  10 sec  Is Power Off
112
113Trigger Warm Reset
114    log to console    "Triggering warm reset"
115    ${data}=   create dictionary   data=@{EMPTY}
116    ${resp}=  openbmc post request
117    ...  ${OPENBMC_BASE_URI}control/bmc0/action/warmReset  data=${data}
118    Should Be Equal As Strings      ${resp.status_code}     ${HTTP_OK}
119    ${session_active}=   Check If warmReset is Initiated
120    Run Keyword If   '${session_active}' == '${True}'
121    ...    Fail   msg=warm reset didn't occur
122
123    Sleep   ${SYSTEM_SHUTDOWN_TIME}min
124    Check If BMC Is Up
125
126Check OS
127    [Documentation]  Attempts to ping the host OS and then checks that the host
128    ...              OS is up by running an SSH command.
129
130    [Arguments]  ${os_host}=${OS_HOST}  ${os_username}=${OS_USERNAME}
131    ...          ${os_password}=${OS_PASSWORD}  ${quiet}=${QUIET}
132    ...          ${print_string}=${EMPTY}
133    [Teardown]  Close Connection
134
135    # os_host           The DNS name/IP of the OS host associated with our BMC.
136    # os_username       The username to be used to sign on to the OS host.
137    # os_password       The password to be used to sign on to the OS host.
138    # quiet             Indicates whether this keyword should write to console.
139    # print_string      A string to be printed before checking the OS.
140
141    rprint  ${print_string}
142
143    # Attempt to ping the OS. Store the return code to check later.
144    ${ping_rc}=  Run Keyword and Return Status  Ping Host  ${os_host}
145
146    Open connection  ${os_host}
147
148    ${status}  ${msg}=  Run Keyword And Ignore Error  Login  ${os_username}
149    ...  ${os_password}
150    ${err_msg1}=  Sprint Error  ${msg}
151    ${err_msg}=  Catenate  SEPARATOR=  \n  ${err_msg1}
152    Run Keyword If  '${status}' == 'FAIL'  Fail  msg=${err_msg}
153    ${output}  ${stderr}  ${rc}=  Execute Command  uptime  return_stderr=True
154    ...        return_rc=True
155
156    ${temp_msg}=  Catenate  Could not execute a command on the operating
157    ...  system.\n
158    ${err_msg1}=  Sprint Error  ${temp_msg}
159    ${err_msg}=  Catenate  SEPARATOR=  \n  ${err_msg1}
160
161    # If the return code returned by "Execute Command" is non-zero, this
162    # keyword will fail.
163    Should Be Equal  ${rc}  ${0}  msg=${err_msg}
164    # We will likewise fail if there is any stderr data.
165    Should Be Empty  ${stderr}
166
167    ${temp_msg}=  Set Variable  Could not ping the operating system.\n
168    ${err_msg1}=  Sprint Error  ${temp_msg}
169    ${err_msg}=  Catenate  SEPARATOR=  \n  ${err_msg1}
170    # We will likewise fail if the OS did not ping, as we could SSH but not
171    # ping
172    Should Be Equal As Strings  ${ping_rc}  ${TRUE}  msg=${err_msg}
173
174Wait for OS
175    [Documentation]  Waits for the host OS to come up via calls to "Check OS".
176    [Arguments]  ${os_host}=${OS_HOST}  ${os_username}=${OS_USERNAME}
177    ...          ${os_password}=${OS_PASSWORD}  ${timeout}=${OS_WAIT_TIMEOUT}
178    ...          ${quiet}=${0}
179    [Teardown]  rprintn
180
181    # os_host           The DNS name or IP of the OS host associated with our
182    #                   BMC.
183    # os_username       The username to be used to sign on to the OS host.
184    # os_password       The password to be used to sign on to the OS host.
185    # timeout           The timeout in seconds indicating how long you're
186    #                   willing to wait for the OS to respond.
187    # quiet             Indicates whether this keyword should write to console.
188
189    # The interval to be used between calls to "Check OS".
190    ${interval}=  Set Variable  5
191
192    ${message}=  Catenate  Checking every ${interval} seconds for up to
193    ...  ${timeout} seconds for the operating system to communicate.
194    rqprint_timen  ${message}
195
196    Wait Until Keyword Succeeds  ${timeout} sec  ${interval}  Check OS
197    ...                          ${os_host}  ${os_username}  ${os_password}
198    ...                          print_string=\#
199
200    rqprintn
201
202    rqprint_timen  The operating system is now communicating.
203
204Get BMC State Deprecated
205    [Documentation]  Returns the state of the BMC as a string. (i.e: BMC_READY)
206    [Arguments]  ${quiet}=${QUIET}
207
208    @{arglist}=  Create List
209    ${args}=  Create Dictionary  data=@{arglist}
210    ${resp}=  Call Method  ${OPENBMC_BASE_URI}managers/System/  getSystemState
211    ...        data=${args}  quiet=${quiet}
212    Should be equal as strings  ${resp.status_code}  ${HTTP_OK}
213    ${content}=  to json  ${resp.content}
214    [Return]  ${content["data"]}
215
216Get Power State
217    [Documentation]  Returns the power state as an integer. Either 0 or 1.
218    [Arguments]  ${quiet}=${QUIET}
219
220    @{arglist}=  Create List
221    ${args}=  Create Dictionary  data=@{arglist}
222
223    ${resp}=  Call Method  ${OPENBMC_BASE_URI}control/chassis0/  getPowerState
224    ...        data=${args}  quiet=${quiet}
225    Should be equal as strings  ${resp.status_code}  ${HTTP_OK}
226    ${content}=  to json  ${resp.content}
227    [Return]  ${content["data"]}
228
229Clear BMC Record Log
230    [Documentation]  Clears all the event logs on the BMC. This would be
231    ...              equivalent to ipmitool sel clear.
232    @{arglist}=   Create List
233    ${args}=     Create Dictionary    data=@{arglist}
234    ${resp}=  Call Method
235    ...  ${OPENBMC_BASE_URI}records/events/  clear  data=${args}
236    should be equal as strings      ${resp.status_code}     ${HTTP_OK}
237
238Copy PNOR to BMC
239    Import Library      SCPLibrary      WITH NAME       scp
240    Open Connection for SCP
241    Log    Copying ${PNOR_IMAGE_PATH} to /tmp
242    scp.Put File    ${PNOR_IMAGE_PATH}   /tmp
243
244Flash PNOR
245    [Documentation]    Calls flash bios update method to flash PNOR image
246    [Arguments]    ${pnor_image}
247    @{arglist}=   Create List    ${pnor_image}
248    ${args}=     Create Dictionary    data=@{arglist}
249    ${resp}=  Call Method  ${OPENBMC_BASE_URI}control/flash/bios/  update
250    ...  data=${args}
251    should be equal as strings      ${resp.status_code}     ${HTTP_OK}
252    Wait Until Keyword Succeeds    2 min   10 sec    Is PNOR Flashing
253
254Get Flash BIOS Status
255    [Documentation]  Returns the status of the flash BIOS API as a string. For
256    ...              example 'Flashing', 'Flash Done', etc
257    ${data}=      Read Properties     ${OPENBMC_BASE_URI}control/flash/bios
258    [Return]    ${data['status']}
259
260Is PNOR Flashing
261    [Documentation]  Get BIOS 'Flashing' status. This indicates that PNOR
262    ...              flashing has started.
263    ${status}=    Get Flash BIOS Status
264    should be equal as strings     ${status}     Flashing
265
266Is PNOR Flash Done
267    [Documentation]  Get BIOS 'Flash Done' status.  This indicates that the
268    ...              PNOR flashing has completed.
269    ${status}=    Get Flash BIOS Status
270    should be equal as strings     ${status}     Flash Done
271
272Is System State Host Booted
273    [Documentation]  Checks whether system state is HOST_BOOTED.
274    ${state}=    Get BMC State Deprecated
275    should be equal as strings     ${state}     HOST_BOOTED
276
277Is OS Starting
278    [Documentation]  Check if boot progress is OS starting.
279    ${boot_progress}=  Get Boot Progress
280    Should Be Equal  ${boot_progress}  FW Progress, Starting OS
281
282Verify Ping and REST Authentication
283    ${l_ping}=   Run Keyword And Return Status
284    ...    Ping Host  ${OPENBMC_HOST}
285    Run Keyword If  '${l_ping}' == '${False}'
286    ...    Fail   msg=Ping Failed
287
288    ${l_rest}=   Run Keyword And Return Status
289    ...    Initialize OpenBMC
290    Run Keyword If  '${l_rest}' == '${False}'
291    ...    Fail   msg=REST Authentication Failed
292
293    # Just to make sure the SSH is working for SCP
294    Open Connection And Log In
295    ${system}   ${stderr}=    Execute Command   hostname   return_stderr=True
296    Should Be Empty     ${stderr}
297
298Check If BMC is Up
299    [Documentation]  Wait for Host to be online. Checks every X seconds
300    ...              interval for Y minutes and fails if timed out.
301    ...              Default MAX timedout is 10 min, interval 10 seconds.
302    [Arguments]      ${max_timeout}=${OPENBMC_REBOOT_TIMEOUT} min
303    ...              ${interval}=10 sec
304
305    Wait Until Keyword Succeeds
306    ...   ${max_timeout}  ${interval}   Verify Ping and REST Authentication
307
308
309Check If warmReset is Initiated
310    [Documentation]  Ping would be still alive, so try SSH to connect
311    ...              if fails the ports are down indicating reboot
312    ...              is in progress
313
314    # Warm reset adds 3 seconds delay before forcing reboot
315    # To minimize race conditions, we wait for 7 seconds
316    Sleep  7s
317    ${alive}=   Run Keyword and Return Status
318    ...    Open Connection And Log In
319    Return From Keyword If   '${alive}' == '${False}'    ${False}
320    [Return]    ${True}
321
322Flush REST Sessions
323    [Documentation]   Removes all the active session objects
324    Delete All Sessions
325
326Initialize DBUS cmd
327    [Documentation]  Initialize dbus string with property string to extract
328    [Arguments]   ${boot_property}
329    ${cmd}=     Catenate  ${dbuscmdBase} ${dbuscmdGet} ${dbuscmdString}
330    ${cmd}=     Catenate  ${cmd}${boot_property}
331    Set Global Variable   ${dbuscmd}     ${cmd}
332
333Create OS Console File Path
334    [Documentation]  Create OS console file path name and return it.
335    [Arguments]  ${log_file_path}=${EMPTY}
336
337    # Description of arguements:
338    # file_path  The caller's candidate value.  If this value is ${EMPTY}, this
339    #            keyword will compose a file path name.  Otherwise, this
340    #            keyword will use the caller's file_path value.  In either
341    #            case, the value will be returned.
342
343    ${default_file_path}=  Catenate  /tmp/${OPENBMC_HOST}_os_console
344    ${log_file_path}=  Set Variable If  '${log_file_path}' == '${EMPTY}'
345    ...  ${default_file_path}  ${log_file_path}
346
347    [Return]  ${log_file_path}
348
349Create OS Console Command String
350    [Documentation]  Return a command string to start OS console logging.
351
352    # First make sure that the ssh_pw program is available.
353    ${cmd_buf}=  Catenate  which ssh_pw 2>&1
354    Rdpissuing  ${cmd_buf}
355    ${rc}  ${output}=  Run And Return Rc And Output  ${cmd_buf}
356    Rdpvars  rc  output
357    Should Be Equal  ${rc}  ${0}  msg=${output}\n
358
359    ${cmd_buf}=  Catenate  ssh_pw ${OPENBMC_PASSWORD} -p 2200
360    ...  ${OPENBMC_USERNAME}@${OPENBMC_HOST}
361
362    [Return]  ${cmd_buf}
363
364Stop SOL Console Logging
365    [Documentation]  Stop system console logging and return log output.
366    [Arguments]  ${log_file_path}=${EMPTY}  ${targ_file_path}=${EMPTY}
367
368    # If there are muliple system console processes, they will all be stopped.
369    # If there is no existing log file this keyword will return an error
370    # message to that effect (and write that message to targ_file_path, if
371    # specified).
372    # NOTE: This keyword will not fail if there is no running system console
373    # process.
374
375    # Description of arguments:
376    # log_file_path   The file path that was used to call "Start SOL
377    #                 Console Logging".  See that keyword (above) for details.
378    # targ_file_path  If specified, the file path to which the source
379    #                 file path (i.e. "log_file_path") should be copied.
380
381    ${log_file_path}=  Create OS Console File Path  ${log_file_path}
382    # Find the pid of the active system console logging session (if any).
383    ${search_string}=  Create OS Console Command String
384    ${cmd_buf}=  Catenate  echo $(ps -ef | egrep '${search_string}'
385    ...  | egrep -v grep | cut -c10-14)
386    Rdpissuing  ${cmd_buf}
387    ${rc}  ${os_con_pid}=  Run And Return Rc And Output  ${cmd_buf}
388    Rdpvars  os_con_pid
389    # If rc is not zero it just means that there is no OS Console process
390    # running.
391
392    ${cmd_buf}=  Catenate  kill -9 ${os_con_pid}
393    Run Keyword If  '${os_con_pid}' != '${EMPTY}'  Rdpissuing  ${cmd_buf}
394    ${rc}  ${output}=  Run Keyword If  '${os_con_pid}' != '${EMPTY}'
395    ...  Run And Return Rc And Output  ${cmd_buf}
396    Run Keyword If  '${os_con_pid}' != '${EMPTY}'  Rdpvars  rc  output
397
398    ${cmd_buf}=  Set Variable  cat ${log_file_path} 2>&1
399    Rdpissuing  ${cmd_buf}
400    ${rc}  ${output}=  Run And Return Rc And Output  ${cmd_buf}
401    Rdpvars  rc
402
403    Run Keyword If  '${targ_file_path}' != '${EMPTY}'
404    ...  Run Keyword And Ignore Error
405    ...  Copy File  ${log_file_path}  ${targ_file_path}
406
407    [Return]  ${output}
408
409Start SOL Console Logging
410    [Documentation]  Start system console log to file.
411    [Arguments]  ${log_file_path}=${EMPTY}
412
413    # This keyword will first call "Stop SOL Console Logging".  Only then will
414    # it start SOL console logging.  The data returned by "Stop SOL Console
415    # Logging" will in turn be returned by this keyword.
416
417    # Description of arguments:
418    # log_file_path  The file path to which system console log data should be
419    #                written.  Note that this path is taken to be a location on
420    #                the machine where this program is running rather than on
421    #                the Open BMC system.
422
423    ${log_file_path}=  Create OS Console File Path  ${log_file_path}
424
425    ${log_output}=  Stop SOL Console Logging  ${log_file_path}
426
427    # Validate by making sure we can create the file.  Problems creating the
428    # file would not be noticed by the subsequent ssh command because we fork
429    # the command.
430    Create File  ${log_file_path}
431    ${sub_cmd_buf}=  Create OS Console Command String
432    # Routing stderr to stdout so that any startup error text will go to the
433    # output file.
434    ${cmd_buf}=  Catenate  ${sub_cmd_buf} > ${log_file_path} 2>&1 &
435    Rdpissuing  ${cmd_buf}
436    ${rc}  ${output}=  Run And Return Rc And Output  ${cmd_buf}
437    # Because we are forking this command, we essentially will never get a
438    # non-zero return code or any output.
439    Should Be Equal  ${rc}  ${0}
440
441    [Return]  ${log_output}
442
443Get Time Stamp
444    [Documentation]     Get the current time stamp data
445    ${cur_time}=    Get Current Date   result_format=%Y%m%d%H%M%S%f
446    [Return]   ${cur_time}
447
448
449Verify BMC State
450    [Documentation]   Get the BMC state and verify if the current
451    ...               BMC state is as expected.
452    [Arguments]       ${expected}
453
454    ${current}=  Get BMC State Deprecated
455    Should Contain  ${expected}   ${current}
456
457Start Journal Log
458    [Documentation]   Start capturing journal log to a file in /tmp using
459    ...               journalctl command. By default journal log is collected
460    ...               at /tmp/journal_log else user input location.
461    ...               The File is appended with datetime.
462    [Arguments]       ${file_path}=/tmp/journal_log
463
464    Open Connection And Log In
465
466    ${cur_time}=    Get Time Stamp
467    Set Global Variable   ${LOG_TIME}   ${cur_time}
468    Start Command
469    ...  journalctl -f > ${file_path}-${LOG_TIME}
470    Log    Journal Log Started: ${file_path}-${LOG_TIME}
471
472
473Stop Journal Log
474    [Documentation]   Stop journalctl process if its running.
475    ...               By default return log from /tmp/journal_log else
476    ...               user input location.
477    [Arguments]       ${file_path}=/tmp/journal_log
478
479    Open Connection And Log In
480
481    ${rc}=
482    ...  Execute Command
483    ...  ps ax | grep journalctl | grep -v grep
484    ...  return_stdout=False  return_rc=True
485
486    Return From Keyword If   '${rc}' == '${1}'
487    ...   No journal log process running
488
489    ${output}  ${stderr}=
490    ...  Execute Command   killall journalctl
491    ...  return_stderr=True
492    Should Be Empty     ${stderr}
493
494    ${journal_log}  ${stderr}=
495    ...  Execute Command
496    ...  cat ${file_path}-${LOG_TIME}
497    ...  return_stderr=True
498    Should Be Empty     ${stderr}
499
500    Log    ${journal_log}
501
502    Execute Command    rm ${file_path}-${LOG_TIME}
503
504    [Return]    ${journal_log}
505
506Mac Address To Hex String
507    [Documentation]   Converts MAC address into hex format.
508    ...               Example
509    ...               Given the following MAC: 00:01:6C:80:02:78
510    ...               This keyword will return: 0x00 0x01 0x6C 0x80 0x02 0x78
511    ...               Description of arguments:
512    ...               i_macaddress  MAC address in the following format
513    ...               00:01:6C:80:02:78
514    [Arguments]    ${i_macaddress}
515
516    ${mac_hex}=  Catenate  0x${i_macaddress.replace(':', ' 0x')}
517    [Return]    ${mac_hex}
518
519IP Address To Hex String
520    [Documentation]   Converts IP address into hex format.
521    ...               Example:
522    ...               Given the following IP: 10.3.164.100
523    ...               This keyword will return: 0xa 0x3 0xa4 0xa0
524    ...               Description of arguments:
525    ...               i_ipaddress  IP address in the following format
526    ...               10.10.10.10
527    [Arguments]    ${i_ipaddress}
528
529    @{ip}=  Split String  ${i_ipaddress}    .
530    ${index}=  Set Variable  ${0}
531
532    :FOR    ${item}     IN      @{ip}
533    \   ${hex}=  Convert To Hex    ${item}    prefix=0x    lowercase=yes
534    \   Set List Value    ${ip}    ${index}    ${hex}
535    \   ${index}=  Set Variable    ${index + 1}
536    ${ip_hex}=  Catenate    @{ip}
537    [Return]    ${ip_hex}
538
539BMC CPU Performance Check
540   [Documentation]   Minimal 10% of proc should be free in this instance
541
542    ${bmc_cpu_usage_output}  ${stderr}=  Execute Command  ${bmc_cpu_usage_cmd}
543    ...                   return_stderr=True
544    Should be empty  ${stderr}
545    ${bmc_cpu_percentage}=  Fetch From Left  ${bmc_cpu_usage_output}  %
546    Should be true  ${bmc_cpu_percentage} < 90
547
548BMC Mem Performance Check
549    [Documentation]   Minimal 10% of memory should be free in this instance
550
551    ${bmc_mem_free_output}  ${stderr}=   Execute Command  ${bmc_mem_free_cmd}
552    ...                   return_stderr=True
553    Should be empty  ${stderr}
554
555    ${bmc_mem_total_output}  ${stderr}=   Execute Command  ${bmc_mem_total_cmd}
556    ...                   return_stderr=True
557    Should be empty  ${stderr}
558
559    ${bmc_mem_percentage}=   Evaluate  ${bmc_mem_free_output}*100
560    ${bmc_mem_percentage}=  Evaluate
561    ...   ${bmc_mem_percentage}/${bmc_mem_total_output}
562    Should be true  ${bmc_mem_percentage} > 10
563
564BMC File System Usage Check
565    [Documentation]   Check the file system space. None should be 100% full
566    ...   except /run/initramfs/ro
567    ${bmc_fs_usage_output}  ${stderr}=   Execute Command
568    ...   ${bmc_file_system_usage_cmd}  return_stderr=True
569    Should Be Empty  ${stderr}
570    Should Be True  ${bmc_fs_usage_output}==0
571
572Check BMC CPU Performance
573    [Documentation]   Minimal 10% of proc should be free in 3 sample
574    :FOR  ${var}  IN Range  1  4
575    \     BMC CPU Performance check
576
577Check BMC Mem Performance
578    [Documentation]   Minimal 10% of memory should be free
579
580    :FOR  ${var}  IN Range  1  4
581    \     BMC Mem Performance check
582
583Check BMC File System Performance
584    [Documentation]  Check for file system usage for 4 times
585
586    :FOR  ${var}  IN Range  1  4
587    \     BMC File System Usage check
588
589Get Endpoint Paths
590    [Documentation]   Returns all url paths ending with given endpoint
591    ...               Example:
592    ...               Given the following endpoint: cpu
593    ...               This keyword will return: list of all urls ending with
594    ...               cpu -
595    ...               /org/openbmc/inventory/system/chassis/motherboard/cpu0,
596    ...               /org/openbmc/inventory/system/chassis/motherboard/cpu1
597    ...               Description of arguments:
598    ...               path       URL path for enumeration
599    ...               endpoint   string for which url path ending
600    [Arguments]   ${path}   ${endpoint}
601
602    ${resp}=   Read Properties   ${path}/enumerate   timeout=30
603    log Dictionary   ${resp}
604
605    ${list}=   Get Dictionary Keys   ${resp}
606    ${resp}=   Get Matches   ${list}   regexp=^.*[0-9a-z_].${endpoint}[0-9]*$
607    [Return]   ${resp}
608
609Check Zombie Process
610    [Documentation]    Check if any defunct process exist or not on BMC
611    ${count}  ${stderr}  ${rc}=  Execute Command  ps -o stat | grep Z | wc -l
612    ...    return_stderr=True  return_rc=True
613    Should Be True    ${count}==0
614    Should Be Empty    ${stderr}
615
616Prune Journal Log
617    [Documentation]   Prune archived journal logs.
618    [Arguments]   ${vacuum_size}=1M
619
620    # This keyword can be used to prevent the journal
621    # log from filling up the /run filesystem.
622    # This command will retain only the latest logs
623    # of the user specified size.
624
625    Open Connection And Log In
626    ${output}  ${stderr}  ${rc}=
627    ...  Execute Command
628    ...  journalctl --vacuum-size=${vacuum_size}
629    ...  return_stderr=True  return_rc=True
630
631    Should Be Equal  ${rc}  ${0}  msg=${stderr}
632    Should Contain   ${stderr}  Vacuuming done
633
634Set BMC Power Policy
635    [Documentation]   Set the given BMC power policy.
636    [arguments]   ${policy}
637
638    ${valueDict}=     create dictionary  data=${policy}
639    Write Attribute    ${HOST_SETTING}    power_policy   data=${valueDict}
640    ${currentPolicy}=  Read Attribute     ${HOST_SETTING}   power_policy
641    Should Be Equal    ${currentPolicy}   ${policy}
642
643Get System Power Policy
644    [Documentation]  Get the BMC power policy.
645    ${currentPolicy}=  Read Attribute  ${HOST_SETTING}  power_policy
646    [Return]  ${currentPolicy}
647
648Get Auto Reboot
649    [Documentation]  Returns auto reboot setting.
650    ${setting}=  Read Attribute  ${HOST_SETTINGS}  auto_reboot
651    [Return]  ${setting}
652
653
654Set Auto Reboot
655    [Documentation]  Set the given auto reboot setting.
656    [Arguments]  ${setting}
657    # setting  auto reboot's setting, i.e. yes or no
658
659    ${valueDict}=  Set Variable  ${setting}
660    ${data}=  Create Dictionary  data=${valueDict}
661    Write Attribute  ${HOST_SETTINGS}  auto_reboot  data=${data}
662    ${current_setting}=  Get Auto Reboot
663    Should Be Equal  ${current_setting}  ${setting}
664
665
666Set BMC Reset Reference Time
667    [Documentation]  Set current boot time as a reference and increment
668    ...              boot count.
669
670    ${cur_btime}=  Get BMC Boot Time
671    Run Keyword If  ${BOOT_TIME} == ${0} and ${BOOT_COUNT} == ${0}
672    ...  Set Global Variable  ${BOOT_TIME}  ${cur_btime}
673    ...  ELSE IF  ${cur_btime} > ${BOOT_TIME}
674    ...  Run Keywords  Set Global Variable  ${BOOT_TIME}  ${cur_btime}
675    ...  AND
676    ...  Set Global Variable  ${BOOT_COUNT}  ${BOOT_COUNT + 1}
677
678Get BMC Boot Time
679    [Documentation]  Get boot time from /proc/stat.
680
681    Open Connection And Log In
682    ${output}  ${stderr}=
683    ...  Execute Command  egrep '^btime ' /proc/stat | cut -f 2 -d ' '
684    ...  return_stderr=True
685    Should Be Empty  ${stderr}
686    ${btime}=  Convert To Integer  ${output}
687    [Return]  ${btime}
688
689Execute Command On BMC
690    [Documentation]  Execute given command on BMC and return output.
691    [Arguments]  ${command}
692    ${stdout}  ${stderr}=  Execute Command  ${command}  return_stderr=True
693    Should Be Empty  ${stderr}
694    [Return]  ${stdout}
695
696