1*** Settings *** 2Resource ../lib/resource.txt 3Resource ../lib/rest_client.robot 4Resource ../lib/connection_client.robot 5Library DateTime 6Library Process 7Library OperatingSystem 8Library gen_print.py 9Library gen_robot_print.py 10 11*** Variables *** 12${SYSTEM_SHUTDOWN_TIME} ${5} 13${dbuscmdBase} 14... dbus-send --system --print-reply --dest=${OPENBMC_BASE_DBUS}.settings.Host 15${dbuscmdGet} 16... ${SETTINGS_URI}host0 org.freedesktop.DBus.Properties.Get 17# Enable when ready with openbmc/openbmc-test-automation#203 18#${dbuscmdString}= string:"xyz.openbmc_project.settings.Host" string: 19${dbuscmdString}= string:"org.openbmc.settings.Host" string: 20 21# Assign default value to QUIET for programs which may not define it. 22${QUIET} ${0} 23${bmc_mem_free_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f4 24${bmc_mem_total_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f2 25${bmc_cpu_usage_cmd}= top -n 1 | grep CPU: | cut -c 7-9 26${HOST_SETTING} ${SETTINGS_URI}host0 27# /run/initramfs/ro associate filesystem should be 100% full always 28${bmc_file_system_usage_cmd}= 29... df -h | grep -v /run/initramfs/ro | cut -c 52-54 | grep 100 | wc -l 30 31${BOOT_TIME} ${0} 32${BOOT_COUNT} ${0} 33${count} ${0} 34${devicetree_base} /sys/firmware/devicetree/base/model 35 36*** Keywords *** 37 38Get BMC System Model 39 [Documentation] Get the BMC model from the device tree. 40 41 Open Connection And Log In 42 ${bmc_model} ${stderr}= Execute Command 43 ... cat ${devicetree_base} | cut -d " " -f 1 return_stderr=True 44 Should Be Empty ${stderr} 45 Should Not Be Empty ${bmc_model} 46 [Return] ${bmc_model} 47 48Verify BMC System Model 49 [Documentation] Verify the BMC model with ${OPENBMC_MODEL}. 50 [Arguments] ${bmc_model} 51 52 ${tmp_bmc_model}= Fetch From Right ${OPENBMC_MODEL} / 53 ${tmp_bmc_model}= Fetch From Left ${tmp_bmc_model} . 54 ${ret}= Run Keyword And Return Status Should Contain ${bmc_model} 55 ... ${tmp_bmc_model} ignore_case=True 56 [Return] ${ret} 57 58Wait For Host To Ping 59 [Arguments] ${host} ${timeout}=${OPENBMC_REBOOT_TIMEOUT}min 60 ... ${interval}=5 sec 61 62 # host The DNS name or IP of the host to ping. 63 # timeout The amount of time after which attempts to ping cease. 64 # interval The amount of time in between attempts to ping. 65 66 Wait Until Keyword Succeeds ${timeout} ${interval} Ping Host ${host} 67 68Ping Host 69 [Arguments] ${host} 70 Should Not Be Empty ${host} msg=No host provided 71 ${RC} ${output}= Run and return RC and Output ping -c 4 ${host} 72 Log RC: ${RC}\nOutput:\n${output} 73 Should be equal ${RC} ${0} 74 75Get Boot Progress 76 [Arguments] ${quiet}=${QUIET} 77 78 ${state}= Read Attribute ${OPENBMC_BASE_URI}sensors/host/BootProgress 79 ... value quiet=${quiet} 80 [Return] ${state} 81 82Is Power On 83 ${state}= Get Power State 84 Should be equal ${state} ${1} 85 86Is Power Off 87 ${state}= Get Power State 88 Should be equal ${state} ${0} 89 90Initiate Power On 91 [Documentation] Initiates the power on and waits until the Is Power On 92 ... keyword returns that the power state has switched to on. 93 [Arguments] ${wait}=${1} 94 95 @{arglist}= Create List 96 ${args}= Create Dictionary data=@{arglist} 97 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOn 98 ... data=${args} 99 should be equal as strings ${resp.status_code} ${HTTP_OK} 100 101 # Does caller want to wait for power on status? 102 Run Keyword If '${wait}' == '${0}' Return From Keyword 103 Wait Until Keyword Succeeds 3 min 10 sec Is Power On 104 105Initiate Power Off 106 [Documentation] Initiates the power off and waits until the Is Power Off 107 ... keyword returns that the power state has switched to off. 108 @{arglist}= Create List 109 ${args}= Create Dictionary data=@{arglist} 110 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOff 111 ... data=${args} 112 should be equal as strings ${resp.status_code} ${HTTP_OK} 113 Wait Until Keyword Succeeds 1 min 10 sec Is Power Off 114 115Initiate OS Host Power Off 116 [Documentation] Initiate an OS reboot. 117 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 118 ... ${os_password}=${OS_PASSWORD} 119 120 # Description of arguments: 121 # os_host The DNS name or IP of the OS. 122 # os_username The username to be used to sign in to the OS. 123 # os_password The password to be used to sign in to the OS. 124 125 SSHLibrary.Open connection ${os_host} 126 Login ${os_username} ${os_password} 127 ${cmd_buf} Catenate shutdown 128 Start Command ${cmd_buf} 129 SSHLibrary.Close Connection 130 131Initiate OS Host Reboot 132 [Documentation] Initiate an OS reboot. 133 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 134 ... ${os_password}=${OS_PASSWORD} 135 136 # Description of arguments: 137 # os_host The DNS name or IP of the OS. 138 # os_username The username to be used to sign in to the OS. 139 # os_password The password to be used to sign in to the OS. 140 141 SSHLibrary.Open connection ${os_host} 142 Login ${os_username} ${os_password} 143 ${cmd_buf} Catenate reboot 144 Start Command ${cmd_buf} 145 SSHLibrary.Close Connection 146 147Initiate Auto Reboot 148 [Documentation] Initiate an auto reboot. 149 150 # Set the auto reboot policy. 151 Set Auto Reboot yes 152 153 SSHLibrary.Open connection ${openbmc_host} 154 Login ${openbmc_username} ${openbmc_password} 155 156 # Set the watchdog timer. Note: 5000 = milliseconds which is 5 seconds. 157 ${cmd_buf}= Catenate /usr/sbin/mapper call /org/openbmc/watchdog/host0 158 ... org.openbmc.Watchdog set i 5000 159 ${output} ${stderr} ${rc}= Execute Command ${cmd_buf} 160 ... return_stderr=True return_rc=True 161 Should Be Empty ${stderr} 162 Should be equal ${rc} ${0} 163 164 # Start the watchdog. 165 ${cmd_buf}= Catenate /usr/sbin/mapper call /org/openbmc/watchdog/host0 166 ... org.openbmc.Watchdog start 167 ${output} ${stderr} ${rc}= Execute Command ${cmd_buf} 168 ... return_stderr=True return_rc=True 169 Should Be Empty ${stderr} 170 Should be equal ${rc} ${0} 171 172Trigger Warm Reset 173 log to console "Triggering warm reset" 174 ${data}= create dictionary data=@{EMPTY} 175 ${resp}= openbmc post request 176 ... ${OPENBMC_BASE_URI}control/bmc0/action/warmReset data=${data} 177 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 178 ${session_active}= Check If warmReset is Initiated 179 Run Keyword If '${session_active}' == '${True}' 180 ... Fail msg=warm reset didn't occur 181 182 Sleep ${SYSTEM_SHUTDOWN_TIME}min 183 Check If BMC Is Up 184 185Check OS 186 [Documentation] Attempts to ping the host OS and then checks that the host 187 ... OS is up by running an SSH command. 188 189 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 190 ... ${os_password}=${OS_PASSWORD} ${quiet}=${QUIET} 191 ... ${print_string}=${EMPTY} 192 [Teardown] SSHLibrary.Close Connection 193 194 # os_host The DNS name/IP of the OS host associated with our BMC. 195 # os_username The username to be used to sign on to the OS host. 196 # os_password The password to be used to sign on to the OS host. 197 # quiet Indicates whether this keyword should write to console. 198 # print_string A string to be printed before checking the OS. 199 200 rprint ${print_string} 201 202 # Attempt to ping the OS. Store the return code to check later. 203 ${ping_rc}= Run Keyword and Return Status Ping Host ${os_host} 204 205 SSHLibrary.Open connection ${os_host} 206 207 ${status} ${msg}= Run Keyword And Ignore Error Login ${os_username} 208 ... ${os_password} 209 ${err_msg1}= Sprint Error ${msg} 210 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 211 Run Keyword If '${status}' == 'FAIL' Fail msg=${err_msg} 212 ${output} ${stderr} ${rc}= Execute Command uptime return_stderr=True 213 ... return_rc=True 214 215 ${temp_msg}= Catenate Could not execute a command on the operating 216 ... system.\n 217 ${err_msg1}= Sprint Error ${temp_msg} 218 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 219 220 # If the return code returned by "Execute Command" is non-zero, this 221 # keyword will fail. 222 Should Be Equal ${rc} ${0} msg=${err_msg} 223 # We will likewise fail if there is any stderr data. 224 Should Be Empty ${stderr} 225 226 ${temp_msg}= Set Variable Could not ping the operating system.\n 227 ${err_msg1}= Sprint Error ${temp_msg} 228 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 229 # We will likewise fail if the OS did not ping, as we could SSH but not 230 # ping 231 Should Be Equal As Strings ${ping_rc} ${TRUE} msg=${err_msg} 232 233Wait for OS 234 [Documentation] Waits for the host OS to come up via calls to "Check OS". 235 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 236 ... ${os_password}=${OS_PASSWORD} ${timeout}=${OS_WAIT_TIMEOUT} 237 ... ${quiet}=${0} 238 [Teardown] rprintn 239 240 # os_host The DNS name or IP of the OS host associated with our 241 # BMC. 242 # os_username The username to be used to sign on to the OS host. 243 # os_password The password to be used to sign on to the OS host. 244 # timeout The timeout in seconds indicating how long you're 245 # willing to wait for the OS to respond. 246 # quiet Indicates whether this keyword should write to console. 247 248 # The interval to be used between calls to "Check OS". 249 ${interval}= Set Variable 5 250 251 ${message}= Catenate Checking every ${interval} seconds for up to 252 ... ${timeout} seconds for the operating system to communicate. 253 rqprint_timen ${message} 254 255 Wait Until Keyword Succeeds ${timeout} sec ${interval} Check OS 256 ... ${os_host} ${os_username} ${os_password} 257 ... print_string=\# 258 259 rqprintn 260 261 rqprint_timen The operating system is now communicating. 262 263Get BMC State Deprecated 264 [Documentation] Returns the state of the BMC as a string. (i.e: BMC_READY) 265 [Arguments] ${quiet}=${QUIET} 266 267 @{arglist}= Create List 268 ${args}= Create Dictionary data=@{arglist} 269 ${resp}= Call Method ${OPENBMC_BASE_URI}managers/System/ getSystemState 270 ... data=${args} quiet=${quiet} 271 Should be equal as strings ${resp.status_code} ${HTTP_OK} 272 ${content}= to json ${resp.content} 273 [Return] ${content["data"]} 274 275Get Power State 276 [Documentation] Returns the power state as an integer. Either 0 or 1. 277 [Arguments] ${quiet}=${QUIET} 278 279 @{arglist}= Create List 280 ${args}= Create Dictionary data=@{arglist} 281 282 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ getPowerState 283 ... data=${args} quiet=${quiet} 284 Should be equal as strings ${resp.status_code} ${HTTP_OK} 285 ${content}= to json ${resp.content} 286 [Return] ${content["data"]} 287 288Clear BMC Record Log 289 [Documentation] Clears all the event logs on the BMC. This would be 290 ... equivalent to ipmitool sel clear. 291 @{arglist}= Create List 292 ${args}= Create Dictionary data=@{arglist} 293 ${resp}= Call Method 294 ... ${OPENBMC_BASE_URI}records/events/ clear data=${args} 295 should be equal as strings ${resp.status_code} ${HTTP_OK} 296 297Copy PNOR to BMC 298 Import Library SCPLibrary WITH NAME scp 299 Open Connection for SCP 300 Log Copying ${PNOR_IMAGE_PATH} to /tmp 301 scp.Put File ${PNOR_IMAGE_PATH} /tmp 302 303Flash PNOR 304 [Documentation] Calls flash bios update method to flash PNOR image 305 [Arguments] ${pnor_image} 306 @{arglist}= Create List ${pnor_image} 307 ${args}= Create Dictionary data=@{arglist} 308 ${resp}= Call Method ${OPENBMC_BASE_URI}control/flash/bios/ update 309 ... data=${args} 310 should be equal as strings ${resp.status_code} ${HTTP_OK} 311 Wait Until Keyword Succeeds 2 min 10 sec Is PNOR Flashing 312 313Get Flash BIOS Status 314 [Documentation] Returns the status of the flash BIOS API as a string. For 315 ... example 'Flashing', 'Flash Done', etc 316 ${data}= Read Properties ${OPENBMC_BASE_URI}control/flash/bios 317 [Return] ${data['status']} 318 319Is PNOR Flashing 320 [Documentation] Get BIOS 'Flashing' status. This indicates that PNOR 321 ... flashing has started. 322 ${status}= Get Flash BIOS Status 323 Should Contain ${status} Flashing 324 325Is PNOR Flash Done 326 [Documentation] Get BIOS 'Flash Done' status. This indicates that the 327 ... PNOR flashing has completed. 328 ${status}= Get Flash BIOS Status 329 should be equal as strings ${status} Flash Done 330 331Is System State Host Booted 332 [Documentation] Checks whether system state is HOST_BOOTED. 333 ${state}= Get BMC State Deprecated 334 should be equal as strings ${state} HOST_BOOTED 335 336Is OS Starting 337 [Documentation] Check if boot progress is OS starting. 338 ${boot_progress}= Get Boot Progress 339 Should Be Equal ${boot_progress} FW Progress, Starting OS 340 341Is OS Off 342 [Documentation] Check if boot progress is "Off". 343 ${boot_progress}= Get Boot Progress 344 Should Be Equal ${boot_progress} Off 345 346Verify Ping and REST Authentication 347 ${l_ping}= Run Keyword And Return Status 348 ... Ping Host ${OPENBMC_HOST} 349 Run Keyword If '${l_ping}' == '${False}' 350 ... Fail msg=Ping Failed 351 352 ${l_rest}= Run Keyword And Return Status 353 ... Initialize OpenBMC 354 Run Keyword If '${l_rest}' == '${False}' 355 ... Fail msg=REST Authentication Failed 356 357 # Just to make sure the SSH is working for SCP 358 Open Connection And Log In 359 ${system} ${stderr}= Execute Command hostname return_stderr=True 360 Should Be Empty ${stderr} 361 362Check If BMC is Up 363 [Documentation] Wait for Host to be online. Checks every X seconds 364 ... interval for Y minutes and fails if timed out. 365 ... Default MAX timedout is 10 min, interval 10 seconds. 366 [Arguments] ${max_timeout}=${OPENBMC_REBOOT_TIMEOUT} min 367 ... ${interval}=10 sec 368 369 Wait Until Keyword Succeeds 370 ... ${max_timeout} ${interval} Verify Ping and REST Authentication 371 372 373Check If warmReset is Initiated 374 [Documentation] Ping would be still alive, so try SSH to connect 375 ... if fails the ports are down indicating reboot 376 ... is in progress 377 378 # Warm reset adds 3 seconds delay before forcing reboot 379 # To minimize race conditions, we wait for 7 seconds 380 Sleep 7s 381 ${alive}= Run Keyword and Return Status 382 ... Open Connection And Log In 383 Return From Keyword If '${alive}' == '${False}' ${False} 384 [Return] ${True} 385 386Flush REST Sessions 387 [Documentation] Removes all the active session objects 388 Delete All Sessions 389 390Initialize DBUS cmd 391 [Documentation] Initialize dbus string with property string to extract 392 [Arguments] ${boot_property} 393 ${cmd}= Catenate ${dbuscmdBase} ${dbuscmdGet} ${dbuscmdString} 394 ${cmd}= Catenate ${cmd}${boot_property} 395 Set Global Variable ${dbuscmd} ${cmd} 396 397Create OS Console File Path 398 [Documentation] Create OS console file path name and return it. 399 [Arguments] ${log_file_path}=${EMPTY} 400 401 # Description of arguements: 402 # file_path The caller's candidate value. If this value is ${EMPTY}, this 403 # keyword will compose a file path name. Otherwise, this 404 # keyword will use the caller's file_path value. In either 405 # case, the value will be returned. 406 407 ${default_file_path}= Catenate /tmp/${OPENBMC_HOST}_os_console 408 ${log_file_path}= Set Variable If '${log_file_path}' == '${EMPTY}' 409 ... ${default_file_path} ${log_file_path} 410 411 [Return] ${log_file_path} 412 413Create OS Console Command String 414 [Documentation] Return a command string to start OS console logging. 415 416 # First make sure that the ssh_pw program is available. 417 ${cmd_buf}= Catenate which ssh_pw 2>&1 418 Rdpissuing ${cmd_buf} 419 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 420 Rdpvars rc output 421 Should Be Equal ${rc} ${0} msg=${output}\n 422 423 ${cmd_buf}= Catenate ssh_pw ${OPENBMC_PASSWORD} -p 2200 424 ... ${OPENBMC_USERNAME}@${OPENBMC_HOST} 425 426 [Return] ${cmd_buf} 427 428Stop SOL Console Logging 429 [Documentation] Stop system console logging and return log output. 430 [Arguments] ${log_file_path}=${EMPTY} ${targ_file_path}=${EMPTY} 431 432 # If there are muliple system console processes, they will all be stopped. 433 # If there is no existing log file this keyword will return an error 434 # message to that effect (and write that message to targ_file_path, if 435 # specified). 436 # NOTE: This keyword will not fail if there is no running system console 437 # process. 438 439 # Description of arguments: 440 # log_file_path The file path that was used to call "Start SOL 441 # Console Logging". See that keyword (above) for details. 442 # targ_file_path If specified, the file path to which the source 443 # file path (i.e. "log_file_path") should be copied. 444 445 ${log_file_path}= Create OS Console File Path ${log_file_path} 446 # Find the pid of the active system console logging session (if any). 447 ${search_string}= Create OS Console Command String 448 ${cmd_buf}= Catenate echo $(ps -ef | egrep '${search_string}' 449 ... | egrep -v grep | cut -c10-14) 450 Rdpissuing ${cmd_buf} 451 ${rc} ${os_con_pid}= Run And Return Rc And Output ${cmd_buf} 452 Rdpvars os_con_pid 453 # If rc is not zero it just means that there is no OS Console process 454 # running. 455 456 ${cmd_buf}= Catenate kill -9 ${os_con_pid} 457 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpissuing ${cmd_buf} 458 ${rc} ${output}= Run Keyword If '${os_con_pid}' != '${EMPTY}' 459 ... Run And Return Rc And Output ${cmd_buf} 460 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpvars rc output 461 462 ${cmd_buf}= Set Variable cat ${log_file_path} 2>&1 463 Rdpissuing ${cmd_buf} 464 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 465 Rdpvars rc 466 467 Run Keyword If '${targ_file_path}' != '${EMPTY}' 468 ... Run Keyword And Ignore Error 469 ... Copy File ${log_file_path} ${targ_file_path} 470 471 [Return] ${output} 472 473Start SOL Console Logging 474 [Documentation] Start system console log to file. 475 [Arguments] ${log_file_path}=${EMPTY} 476 477 # This keyword will first call "Stop SOL Console Logging". Only then will 478 # it start SOL console logging. The data returned by "Stop SOL Console 479 # Logging" will in turn be returned by this keyword. 480 481 # Description of arguments: 482 # log_file_path The file path to which system console log data should be 483 # written. Note that this path is taken to be a location on 484 # the machine where this program is running rather than on 485 # the Open BMC system. 486 487 ${log_file_path}= Create OS Console File Path ${log_file_path} 488 489 ${log_output}= Stop SOL Console Logging ${log_file_path} 490 491 # Validate by making sure we can create the file. Problems creating the 492 # file would not be noticed by the subsequent ssh command because we fork 493 # the command. 494 Create File ${log_file_path} 495 ${sub_cmd_buf}= Create OS Console Command String 496 # Routing stderr to stdout so that any startup error text will go to the 497 # output file. 498 ${cmd_buf}= Catenate ${sub_cmd_buf} > ${log_file_path} 2>&1 & 499 Rdpissuing ${cmd_buf} 500 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 501 # Because we are forking this command, we essentially will never get a 502 # non-zero return code or any output. 503 Should Be Equal ${rc} ${0} 504 505 [Return] ${log_output} 506 507Get Time Stamp 508 [Documentation] Get the current time stamp data 509 ${cur_time}= Get Current Date result_format=%Y%m%d%H%M%S%f 510 [Return] ${cur_time} 511 512 513Verify BMC State 514 [Documentation] Get the BMC state and verify if the current 515 ... BMC state is as expected. 516 [Arguments] ${expected} 517 518 ${current}= Get BMC State Deprecated 519 Should Contain ${expected} ${current} 520 521Start Journal Log 522 [Documentation] Start capturing journal log to a file in /tmp using 523 ... journalctl command. By default journal log is collected 524 ... at /tmp/journal_log else user input location. 525 ... The File is appended with datetime. 526 [Arguments] ${file_path}=/tmp/journal_log 527 528 Open Connection And Log In 529 530 ${cur_time}= Get Time Stamp 531 Set Global Variable ${LOG_TIME} ${cur_time} 532 Start Command 533 ... journalctl -f > ${file_path}-${LOG_TIME} 534 Log Journal Log Started: ${file_path}-${LOG_TIME} 535 536 537Stop Journal Log 538 [Documentation] Stop journalctl process if its running. 539 ... By default return log from /tmp/journal_log else 540 ... user input location. 541 [Arguments] ${file_path}=/tmp/journal_log 542 543 Open Connection And Log In 544 545 ${rc}= 546 ... Execute Command 547 ... ps ax | grep journalctl | grep -v grep 548 ... return_stdout=False return_rc=True 549 550 Return From Keyword If '${rc}' == '${1}' 551 ... No journal log process running 552 553 ${output} ${stderr}= 554 ... Execute Command killall journalctl 555 ... return_stderr=True 556 Should Be Empty ${stderr} 557 558 ${journal_log} ${stderr}= 559 ... Execute Command 560 ... cat ${file_path}-${LOG_TIME} 561 ... return_stderr=True 562 Should Be Empty ${stderr} 563 564 Log ${journal_log} 565 566 Execute Command rm ${file_path}-${LOG_TIME} 567 568 [Return] ${journal_log} 569 570Mac Address To Hex String 571 [Documentation] Converts MAC address into hex format. 572 ... Example 573 ... Given the following MAC: 00:01:6C:80:02:78 574 ... This keyword will return: 0x00 0x01 0x6C 0x80 0x02 0x78 575 ... Description of arguments: 576 ... i_macaddress MAC address in the following format 577 ... 00:01:6C:80:02:78 578 [Arguments] ${i_macaddress} 579 580 ${mac_hex}= Catenate 0x${i_macaddress.replace(':', ' 0x')} 581 [Return] ${mac_hex} 582 583IP Address To Hex String 584 [Documentation] Converts IP address into hex format. 585 ... Example: 586 ... Given the following IP: 10.3.164.100 587 ... This keyword will return: 0xa 0x3 0xa4 0xa0 588 ... Description of arguments: 589 ... i_ipaddress IP address in the following format 590 ... 10.10.10.10 591 [Arguments] ${i_ipaddress} 592 593 @{ip}= Split String ${i_ipaddress} . 594 ${index}= Set Variable ${0} 595 596 :FOR ${item} IN @{ip} 597 \ ${hex}= Convert To Hex ${item} prefix=0x lowercase=yes 598 \ Set List Value ${ip} ${index} ${hex} 599 \ ${index}= Set Variable ${index + 1} 600 ${ip_hex}= Catenate @{ip} 601 [Return] ${ip_hex} 602 603BMC CPU Performance Check 604 [Documentation] Minimal 10% of proc should be free in this instance 605 606 ${bmc_cpu_usage_output} ${stderr}= Execute Command ${bmc_cpu_usage_cmd} 607 ... return_stderr=True 608 Should be empty ${stderr} 609 ${bmc_cpu_percentage}= Fetch From Left ${bmc_cpu_usage_output} % 610 Should be true ${bmc_cpu_percentage} < 90 611 612BMC Mem Performance Check 613 [Documentation] Minimal 10% of memory should be free in this instance 614 615 ${bmc_mem_free_output} ${stderr}= Execute Command ${bmc_mem_free_cmd} 616 ... return_stderr=True 617 Should be empty ${stderr} 618 619 ${bmc_mem_total_output} ${stderr}= Execute Command ${bmc_mem_total_cmd} 620 ... return_stderr=True 621 Should be empty ${stderr} 622 623 ${bmc_mem_percentage}= Evaluate ${bmc_mem_free_output}*100 624 ${bmc_mem_percentage}= Evaluate 625 ... ${bmc_mem_percentage}/${bmc_mem_total_output} 626 Should be true ${bmc_mem_percentage} > 10 627 628BMC File System Usage Check 629 [Documentation] Check the file system space. None should be 100% full 630 ... except /run/initramfs/ro 631 ${bmc_fs_usage_output} ${stderr}= Execute Command 632 ... ${bmc_file_system_usage_cmd} return_stderr=True 633 Should Be Empty ${stderr} 634 Should Be True ${bmc_fs_usage_output}==0 635 636Check BMC CPU Performance 637 [Documentation] Minimal 10% of proc should be free in 3 sample 638 :FOR ${var} IN Range 1 4 639 \ BMC CPU Performance check 640 641Check BMC Mem Performance 642 [Documentation] Minimal 10% of memory should be free 643 644 :FOR ${var} IN Range 1 4 645 \ BMC Mem Performance check 646 647Check BMC File System Performance 648 [Documentation] Check for file system usage for 4 times 649 650 :FOR ${var} IN Range 1 4 651 \ BMC File System Usage check 652 653Get URL List 654 [Documentation] Return list of URLs under given URL. 655 [Arguments] ${openbmc_url} 656 # Description of argument(s): 657 # openbmc_url URL for list operation (e.g. /xyz/openbmc_project/inventory). 658 659 ${url_list}= Read Properties ${openbmc_url}/list 660 [Return] ${url_list} 661 662Get Endpoint Paths 663 [Documentation] Returns all url paths ending with given endpoint 664 ... Example: 665 ... Given the following endpoint: cpu 666 ... This keyword will return: list of all urls ending with 667 ... cpu - 668 ... /org/openbmc/inventory/system/chassis/motherboard/cpu0, 669 ... /org/openbmc/inventory/system/chassis/motherboard/cpu1 670 ... Description of arguments: 671 ... path URL path for enumeration 672 ... endpoint string for which url path ending 673 [Arguments] ${path} ${endpoint} 674 675 ${resp}= Read Properties ${path}/enumerate timeout=30 676 log Dictionary ${resp} 677 678 ${list}= Get Dictionary Keys ${resp} 679 ${resp}= Get Matches ${list} regexp=^.*[0-9a-z_].${endpoint}[0-9]*$ 680 [Return] ${resp} 681 682Check Zombie Process 683 [Documentation] Check if any defunct process exist or not on BMC 684 ${count} ${stderr} ${rc}= Execute Command ps -o stat | grep Z | wc -l 685 ... return_stderr=True return_rc=True 686 Should Be True ${count}==0 687 Should Be Empty ${stderr} 688 689Prune Journal Log 690 [Documentation] Prune archived journal logs. 691 [Arguments] ${vacuum_size}=1M 692 693 # This keyword can be used to prevent the journal 694 # log from filling up the /run filesystem. 695 # This command will retain only the latest logs 696 # of the user specified size. 697 698 Open Connection And Log In 699 ${output} ${stderr} ${rc}= 700 ... Execute Command 701 ... journalctl --vacuum-size=${vacuum_size} 702 ... return_stderr=True return_rc=True 703 704 Should Be Equal ${rc} ${0} msg=${stderr} 705 706Set BMC Power Policy 707 [Documentation] Set the given BMC power policy. 708 [arguments] ${policy} 709 710 ${valueDict}= create dictionary data=${policy} 711 Write Attribute ${HOST_SETTING} power_policy data=${valueDict} 712 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 713 Should Be Equal ${currentPolicy} ${policy} 714 715Get System Power Policy 716 [Documentation] Get the BMC power policy. 717 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 718 [Return] ${currentPolicy} 719 720Get Auto Reboot 721 [Documentation] Returns auto reboot setting. 722 ${setting}= Read Attribute ${HOST_SETTING} auto_reboot 723 [Return] ${setting} 724 725 726Set Auto Reboot 727 [Documentation] Set the given auto reboot setting. 728 [Arguments] ${setting} 729 # setting auto reboot's setting, i.e. yes or no 730 731 ${valueDict}= Set Variable ${setting} 732 ${data}= Create Dictionary data=${valueDict} 733 Write Attribute ${HOST_SETTING} auto_reboot data=${data} 734 ${current_setting}= Get Auto Reboot 735 Should Be Equal ${current_setting} ${setting} 736 737 738Set BMC Reset Reference Time 739 [Documentation] Set current boot time as a reference and increment 740 ... boot count. 741 742 ${cur_btime}= Get BMC Boot Time 743 Run Keyword If ${BOOT_TIME} == ${0} and ${BOOT_COUNT} == ${0} 744 ... Set Global Variable ${BOOT_TIME} ${cur_btime} 745 ... ELSE IF ${cur_btime} > ${BOOT_TIME} 746 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 747 ... AND 748 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 749 750Get BMC Boot Time 751 [Documentation] Get boot time from /proc/stat. 752 753 Open Connection And Log In 754 ${output} ${stderr}= 755 ... Execute Command egrep '^btime ' /proc/stat | cut -f 2 -d ' ' 756 ... return_stderr=True 757 Should Be Empty ${stderr} 758 ${btime}= Convert To Integer ${output} 759 [Return] ${btime} 760 761Execute Command On BMC 762 [Documentation] Execute given command on BMC and return output. 763 [Arguments] ${command} 764 ${stdout} ${stderr}= Execute Command ${command} return_stderr=True 765 Should Be Empty ${stderr} 766 [Return] ${stdout} 767 768 769Enable Core Dump On BMC 770 [Documentation] Enable core dump collection. 771 Open Connection And Log In 772 ${core_pattern}= Execute Command On BMC 773 ... echo '/tmp/core_%e.%p' | tee /proc/sys/kernel/core_pattern 774 Should Be Equal As Strings ${core_pattern} /tmp/core_%e.%p 775 776Get Number Of BMC Core Dump Files 777 [Documentation] Get number of core dump files on BMC. 778 Open Connection And Log In 779 ${num_of_core_dump}= Execute Command 780 ... ls /tmp/core* 2>/dev/null | wc -l 781 [Return] ${num_of_core_dump} 782 783Set Core Dump File Size Unlimited 784 [Documentation] Set core dump file size to unlimited. 785 Open Connection And Log In 786 Execute Command On BMC 787 ... ulimit -c unlimited 788 789Check For Core Dumps 790 [Documentation] Check for any core dumps exist. 791 ${output}= Get Number Of BMC Core Dump Files 792 Run Keyword If ${output} > 0 793 ... Log **Warning** BMC core dump files exist level=WARN 794 795Trigger Host Watchdog Error 796 [Documentation] Inject host watchdog error using BMC. 797 [Arguments] ${milliseconds}=1000 ${sleep_time}=5s 798 # Description of argument(s): 799 # milliseconds The time watchdog timer value in milliseconds (e.g. 1000 = 800 # 1 second). 801 # sleep_time Time delay for host watchdog error to get injected. 802 # Default is 5 seconds. 803 804 Execute Command On BMC 805 ... /usr/sbin/mapper call /org/openbmc/watchdog/host0 org.openbmc.Watchdog set i ${milliseconds} 806 Execute Command On BMC 807 ... /usr/sbin/mapper call /org/openbmc/watchdog/host0 org.openbmc.Watchdog start 808 Sleep ${sleep_time} 809 810Login To OS Host 811 [Documentation] Login to OS Host. 812 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 813 ... ${os_password}=${OS_PASSWORD} 814 # Desription of arguments: 815 # ${os_host} IP address of the OS Host. 816 # ${os_username} OS Host Login user name. 817 # ${os_password} OS Host Login passwrd. 818 819 ${os_state}= Run Keyword And Return Status Ping Host ${os_host} 820 Run Keyword If '${os_state}' == 'False' 821 ... Run Keywords Initiate Host Reboot AND 822 ... Is Host Running AND 823 ... Wait for OS ${os_host} ${os_username} ${os_password} 824 825 Open Connection ${os_host} 826 ${resp}= Login ${os_username} ${os_password} 827 [Return] ${resp} 828 829Configure Initial Settings 830 [Documentation] Restore old IP and route. 831 ... This keyword requires initial settings viz IP address, 832 ... Network Mask, default gatway and serial console IP and port 833 ... information which should be provided in command line. 834 835 [Arguments] ${host}=${OPENBMC_HOST} ${mask}=${NET_MASK} 836 ... ${gw_ip}=${GW_IP} 837 838 # Open telnet connection and ignore the error, in case telnet session is 839 # already opened by the program calling this keyword. 840 841 Run Keyword And Ignore Error Open Telnet Connection to BMC Serial Console 842 Telnet.write ifconfig eth0 ${host} netmask ${mask} 843 Telnet.write route add default gw ${gw_ip} 844 845Install Debug Tarball On BMC 846 [Documentation] Copy the tar file to BMC and install in "/usr/". 847 [Arguments] ${tarball_path} 848 # Description of arguments: 849 # tarball_path Absolute path of the debug tarball file. 850 # The tar file is downloaded from the build page 851 # https://openpower.xyz/job/openbmc-build/665/distro=ubuntu, 852 # target=witherspoon/artifact/images/witherspoon/ 853 # obmc-phosphor-debug-tarball-witherspoon.tar.xz 854 855 OperatingSystem.File Should Exist ${tarball_path} 856 ... msg=${tarball_path} doesn't exist. 857 Import Library SCPLibrary WITH NAME scp 858 Open Connection for SCP 859 scp.Put File ${tarball_path} /tmp/debug-tarball.tar.xz 860 Open Connection And Log In 861 Execute Command On BMC tar -xf /tmp/debug-tarball.tar.xz -C /usr/ 862 # Remove the tarball file from BMC 863 Execute Command On BMC rm /tmp/debug-tarball.tar.xz 864 865Get BMC Boot Count 866 [Documentation] Get BMC boot count based on boot time. 867 ${cur_btime}= Get BMC Boot Time 868 869 # Set global variable BOOT_TIME to current boot time if current boot time 870 # is changed. Also increase value of global variable BOOT_COUNT by 1. 871 Run Keyword If ${cur_btime} > ${BOOT_TIME} 872 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 873 ... AND 874 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 875 [Return] ${BOOT_COUNT} 876 877Set BMC Boot Count 878 [Documentation] Set BMC boot count to given value. 879 [Arguments] ${count} 880 881 # Description of arguments: 882 # count boot count value. 883 ${cur_btime}= Get BMC Boot Time 884 885 # Set global variable BOOT_COUNT to given value. 886 Set Global Variable ${BOOT_COUNT} ${count} 887 888 # Set BOOT_TIME variable to current boot time. 889 Set Global Variable ${BOOT_COUNT} ${count} 890 891############################################################################### 892Delete Error logs 893 [Documentation] Delete error logs. 894 895 # The REST method to delete error openbmc/openbmc#1327 896 # until then using logging restart. 897 Open Connection And Log In 898 Execute Command On BMC 899 ... systemctl restart xyz.openbmc_project.Logging.service 900 Sleep 10s reason=Wait for logging service to restart properly. 901 902############################################################################### 903Delete Error log Entry 904 [Documentation] Delete error log entry. 905 [Arguments] ${entry_path} 906 907 # Description of argument(s): 908 # entry_path Delete an error log entry. 909 # Ex. /xyz/openbmc_project/logging/entry/1 910 911 ${data}= Create Dictionary data=@{EMPTY} 912 ${resp}= Openbmc Delete Request ${entry_path} data=${data} 913 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 914