1*** Settings *** 2Resource ../lib/resource.txt 3Resource ../lib/rest_client.robot 4Resource ../lib/connection_client.robot 5Library DateTime 6Library Process 7Library OperatingSystem 8Library gen_print.py 9Library gen_robot_print.py 10Library gen_cmd.py 11Library gen_robot_keyword.py 12Library bmc_ssh_utils.py 13 14*** Variables *** 15${pflash_cmd} /usr/sbin/pflash -r /dev/stdout -P VERSION 16${SYSTEM_SHUTDOWN_TIME} ${5} 17${dbuscmdBase} 18... dbus-send --system --print-reply --dest=${OPENBMC_BASE_DBUS}.settings.Host 19${dbuscmdGet} 20... ${SETTINGS_URI}host0 org.freedesktop.DBus.Properties.Get 21# Enable when ready with openbmc/openbmc-test-automation#203 22#${dbuscmdString}= string:"xyz.openbmc_project.settings.Host" string: 23${dbuscmdString}= string:"org.openbmc.settings.Host" string: 24 25# Assign default value to QUIET for programs which may not define it. 26${QUIET} ${0} 27${bmc_mem_free_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f4 28${bmc_mem_total_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f2 29${bmc_cpu_usage_cmd}= top -n 1 | grep CPU: | cut -c 7-9 30${HOST_SETTING} ${SETTINGS_URI}host0 31# /run/initramfs/ro associate filesystem should be 100% full always 32${bmc_file_system_usage_cmd}= 33... df -h | grep -v /run/initramfs/ro | cut -c 52-54 | grep 100 | wc -l 34 35${BOOT_TIME} ${0} 36${BOOT_COUNT} ${0} 37${count} ${0} 38${devicetree_base} /sys/firmware/devicetree/base/model 39 40# Initialize default debug value to 0. 41${DEBUG} ${0} 42 43*** Keywords *** 44 45Verify PNOR Update 46 [Documentation] Verify that the PNOR is not corrupted. 47 # Example: 48 # FFS: Flash header not found. Code: 100 49 # Error 100 opening ffs ! 50 51 Open Connection And Log In 52 ${pnor_info}= Execute Command On BMC ${pflash_cmd} 53 Should Not Contain Any ${pnor_info} Flash header not found Error 54 55Get BMC System Model 56 [Documentation] Get the BMC model from the device tree. 57 58 ${bmc_model} ${stderr} ${rc}= BMC Execute Command 59 ... cat ${devicetree_base} | cut -d " " -f 1 return_stderr=True 60 Should Be Empty ${stderr} 61 Should Not Be Empty ${bmc_model} 62 [Return] ${bmc_model} 63 64Verify BMC System Model 65 [Documentation] Verify the BMC model with ${OPENBMC_MODEL}. 66 [Arguments] ${bmc_model} 67 68 ${tmp_bmc_model}= Fetch From Right ${OPENBMC_MODEL} / 69 ${tmp_bmc_model}= Fetch From Left ${tmp_bmc_model} . 70 ${ret}= Run Keyword And Return Status Should Contain ${bmc_model} 71 ... ${tmp_bmc_model} ignore_case=True 72 [Return] ${ret} 73 74Wait For Host To Ping 75 [Arguments] ${host} ${timeout}=${OPENBMC_REBOOT_TIMEOUT}min 76 ... ${interval}=5 sec 77 78 # host The DNS name or IP of the host to ping. 79 # timeout The amount of time after which attempts to ping cease. 80 # interval The amount of time in between attempts to ping. 81 82 Wait Until Keyword Succeeds ${timeout} ${interval} Ping Host ${host} 83 84Ping Host 85 [Arguments] ${host} 86 Should Not Be Empty ${host} msg=No host provided 87 ${RC} ${output}= Run and return RC and Output ping -c 4 ${host} 88 Log RC: ${RC}\nOutput:\n${output} 89 Should be equal ${RC} ${0} 90 91Get Boot Progress 92 [Arguments] ${quiet}=${QUIET} 93 94 ${state}= Read Attribute ${OPENBMC_BASE_URI}sensors/host/BootProgress 95 ... value quiet=${quiet} 96 [Return] ${state} 97 98Is Power On 99 ${state}= Get Power State 100 Should be equal ${state} ${1} 101 102Is Power Off 103 ${state}= Get Power State 104 Should be equal ${state} ${0} 105 106Initiate Power On 107 [Documentation] Initiates the power on and waits until the Is Power On 108 ... keyword returns that the power state has switched to on. 109 [Arguments] ${wait}=${1} 110 111 @{arglist}= Create List 112 ${args}= Create Dictionary data=@{arglist} 113 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOn 114 ... data=${args} 115 should be equal as strings ${resp.status_code} ${HTTP_OK} 116 117 # Does caller want to wait for power on status? 118 Run Keyword If '${wait}' == '${0}' Return From Keyword 119 Wait Until Keyword Succeeds 3 min 10 sec Is Power On 120 121Initiate Power Off 122 [Documentation] Initiates the power off and waits until the Is Power Off 123 ... keyword returns that the power state has switched to off. 124 @{arglist}= Create List 125 ${args}= Create Dictionary data=@{arglist} 126 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOff 127 ... data=${args} 128 should be equal as strings ${resp.status_code} ${HTTP_OK} 129 Wait Until Keyword Succeeds 1 min 10 sec Is Power Off 130 131Initiate OS Host Power Off 132 [Documentation] Initiate an OS reboot. 133 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 134 ... ${os_password}=${OS_PASSWORD} 135 136 # Description of arguments: 137 # os_host The DNS name or IP of the OS. 138 # os_username The username to be used to sign in to the OS. 139 # os_password The password to be used to sign in to the OS. 140 141 SSHLibrary.Open connection ${os_host} 142 Login ${os_username} ${os_password} 143 ${cmd_buf} Catenate shutdown 144 Start Command ${cmd_buf} 145 SSHLibrary.Close Connection 146 147Initiate OS Host Reboot 148 [Documentation] Initiate an OS reboot. 149 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 150 ... ${os_password}=${OS_PASSWORD} 151 152 # Description of arguments: 153 # os_host The DNS name or IP of the OS. 154 # os_username The username to be used to sign in to the OS. 155 # os_password The password to be used to sign in to the OS. 156 157 SSHLibrary.Open connection ${os_host} 158 Login ${os_username} ${os_password} 159 ${cmd_buf} Catenate reboot 160 Start Command ${cmd_buf} 161 SSHLibrary.Close Connection 162 163Initiate Auto Reboot 164 [Documentation] Initiate an auto reboot. 165 166 # Set the auto reboot policy. 167 Set Auto Reboot yes 168 # Set the watchdog timer. Note: 5000 = milliseconds which is 5 seconds. 169 Trigger Host Watchdog Error 5000 170 171Trigger Warm Reset 172 log to console "Triggering warm reset" 173 ${data}= create dictionary data=@{EMPTY} 174 ${resp}= openbmc post request 175 ... ${OPENBMC_BASE_URI}control/bmc0/action/warmReset data=${data} 176 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 177 ${session_active}= Check If warmReset is Initiated 178 Run Keyword If '${session_active}' == '${True}' 179 ... Fail msg=warm reset didn't occur 180 181 Sleep ${SYSTEM_SHUTDOWN_TIME}min 182 Check If BMC Is Up 183 184Check OS 185 [Documentation] Attempts to ping the host OS and then checks that the host 186 ... OS is up by running an SSH command. 187 188 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 189 ... ${os_password}=${OS_PASSWORD} ${quiet}=${QUIET} 190 ... ${print_string}=${EMPTY} 191 [Teardown] SSHLibrary.Close Connection 192 193 # os_host The DNS name/IP of the OS host associated with our BMC. 194 # os_username The username to be used to sign on to the OS host. 195 # os_password The password to be used to sign on to the OS host. 196 # quiet Indicates whether this keyword should write to console. 197 # print_string A string to be printed before checking the OS. 198 199 rprint ${print_string} 200 201 # Attempt to ping the OS. Store the return code to check later. 202 ${ping_rc}= Run Keyword and Return Status Ping Host ${os_host} 203 204 SSHLibrary.Open connection ${os_host} 205 206 ${status} ${msg}= Run Keyword And Ignore Error Login ${os_username} 207 ... ${os_password} 208 ${err_msg1}= Sprint Error ${msg} 209 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 210 Run Keyword If '${status}' == 'FAIL' Fail msg=${err_msg} 211 ${output} ${stderr} ${rc}= Execute Command uptime return_stderr=True 212 ... return_rc=True 213 214 ${temp_msg}= Catenate Could not execute a command on the operating 215 ... system.\n 216 ${err_msg1}= Sprint Error ${temp_msg} 217 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 218 219 # If the return code returned by "Execute Command" is non-zero, this 220 # keyword will fail. 221 Should Be Equal ${rc} ${0} msg=${err_msg} 222 # We will likewise fail if there is any stderr data. 223 Should Be Empty ${stderr} 224 225 ${temp_msg}= Set Variable Could not ping the operating system.\n 226 ${err_msg1}= Sprint Error ${temp_msg} 227 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 228 # We will likewise fail if the OS did not ping, as we could SSH but not 229 # ping 230 Should Be Equal As Strings ${ping_rc} ${TRUE} msg=${err_msg} 231 232Wait for OS 233 [Documentation] Waits for the host OS to come up via calls to "Check OS". 234 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 235 ... ${os_password}=${OS_PASSWORD} ${timeout}=${OS_WAIT_TIMEOUT} 236 ... ${quiet}=${0} 237 [Teardown] rprintn 238 239 # os_host The DNS name or IP of the OS host associated with our 240 # BMC. 241 # os_username The username to be used to sign on to the OS host. 242 # os_password The password to be used to sign on to the OS host. 243 # timeout The timeout in seconds indicating how long you're 244 # willing to wait for the OS to respond. 245 # quiet Indicates whether this keyword should write to console. 246 247 # The interval to be used between calls to "Check OS". 248 ${interval}= Set Variable 5 249 250 ${message}= Catenate Checking every ${interval} seconds for up to 251 ... ${timeout} seconds for the operating system to communicate. 252 rqprint_timen ${message} 253 254 Wait Until Keyword Succeeds ${timeout} sec ${interval} Check OS 255 ... ${os_host} ${os_username} ${os_password} 256 ... print_string=\# 257 258 rqprintn 259 260 rqprint_timen The operating system is now communicating. 261 262Get BMC State Deprecated 263 [Documentation] Returns the state of the BMC as a string. (i.e: BMC_READY) 264 [Arguments] ${quiet}=${QUIET} 265 266 @{arglist}= Create List 267 ${args}= Create Dictionary data=@{arglist} 268 ${resp}= Call Method ${OPENBMC_BASE_URI}managers/System/ getSystemState 269 ... data=${args} quiet=${quiet} 270 Should be equal as strings ${resp.status_code} ${HTTP_OK} 271 ${content}= to json ${resp.content} 272 [Return] ${content["data"]} 273 274Get Power State 275 [Documentation] Returns the power state as an integer. Either 0 or 1. 276 [Arguments] ${quiet}=${QUIET} 277 278 @{arglist}= Create List 279 ${args}= Create Dictionary data=@{arglist} 280 281 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ getPowerState 282 ... data=${args} quiet=${quiet} 283 Should be equal as strings ${resp.status_code} ${HTTP_OK} 284 ${content}= to json ${resp.content} 285 [Return] ${content["data"]} 286 287Clear BMC Record Log 288 [Documentation] Clears all the event logs on the BMC. This would be 289 ... equivalent to ipmitool sel clear. 290 @{arglist}= Create List 291 ${args}= Create Dictionary data=@{arglist} 292 ${resp}= Call Method 293 ... ${OPENBMC_BASE_URI}records/events/ clear data=${args} 294 should be equal as strings ${resp.status_code} ${HTTP_OK} 295 296Copy PNOR to BMC 297 Import Library SCPLibrary WITH NAME scp 298 Open Connection for SCP 299 Log Copying ${PNOR_IMAGE_PATH} to /tmp 300 scp.Put File ${PNOR_IMAGE_PATH} /tmp 301 302Flash PNOR 303 [Documentation] Calls flash bios update method to flash PNOR image 304 [Arguments] ${pnor_image} 305 @{arglist}= Create List ${pnor_image} 306 ${args}= Create Dictionary data=@{arglist} 307 ${resp}= Call Method ${OPENBMC_BASE_URI}control/flash/bios/ update 308 ... data=${args} 309 should be equal as strings ${resp.status_code} ${HTTP_OK} 310 Wait Until Keyword Succeeds 2 min 10 sec Is PNOR Flashing 311 312Get Flash BIOS Status 313 [Documentation] Returns the status of the flash BIOS API as a string. For 314 ... example 'Flashing', 'Flash Done', etc 315 ${data}= Read Properties ${OPENBMC_BASE_URI}control/flash/bios 316 [Return] ${data['status']} 317 318Is PNOR Flashing 319 [Documentation] Get BIOS 'Flashing' status. This indicates that PNOR 320 ... flashing has started. 321 ${status}= Get Flash BIOS Status 322 Should Contain ${status} Flashing 323 324Is PNOR Flash Done 325 [Documentation] Get BIOS 'Flash Done' status. This indicates that the 326 ... PNOR flashing has completed. 327 ${status}= Get Flash BIOS Status 328 should be equal as strings ${status} Flash Done 329 330Is System State Host Booted 331 [Documentation] Checks whether system state is HOST_BOOTED. 332 ${state}= Get BMC State Deprecated 333 should be equal as strings ${state} HOST_BOOTED 334 335Is OS Starting 336 [Documentation] Check if boot progress is OS starting. 337 ${boot_progress}= Get Boot Progress 338 Should Be Equal ${boot_progress} FW Progress, Starting OS 339 340Is OS Off 341 [Documentation] Check if boot progress is "Off". 342 ${boot_progress}= Get Boot Progress 343 Should Be Equal ${boot_progress} Off 344 345Verify Ping and REST Authentication 346 ${l_ping}= Run Keyword And Return Status 347 ... Ping Host ${OPENBMC_HOST} 348 Run Keyword If '${l_ping}' == '${False}' 349 ... Fail msg=Ping Failed 350 351 ${l_rest}= Run Keyword And Return Status 352 ... Initialize OpenBMC 353 Run Keyword If '${l_rest}' == '${False}' 354 ... Fail msg=REST Authentication Failed 355 356 # Just to make sure the SSH is working for SCP 357 Open Connection And Log In 358 ${system} ${stderr}= Execute Command hostname return_stderr=True 359 Should Be Empty ${stderr} 360 361Check If BMC is Up 362 [Documentation] Wait for Host to be online. Checks every X seconds 363 ... interval for Y minutes and fails if timed out. 364 ... Default MAX timedout is 10 min, interval 10 seconds. 365 [Arguments] ${max_timeout}=${OPENBMC_REBOOT_TIMEOUT} min 366 ... ${interval}=10 sec 367 368 Wait Until Keyword Succeeds 369 ... ${max_timeout} ${interval} Verify Ping and REST Authentication 370 371 372Check If warmReset is Initiated 373 [Documentation] Ping would be still alive, so try SSH to connect 374 ... if fails the ports are down indicating reboot 375 ... is in progress 376 377 # Warm reset adds 3 seconds delay before forcing reboot 378 # To minimize race conditions, we wait for 7 seconds 379 Sleep 7s 380 ${alive}= Run Keyword and Return Status 381 ... Open Connection And Log In 382 Return From Keyword If '${alive}' == '${False}' ${False} 383 [Return] ${True} 384 385Flush REST Sessions 386 [Documentation] Removes all the active session objects 387 Delete All Sessions 388 389Initialize DBUS cmd 390 [Documentation] Initialize dbus string with property string to extract 391 [Arguments] ${boot_property} 392 ${cmd}= Catenate ${dbuscmdBase} ${dbuscmdGet} ${dbuscmdString} 393 ${cmd}= Catenate ${cmd}${boot_property} 394 Set Global Variable ${dbuscmd} ${cmd} 395 396Create OS Console File Path 397 [Documentation] Create OS console file path name and return it. 398 [Arguments] ${log_file_path}=${EMPTY} 399 400 # Description of arguements: 401 # file_path The caller's candidate value. If this value is ${EMPTY}, this 402 # keyword will compose a file path name. Otherwise, this 403 # keyword will use the caller's file_path value. In either 404 # case, the value will be returned. 405 406 ${default_file_path}= Catenate /tmp/${OPENBMC_HOST}_os_console 407 ${log_file_path}= Set Variable If '${log_file_path}' == '${EMPTY}' 408 ... ${default_file_path} ${log_file_path} 409 410 [Return] ${log_file_path} 411 412Create OS Console Command String 413 [Documentation] Return a command string to start OS console logging. 414 415 # First make sure that the ssh_pw program is available. 416 ${cmd_buf}= Catenate which ssh_pw 2>&1 417 Rdpissuing ${cmd_buf} 418 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 419 Rdpvars rc output 420 Should Be Equal ${rc} ${0} msg=${output}\n 421 422 ${cmd_buf}= Catenate ssh_pw ${OPENBMC_PASSWORD} -p 2200 423 ... -o "StrictHostKeyChecking no" ${OPENBMC_USERNAME}@${OPENBMC_HOST} 424 425 [Return] ${cmd_buf} 426 427Stop SOL Console Logging 428 [Documentation] Stop system console logging and return log output. 429 [Arguments] ${log_file_path}=${EMPTY} 430 ... ${targ_file_path}=${EXECDIR}${/}logs${/} 431 ... ${return_data}=${1} 432 433 # If there are muliple system console processes, they will all be stopped. 434 # If there is no existing log file this keyword will return an error 435 # message to that effect (and write that message to targ_file_path, if 436 # specified). 437 # NOTE: This keyword will not fail if there is no running system console 438 # process. 439 440 # Description of arguments: 441 # log_file_path The file path that was used to call "Start SOL 442 # Console Logging". See that keyword (above) for details. 443 # targ_file_path If specified, the file path to which the source 444 # file path (i.e. "log_file_path") should be copied. 445 # return_data If this is set to ${1}, this keyword will return the SOL 446 # data to the caller as a unicode string. 447 448 ${log_file_path}= Create OS Console File Path ${log_file_path} 449 # Find the pid of the active system console logging session (if any). 450 ${search_string}= Create OS Console Command String 451 ${cmd_buf}= Catenate echo $(ps -ef | egrep '${search_string}' 452 ... | egrep -v grep | cut -c10-14) 453 Rdpissuing ${cmd_buf} 454 ${rc} ${os_con_pid}= Run And Return Rc And Output ${cmd_buf} 455 Rdpvars os_con_pid 456 # If rc is not zero it just means that there is no OS Console process 457 # running. 458 459 ${cmd_buf}= Catenate kill -9 ${os_con_pid} 460 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpissuing ${cmd_buf} 461 ${rc} ${output}= Run Keyword If '${os_con_pid}' != '${EMPTY}' 462 ... Run And Return Rc And Output ${cmd_buf} 463 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpvars rc output 464 465 Run Keyword If '${targ_file_path}' != '${EMPTY}' 466 ... Run Keyword And Ignore Error 467 ... Copy File ${log_file_path} ${targ_file_path} 468 469 ${output}= Set Variable ${EMPTY} 470 ${loc_quiet}= Evaluate ${debug}^1 471 ${rc} ${output}= Run Keyword If '${return_data}' == '${1}' 472 ... Cmd Fnc cat ${log_file_path} quiet=${loc_quiet} print_output=${0} 473 474 [Return] ${output} 475 476Start SOL Console Logging 477 [Documentation] Start system console log to file. 478 [Arguments] ${log_file_path}=${EMPTY} ${return_data}=${1} 479 480 # This keyword will first call "Stop SOL Console Logging". Only then will 481 # it start SOL console logging. The data returned by "Stop SOL Console 482 # Logging" will in turn be returned by this keyword. 483 484 # Description of arguments: 485 # log_file_path The file path to which system console log data should be 486 # written. Note that this path is taken to be a location 487 # on the machine where this program is running rather than 488 # on the Open BMC system. 489 # return_data If this is set to ${1}, this keyword will return any SOL 490 # data to the caller as a unicode string. 491 492 ${log_file_path}= Create OS Console File Path ${log_file_path} 493 494 ${log_output}= Stop SOL Console Logging ${log_file_path} 495 ... return_data=${return_data} 496 497 # Validate by making sure we can create the file. Problems creating the 498 # file would not be noticed by the subsequent ssh command because we fork 499 # the command. 500 Create File ${log_file_path} 501 ${sub_cmd_buf}= Create OS Console Command String 502 # Routing stderr to stdout so that any startup error text will go to the 503 # output file. 504 ${cmd_buf}= Catenate ${sub_cmd_buf} > ${log_file_path} 2>&1 & 505 Rdpissuing ${cmd_buf} 506 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 507 # Because we are forking this command, we essentially will never get a 508 # non-zero return code or any output. 509 Should Be Equal ${rc} ${0} 510 511 [Return] ${log_output} 512 513Get Time Stamp 514 [Documentation] Get the current time stamp data 515 ${cur_time}= Get Current Date result_format=%Y%m%d%H%M%S%f 516 [Return] ${cur_time} 517 518 519Verify BMC State 520 [Documentation] Get the BMC state and verify if the current 521 ... BMC state is as expected. 522 [Arguments] ${expected} 523 524 ${current}= Get BMC State Deprecated 525 Should Contain ${expected} ${current} 526 527Start Journal Log 528 [Documentation] Start capturing journal log to a file in /tmp using 529 ... journalctl command. By default journal log is collected 530 ... at /tmp/journal_log else user input location. 531 ... The File is appended with datetime. 532 [Arguments] ${file_path}=/tmp/journal_log 533 534 Open Connection And Log In 535 536 ${cur_time}= Get Time Stamp 537 Set Global Variable ${LOG_TIME} ${cur_time} 538 Start Command 539 ... journalctl -f > ${file_path}-${LOG_TIME} 540 Log Journal Log Started: ${file_path}-${LOG_TIME} 541 542 543Stop Journal Log 544 [Documentation] Stop journalctl process if its running. 545 ... By default return log from /tmp/journal_log else 546 ... user input location. 547 [Arguments] ${file_path}=/tmp/journal_log 548 549 Open Connection And Log In 550 551 ${rc}= 552 ... Execute Command 553 ... ps ax | grep journalctl | grep -v grep 554 ... return_stdout=False return_rc=True 555 556 Return From Keyword If '${rc}' == '${1}' 557 ... No journal log process running 558 559 ${output} ${stderr}= 560 ... Execute Command killall journalctl 561 ... return_stderr=True 562 Should Be Empty ${stderr} 563 564 ${journal_log} ${stderr}= 565 ... Execute Command 566 ... cat ${file_path}-${LOG_TIME} 567 ... return_stderr=True 568 Should Be Empty ${stderr} 569 570 Log ${journal_log} 571 572 Execute Command rm ${file_path}-${LOG_TIME} 573 574 [Return] ${journal_log} 575 576Mac Address To Hex String 577 [Documentation] Converts MAC address into hex format. 578 ... Example 579 ... Given the following MAC: 00:01:6C:80:02:78 580 ... This keyword will return: 0x00 0x01 0x6C 0x80 0x02 0x78 581 ... Description of arguments: 582 ... i_macaddress MAC address in the following format 583 ... 00:01:6C:80:02:78 584 [Arguments] ${i_macaddress} 585 586 ${mac_hex}= Catenate 0x${i_macaddress.replace(':', ' 0x')} 587 [Return] ${mac_hex} 588 589IP Address To Hex String 590 [Documentation] Converts IP address into hex format. 591 ... Example: 592 ... Given the following IP: 10.3.164.100 593 ... This keyword will return: 0xa 0x3 0xa4 0xa0 594 ... Description of arguments: 595 ... i_ipaddress IP address in the following format 596 ... 10.10.10.10 597 [Arguments] ${i_ipaddress} 598 599 @{ip}= Split String ${i_ipaddress} . 600 ${index}= Set Variable ${0} 601 602 :FOR ${item} IN @{ip} 603 \ ${hex}= Convert To Hex ${item} prefix=0x lowercase=yes 604 \ Set List Value ${ip} ${index} ${hex} 605 \ ${index}= Set Variable ${index + 1} 606 ${ip_hex}= Catenate @{ip} 607 [Return] ${ip_hex} 608 609BMC CPU Performance Check 610 [Documentation] Minimal 10% of proc should be free in this instance 611 612 ${bmc_cpu_usage_output} ${stderr}= Execute Command ${bmc_cpu_usage_cmd} 613 ... return_stderr=True 614 Should be empty ${stderr} 615 ${bmc_cpu_percentage}= Fetch From Left ${bmc_cpu_usage_output} % 616 Should be true ${bmc_cpu_percentage} < 90 617 618BMC Mem Performance Check 619 [Documentation] Minimal 10% of memory should be free in this instance 620 621 ${bmc_mem_free_output} ${stderr}= Execute Command ${bmc_mem_free_cmd} 622 ... return_stderr=True 623 Should be empty ${stderr} 624 625 ${bmc_mem_total_output} ${stderr}= Execute Command ${bmc_mem_total_cmd} 626 ... return_stderr=True 627 Should be empty ${stderr} 628 629 ${bmc_mem_percentage}= Evaluate ${bmc_mem_free_output}*100 630 ${bmc_mem_percentage}= Evaluate 631 ... ${bmc_mem_percentage}/${bmc_mem_total_output} 632 Should be true ${bmc_mem_percentage} > 10 633 634BMC File System Usage Check 635 [Documentation] Check the file system space. None should be 100% full 636 ... except /run/initramfs/ro 637 ${bmc_fs_usage_output} ${stderr}= Execute Command 638 ... ${bmc_file_system_usage_cmd} return_stderr=True 639 Should Be Empty ${stderr} 640 Should Be True ${bmc_fs_usage_output}==0 641 642Check BMC CPU Performance 643 [Documentation] Minimal 10% of proc should be free in 3 sample 644 :FOR ${var} IN Range 1 4 645 \ BMC CPU Performance check 646 647Check BMC Mem Performance 648 [Documentation] Minimal 10% of memory should be free 649 650 :FOR ${var} IN Range 1 4 651 \ BMC Mem Performance check 652 653Check BMC File System Performance 654 [Documentation] Check for file system usage for 4 times 655 656 :FOR ${var} IN Range 1 4 657 \ BMC File System Usage check 658 659Get URL List 660 [Documentation] Return list of URLs under given URL. 661 [Arguments] ${openbmc_url} 662 # Description of argument(s): 663 # openbmc_url URL for list operation (e.g. 664 # /xyz/openbmc_project/inventory). 665 666 ${url_list}= Read Properties ${openbmc_url}/list quiet=${1} 667 Sort List ${url_list} 668 [Return] ${url_list} 669 670Get Endpoint Paths 671 [Documentation] Returns all url paths ending with given endpoint 672 ... Example: 673 ... Given the following endpoint: cpu 674 ... This keyword will return: list of all urls ending with 675 ... cpu - 676 ... /org/openbmc/inventory/system/chassis/motherboard/cpu0, 677 ... /org/openbmc/inventory/system/chassis/motherboard/cpu1 678 ... Description of arguments: 679 ... path URL path for enumeration 680 ... endpoint string for which url path ending 681 [Arguments] ${path} ${endpoint} 682 683 ${resp}= Read Properties ${path}/enumerate timeout=30 684 log Dictionary ${resp} 685 686 ${list}= Get Dictionary Keys ${resp} 687 ${resp}= Get Matches ${list} regexp=^.*[0-9a-z_].${endpoint}[0-9]*$ 688 [Return] ${resp} 689 690Check Zombie Process 691 [Documentation] Check if any defunct process exist or not on BMC 692 ${count} ${stderr} ${rc}= Execute Command ps -o stat | grep Z | wc -l 693 ... return_stderr=True return_rc=True 694 Should Be True ${count}==0 695 Should Be Empty ${stderr} 696 697Prune Journal Log 698 [Documentation] Prune archived journal logs. 699 [Arguments] ${vacuum_size}=1M 700 701 # This keyword can be used to prevent the journal 702 # log from filling up the /run filesystem. 703 # This command will retain only the latest logs 704 # of the user specified size. 705 706 Open Connection And Log In 707 ${output} ${stderr} ${rc}= 708 ... Execute Command 709 ... journalctl --vacuum-size=${vacuum_size} 710 ... return_stderr=True return_rc=True 711 712 Should Be Equal ${rc} ${0} msg=${stderr} 713 714Set BMC Power Policy 715 [Documentation] Set the given BMC power policy. 716 [arguments] ${policy} 717 718 ${valueDict}= create dictionary data=${policy} 719 Write Attribute ${HOST_SETTING} power_policy data=${valueDict} 720 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 721 Should Be Equal ${currentPolicy} ${policy} 722 723Get System Power Policy 724 [Documentation] Get the BMC power policy. 725 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 726 [Return] ${currentPolicy} 727 728Get Auto Reboot 729 [Documentation] Returns auto reboot setting. 730 ${setting}= Read Attribute ${HOST_SETTING} auto_reboot 731 [Return] ${setting} 732 733 734Set Auto Reboot 735 [Documentation] Set the given auto reboot setting. 736 [Arguments] ${setting} 737 # setting auto reboot's setting, i.e. yes or no 738 739 ${valueDict}= Set Variable ${setting} 740 ${data}= Create Dictionary data=${valueDict} 741 Write Attribute ${HOST_SETTING} auto_reboot data=${data} 742 ${current_setting}= Get Auto Reboot 743 Should Be Equal ${current_setting} ${setting} 744 745 746Set BMC Reset Reference Time 747 [Documentation] Set current boot time as a reference and increment 748 ... boot count. 749 750 ${cur_btime}= Get BMC Boot Time 751 Run Keyword If ${BOOT_TIME} == ${0} and ${BOOT_COUNT} == ${0} 752 ... Set Global Variable ${BOOT_TIME} ${cur_btime} 753 ... ELSE IF ${cur_btime} > ${BOOT_TIME} 754 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 755 ... AND 756 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 757 758Get BMC Boot Time 759 [Documentation] Get boot time from /proc/stat. 760 761 Open Connection And Log In 762 ${output} ${stderr}= 763 ... Execute Command egrep '^btime ' /proc/stat | cut -f 2 -d ' ' 764 ... return_stderr=True 765 Should Be Empty ${stderr} 766 ${btime}= Convert To Integer ${output} 767 [Return] ${btime} 768 769Execute Command On BMC 770 [Documentation] Execute given command on BMC and return output. 771 [Arguments] ${command} 772 ${stdout} ${stderr}= Execute Command ${command} return_stderr=True 773 Should Be Empty ${stderr} 774 [Return] ${stdout} 775 776 777Enable Core Dump On BMC 778 [Documentation] Enable core dump collection. 779 Open Connection And Log In 780 ${core_pattern}= Execute Command On BMC 781 ... echo '/tmp/core_%e.%p' | tee /proc/sys/kernel/core_pattern 782 Should Be Equal As Strings ${core_pattern} /tmp/core_%e.%p 783 784Get Number Of BMC Core Dump Files 785 [Documentation] Get number of core dump files on BMC. 786 Open Connection And Log In 787 ${num_of_core_dump}= Execute Command 788 ... ls /tmp/core* 2>/dev/null | wc -l 789 [Return] ${num_of_core_dump} 790 791Set Core Dump File Size Unlimited 792 [Documentation] Set core dump file size to unlimited. 793 Open Connection And Log In 794 Execute Command On BMC 795 ... ulimit -c unlimited 796 797Check For Core Dumps 798 [Documentation] Check for any core dumps exist. 799 ${output}= Get Number Of BMC Core Dump Files 800 Run Keyword If ${output} > 0 801 ... Log **Warning** BMC core dump files exist level=WARN 802 803Trigger Host Watchdog Error 804 [Documentation] Inject host watchdog timeout error via REST. 805 [Arguments] ${milliseconds}=1000 ${sleep_time}=5s 806 # Description of argument(s): 807 # milliseconds The time watchdog timer value in milliseconds (e.g. 1000 = 808 # 1 second). 809 # sleep_time Time delay for host watchdog error to get injected. 810 # Default is 5 seconds. 811 812 ${data}= Create Dictionary data=${True} 813 Write Attribute /xyz/openbmc_project/watchdog/host0 Enabled data=${data} 814 815 ${data}= Create Dictionary data=${milliseconds} 816 Write Attribute /xyz/openbmc_project/watchdog/host0 TimeRemaining data=${data} 817 818 Sleep ${sleep_time} 819 820Login To OS Host 821 [Documentation] Login to OS Host. 822 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 823 ... ${os_password}=${OS_PASSWORD} 824 # Desription of arguments: 825 # ${os_host} IP address of the OS Host. 826 # ${os_username} OS Host Login user name. 827 # ${os_password} OS Host Login passwrd. 828 829 ${os_state}= Run Keyword And Return Status Ping Host ${os_host} 830 Run Keyword If '${os_state}' == 'False' 831 ... Run Keywords Initiate Host Reboot AND 832 ... Is Host Running AND 833 ... Wait for OS ${os_host} ${os_username} ${os_password} 834 835 SSHLibrary.Open Connection ${os_host} 836 ${resp}= Login ${os_username} ${os_password} 837 [Return] ${resp} 838 839Configure Initial Settings 840 [Documentation] Restore old IP and route. 841 ... This keyword requires initial settings viz IP address, 842 ... Network Mask, default gatway and serial console IP and port 843 ... information which should be provided in command line. 844 845 [Arguments] ${host}=${OPENBMC_HOST} ${mask}=${NET_MASK} 846 ... ${gw_ip}=${GW_IP} 847 848 # Open telnet connection and ignore the error, in case telnet session is 849 # already opened by the program calling this keyword. 850 851 Run Keyword And Ignore Error Open Telnet Connection to BMC Serial Console 852 Telnet.write ifconfig eth0 ${host} netmask ${mask} 853 Telnet.write route add default gw ${gw_ip} 854 855Install Debug Tarball On BMC 856 [Documentation] Copy the debug tar file to BMC and install. 857 [Arguments] ${tarball_file_path}=${EXECDIR}/obmc-phosphor-debug-tarball-witherspoon.tar.xz 858 ... ${targ_tarball_dir_path}=/home/root/tarball/ 859 860 # Description of arguments: 861 # tarball_file_path Path of the debug tarball file. 862 # The tar file is downloaded from the build page 863 # https://openpower.xyz/job/openbmc-build/ 864 # obmc-phosphor-debug-tarball-witherspoon.tar.xz 865 # 866 # targ_tarball_dir_path The directory path where the tarball is to be 867 # installed. 868 869 OperatingSystem.File Should Exist ${tarball_file_path} 870 ... msg=${tarball_file_path} doesn't exist. 871 872 # Upload the file to BMC. 873 Import Library SCPLibrary WITH NAME scp 874 Open Connection for SCP 875 scp.Put File ${tarball_file_path} /tmp/debug-tarball.tar.xz 876 877 # Create tarball directory and install. 878 Open Connection And Log In 879 Execute Command On BMC mkdir -p ${targ_tarball_dir_path} 880 Execute Command On BMC 881 ... tar -xf /tmp/debug-tarball.tar.xz -C ${targ_tarball_dir_path} 882 883 # Create symlink to callout-test binary. 884 Execute Command On BMC 885 ... ln -s ${targ_tarball_dir_path}/bin/callout-test /usr/bin/callout-test 886 887 # Create symlink to logging-test binary. 888 Execute Command On BMC 889 ... ln -s ${targ_tarball_dir_path}/bin/logging-test /usr/bin/logging-test 890 891 # Remove the tarball file from BMC 892 Execute Command On BMC rm /tmp/debug-tarball.tar.xz 893 894 895Get BMC Boot Count 896 [Documentation] Get BMC boot count based on boot time. 897 ${cur_btime}= Get BMC Boot Time 898 899 # Set global variable BOOT_TIME to current boot time if current boot time 900 # is changed. Also increase value of global variable BOOT_COUNT by 1. 901 Run Keyword If ${cur_btime} > ${BOOT_TIME} 902 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 903 ... AND 904 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 905 [Return] ${BOOT_COUNT} 906 907Set BMC Boot Count 908 [Documentation] Set BMC boot count to given value. 909 [Arguments] ${count} 910 911 # Description of arguments: 912 # count boot count value. 913 ${cur_btime}= Get BMC Boot Time 914 915 # Set global variable BOOT_COUNT to given value. 916 Set Global Variable ${BOOT_COUNT} ${count} 917 918 # Set BOOT_TIME variable to current boot time. 919 Set Global Variable ${BOOT_COUNT} ${count} 920 921Get System LED State 922 [Documentation] Return the state of given system LED. 923 [Arguments] ${led_name} 924 925 # Description of argument(s): 926 # led_name System LED name (e.g. heartbeat, identify, beep). 927 928 ${state}= Read Attribute ${LED_PHYSICAL_URI}${led_name} State 929 [Return] ${state.rsplit('.', 1)[1]} 930 931############################################################################### 932Delete Error Logs 933 [Documentation] Delete error logs. 934 935 # Check if error logs entries exist, if not return. 936 ${resp}= OpenBMC Get Request ${BMC_LOGGING_ENTRY}${/}list quiet=${1} 937 Return From Keyword If ${resp.status_code} == ${HTTP_NOT_FOUND} 938 939 # Get the list of error logs entries and delete them all. 940 ${elog_entries}= Get URL List ${BMC_LOGGING_ENTRY} 941 :FOR ${entry} IN @{elog_entries} 942 \ Delete Error Log Entry ${entry} 943 944############################################################################### 945Delete Error Log Entry 946 [Documentation] Delete error log entry. 947 [Arguments] ${entry_path} 948 949 # Description of argument(s): 950 # entry_path Delete an error log entry. 951 # Ex. /xyz/openbmc_project/logging/entry/1 952 953 # Skip delete if entry URI is a callout. 954 # Example: /xyz/openbmc_project/logging/entry/1/callout 955 Return From Keyword If '${entry_path.rsplit('/', 1)[1]}' == 'callout' 956 957 ${data}= Create Dictionary data=@{EMPTY} 958 ${resp}= Openbmc Delete Request ${entry_path} data=${data} 959 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 960 961 962############################################################################### 963Get LED State XYZ 964 [Documentation] Returns state of given LED. 965 [Arguments] ${led_name} 966 # Description of argument(s): 967 # led_name Name of LED 968 969 ${state}= Read Attribute ${LED_GROUPS_URI}${led_name} Asserted 970 [Return] ${state} 971 972 973############################################################################### 974Get BMC Version 975 [Documentation] Returns BMC version from /etc/os-release. 976 ... e.g. "v1.99.6-141-ge662190" 977 978 Open Connection And Log In 979 ${cmd}= Set Variable grep ^VERSION_ID= /etc/os-release | cut -f 2 -d '=' 980 ${output}= Execute Command On BMC ${cmd} 981 [Return] ${output} 982 983 984############################################################################### 985Get Elog URL List 986 [Documentation] Return error log entry list of URLs. 987 988 ${url_list}= Read Properties /xyz/openbmc_project/logging/entry/ 989 Sort List ${url_list} 990 [Return] ${url_list} 991 992 993############################################################################### 994