1*** Settings *** 2Resource ../lib/resource.txt 3Resource ../lib/rest_client.robot 4Resource ../lib/connection_client.robot 5Library DateTime 6Library Process 7Library OperatingSystem 8Library gen_print.py 9Library gen_robot_print.py 10Library gen_cmd.py 11 12*** Variables *** 13${SYSTEM_SHUTDOWN_TIME} ${5} 14${dbuscmdBase} 15... dbus-send --system --print-reply --dest=${OPENBMC_BASE_DBUS}.settings.Host 16${dbuscmdGet} 17... ${SETTINGS_URI}host0 org.freedesktop.DBus.Properties.Get 18# Enable when ready with openbmc/openbmc-test-automation#203 19#${dbuscmdString}= string:"xyz.openbmc_project.settings.Host" string: 20${dbuscmdString}= string:"org.openbmc.settings.Host" string: 21 22# Assign default value to QUIET for programs which may not define it. 23${QUIET} ${0} 24${bmc_mem_free_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f4 25${bmc_mem_total_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f2 26${bmc_cpu_usage_cmd}= top -n 1 | grep CPU: | cut -c 7-9 27${HOST_SETTING} ${SETTINGS_URI}host0 28# /run/initramfs/ro associate filesystem should be 100% full always 29${bmc_file_system_usage_cmd}= 30... df -h | grep -v /run/initramfs/ro | cut -c 52-54 | grep 100 | wc -l 31 32${BOOT_TIME} ${0} 33${BOOT_COUNT} ${0} 34${count} ${0} 35${devicetree_base} /sys/firmware/devicetree/base/model 36 37# Initialize default debug value to 0. 38${DEBUG} ${0} 39 40*** Keywords *** 41 42Get BMC System Model 43 [Documentation] Get the BMC model from the device tree. 44 45 Open Connection And Log In 46 ${bmc_model} ${stderr}= Execute Command 47 ... cat ${devicetree_base} | cut -d " " -f 1 return_stderr=True 48 Should Be Empty ${stderr} 49 Should Not Be Empty ${bmc_model} 50 [Return] ${bmc_model} 51 52Verify BMC System Model 53 [Documentation] Verify the BMC model with ${OPENBMC_MODEL}. 54 [Arguments] ${bmc_model} 55 56 ${tmp_bmc_model}= Fetch From Right ${OPENBMC_MODEL} / 57 ${tmp_bmc_model}= Fetch From Left ${tmp_bmc_model} . 58 ${ret}= Run Keyword And Return Status Should Contain ${bmc_model} 59 ... ${tmp_bmc_model} ignore_case=True 60 [Return] ${ret} 61 62Wait For Host To Ping 63 [Arguments] ${host} ${timeout}=${OPENBMC_REBOOT_TIMEOUT}min 64 ... ${interval}=5 sec 65 66 # host The DNS name or IP of the host to ping. 67 # timeout The amount of time after which attempts to ping cease. 68 # interval The amount of time in between attempts to ping. 69 70 Wait Until Keyword Succeeds ${timeout} ${interval} Ping Host ${host} 71 72Ping Host 73 [Arguments] ${host} 74 Should Not Be Empty ${host} msg=No host provided 75 ${RC} ${output}= Run and return RC and Output ping -c 4 ${host} 76 Log RC: ${RC}\nOutput:\n${output} 77 Should be equal ${RC} ${0} 78 79Get Boot Progress 80 [Arguments] ${quiet}=${QUIET} 81 82 ${state}= Read Attribute ${OPENBMC_BASE_URI}sensors/host/BootProgress 83 ... value quiet=${quiet} 84 [Return] ${state} 85 86Is Power On 87 ${state}= Get Power State 88 Should be equal ${state} ${1} 89 90Is Power Off 91 ${state}= Get Power State 92 Should be equal ${state} ${0} 93 94Initiate Power On 95 [Documentation] Initiates the power on and waits until the Is Power On 96 ... keyword returns that the power state has switched to on. 97 [Arguments] ${wait}=${1} 98 99 @{arglist}= Create List 100 ${args}= Create Dictionary data=@{arglist} 101 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOn 102 ... data=${args} 103 should be equal as strings ${resp.status_code} ${HTTP_OK} 104 105 # Does caller want to wait for power on status? 106 Run Keyword If '${wait}' == '${0}' Return From Keyword 107 Wait Until Keyword Succeeds 3 min 10 sec Is Power On 108 109Initiate Power Off 110 [Documentation] Initiates the power off and waits until the Is Power Off 111 ... keyword returns that the power state has switched to off. 112 @{arglist}= Create List 113 ${args}= Create Dictionary data=@{arglist} 114 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOff 115 ... data=${args} 116 should be equal as strings ${resp.status_code} ${HTTP_OK} 117 Wait Until Keyword Succeeds 1 min 10 sec Is Power Off 118 119Initiate OS Host Power Off 120 [Documentation] Initiate an OS reboot. 121 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 122 ... ${os_password}=${OS_PASSWORD} 123 124 # Description of arguments: 125 # os_host The DNS name or IP of the OS. 126 # os_username The username to be used to sign in to the OS. 127 # os_password The password to be used to sign in to the OS. 128 129 SSHLibrary.Open connection ${os_host} 130 Login ${os_username} ${os_password} 131 ${cmd_buf} Catenate shutdown 132 Start Command ${cmd_buf} 133 SSHLibrary.Close Connection 134 135Initiate OS Host Reboot 136 [Documentation] Initiate an OS reboot. 137 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 138 ... ${os_password}=${OS_PASSWORD} 139 140 # Description of arguments: 141 # os_host The DNS name or IP of the OS. 142 # os_username The username to be used to sign in to the OS. 143 # os_password The password to be used to sign in to the OS. 144 145 SSHLibrary.Open connection ${os_host} 146 Login ${os_username} ${os_password} 147 ${cmd_buf} Catenate reboot 148 Start Command ${cmd_buf} 149 SSHLibrary.Close Connection 150 151Initiate Auto Reboot 152 [Documentation] Initiate an auto reboot. 153 154 # Set the auto reboot policy. 155 Set Auto Reboot yes 156 157 SSHLibrary.Open connection ${openbmc_host} 158 Login ${openbmc_username} ${openbmc_password} 159 160 # Set the watchdog timer. Note: 5000 = milliseconds which is 5 seconds. 161 ${cmd_buf}= Catenate /usr/sbin/mapper call /org/openbmc/watchdog/host0 162 ... org.openbmc.Watchdog set i 5000 163 ${output} ${stderr} ${rc}= Execute Command ${cmd_buf} 164 ... return_stderr=True return_rc=True 165 Should Be Empty ${stderr} 166 Should be equal ${rc} ${0} 167 168 # Start the watchdog. 169 ${cmd_buf}= Catenate /usr/sbin/mapper call /org/openbmc/watchdog/host0 170 ... org.openbmc.Watchdog start 171 ${output} ${stderr} ${rc}= Execute Command ${cmd_buf} 172 ... return_stderr=True return_rc=True 173 Should Be Empty ${stderr} 174 Should be equal ${rc} ${0} 175 176Trigger Warm Reset 177 log to console "Triggering warm reset" 178 ${data}= create dictionary data=@{EMPTY} 179 ${resp}= openbmc post request 180 ... ${OPENBMC_BASE_URI}control/bmc0/action/warmReset data=${data} 181 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 182 ${session_active}= Check If warmReset is Initiated 183 Run Keyword If '${session_active}' == '${True}' 184 ... Fail msg=warm reset didn't occur 185 186 Sleep ${SYSTEM_SHUTDOWN_TIME}min 187 Check If BMC Is Up 188 189Check OS 190 [Documentation] Attempts to ping the host OS and then checks that the host 191 ... OS is up by running an SSH command. 192 193 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 194 ... ${os_password}=${OS_PASSWORD} ${quiet}=${QUIET} 195 ... ${print_string}=${EMPTY} 196 [Teardown] SSHLibrary.Close Connection 197 198 # os_host The DNS name/IP of the OS host associated with our BMC. 199 # os_username The username to be used to sign on to the OS host. 200 # os_password The password to be used to sign on to the OS host. 201 # quiet Indicates whether this keyword should write to console. 202 # print_string A string to be printed before checking the OS. 203 204 rprint ${print_string} 205 206 # Attempt to ping the OS. Store the return code to check later. 207 ${ping_rc}= Run Keyword and Return Status Ping Host ${os_host} 208 209 SSHLibrary.Open connection ${os_host} 210 211 ${status} ${msg}= Run Keyword And Ignore Error Login ${os_username} 212 ... ${os_password} 213 ${err_msg1}= Sprint Error ${msg} 214 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 215 Run Keyword If '${status}' == 'FAIL' Fail msg=${err_msg} 216 ${output} ${stderr} ${rc}= Execute Command uptime return_stderr=True 217 ... return_rc=True 218 219 ${temp_msg}= Catenate Could not execute a command on the operating 220 ... system.\n 221 ${err_msg1}= Sprint Error ${temp_msg} 222 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 223 224 # If the return code returned by "Execute Command" is non-zero, this 225 # keyword will fail. 226 Should Be Equal ${rc} ${0} msg=${err_msg} 227 # We will likewise fail if there is any stderr data. 228 Should Be Empty ${stderr} 229 230 ${temp_msg}= Set Variable Could not ping the operating system.\n 231 ${err_msg1}= Sprint Error ${temp_msg} 232 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 233 # We will likewise fail if the OS did not ping, as we could SSH but not 234 # ping 235 Should Be Equal As Strings ${ping_rc} ${TRUE} msg=${err_msg} 236 237Wait for OS 238 [Documentation] Waits for the host OS to come up via calls to "Check OS". 239 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 240 ... ${os_password}=${OS_PASSWORD} ${timeout}=${OS_WAIT_TIMEOUT} 241 ... ${quiet}=${0} 242 [Teardown] rprintn 243 244 # os_host The DNS name or IP of the OS host associated with our 245 # BMC. 246 # os_username The username to be used to sign on to the OS host. 247 # os_password The password to be used to sign on to the OS host. 248 # timeout The timeout in seconds indicating how long you're 249 # willing to wait for the OS to respond. 250 # quiet Indicates whether this keyword should write to console. 251 252 # The interval to be used between calls to "Check OS". 253 ${interval}= Set Variable 5 254 255 ${message}= Catenate Checking every ${interval} seconds for up to 256 ... ${timeout} seconds for the operating system to communicate. 257 rqprint_timen ${message} 258 259 Wait Until Keyword Succeeds ${timeout} sec ${interval} Check OS 260 ... ${os_host} ${os_username} ${os_password} 261 ... print_string=\# 262 263 rqprintn 264 265 rqprint_timen The operating system is now communicating. 266 267Get BMC State Deprecated 268 [Documentation] Returns the state of the BMC as a string. (i.e: BMC_READY) 269 [Arguments] ${quiet}=${QUIET} 270 271 @{arglist}= Create List 272 ${args}= Create Dictionary data=@{arglist} 273 ${resp}= Call Method ${OPENBMC_BASE_URI}managers/System/ getSystemState 274 ... data=${args} quiet=${quiet} 275 Should be equal as strings ${resp.status_code} ${HTTP_OK} 276 ${content}= to json ${resp.content} 277 [Return] ${content["data"]} 278 279Get Power State 280 [Documentation] Returns the power state as an integer. Either 0 or 1. 281 [Arguments] ${quiet}=${QUIET} 282 283 @{arglist}= Create List 284 ${args}= Create Dictionary data=@{arglist} 285 286 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ getPowerState 287 ... data=${args} quiet=${quiet} 288 Should be equal as strings ${resp.status_code} ${HTTP_OK} 289 ${content}= to json ${resp.content} 290 [Return] ${content["data"]} 291 292Clear BMC Record Log 293 [Documentation] Clears all the event logs on the BMC. This would be 294 ... equivalent to ipmitool sel clear. 295 @{arglist}= Create List 296 ${args}= Create Dictionary data=@{arglist} 297 ${resp}= Call Method 298 ... ${OPENBMC_BASE_URI}records/events/ clear data=${args} 299 should be equal as strings ${resp.status_code} ${HTTP_OK} 300 301Copy PNOR to BMC 302 Import Library SCPLibrary WITH NAME scp 303 Open Connection for SCP 304 Log Copying ${PNOR_IMAGE_PATH} to /tmp 305 scp.Put File ${PNOR_IMAGE_PATH} /tmp 306 307Flash PNOR 308 [Documentation] Calls flash bios update method to flash PNOR image 309 [Arguments] ${pnor_image} 310 @{arglist}= Create List ${pnor_image} 311 ${args}= Create Dictionary data=@{arglist} 312 ${resp}= Call Method ${OPENBMC_BASE_URI}control/flash/bios/ update 313 ... data=${args} 314 should be equal as strings ${resp.status_code} ${HTTP_OK} 315 Wait Until Keyword Succeeds 2 min 10 sec Is PNOR Flashing 316 317Get Flash BIOS Status 318 [Documentation] Returns the status of the flash BIOS API as a string. For 319 ... example 'Flashing', 'Flash Done', etc 320 ${data}= Read Properties ${OPENBMC_BASE_URI}control/flash/bios 321 [Return] ${data['status']} 322 323Is PNOR Flashing 324 [Documentation] Get BIOS 'Flashing' status. This indicates that PNOR 325 ... flashing has started. 326 ${status}= Get Flash BIOS Status 327 Should Contain ${status} Flashing 328 329Is PNOR Flash Done 330 [Documentation] Get BIOS 'Flash Done' status. This indicates that the 331 ... PNOR flashing has completed. 332 ${status}= Get Flash BIOS Status 333 should be equal as strings ${status} Flash Done 334 335Is System State Host Booted 336 [Documentation] Checks whether system state is HOST_BOOTED. 337 ${state}= Get BMC State Deprecated 338 should be equal as strings ${state} HOST_BOOTED 339 340Is OS Starting 341 [Documentation] Check if boot progress is OS starting. 342 ${boot_progress}= Get Boot Progress 343 Should Be Equal ${boot_progress} FW Progress, Starting OS 344 345Is OS Off 346 [Documentation] Check if boot progress is "Off". 347 ${boot_progress}= Get Boot Progress 348 Should Be Equal ${boot_progress} Off 349 350Verify Ping and REST Authentication 351 ${l_ping}= Run Keyword And Return Status 352 ... Ping Host ${OPENBMC_HOST} 353 Run Keyword If '${l_ping}' == '${False}' 354 ... Fail msg=Ping Failed 355 356 ${l_rest}= Run Keyword And Return Status 357 ... Initialize OpenBMC 358 Run Keyword If '${l_rest}' == '${False}' 359 ... Fail msg=REST Authentication Failed 360 361 # Just to make sure the SSH is working for SCP 362 Open Connection And Log In 363 ${system} ${stderr}= Execute Command hostname return_stderr=True 364 Should Be Empty ${stderr} 365 366Check If BMC is Up 367 [Documentation] Wait for Host to be online. Checks every X seconds 368 ... interval for Y minutes and fails if timed out. 369 ... Default MAX timedout is 10 min, interval 10 seconds. 370 [Arguments] ${max_timeout}=${OPENBMC_REBOOT_TIMEOUT} min 371 ... ${interval}=10 sec 372 373 Wait Until Keyword Succeeds 374 ... ${max_timeout} ${interval} Verify Ping and REST Authentication 375 376 377Check If warmReset is Initiated 378 [Documentation] Ping would be still alive, so try SSH to connect 379 ... if fails the ports are down indicating reboot 380 ... is in progress 381 382 # Warm reset adds 3 seconds delay before forcing reboot 383 # To minimize race conditions, we wait for 7 seconds 384 Sleep 7s 385 ${alive}= Run Keyword and Return Status 386 ... Open Connection And Log In 387 Return From Keyword If '${alive}' == '${False}' ${False} 388 [Return] ${True} 389 390Flush REST Sessions 391 [Documentation] Removes all the active session objects 392 Delete All Sessions 393 394Initialize DBUS cmd 395 [Documentation] Initialize dbus string with property string to extract 396 [Arguments] ${boot_property} 397 ${cmd}= Catenate ${dbuscmdBase} ${dbuscmdGet} ${dbuscmdString} 398 ${cmd}= Catenate ${cmd}${boot_property} 399 Set Global Variable ${dbuscmd} ${cmd} 400 401Create OS Console File Path 402 [Documentation] Create OS console file path name and return it. 403 [Arguments] ${log_file_path}=${EMPTY} 404 405 # Description of arguements: 406 # file_path The caller's candidate value. If this value is ${EMPTY}, this 407 # keyword will compose a file path name. Otherwise, this 408 # keyword will use the caller's file_path value. In either 409 # case, the value will be returned. 410 411 ${default_file_path}= Catenate /tmp/${OPENBMC_HOST}_os_console 412 ${log_file_path}= Set Variable If '${log_file_path}' == '${EMPTY}' 413 ... ${default_file_path} ${log_file_path} 414 415 [Return] ${log_file_path} 416 417Create OS Console Command String 418 [Documentation] Return a command string to start OS console logging. 419 420 # First make sure that the ssh_pw program is available. 421 ${cmd_buf}= Catenate which ssh_pw 2>&1 422 Rdpissuing ${cmd_buf} 423 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 424 Rdpvars rc output 425 Should Be Equal ${rc} ${0} msg=${output}\n 426 427 ${cmd_buf}= Catenate ssh_pw ${OPENBMC_PASSWORD} -p 2200 428 ... ${OPENBMC_USERNAME}@${OPENBMC_HOST} 429 430 [Return] ${cmd_buf} 431 432Stop SOL Console Logging 433 [Documentation] Stop system console logging and return log output. 434 [Arguments] ${log_file_path}=${EMPTY} ${targ_file_path}=${EMPTY} 435 ... ${return_data}=${1} 436 437 # If there are muliple system console processes, they will all be stopped. 438 # If there is no existing log file this keyword will return an error 439 # message to that effect (and write that message to targ_file_path, if 440 # specified). 441 # NOTE: This keyword will not fail if there is no running system console 442 # process. 443 444 # Description of arguments: 445 # log_file_path The file path that was used to call "Start SOL 446 # Console Logging". See that keyword (above) for details. 447 # targ_file_path If specified, the file path to which the source 448 # file path (i.e. "log_file_path") should be copied. 449 # return_data If this is set to ${1}, this keyword will return the SOL 450 # data to the caller as a unicode string. 451 452 ${log_file_path}= Create OS Console File Path ${log_file_path} 453 # Find the pid of the active system console logging session (if any). 454 ${search_string}= Create OS Console Command String 455 ${cmd_buf}= Catenate echo $(ps -ef | egrep '${search_string}' 456 ... | egrep -v grep | cut -c10-14) 457 Rdpissuing ${cmd_buf} 458 ${rc} ${os_con_pid}= Run And Return Rc And Output ${cmd_buf} 459 Rdpvars os_con_pid 460 # If rc is not zero it just means that there is no OS Console process 461 # running. 462 463 ${cmd_buf}= Catenate kill -9 ${os_con_pid} 464 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpissuing ${cmd_buf} 465 ${rc} ${output}= Run Keyword If '${os_con_pid}' != '${EMPTY}' 466 ... Run And Return Rc And Output ${cmd_buf} 467 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpvars rc output 468 469 Run Keyword If '${targ_file_path}' != '${EMPTY}' 470 ... Run Keyword And Ignore Error 471 ... Copy File ${log_file_path} ${targ_file_path} 472 473 ${output}= Set Variable ${EMPTY} 474 ${loc_quiet}= Evaluate ${debug}^1 475 ${rc} ${output}= Run Keyword If '${return_data}' == '${1}' 476 ... Cmd Fnc cat ${log_file_path} quiet=${loc_quiet} print_output=${0} 477 478 [Return] ${output} 479 480Start SOL Console Logging 481 [Documentation] Start system console log to file. 482 [Arguments] ${log_file_path}=${EMPTY} ${return_data}=${1} 483 484 # This keyword will first call "Stop SOL Console Logging". Only then will 485 # it start SOL console logging. The data returned by "Stop SOL Console 486 # Logging" will in turn be returned by this keyword. 487 488 # Description of arguments: 489 # log_file_path The file path to which system console log data should be 490 # written. Note that this path is taken to be a location 491 # on the machine where this program is running rather than 492 # on the Open BMC system. 493 # return_data If this is set to ${1}, this keyword will return any SOL 494 # data to the caller as a unicode string. 495 496 ${log_file_path}= Create OS Console File Path ${log_file_path} 497 498 ${log_output}= Stop SOL Console Logging ${log_file_path} 499 ... return_data=${return_data} 500 501 # Validate by making sure we can create the file. Problems creating the 502 # file would not be noticed by the subsequent ssh command because we fork 503 # the command. 504 Create File ${log_file_path} 505 ${sub_cmd_buf}= Create OS Console Command String 506 # Routing stderr to stdout so that any startup error text will go to the 507 # output file. 508 ${cmd_buf}= Catenate ${sub_cmd_buf} > ${log_file_path} 2>&1 & 509 Rdpissuing ${cmd_buf} 510 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 511 # Because we are forking this command, we essentially will never get a 512 # non-zero return code or any output. 513 Should Be Equal ${rc} ${0} 514 515 [Return] ${log_output} 516 517Get Time Stamp 518 [Documentation] Get the current time stamp data 519 ${cur_time}= Get Current Date result_format=%Y%m%d%H%M%S%f 520 [Return] ${cur_time} 521 522 523Verify BMC State 524 [Documentation] Get the BMC state and verify if the current 525 ... BMC state is as expected. 526 [Arguments] ${expected} 527 528 ${current}= Get BMC State Deprecated 529 Should Contain ${expected} ${current} 530 531Start Journal Log 532 [Documentation] Start capturing journal log to a file in /tmp using 533 ... journalctl command. By default journal log is collected 534 ... at /tmp/journal_log else user input location. 535 ... The File is appended with datetime. 536 [Arguments] ${file_path}=/tmp/journal_log 537 538 Open Connection And Log In 539 540 ${cur_time}= Get Time Stamp 541 Set Global Variable ${LOG_TIME} ${cur_time} 542 Start Command 543 ... journalctl -f > ${file_path}-${LOG_TIME} 544 Log Journal Log Started: ${file_path}-${LOG_TIME} 545 546 547Stop Journal Log 548 [Documentation] Stop journalctl process if its running. 549 ... By default return log from /tmp/journal_log else 550 ... user input location. 551 [Arguments] ${file_path}=/tmp/journal_log 552 553 Open Connection And Log In 554 555 ${rc}= 556 ... Execute Command 557 ... ps ax | grep journalctl | grep -v grep 558 ... return_stdout=False return_rc=True 559 560 Return From Keyword If '${rc}' == '${1}' 561 ... No journal log process running 562 563 ${output} ${stderr}= 564 ... Execute Command killall journalctl 565 ... return_stderr=True 566 Should Be Empty ${stderr} 567 568 ${journal_log} ${stderr}= 569 ... Execute Command 570 ... cat ${file_path}-${LOG_TIME} 571 ... return_stderr=True 572 Should Be Empty ${stderr} 573 574 Log ${journal_log} 575 576 Execute Command rm ${file_path}-${LOG_TIME} 577 578 [Return] ${journal_log} 579 580Mac Address To Hex String 581 [Documentation] Converts MAC address into hex format. 582 ... Example 583 ... Given the following MAC: 00:01:6C:80:02:78 584 ... This keyword will return: 0x00 0x01 0x6C 0x80 0x02 0x78 585 ... Description of arguments: 586 ... i_macaddress MAC address in the following format 587 ... 00:01:6C:80:02:78 588 [Arguments] ${i_macaddress} 589 590 ${mac_hex}= Catenate 0x${i_macaddress.replace(':', ' 0x')} 591 [Return] ${mac_hex} 592 593IP Address To Hex String 594 [Documentation] Converts IP address into hex format. 595 ... Example: 596 ... Given the following IP: 10.3.164.100 597 ... This keyword will return: 0xa 0x3 0xa4 0xa0 598 ... Description of arguments: 599 ... i_ipaddress IP address in the following format 600 ... 10.10.10.10 601 [Arguments] ${i_ipaddress} 602 603 @{ip}= Split String ${i_ipaddress} . 604 ${index}= Set Variable ${0} 605 606 :FOR ${item} IN @{ip} 607 \ ${hex}= Convert To Hex ${item} prefix=0x lowercase=yes 608 \ Set List Value ${ip} ${index} ${hex} 609 \ ${index}= Set Variable ${index + 1} 610 ${ip_hex}= Catenate @{ip} 611 [Return] ${ip_hex} 612 613BMC CPU Performance Check 614 [Documentation] Minimal 10% of proc should be free in this instance 615 616 ${bmc_cpu_usage_output} ${stderr}= Execute Command ${bmc_cpu_usage_cmd} 617 ... return_stderr=True 618 Should be empty ${stderr} 619 ${bmc_cpu_percentage}= Fetch From Left ${bmc_cpu_usage_output} % 620 Should be true ${bmc_cpu_percentage} < 90 621 622BMC Mem Performance Check 623 [Documentation] Minimal 10% of memory should be free in this instance 624 625 ${bmc_mem_free_output} ${stderr}= Execute Command ${bmc_mem_free_cmd} 626 ... return_stderr=True 627 Should be empty ${stderr} 628 629 ${bmc_mem_total_output} ${stderr}= Execute Command ${bmc_mem_total_cmd} 630 ... return_stderr=True 631 Should be empty ${stderr} 632 633 ${bmc_mem_percentage}= Evaluate ${bmc_mem_free_output}*100 634 ${bmc_mem_percentage}= Evaluate 635 ... ${bmc_mem_percentage}/${bmc_mem_total_output} 636 Should be true ${bmc_mem_percentage} > 10 637 638BMC File System Usage Check 639 [Documentation] Check the file system space. None should be 100% full 640 ... except /run/initramfs/ro 641 ${bmc_fs_usage_output} ${stderr}= Execute Command 642 ... ${bmc_file_system_usage_cmd} return_stderr=True 643 Should Be Empty ${stderr} 644 Should Be True ${bmc_fs_usage_output}==0 645 646Check BMC CPU Performance 647 [Documentation] Minimal 10% of proc should be free in 3 sample 648 :FOR ${var} IN Range 1 4 649 \ BMC CPU Performance check 650 651Check BMC Mem Performance 652 [Documentation] Minimal 10% of memory should be free 653 654 :FOR ${var} IN Range 1 4 655 \ BMC Mem Performance check 656 657Check BMC File System Performance 658 [Documentation] Check for file system usage for 4 times 659 660 :FOR ${var} IN Range 1 4 661 \ BMC File System Usage check 662 663Get URL List 664 [Documentation] Return list of URLs under given URL. 665 [Arguments] ${openbmc_url} 666 # Description of argument(s): 667 # openbmc_url URL for list operation (e.g. 668 # /xyz/openbmc_project/inventory). 669 670 ${url_list}= Read Properties ${openbmc_url}/list 671 [Return] ${url_list} 672 673Get Endpoint Paths 674 [Documentation] Returns all url paths ending with given endpoint 675 ... Example: 676 ... Given the following endpoint: cpu 677 ... This keyword will return: list of all urls ending with 678 ... cpu - 679 ... /org/openbmc/inventory/system/chassis/motherboard/cpu0, 680 ... /org/openbmc/inventory/system/chassis/motherboard/cpu1 681 ... Description of arguments: 682 ... path URL path for enumeration 683 ... endpoint string for which url path ending 684 [Arguments] ${path} ${endpoint} 685 686 ${resp}= Read Properties ${path}/enumerate timeout=30 687 log Dictionary ${resp} 688 689 ${list}= Get Dictionary Keys ${resp} 690 ${resp}= Get Matches ${list} regexp=^.*[0-9a-z_].${endpoint}[0-9]*$ 691 [Return] ${resp} 692 693Check Zombie Process 694 [Documentation] Check if any defunct process exist or not on BMC 695 ${count} ${stderr} ${rc}= Execute Command ps -o stat | grep Z | wc -l 696 ... return_stderr=True return_rc=True 697 Should Be True ${count}==0 698 Should Be Empty ${stderr} 699 700Prune Journal Log 701 [Documentation] Prune archived journal logs. 702 [Arguments] ${vacuum_size}=1M 703 704 # This keyword can be used to prevent the journal 705 # log from filling up the /run filesystem. 706 # This command will retain only the latest logs 707 # of the user specified size. 708 709 Open Connection And Log In 710 ${output} ${stderr} ${rc}= 711 ... Execute Command 712 ... journalctl --vacuum-size=${vacuum_size} 713 ... return_stderr=True return_rc=True 714 715 Should Be Equal ${rc} ${0} msg=${stderr} 716 717Set BMC Power Policy 718 [Documentation] Set the given BMC power policy. 719 [arguments] ${policy} 720 721 ${valueDict}= create dictionary data=${policy} 722 Write Attribute ${HOST_SETTING} power_policy data=${valueDict} 723 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 724 Should Be Equal ${currentPolicy} ${policy} 725 726Get System Power Policy 727 [Documentation] Get the BMC power policy. 728 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 729 [Return] ${currentPolicy} 730 731Get Auto Reboot 732 [Documentation] Returns auto reboot setting. 733 ${setting}= Read Attribute ${HOST_SETTING} auto_reboot 734 [Return] ${setting} 735 736 737Set Auto Reboot 738 [Documentation] Set the given auto reboot setting. 739 [Arguments] ${setting} 740 # setting auto reboot's setting, i.e. yes or no 741 742 ${valueDict}= Set Variable ${setting} 743 ${data}= Create Dictionary data=${valueDict} 744 Write Attribute ${HOST_SETTING} auto_reboot data=${data} 745 ${current_setting}= Get Auto Reboot 746 Should Be Equal ${current_setting} ${setting} 747 748 749Set BMC Reset Reference Time 750 [Documentation] Set current boot time as a reference and increment 751 ... boot count. 752 753 ${cur_btime}= Get BMC Boot Time 754 Run Keyword If ${BOOT_TIME} == ${0} and ${BOOT_COUNT} == ${0} 755 ... Set Global Variable ${BOOT_TIME} ${cur_btime} 756 ... ELSE IF ${cur_btime} > ${BOOT_TIME} 757 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 758 ... AND 759 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 760 761Get BMC Boot Time 762 [Documentation] Get boot time from /proc/stat. 763 764 Open Connection And Log In 765 ${output} ${stderr}= 766 ... Execute Command egrep '^btime ' /proc/stat | cut -f 2 -d ' ' 767 ... return_stderr=True 768 Should Be Empty ${stderr} 769 ${btime}= Convert To Integer ${output} 770 [Return] ${btime} 771 772Execute Command On BMC 773 [Documentation] Execute given command on BMC and return output. 774 [Arguments] ${command} 775 ${stdout} ${stderr}= Execute Command ${command} return_stderr=True 776 Should Be Empty ${stderr} 777 [Return] ${stdout} 778 779 780Enable Core Dump On BMC 781 [Documentation] Enable core dump collection. 782 Open Connection And Log In 783 ${core_pattern}= Execute Command On BMC 784 ... echo '/tmp/core_%e.%p' | tee /proc/sys/kernel/core_pattern 785 Should Be Equal As Strings ${core_pattern} /tmp/core_%e.%p 786 787Get Number Of BMC Core Dump Files 788 [Documentation] Get number of core dump files on BMC. 789 Open Connection And Log In 790 ${num_of_core_dump}= Execute Command 791 ... ls /tmp/core* 2>/dev/null | wc -l 792 [Return] ${num_of_core_dump} 793 794Set Core Dump File Size Unlimited 795 [Documentation] Set core dump file size to unlimited. 796 Open Connection And Log In 797 Execute Command On BMC 798 ... ulimit -c unlimited 799 800Check For Core Dumps 801 [Documentation] Check for any core dumps exist. 802 ${output}= Get Number Of BMC Core Dump Files 803 Run Keyword If ${output} > 0 804 ... Log **Warning** BMC core dump files exist level=WARN 805 806Trigger Host Watchdog Error 807 [Documentation] Inject host watchdog error using BMC. 808 [Arguments] ${milliseconds}=1000 ${sleep_time}=5s 809 # Description of argument(s): 810 # milliseconds The time watchdog timer value in milliseconds (e.g. 1000 = 811 # 1 second). 812 # sleep_time Time delay for host watchdog error to get injected. 813 # Default is 5 seconds. 814 815 Execute Command On BMC 816 ... /usr/sbin/mapper call /org/openbmc/watchdog/host0 org.openbmc.Watchdog set i ${milliseconds} 817 Execute Command On BMC 818 ... /usr/sbin/mapper call /org/openbmc/watchdog/host0 org.openbmc.Watchdog start 819 Sleep ${sleep_time} 820 821Login To OS Host 822 [Documentation] Login to OS Host. 823 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 824 ... ${os_password}=${OS_PASSWORD} 825 # Desription of arguments: 826 # ${os_host} IP address of the OS Host. 827 # ${os_username} OS Host Login user name. 828 # ${os_password} OS Host Login passwrd. 829 830 ${os_state}= Run Keyword And Return Status Ping Host ${os_host} 831 Run Keyword If '${os_state}' == 'False' 832 ... Run Keywords Initiate Host Reboot AND 833 ... Is Host Running AND 834 ... Wait for OS ${os_host} ${os_username} ${os_password} 835 836 Open Connection ${os_host} 837 ${resp}= Login ${os_username} ${os_password} 838 [Return] ${resp} 839 840Configure Initial Settings 841 [Documentation] Restore old IP and route. 842 ... This keyword requires initial settings viz IP address, 843 ... Network Mask, default gatway and serial console IP and port 844 ... information which should be provided in command line. 845 846 [Arguments] ${host}=${OPENBMC_HOST} ${mask}=${NET_MASK} 847 ... ${gw_ip}=${GW_IP} 848 849 # Open telnet connection and ignore the error, in case telnet session is 850 # already opened by the program calling this keyword. 851 852 Run Keyword And Ignore Error Open Telnet Connection to BMC Serial Console 853 Telnet.write ifconfig eth0 ${host} netmask ${mask} 854 Telnet.write route add default gw ${gw_ip} 855 856Install Debug Tarball On BMC 857 [Documentation] Copy the debug tar file to BMC and install. 858 [Arguments] ${tarball_file_path}=${EXECDIR}/obmc-phosphor-debug-tarball-witherspoon.tar.xz 859 ... ${targ_tarball_dir_path}=/home/root/tarball/ 860 861 # Description of arguments: 862 # tarball_file_path Path of the debug tarball file. 863 # The tar file is downloaded from the build page 864 # https://openpower.xyz/job/openbmc-build/ 865 # obmc-phosphor-debug-tarball-witherspoon.tar.xz 866 # 867 # targ_tarball_dir_path The directory path where the tarball is to be 868 # installed. 869 870 OperatingSystem.File Should Exist ${tarball_file_path} 871 ... msg=${tarball_file_path} doesn't exist. 872 873 # Upload the file to BMC. 874 Import Library SCPLibrary WITH NAME scp 875 Open Connection for SCP 876 scp.Put File ${tarball_file_path} /tmp/debug-tarball.tar.xz 877 878 # Create tarball directory and install. 879 Open Connection And Log In 880 Execute Command On BMC mkdir -p ${targ_tarball_dir_path} 881 Execute Command On BMC 882 ... tar -xf /tmp/debug-tarball.tar.xz -C ${targ_tarball_dir_path} 883 884 # Create symlink to callout-test binary. 885 Execute Command On BMC 886 ... ln -s ${targ_tarball_dir_path}/bin/callout-test /usr/bin/callout-test 887 888 # Remove the tarball file from BMC 889 Execute Command On BMC rm /tmp/debug-tarball.tar.xz 890 891 892Get BMC Boot Count 893 [Documentation] Get BMC boot count based on boot time. 894 ${cur_btime}= Get BMC Boot Time 895 896 # Set global variable BOOT_TIME to current boot time if current boot time 897 # is changed. Also increase value of global variable BOOT_COUNT by 1. 898 Run Keyword If ${cur_btime} > ${BOOT_TIME} 899 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 900 ... AND 901 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 902 [Return] ${BOOT_COUNT} 903 904Set BMC Boot Count 905 [Documentation] Set BMC boot count to given value. 906 [Arguments] ${count} 907 908 # Description of arguments: 909 # count boot count value. 910 ${cur_btime}= Get BMC Boot Time 911 912 # Set global variable BOOT_COUNT to given value. 913 Set Global Variable ${BOOT_COUNT} ${count} 914 915 # Set BOOT_TIME variable to current boot time. 916 Set Global Variable ${BOOT_COUNT} ${count} 917 918############################################################################### 919Delete Error logs 920 [Documentation] Delete error logs. 921 922 # The REST method to delete error openbmc/openbmc#1327 923 # until then using logging restart. 924 Open Connection And Log In 925 Execute Command On BMC 926 ... systemctl restart xyz.openbmc_project.Logging.service 927 Sleep 10s reason=Wait for logging service to restart properly. 928 929############################################################################### 930Delete Error log Entry 931 [Documentation] Delete error log entry. 932 [Arguments] ${entry_path} 933 934 # Description of argument(s): 935 # entry_path Delete an error log entry. 936 # Ex. /xyz/openbmc_project/logging/entry/1 937 938 ${data}= Create Dictionary data=@{EMPTY} 939 ${resp}= Openbmc Delete Request ${entry_path} data=${data} 940 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 941