1*** Settings *** 2Resource ../lib/resource.txt 3Resource ../lib/rest_client.robot 4Resource ../lib/connection_client.robot 5Library DateTime 6Library Process 7Library OperatingSystem 8Library gen_print.py 9Library gen_robot_print.py 10Library gen_cmd.py 11Library gen_robot_keyword.py 12Library bmc_ssh_utils.py 13 14*** Variables *** 15${pflash_cmd} /usr/sbin/pflash -r /dev/stdout -P VERSION 16${SYSTEM_SHUTDOWN_TIME} ${5} 17${dbuscmdBase} 18... dbus-send --system --print-reply --dest=${OPENBMC_BASE_DBUS}.settings.Host 19${dbuscmdGet} 20... ${SETTINGS_URI}host0 org.freedesktop.DBus.Properties.Get 21# Enable when ready with openbmc/openbmc-test-automation#203 22#${dbuscmdString}= string:"xyz.openbmc_project.settings.Host" string: 23${dbuscmdString}= string:"org.openbmc.settings.Host" string: 24 25# Assign default value to QUIET for programs which may not define it. 26${QUIET} ${0} 27${bmc_mem_free_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f4 28${bmc_mem_total_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f2 29${bmc_cpu_usage_cmd}= top -n 1 | grep CPU: | cut -c 7-9 30${HOST_SETTING} ${SETTINGS_URI}host0 31# /run/initramfs/ro associate filesystem should be 100% full always 32${bmc_file_system_usage_cmd}= 33... df -h | grep -v /run/initramfs/ro | cut -c 52-54 | grep 100 | wc -l 34 35${BOOT_TIME} ${0} 36${BOOT_COUNT} ${0} 37${count} ${0} 38${devicetree_base} /sys/firmware/devicetree/base/model 39 40# Initialize default debug value to 0. 41${DEBUG} ${0} 42 43*** Keywords *** 44 45Verify PNOR Update 46 [Documentation] Verify that the PNOR is not corrupted. 47 # Example: 48 # FFS: Flash header not found. Code: 100 49 # Error 100 opening ffs ! 50 51 Open Connection And Log In 52 ${pnor_info}= Execute Command On BMC ${pflash_cmd} 53 Should Not Contain Any ${pnor_info} Flash header not found Error 54 55Get BMC System Model 56 [Documentation] Get the BMC model from the device tree. 57 58 ${bmc_model} ${stderr} ${rc}= BMC Execute Command 59 ... cat ${devicetree_base} | cut -d " " -f 1 return_stderr=True 60 Should Be Empty ${stderr} 61 Should Not Be Empty ${bmc_model} 62 [Return] ${bmc_model} 63 64Verify BMC System Model 65 [Documentation] Verify the BMC model with ${OPENBMC_MODEL}. 66 [Arguments] ${bmc_model} 67 68 ${tmp_bmc_model}= Fetch From Right ${OPENBMC_MODEL} / 69 ${tmp_bmc_model}= Fetch From Left ${tmp_bmc_model} . 70 ${ret}= Run Keyword And Return Status Should Contain ${bmc_model} 71 ... ${tmp_bmc_model} ignore_case=True 72 [Return] ${ret} 73 74Wait For Host To Ping 75 [Arguments] ${host} ${timeout}=${OPENBMC_REBOOT_TIMEOUT}min 76 ... ${interval}=5 sec 77 78 # host The DNS name or IP of the host to ping. 79 # timeout The amount of time after which attempts to ping cease. 80 # interval The amount of time in between attempts to ping. 81 82 Wait Until Keyword Succeeds ${timeout} ${interval} Ping Host ${host} 83 84Ping Host 85 [Arguments] ${host} 86 Should Not Be Empty ${host} msg=No host provided 87 ${RC} ${output}= Run and return RC and Output ping -c 4 ${host} 88 Log RC: ${RC}\nOutput:\n${output} 89 Should be equal ${RC} ${0} 90 91Get Boot Progress 92 [Arguments] ${quiet}=${QUIET} 93 94 ${state}= Read Attribute ${OPENBMC_BASE_URI}sensors/host/BootProgress 95 ... value quiet=${quiet} 96 [Return] ${state} 97 98Is Power On 99 ${state}= Get Power State 100 Should be equal ${state} ${1} 101 102Is Power Off 103 ${state}= Get Power State 104 Should be equal ${state} ${0} 105 106Initiate Power On 107 [Documentation] Initiates the power on and waits until the Is Power On 108 ... keyword returns that the power state has switched to on. 109 [Arguments] ${wait}=${1} 110 111 @{arglist}= Create List 112 ${args}= Create Dictionary data=@{arglist} 113 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOn 114 ... data=${args} 115 should be equal as strings ${resp.status_code} ${HTTP_OK} 116 117 # Does caller want to wait for power on status? 118 Run Keyword If '${wait}' == '${0}' Return From Keyword 119 Wait Until Keyword Succeeds 3 min 10 sec Is Power On 120 121Initiate Power Off 122 [Documentation] Initiates the power off and waits until the Is Power Off 123 ... keyword returns that the power state has switched to off. 124 @{arglist}= Create List 125 ${args}= Create Dictionary data=@{arglist} 126 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOff 127 ... data=${args} 128 should be equal as strings ${resp.status_code} ${HTTP_OK} 129 Wait Until Keyword Succeeds 1 min 10 sec Is Power Off 130 131Initiate OS Host Power Off 132 [Documentation] Initiate an OS reboot. 133 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 134 ... ${os_password}=${OS_PASSWORD} 135 136 # Description of arguments: 137 # os_host The DNS name or IP of the OS. 138 # os_username The username to be used to sign in to the OS. 139 # os_password The password to be used to sign in to the OS. 140 141 SSHLibrary.Open connection ${os_host} 142 Login ${os_username} ${os_password} 143 ${cmd_buf} Catenate shutdown 144 Start Command ${cmd_buf} 145 SSHLibrary.Close Connection 146 147Initiate OS Host Reboot 148 [Documentation] Initiate an OS reboot. 149 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 150 ... ${os_password}=${OS_PASSWORD} 151 152 # Description of arguments: 153 # os_host The DNS name or IP of the OS. 154 # os_username The username to be used to sign in to the OS. 155 # os_password The password to be used to sign in to the OS. 156 157 SSHLibrary.Open connection ${os_host} 158 Login ${os_username} ${os_password} 159 ${cmd_buf} Catenate reboot 160 Start Command ${cmd_buf} 161 SSHLibrary.Close Connection 162 163Initiate Auto Reboot 164 [Documentation] Initiate an auto reboot. 165 166 # Set the auto reboot policy. 167 Set Auto Reboot yes 168 # Set the watchdog timer. Note: 5000 = milliseconds which is 5 seconds. 169 Trigger Host Watchdog Error 5000 170 171Trigger Warm Reset 172 log to console "Triggering warm reset" 173 ${data}= create dictionary data=@{EMPTY} 174 ${resp}= openbmc post request 175 ... ${OPENBMC_BASE_URI}control/bmc0/action/warmReset data=${data} 176 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 177 ${session_active}= Check If warmReset is Initiated 178 Run Keyword If '${session_active}' == '${True}' 179 ... Fail msg=warm reset didn't occur 180 181 Sleep ${SYSTEM_SHUTDOWN_TIME}min 182 Check If BMC Is Up 183 184Check OS 185 [Documentation] Attempts to ping the host OS and then checks that the host 186 ... OS is up by running an SSH command. 187 188 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 189 ... ${os_password}=${OS_PASSWORD} ${quiet}=${QUIET} 190 ... ${print_string}=${EMPTY} 191 [Teardown] SSHLibrary.Close Connection 192 193 # os_host The DNS name/IP of the OS host associated with our BMC. 194 # os_username The username to be used to sign on to the OS host. 195 # os_password The password to be used to sign on to the OS host. 196 # quiet Indicates whether this keyword should write to console. 197 # print_string A string to be printed before checking the OS. 198 199 rprint ${print_string} 200 201 # Attempt to ping the OS. Store the return code to check later. 202 ${ping_rc}= Run Keyword and Return Status Ping Host ${os_host} 203 204 SSHLibrary.Open connection ${os_host} 205 206 ${status} ${msg}= Run Keyword And Ignore Error Login ${os_username} 207 ... ${os_password} 208 ${err_msg1}= Sprint Error ${msg} 209 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 210 Run Keyword If '${status}' == 'FAIL' Fail msg=${err_msg} 211 ${output} ${stderr} ${rc}= Execute Command uptime return_stderr=True 212 ... return_rc=True 213 214 ${temp_msg}= Catenate Could not execute a command on the operating 215 ... system.\n 216 ${err_msg1}= Sprint Error ${temp_msg} 217 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 218 219 # If the return code returned by "Execute Command" is non-zero, this 220 # keyword will fail. 221 Should Be Equal ${rc} ${0} msg=${err_msg} 222 # We will likewise fail if there is any stderr data. 223 Should Be Empty ${stderr} 224 225 ${temp_msg}= Set Variable Could not ping the operating system.\n 226 ${err_msg1}= Sprint Error ${temp_msg} 227 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 228 # We will likewise fail if the OS did not ping, as we could SSH but not 229 # ping 230 Should Be Equal As Strings ${ping_rc} ${TRUE} msg=${err_msg} 231 232Wait for OS 233 [Documentation] Waits for the host OS to come up via calls to "Check OS". 234 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 235 ... ${os_password}=${OS_PASSWORD} ${timeout}=${OS_WAIT_TIMEOUT} 236 ... ${quiet}=${0} 237 [Teardown] rprintn 238 239 # os_host The DNS name or IP of the OS host associated with our 240 # BMC. 241 # os_username The username to be used to sign on to the OS host. 242 # os_password The password to be used to sign on to the OS host. 243 # timeout The timeout in seconds indicating how long you're 244 # willing to wait for the OS to respond. 245 # quiet Indicates whether this keyword should write to console. 246 247 # The interval to be used between calls to "Check OS". 248 ${interval}= Set Variable 5 249 250 ${message}= Catenate Checking every ${interval} seconds for up to 251 ... ${timeout} seconds for the operating system to communicate. 252 rqprint_timen ${message} 253 254 Wait Until Keyword Succeeds ${timeout} sec ${interval} Check OS 255 ... ${os_host} ${os_username} ${os_password} 256 ... print_string=\# 257 258 rqprintn 259 260 rqprint_timen The operating system is now communicating. 261 262Get BMC State Deprecated 263 [Documentation] Returns the state of the BMC as a string. (i.e: BMC_READY) 264 [Arguments] ${quiet}=${QUIET} 265 266 @{arglist}= Create List 267 ${args}= Create Dictionary data=@{arglist} 268 ${resp}= Call Method ${OPENBMC_BASE_URI}managers/System/ getSystemState 269 ... data=${args} quiet=${quiet} 270 Should be equal as strings ${resp.status_code} ${HTTP_OK} 271 ${content}= to json ${resp.content} 272 [Return] ${content["data"]} 273 274Get Power State 275 [Documentation] Returns the power state as an integer. Either 0 or 1. 276 [Arguments] ${quiet}=${QUIET} 277 278 @{arglist}= Create List 279 ${args}= Create Dictionary data=@{arglist} 280 281 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ getPowerState 282 ... data=${args} quiet=${quiet} 283 Should be equal as strings ${resp.status_code} ${HTTP_OK} 284 ${content}= to json ${resp.content} 285 [Return] ${content["data"]} 286 287Clear BMC Record Log 288 [Documentation] Clears all the event logs on the BMC. This would be 289 ... equivalent to ipmitool sel clear. 290 @{arglist}= Create List 291 ${args}= Create Dictionary data=@{arglist} 292 ${resp}= Call Method 293 ... ${OPENBMC_BASE_URI}records/events/ clear data=${args} 294 should be equal as strings ${resp.status_code} ${HTTP_OK} 295 296Copy PNOR to BMC 297 Import Library SCPLibrary WITH NAME scp 298 Open Connection for SCP 299 Log Copying ${PNOR_IMAGE_PATH} to /tmp 300 scp.Put File ${PNOR_IMAGE_PATH} /tmp 301 302Flash PNOR 303 [Documentation] Calls flash bios update method to flash PNOR image 304 [Arguments] ${pnor_image} 305 @{arglist}= Create List ${pnor_image} 306 ${args}= Create Dictionary data=@{arglist} 307 ${resp}= Call Method ${OPENBMC_BASE_URI}control/flash/bios/ update 308 ... data=${args} 309 should be equal as strings ${resp.status_code} ${HTTP_OK} 310 Wait Until Keyword Succeeds 2 min 10 sec Is PNOR Flashing 311 312Get Flash BIOS Status 313 [Documentation] Returns the status of the flash BIOS API as a string. For 314 ... example 'Flashing', 'Flash Done', etc 315 ${data}= Read Properties ${OPENBMC_BASE_URI}control/flash/bios 316 [Return] ${data['status']} 317 318Is PNOR Flashing 319 [Documentation] Get BIOS 'Flashing' status. This indicates that PNOR 320 ... flashing has started. 321 ${status}= Get Flash BIOS Status 322 Should Contain ${status} Flashing 323 324Is PNOR Flash Done 325 [Documentation] Get BIOS 'Flash Done' status. This indicates that the 326 ... PNOR flashing has completed. 327 ${status}= Get Flash BIOS Status 328 should be equal as strings ${status} Flash Done 329 330Is System State Host Booted 331 [Documentation] Checks whether system state is HOST_BOOTED. 332 ${state}= Get BMC State Deprecated 333 should be equal as strings ${state} HOST_BOOTED 334 335Is OS Starting 336 [Documentation] Check if boot progress is OS starting. 337 ${boot_progress}= Get Boot Progress 338 Should Be Equal ${boot_progress} FW Progress, Starting OS 339 340Is OS Off 341 [Documentation] Check if boot progress is "Off". 342 ${boot_progress}= Get Boot Progress 343 Should Be Equal ${boot_progress} Off 344 345Get Boot Progress To OS Starting State 346 [Documentation] Get the system to a boot progress state of 'FW Progress, 347 ... Starting OS'. 348 349 ${boot_progress}= Get Boot Progress 350 Run Keyword If '${boot_progress}' == 'FW Progress, Starting OS' 351 ... Log Host is already in OS starting state 352 ... ELSE 353 ... Run Keywords Initiate Host PowerOff AND Initiate Host Boot 354 ... AND Wait Until Keyword Succeeds 10 min 10 sec Is OS Starting 355 356Verify Ping and REST Authentication 357 ${l_ping}= Run Keyword And Return Status 358 ... Ping Host ${OPENBMC_HOST} 359 Run Keyword If '${l_ping}' == '${False}' 360 ... Fail msg=Ping Failed 361 362 ${l_rest}= Run Keyword And Return Status 363 ... Initialize OpenBMC 364 Run Keyword If '${l_rest}' == '${False}' 365 ... Fail msg=REST Authentication Failed 366 367 # Just to make sure the SSH is working for SCP 368 Open Connection And Log In 369 ${system} ${stderr}= Execute Command hostname return_stderr=True 370 Should Be Empty ${stderr} 371 372Check If BMC is Up 373 [Documentation] Wait for Host to be online. Checks every X seconds 374 ... interval for Y minutes and fails if timed out. 375 ... Default MAX timedout is 10 min, interval 10 seconds. 376 [Arguments] ${max_timeout}=${OPENBMC_REBOOT_TIMEOUT} min 377 ... ${interval}=10 sec 378 379 Wait Until Keyword Succeeds 380 ... ${max_timeout} ${interval} Verify Ping and REST Authentication 381 382 383Check If warmReset is Initiated 384 [Documentation] Ping would be still alive, so try SSH to connect 385 ... if fails the ports are down indicating reboot 386 ... is in progress 387 388 # Warm reset adds 3 seconds delay before forcing reboot 389 # To minimize race conditions, we wait for 7 seconds 390 Sleep 7s 391 ${alive}= Run Keyword and Return Status 392 ... Open Connection And Log In 393 Return From Keyword If '${alive}' == '${False}' ${False} 394 [Return] ${True} 395 396Flush REST Sessions 397 [Documentation] Removes all the active session objects 398 Delete All Sessions 399 400Initialize DBUS cmd 401 [Documentation] Initialize dbus string with property string to extract 402 [Arguments] ${boot_property} 403 ${cmd}= Catenate ${dbuscmdBase} ${dbuscmdGet} ${dbuscmdString} 404 ${cmd}= Catenate ${cmd}${boot_property} 405 Set Global Variable ${dbuscmd} ${cmd} 406 407Create OS Console File Path 408 [Documentation] Create OS console file path name and return it. 409 [Arguments] ${log_file_path}=${EMPTY} 410 411 # Description of arguements: 412 # file_path The caller's candidate value. If this value is ${EMPTY}, this 413 # keyword will compose a file path name. Otherwise, this 414 # keyword will use the caller's file_path value. In either 415 # case, the value will be returned. 416 417 ${default_file_path}= Catenate /tmp/${OPENBMC_HOST}_os_console 418 ${log_file_path}= Set Variable If '${log_file_path}' == '${EMPTY}' 419 ... ${default_file_path} ${log_file_path} 420 421 [Return] ${log_file_path} 422 423Create OS Console Command String 424 [Documentation] Return a command string to start OS console logging. 425 426 # First make sure that the ssh_pw program is available. 427 ${cmd_buf}= Catenate which ssh_pw 2>&1 428 Rdpissuing ${cmd_buf} 429 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 430 Rdpvars rc output 431 Should Be Equal ${rc} ${0} msg=${output}\n 432 433 ${cmd_buf}= Catenate ssh_pw ${OPENBMC_PASSWORD} -p 2200 434 ... -o "StrictHostKeyChecking no" ${OPENBMC_USERNAME}@${OPENBMC_HOST} 435 436 [Return] ${cmd_buf} 437 438Stop SOL Console Logging 439 [Documentation] Stop system console logging and return log output. 440 [Arguments] ${log_file_path}=${EMPTY} 441 ... ${targ_file_path}=${EXECDIR}${/}logs${/} 442 ... ${return_data}=${1} 443 444 # If there are muliple system console processes, they will all be stopped. 445 # If there is no existing log file this keyword will return an error 446 # message to that effect (and write that message to targ_file_path, if 447 # specified). 448 # NOTE: This keyword will not fail if there is no running system console 449 # process. 450 451 # Description of arguments: 452 # log_file_path The file path that was used to call "Start SOL 453 # Console Logging". See that keyword (above) for details. 454 # targ_file_path If specified, the file path to which the source 455 # file path (i.e. "log_file_path") should be copied. 456 # return_data If this is set to ${1}, this keyword will return the SOL 457 # data to the caller as a unicode string. 458 459 ${log_file_path}= Create OS Console File Path ${log_file_path} 460 # Find the pid of the active system console logging session (if any). 461 ${search_string}= Create OS Console Command String 462 ${cmd_buf}= Catenate echo $(ps -ef | egrep '${search_string}' 463 ... | egrep -v grep | cut -c10-14) 464 Rdpissuing ${cmd_buf} 465 ${rc} ${os_con_pid}= Run And Return Rc And Output ${cmd_buf} 466 Rdpvars os_con_pid 467 # If rc is not zero it just means that there is no OS Console process 468 # running. 469 470 ${cmd_buf}= Catenate kill -9 ${os_con_pid} 471 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpissuing ${cmd_buf} 472 ${rc} ${output}= Run Keyword If '${os_con_pid}' != '${EMPTY}' 473 ... Run And Return Rc And Output ${cmd_buf} 474 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpvars rc output 475 476 Run Keyword If '${targ_file_path}' != '${EMPTY}' 477 ... Run Keyword And Ignore Error 478 ... Copy File ${log_file_path} ${targ_file_path} 479 480 ${output}= Set Variable ${EMPTY} 481 ${loc_quiet}= Evaluate ${debug}^1 482 ${rc} ${output}= Run Keyword If '${return_data}' == '${1}' 483 ... Cmd Fnc cat ${log_file_path} quiet=${loc_quiet} print_output=${0} 484 485 [Return] ${output} 486 487Start SOL Console Logging 488 [Documentation] Start system console log to file. 489 [Arguments] ${log_file_path}=${EMPTY} ${return_data}=${1} 490 491 # This keyword will first call "Stop SOL Console Logging". Only then will 492 # it start SOL console logging. The data returned by "Stop SOL Console 493 # Logging" will in turn be returned by this keyword. 494 495 # Description of arguments: 496 # log_file_path The file path to which system console log data should be 497 # written. Note that this path is taken to be a location 498 # on the machine where this program is running rather than 499 # on the Open BMC system. 500 # return_data If this is set to ${1}, this keyword will return any SOL 501 # data to the caller as a unicode string. 502 503 ${log_file_path}= Create OS Console File Path ${log_file_path} 504 505 ${log_output}= Stop SOL Console Logging ${log_file_path} 506 ... return_data=${return_data} 507 508 # Validate by making sure we can create the file. Problems creating the 509 # file would not be noticed by the subsequent ssh command because we fork 510 # the command. 511 Create File ${log_file_path} 512 ${sub_cmd_buf}= Create OS Console Command String 513 # Routing stderr to stdout so that any startup error text will go to the 514 # output file. 515 ${cmd_buf}= Catenate ${sub_cmd_buf} > ${log_file_path} 2>&1 & 516 Rdpissuing ${cmd_buf} 517 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 518 # Because we are forking this command, we essentially will never get a 519 # non-zero return code or any output. 520 Should Be Equal ${rc} ${0} 521 522 [Return] ${log_output} 523 524Get Time Stamp 525 [Documentation] Get the current time stamp data 526 ${cur_time}= Get Current Date result_format=%Y%m%d%H%M%S%f 527 [Return] ${cur_time} 528 529 530Verify BMC State 531 [Documentation] Get the BMC state and verify if the current 532 ... BMC state is as expected. 533 [Arguments] ${expected} 534 535 ${current}= Get BMC State Deprecated 536 Should Contain ${expected} ${current} 537 538Start Journal Log 539 [Documentation] Start capturing journal log to a file in /tmp using 540 ... journalctl command. By default journal log is collected 541 ... at /tmp/journal_log else user input location. 542 ... The File is appended with datetime. 543 [Arguments] ${file_path}=/tmp/journal_log 544 545 Open Connection And Log In 546 547 ${cur_time}= Get Time Stamp 548 Set Global Variable ${LOG_TIME} ${cur_time} 549 Start Command 550 ... journalctl -f > ${file_path}-${LOG_TIME} 551 Log Journal Log Started: ${file_path}-${LOG_TIME} 552 553 554Stop Journal Log 555 [Documentation] Stop journalctl process if its running. 556 ... By default return log from /tmp/journal_log else 557 ... user input location. 558 [Arguments] ${file_path}=/tmp/journal_log 559 560 Open Connection And Log In 561 562 ${rc}= 563 ... Execute Command 564 ... ps ax | grep journalctl | grep -v grep 565 ... return_stdout=False return_rc=True 566 567 Return From Keyword If '${rc}' == '${1}' 568 ... No journal log process running 569 570 ${output} ${stderr}= 571 ... Execute Command killall journalctl 572 ... return_stderr=True 573 Should Be Empty ${stderr} 574 575 ${journal_log} ${stderr}= 576 ... Execute Command 577 ... cat ${file_path}-${LOG_TIME} 578 ... return_stderr=True 579 Should Be Empty ${stderr} 580 581 Log ${journal_log} 582 583 Execute Command rm ${file_path}-${LOG_TIME} 584 585 [Return] ${journal_log} 586 587Mac Address To Hex String 588 [Documentation] Converts MAC address into hex format. 589 ... Example 590 ... Given the following MAC: 00:01:6C:80:02:78 591 ... This keyword will return: 0x00 0x01 0x6C 0x80 0x02 0x78 592 ... Description of arguments: 593 ... i_macaddress MAC address in the following format 594 ... 00:01:6C:80:02:78 595 [Arguments] ${i_macaddress} 596 597 ${mac_hex}= Catenate 0x${i_macaddress.replace(':', ' 0x')} 598 [Return] ${mac_hex} 599 600IP Address To Hex String 601 [Documentation] Converts IP address into hex format. 602 ... Example: 603 ... Given the following IP: 10.3.164.100 604 ... This keyword will return: 0xa 0x3 0xa4 0xa0 605 ... Description of arguments: 606 ... i_ipaddress IP address in the following format 607 ... 10.10.10.10 608 [Arguments] ${i_ipaddress} 609 610 @{ip}= Split String ${i_ipaddress} . 611 ${index}= Set Variable ${0} 612 613 :FOR ${item} IN @{ip} 614 \ ${hex}= Convert To Hex ${item} prefix=0x lowercase=yes 615 \ Set List Value ${ip} ${index} ${hex} 616 \ ${index}= Set Variable ${index + 1} 617 ${ip_hex}= Catenate @{ip} 618 [Return] ${ip_hex} 619 620BMC CPU Performance Check 621 [Documentation] Minimal 10% of proc should be free in this instance 622 623 ${bmc_cpu_usage_output} ${stderr}= Execute Command ${bmc_cpu_usage_cmd} 624 ... return_stderr=True 625 Should be empty ${stderr} 626 ${bmc_cpu_percentage}= Fetch From Left ${bmc_cpu_usage_output} % 627 Should be true ${bmc_cpu_percentage} < 90 628 629BMC Mem Performance Check 630 [Documentation] Minimal 10% of memory should be free in this instance 631 632 ${bmc_mem_free_output} ${stderr}= Execute Command ${bmc_mem_free_cmd} 633 ... return_stderr=True 634 Should be empty ${stderr} 635 636 ${bmc_mem_total_output} ${stderr}= Execute Command ${bmc_mem_total_cmd} 637 ... return_stderr=True 638 Should be empty ${stderr} 639 640 ${bmc_mem_percentage}= Evaluate ${bmc_mem_free_output}*100 641 ${bmc_mem_percentage}= Evaluate 642 ... ${bmc_mem_percentage}/${bmc_mem_total_output} 643 Should be true ${bmc_mem_percentage} > 10 644 645BMC File System Usage Check 646 [Documentation] Check the file system space. None should be 100% full 647 ... except /run/initramfs/ro 648 ${bmc_fs_usage_output} ${stderr}= Execute Command 649 ... ${bmc_file_system_usage_cmd} return_stderr=True 650 Should Be Empty ${stderr} 651 Should Be True ${bmc_fs_usage_output}==0 652 653Check BMC CPU Performance 654 [Documentation] Minimal 10% of proc should be free in 3 sample 655 :FOR ${var} IN Range 1 4 656 \ BMC CPU Performance check 657 658Check BMC Mem Performance 659 [Documentation] Minimal 10% of memory should be free 660 661 :FOR ${var} IN Range 1 4 662 \ BMC Mem Performance check 663 664Check BMC File System Performance 665 [Documentation] Check for file system usage for 4 times 666 667 :FOR ${var} IN Range 1 4 668 \ BMC File System Usage check 669 670Get URL List 671 [Documentation] Return list of URLs under given URL. 672 [Arguments] ${openbmc_url} 673 # Description of argument(s): 674 # openbmc_url URL for list operation (e.g. 675 # /xyz/openbmc_project/inventory). 676 677 ${url_list}= Read Properties ${openbmc_url}/list quiet=${1} 678 Sort List ${url_list} 679 [Return] ${url_list} 680 681Get Endpoint Paths 682 [Documentation] Returns all url paths ending with given endpoint 683 ... Example: 684 ... Given the following endpoint: cpu 685 ... This keyword will return: list of all urls ending with 686 ... cpu - 687 ... /org/openbmc/inventory/system/chassis/motherboard/cpu0, 688 ... /org/openbmc/inventory/system/chassis/motherboard/cpu1 689 ... Description of arguments: 690 ... path URL path for enumeration 691 ... endpoint string for which url path ending 692 [Arguments] ${path} ${endpoint} 693 694 ${resp}= Read Properties ${path}/enumerate timeout=30 695 log Dictionary ${resp} 696 697 ${list}= Get Dictionary Keys ${resp} 698 ${resp}= Get Matches ${list} regexp=^.*[0-9a-z_].${endpoint}[0-9]*$ 699 [Return] ${resp} 700 701Check Zombie Process 702 [Documentation] Check if any defunct process exist or not on BMC 703 ${count} ${stderr} ${rc}= Execute Command ps -o stat | grep Z | wc -l 704 ... return_stderr=True return_rc=True 705 Should Be True ${count}==0 706 Should Be Empty ${stderr} 707 708Prune Journal Log 709 [Documentation] Prune archived journal logs. 710 [Arguments] ${vacuum_size}=1M 711 712 # This keyword can be used to prevent the journal 713 # log from filling up the /run filesystem. 714 # This command will retain only the latest logs 715 # of the user specified size. 716 717 Open Connection And Log In 718 ${output} ${stderr} ${rc}= 719 ... Execute Command 720 ... journalctl --vacuum-size=${vacuum_size} 721 ... return_stderr=True return_rc=True 722 723 Should Be Equal ${rc} ${0} msg=${stderr} 724 725Set BMC Power Policy 726 [Documentation] Set the given BMC power policy. 727 [arguments] ${policy} 728 729 ${valueDict}= create dictionary data=${policy} 730 Write Attribute ${HOST_SETTING} power_policy data=${valueDict} 731 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 732 Should Be Equal ${currentPolicy} ${policy} 733 734Get System Power Policy 735 [Documentation] Get the BMC power policy. 736 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 737 [Return] ${currentPolicy} 738 739Get Auto Reboot 740 [Documentation] Returns auto reboot setting. 741 ${setting}= Read Attribute ${HOST_SETTING} auto_reboot 742 [Return] ${setting} 743 744 745Set Auto Reboot 746 [Documentation] Set the given auto reboot setting. 747 [Arguments] ${setting} 748 # setting auto reboot's setting, i.e. yes or no 749 750 ${valueDict}= Set Variable ${setting} 751 ${data}= Create Dictionary data=${valueDict} 752 Write Attribute ${HOST_SETTING} auto_reboot data=${data} 753 ${current_setting}= Get Auto Reboot 754 Should Be Equal ${current_setting} ${setting} 755 756 757Set BMC Reset Reference Time 758 [Documentation] Set current boot time as a reference and increment 759 ... boot count. 760 761 ${cur_btime}= Get BMC Boot Time 762 Run Keyword If ${BOOT_TIME} == ${0} and ${BOOT_COUNT} == ${0} 763 ... Set Global Variable ${BOOT_TIME} ${cur_btime} 764 ... ELSE IF ${cur_btime} > ${BOOT_TIME} 765 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 766 ... AND 767 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 768 769Get BMC Boot Time 770 [Documentation] Get boot time from /proc/stat. 771 772 Open Connection And Log In 773 ${output} ${stderr}= 774 ... Execute Command egrep '^btime ' /proc/stat | cut -f 2 -d ' ' 775 ... return_stderr=True 776 Should Be Empty ${stderr} 777 ${btime}= Convert To Integer ${output} 778 [Return] ${btime} 779 780Execute Command On BMC 781 [Documentation] Execute given command on BMC and return output. 782 [Arguments] ${command} 783 ${stdout} ${stderr}= Execute Command ${command} return_stderr=True 784 Should Be Empty ${stderr} 785 [Return] ${stdout} 786 787 788Enable Core Dump On BMC 789 [Documentation] Enable core dump collection. 790 Open Connection And Log In 791 ${core_pattern}= Execute Command On BMC 792 ... echo '/tmp/core_%e.%p' | tee /proc/sys/kernel/core_pattern 793 Should Be Equal As Strings ${core_pattern} /tmp/core_%e.%p 794 795Get Number Of BMC Core Dump Files 796 [Documentation] Get number of core dump files on BMC. 797 Open Connection And Log In 798 ${num_of_core_dump}= Execute Command 799 ... ls /tmp/core* 2>/dev/null | wc -l 800 [Return] ${num_of_core_dump} 801 802Set Core Dump File Size Unlimited 803 [Documentation] Set core dump file size to unlimited. 804 Open Connection And Log In 805 Execute Command On BMC 806 ... ulimit -c unlimited 807 808Check For Core Dumps 809 [Documentation] Check for any core dumps exist. 810 ${output}= Get Number Of BMC Core Dump Files 811 Run Keyword If ${output} > 0 812 ... Log **Warning** BMC core dump files exist level=WARN 813 814Trigger Host Watchdog Error 815 [Documentation] Inject host watchdog timeout error via REST. 816 [Arguments] ${milliseconds}=1000 ${sleep_time}=5s 817 # Description of argument(s): 818 # milliseconds The time watchdog timer value in milliseconds (e.g. 1000 = 819 # 1 second). 820 # sleep_time Time delay for host watchdog error to get injected. 821 # Default is 5 seconds. 822 823 ${data}= Create Dictionary data=${True} 824 Write Attribute /xyz/openbmc_project/watchdog/host0 Enabled data=${data} 825 826 ${data}= Create Dictionary data=${milliseconds} 827 Write Attribute /xyz/openbmc_project/watchdog/host0 TimeRemaining data=${data} 828 829 Sleep ${sleep_time} 830 831Login To OS Host 832 [Documentation] Login to OS Host. 833 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 834 ... ${os_password}=${OS_PASSWORD} 835 # Desription of arguments: 836 # ${os_host} IP address of the OS Host. 837 # ${os_username} OS Host Login user name. 838 # ${os_password} OS Host Login passwrd. 839 840 ${os_state}= Run Keyword And Return Status Ping Host ${os_host} 841 Run Keyword If '${os_state}' == 'False' 842 ... Run Keywords Initiate Host Reboot AND 843 ... Is Host Running AND 844 ... Wait for OS ${os_host} ${os_username} ${os_password} 845 846 SSHLibrary.Open Connection ${os_host} 847 ${resp}= Login ${os_username} ${os_password} 848 [Return] ${resp} 849 850Configure Initial Settings 851 [Documentation] Restore old IP and route. 852 ... This keyword requires initial settings viz IP address, 853 ... Network Mask, default gatway and serial console IP and port 854 ... information which should be provided in command line. 855 856 [Arguments] ${host}=${OPENBMC_HOST} ${mask}=${NET_MASK} 857 ... ${gw_ip}=${GW_IP} 858 859 # Open telnet connection and ignore the error, in case telnet session is 860 # already opened by the program calling this keyword. 861 862 Run Keyword And Ignore Error Open Telnet Connection to BMC Serial Console 863 Telnet.write ifconfig eth0 ${host} netmask ${mask} 864 Telnet.write route add default gw ${gw_ip} 865 866Install Debug Tarball On BMC 867 [Documentation] Copy the debug tar file to BMC and install. 868 [Arguments] ${tarball_file_path}=${EXECDIR}/obmc-phosphor-debug-tarball-witherspoon.tar.xz 869 ... ${targ_tarball_dir_path}=/home/root/tarball/ 870 871 # Description of arguments: 872 # tarball_file_path Path of the debug tarball file. 873 # The tar file is downloaded from the build page 874 # https://openpower.xyz/job/openbmc-build/ 875 # obmc-phosphor-debug-tarball-witherspoon.tar.xz 876 # 877 # targ_tarball_dir_path The directory path where the tarball is to be 878 # installed. 879 880 OperatingSystem.File Should Exist ${tarball_file_path} 881 ... msg=${tarball_file_path} doesn't exist. 882 883 # Upload the file to BMC. 884 Import Library SCPLibrary WITH NAME scp 885 Open Connection for SCP 886 scp.Put File ${tarball_file_path} /tmp/debug-tarball.tar.xz 887 888 # Create tarball directory and install. 889 Open Connection And Log In 890 Execute Command On BMC mkdir -p ${targ_tarball_dir_path} 891 Execute Command On BMC 892 ... tar -xf /tmp/debug-tarball.tar.xz -C ${targ_tarball_dir_path} 893 894 # Create symlink to callout-test binary. 895 Execute Command On BMC 896 ... ln -s ${targ_tarball_dir_path}/bin/callout-test /usr/bin/callout-test 897 898 # Create symlink to logging-test binary. 899 Execute Command On BMC 900 ... ln -s ${targ_tarball_dir_path}/bin/logging-test /usr/bin/logging-test 901 902 # Remove the tarball file from BMC 903 Execute Command On BMC rm /tmp/debug-tarball.tar.xz 904 905 906Get BMC Boot Count 907 [Documentation] Get BMC boot count based on boot time. 908 ${cur_btime}= Get BMC Boot Time 909 910 # Set global variable BOOT_TIME to current boot time if current boot time 911 # is changed. Also increase value of global variable BOOT_COUNT by 1. 912 Run Keyword If ${cur_btime} > ${BOOT_TIME} 913 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 914 ... AND 915 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 916 [Return] ${BOOT_COUNT} 917 918Set BMC Boot Count 919 [Documentation] Set BMC boot count to given value. 920 [Arguments] ${count} 921 922 # Description of arguments: 923 # count boot count value. 924 ${cur_btime}= Get BMC Boot Time 925 926 # Set global variable BOOT_COUNT to given value. 927 Set Global Variable ${BOOT_COUNT} ${count} 928 929 # Set BOOT_TIME variable to current boot time. 930 Set Global Variable ${BOOT_COUNT} ${count} 931 932Get System LED State 933 [Documentation] Return the state of given system LED. 934 [Arguments] ${led_name} 935 936 # Description of argument(s): 937 # led_name System LED name (e.g. heartbeat, identify, beep). 938 939 ${state}= Read Attribute ${LED_PHYSICAL_URI}${led_name} State 940 [Return] ${state.rsplit('.', 1)[1]} 941 942############################################################################### 943Delete Error Logs 944 [Documentation] Delete error logs. 945 946 # Check if error logs entries exist, if not return. 947 ${resp}= OpenBMC Get Request ${BMC_LOGGING_ENTRY}${/}list quiet=${1} 948 Return From Keyword If ${resp.status_code} == ${HTTP_NOT_FOUND} 949 950 # Get the list of error logs entries and delete them all. 951 ${elog_entries}= Get URL List ${BMC_LOGGING_ENTRY} 952 :FOR ${entry} IN @{elog_entries} 953 \ Delete Error Log Entry ${entry} 954 955############################################################################### 956Delete Error Log Entry 957 [Documentation] Delete error log entry. 958 [Arguments] ${entry_path} 959 960 # Description of argument(s): 961 # entry_path Delete an error log entry. 962 # Ex. /xyz/openbmc_project/logging/entry/1 963 964 # Skip delete if entry URI is a callout. 965 # Example: /xyz/openbmc_project/logging/entry/1/callout 966 Return From Keyword If '${entry_path.rsplit('/', 1)[1]}' == 'callout' 967 968 ${data}= Create Dictionary data=@{EMPTY} 969 ${resp}= Openbmc Delete Request ${entry_path} data=${data} 970 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 971 972 973############################################################################### 974Get LED State XYZ 975 [Documentation] Returns state of given LED. 976 [Arguments] ${led_name} 977 # Description of argument(s): 978 # led_name Name of LED 979 980 ${state}= Read Attribute ${LED_GROUPS_URI}${led_name} Asserted 981 [Return] ${state} 982 983 984############################################################################### 985Get BMC Version 986 [Documentation] Returns BMC version from /etc/os-release. 987 ... e.g. "v1.99.6-141-ge662190" 988 989 Open Connection And Log In 990 ${cmd}= Set Variable grep ^VERSION_ID= /etc/os-release | cut -f 2 -d '=' 991 ${output}= Execute Command On BMC ${cmd} 992 [Return] ${output} 993 994 995############################################################################### 996Get Elog URL List 997 [Documentation] Return error log entry list of URLs. 998 999 ${url_list}= Read Properties /xyz/openbmc_project/logging/entry/ 1000 Sort List ${url_list} 1001 [Return] ${url_list} 1002 1003 1004############################################################################### 1005