1*** Settings *** 2Resource ../lib/resource.txt 3Resource ../lib/rest_client.robot 4Resource ../lib/connection_client.robot 5Library DateTime 6Library Process 7Library OperatingSystem 8Library gen_print.py 9Library gen_robot_print.py 10Library gen_cmd.py 11Library gen_robot_keyword.py 12 13*** Variables *** 14${pflash_cmd} /usr/sbin/pflash -r /dev/stdout -P VERSION 15${SYSTEM_SHUTDOWN_TIME} ${5} 16${dbuscmdBase} 17... dbus-send --system --print-reply --dest=${OPENBMC_BASE_DBUS}.settings.Host 18${dbuscmdGet} 19... ${SETTINGS_URI}host0 org.freedesktop.DBus.Properties.Get 20# Enable when ready with openbmc/openbmc-test-automation#203 21#${dbuscmdString}= string:"xyz.openbmc_project.settings.Host" string: 22${dbuscmdString}= string:"org.openbmc.settings.Host" string: 23 24# Assign default value to QUIET for programs which may not define it. 25${QUIET} ${0} 26${bmc_mem_free_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f4 27${bmc_mem_total_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f2 28${bmc_cpu_usage_cmd}= top -n 1 | grep CPU: | cut -c 7-9 29${HOST_SETTING} ${SETTINGS_URI}host0 30# /run/initramfs/ro associate filesystem should be 100% full always 31${bmc_file_system_usage_cmd}= 32... df -h | grep -v /run/initramfs/ro | cut -c 52-54 | grep 100 | wc -l 33 34${BOOT_TIME} ${0} 35${BOOT_COUNT} ${0} 36${count} ${0} 37${devicetree_base} /sys/firmware/devicetree/base/model 38 39# Initialize default debug value to 0. 40${DEBUG} ${0} 41 42*** Keywords *** 43 44Verify PNOR Update 45 [Documentation] Verify that the PNOR is not corrupted. 46 # Example: 47 # FFS: Flash header not found. Code: 100 48 # Error 100 opening ffs ! 49 50 Open Connection And Log In 51 ${pnor_info}= Execute Command On BMC ${pflash_cmd} 52 Should Not Contain Any ${pnor_info} Flash header not found Error 53 54Get BMC System Model 55 [Documentation] Get the BMC model from the device tree. 56 57 Open Connection And Log In 58 ${bmc_model} ${stderr}= Execute Command 59 ... cat ${devicetree_base} | cut -d " " -f 1 return_stderr=True 60 Should Be Empty ${stderr} 61 Should Not Be Empty ${bmc_model} 62 [Return] ${bmc_model} 63 64Verify BMC System Model 65 [Documentation] Verify the BMC model with ${OPENBMC_MODEL}. 66 [Arguments] ${bmc_model} 67 68 ${tmp_bmc_model}= Fetch From Right ${OPENBMC_MODEL} / 69 ${tmp_bmc_model}= Fetch From Left ${tmp_bmc_model} . 70 ${ret}= Run Keyword And Return Status Should Contain ${bmc_model} 71 ... ${tmp_bmc_model} ignore_case=True 72 [Return] ${ret} 73 74Wait For Host To Ping 75 [Arguments] ${host} ${timeout}=${OPENBMC_REBOOT_TIMEOUT}min 76 ... ${interval}=5 sec 77 78 # host The DNS name or IP of the host to ping. 79 # timeout The amount of time after which attempts to ping cease. 80 # interval The amount of time in between attempts to ping. 81 82 Wait Until Keyword Succeeds ${timeout} ${interval} Ping Host ${host} 83 84Ping Host 85 [Arguments] ${host} 86 Should Not Be Empty ${host} msg=No host provided 87 ${RC} ${output}= Run and return RC and Output ping -c 4 ${host} 88 Log RC: ${RC}\nOutput:\n${output} 89 Should be equal ${RC} ${0} 90 91Get Boot Progress 92 [Arguments] ${quiet}=${QUIET} 93 94 ${state}= Read Attribute ${OPENBMC_BASE_URI}sensors/host/BootProgress 95 ... value quiet=${quiet} 96 [Return] ${state} 97 98Is Power On 99 ${state}= Get Power State 100 Should be equal ${state} ${1} 101 102Is Power Off 103 ${state}= Get Power State 104 Should be equal ${state} ${0} 105 106Initiate Power On 107 [Documentation] Initiates the power on and waits until the Is Power On 108 ... keyword returns that the power state has switched to on. 109 [Arguments] ${wait}=${1} 110 111 @{arglist}= Create List 112 ${args}= Create Dictionary data=@{arglist} 113 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOn 114 ... data=${args} 115 should be equal as strings ${resp.status_code} ${HTTP_OK} 116 117 # Does caller want to wait for power on status? 118 Run Keyword If '${wait}' == '${0}' Return From Keyword 119 Wait Until Keyword Succeeds 3 min 10 sec Is Power On 120 121Initiate Power Off 122 [Documentation] Initiates the power off and waits until the Is Power Off 123 ... keyword returns that the power state has switched to off. 124 @{arglist}= Create List 125 ${args}= Create Dictionary data=@{arglist} 126 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOff 127 ... data=${args} 128 should be equal as strings ${resp.status_code} ${HTTP_OK} 129 Wait Until Keyword Succeeds 1 min 10 sec Is Power Off 130 131Initiate OS Host Power Off 132 [Documentation] Initiate an OS reboot. 133 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 134 ... ${os_password}=${OS_PASSWORD} 135 136 # Description of arguments: 137 # os_host The DNS name or IP of the OS. 138 # os_username The username to be used to sign in to the OS. 139 # os_password The password to be used to sign in to the OS. 140 141 SSHLibrary.Open connection ${os_host} 142 Login ${os_username} ${os_password} 143 ${cmd_buf} Catenate shutdown 144 Start Command ${cmd_buf} 145 SSHLibrary.Close Connection 146 147Initiate OS Host Reboot 148 [Documentation] Initiate an OS reboot. 149 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 150 ... ${os_password}=${OS_PASSWORD} 151 152 # Description of arguments: 153 # os_host The DNS name or IP of the OS. 154 # os_username The username to be used to sign in to the OS. 155 # os_password The password to be used to sign in to the OS. 156 157 SSHLibrary.Open connection ${os_host} 158 Login ${os_username} ${os_password} 159 ${cmd_buf} Catenate reboot 160 Start Command ${cmd_buf} 161 SSHLibrary.Close Connection 162 163Initiate Auto Reboot 164 [Documentation] Initiate an auto reboot. 165 166 # Set the auto reboot policy. 167 Set Auto Reboot yes 168 169 SSHLibrary.Open connection ${openbmc_host} 170 Login ${openbmc_username} ${openbmc_password} 171 172 # Set the watchdog timer. Note: 5000 = milliseconds which is 5 seconds. 173 ${cmd_buf}= Catenate /usr/sbin/mapper call /org/openbmc/watchdog/host0 174 ... org.openbmc.Watchdog set i 5000 175 ${output} ${stderr} ${rc}= Execute Command ${cmd_buf} 176 ... return_stderr=True return_rc=True 177 Should Be Empty ${stderr} 178 Should be equal ${rc} ${0} 179 180 # Start the watchdog. 181 ${cmd_buf}= Catenate /usr/sbin/mapper call /org/openbmc/watchdog/host0 182 ... org.openbmc.Watchdog start 183 ${output} ${stderr} ${rc}= Execute Command ${cmd_buf} 184 ... return_stderr=True return_rc=True 185 Should Be Empty ${stderr} 186 Should be equal ${rc} ${0} 187 188Trigger Warm Reset 189 log to console "Triggering warm reset" 190 ${data}= create dictionary data=@{EMPTY} 191 ${resp}= openbmc post request 192 ... ${OPENBMC_BASE_URI}control/bmc0/action/warmReset data=${data} 193 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 194 ${session_active}= Check If warmReset is Initiated 195 Run Keyword If '${session_active}' == '${True}' 196 ... Fail msg=warm reset didn't occur 197 198 Sleep ${SYSTEM_SHUTDOWN_TIME}min 199 Check If BMC Is Up 200 201Check OS 202 [Documentation] Attempts to ping the host OS and then checks that the host 203 ... OS is up by running an SSH command. 204 205 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 206 ... ${os_password}=${OS_PASSWORD} ${quiet}=${QUIET} 207 ... ${print_string}=${EMPTY} 208 [Teardown] SSHLibrary.Close Connection 209 210 # os_host The DNS name/IP of the OS host associated with our BMC. 211 # os_username The username to be used to sign on to the OS host. 212 # os_password The password to be used to sign on to the OS host. 213 # quiet Indicates whether this keyword should write to console. 214 # print_string A string to be printed before checking the OS. 215 216 rprint ${print_string} 217 218 # Attempt to ping the OS. Store the return code to check later. 219 ${ping_rc}= Run Keyword and Return Status Ping Host ${os_host} 220 221 SSHLibrary.Open connection ${os_host} 222 223 ${status} ${msg}= Run Keyword And Ignore Error Login ${os_username} 224 ... ${os_password} 225 ${err_msg1}= Sprint Error ${msg} 226 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 227 Run Keyword If '${status}' == 'FAIL' Fail msg=${err_msg} 228 ${output} ${stderr} ${rc}= Execute Command uptime return_stderr=True 229 ... return_rc=True 230 231 ${temp_msg}= Catenate Could not execute a command on the operating 232 ... system.\n 233 ${err_msg1}= Sprint Error ${temp_msg} 234 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 235 236 # If the return code returned by "Execute Command" is non-zero, this 237 # keyword will fail. 238 Should Be Equal ${rc} ${0} msg=${err_msg} 239 # We will likewise fail if there is any stderr data. 240 Should Be Empty ${stderr} 241 242 ${temp_msg}= Set Variable Could not ping the operating system.\n 243 ${err_msg1}= Sprint Error ${temp_msg} 244 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 245 # We will likewise fail if the OS did not ping, as we could SSH but not 246 # ping 247 Should Be Equal As Strings ${ping_rc} ${TRUE} msg=${err_msg} 248 249Wait for OS 250 [Documentation] Waits for the host OS to come up via calls to "Check OS". 251 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 252 ... ${os_password}=${OS_PASSWORD} ${timeout}=${OS_WAIT_TIMEOUT} 253 ... ${quiet}=${0} 254 [Teardown] rprintn 255 256 # os_host The DNS name or IP of the OS host associated with our 257 # BMC. 258 # os_username The username to be used to sign on to the OS host. 259 # os_password The password to be used to sign on to the OS host. 260 # timeout The timeout in seconds indicating how long you're 261 # willing to wait for the OS to respond. 262 # quiet Indicates whether this keyword should write to console. 263 264 # The interval to be used between calls to "Check OS". 265 ${interval}= Set Variable 5 266 267 ${message}= Catenate Checking every ${interval} seconds for up to 268 ... ${timeout} seconds for the operating system to communicate. 269 rqprint_timen ${message} 270 271 Wait Until Keyword Succeeds ${timeout} sec ${interval} Check OS 272 ... ${os_host} ${os_username} ${os_password} 273 ... print_string=\# 274 275 rqprintn 276 277 rqprint_timen The operating system is now communicating. 278 279Get BMC State Deprecated 280 [Documentation] Returns the state of the BMC as a string. (i.e: BMC_READY) 281 [Arguments] ${quiet}=${QUIET} 282 283 @{arglist}= Create List 284 ${args}= Create Dictionary data=@{arglist} 285 ${resp}= Call Method ${OPENBMC_BASE_URI}managers/System/ getSystemState 286 ... data=${args} quiet=${quiet} 287 Should be equal as strings ${resp.status_code} ${HTTP_OK} 288 ${content}= to json ${resp.content} 289 [Return] ${content["data"]} 290 291Get Power State 292 [Documentation] Returns the power state as an integer. Either 0 or 1. 293 [Arguments] ${quiet}=${QUIET} 294 295 @{arglist}= Create List 296 ${args}= Create Dictionary data=@{arglist} 297 298 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ getPowerState 299 ... data=${args} quiet=${quiet} 300 Should be equal as strings ${resp.status_code} ${HTTP_OK} 301 ${content}= to json ${resp.content} 302 [Return] ${content["data"]} 303 304Clear BMC Record Log 305 [Documentation] Clears all the event logs on the BMC. This would be 306 ... equivalent to ipmitool sel clear. 307 @{arglist}= Create List 308 ${args}= Create Dictionary data=@{arglist} 309 ${resp}= Call Method 310 ... ${OPENBMC_BASE_URI}records/events/ clear data=${args} 311 should be equal as strings ${resp.status_code} ${HTTP_OK} 312 313Copy PNOR to BMC 314 Import Library SCPLibrary WITH NAME scp 315 Open Connection for SCP 316 Log Copying ${PNOR_IMAGE_PATH} to /tmp 317 scp.Put File ${PNOR_IMAGE_PATH} /tmp 318 319Flash PNOR 320 [Documentation] Calls flash bios update method to flash PNOR image 321 [Arguments] ${pnor_image} 322 @{arglist}= Create List ${pnor_image} 323 ${args}= Create Dictionary data=@{arglist} 324 ${resp}= Call Method ${OPENBMC_BASE_URI}control/flash/bios/ update 325 ... data=${args} 326 should be equal as strings ${resp.status_code} ${HTTP_OK} 327 Wait Until Keyword Succeeds 2 min 10 sec Is PNOR Flashing 328 329Get Flash BIOS Status 330 [Documentation] Returns the status of the flash BIOS API as a string. For 331 ... example 'Flashing', 'Flash Done', etc 332 ${data}= Read Properties ${OPENBMC_BASE_URI}control/flash/bios 333 [Return] ${data['status']} 334 335Is PNOR Flashing 336 [Documentation] Get BIOS 'Flashing' status. This indicates that PNOR 337 ... flashing has started. 338 ${status}= Get Flash BIOS Status 339 Should Contain ${status} Flashing 340 341Is PNOR Flash Done 342 [Documentation] Get BIOS 'Flash Done' status. This indicates that the 343 ... PNOR flashing has completed. 344 ${status}= Get Flash BIOS Status 345 should be equal as strings ${status} Flash Done 346 347Is System State Host Booted 348 [Documentation] Checks whether system state is HOST_BOOTED. 349 ${state}= Get BMC State Deprecated 350 should be equal as strings ${state} HOST_BOOTED 351 352Is OS Starting 353 [Documentation] Check if boot progress is OS starting. 354 ${boot_progress}= Get Boot Progress 355 Should Be Equal ${boot_progress} FW Progress, Starting OS 356 357Is OS Off 358 [Documentation] Check if boot progress is "Off". 359 ${boot_progress}= Get Boot Progress 360 Should Be Equal ${boot_progress} Off 361 362Verify Ping and REST Authentication 363 ${l_ping}= Run Keyword And Return Status 364 ... Ping Host ${OPENBMC_HOST} 365 Run Keyword If '${l_ping}' == '${False}' 366 ... Fail msg=Ping Failed 367 368 ${l_rest}= Run Keyword And Return Status 369 ... Initialize OpenBMC 370 Run Keyword If '${l_rest}' == '${False}' 371 ... Fail msg=REST Authentication Failed 372 373 # Just to make sure the SSH is working for SCP 374 Open Connection And Log In 375 ${system} ${stderr}= Execute Command hostname return_stderr=True 376 Should Be Empty ${stderr} 377 378Check If BMC is Up 379 [Documentation] Wait for Host to be online. Checks every X seconds 380 ... interval for Y minutes and fails if timed out. 381 ... Default MAX timedout is 10 min, interval 10 seconds. 382 [Arguments] ${max_timeout}=${OPENBMC_REBOOT_TIMEOUT} min 383 ... ${interval}=10 sec 384 385 Wait Until Keyword Succeeds 386 ... ${max_timeout} ${interval} Verify Ping and REST Authentication 387 388 389Check If warmReset is Initiated 390 [Documentation] Ping would be still alive, so try SSH to connect 391 ... if fails the ports are down indicating reboot 392 ... is in progress 393 394 # Warm reset adds 3 seconds delay before forcing reboot 395 # To minimize race conditions, we wait for 7 seconds 396 Sleep 7s 397 ${alive}= Run Keyword and Return Status 398 ... Open Connection And Log In 399 Return From Keyword If '${alive}' == '${False}' ${False} 400 [Return] ${True} 401 402Flush REST Sessions 403 [Documentation] Removes all the active session objects 404 Delete All Sessions 405 406Initialize DBUS cmd 407 [Documentation] Initialize dbus string with property string to extract 408 [Arguments] ${boot_property} 409 ${cmd}= Catenate ${dbuscmdBase} ${dbuscmdGet} ${dbuscmdString} 410 ${cmd}= Catenate ${cmd}${boot_property} 411 Set Global Variable ${dbuscmd} ${cmd} 412 413Create OS Console File Path 414 [Documentation] Create OS console file path name and return it. 415 [Arguments] ${log_file_path}=${EMPTY} 416 417 # Description of arguements: 418 # file_path The caller's candidate value. If this value is ${EMPTY}, this 419 # keyword will compose a file path name. Otherwise, this 420 # keyword will use the caller's file_path value. In either 421 # case, the value will be returned. 422 423 ${default_file_path}= Catenate /tmp/${OPENBMC_HOST}_os_console 424 ${log_file_path}= Set Variable If '${log_file_path}' == '${EMPTY}' 425 ... ${default_file_path} ${log_file_path} 426 427 [Return] ${log_file_path} 428 429Create OS Console Command String 430 [Documentation] Return a command string to start OS console logging. 431 432 # First make sure that the ssh_pw program is available. 433 ${cmd_buf}= Catenate which ssh_pw 2>&1 434 Rdpissuing ${cmd_buf} 435 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 436 Rdpvars rc output 437 Should Be Equal ${rc} ${0} msg=${output}\n 438 439 ${cmd_buf}= Catenate ssh_pw ${OPENBMC_PASSWORD} -p 2200 440 ... -o "StrictHostKeyChecking no" ${OPENBMC_USERNAME}@${OPENBMC_HOST} 441 442 [Return] ${cmd_buf} 443 444Stop SOL Console Logging 445 [Documentation] Stop system console logging and return log output. 446 [Arguments] ${log_file_path}=${EMPTY} ${targ_file_path}=${EMPTY} 447 ... ${return_data}=${1} 448 449 # If there are muliple system console processes, they will all be stopped. 450 # If there is no existing log file this keyword will return an error 451 # message to that effect (and write that message to targ_file_path, if 452 # specified). 453 # NOTE: This keyword will not fail if there is no running system console 454 # process. 455 456 # Description of arguments: 457 # log_file_path The file path that was used to call "Start SOL 458 # Console Logging". See that keyword (above) for details. 459 # targ_file_path If specified, the file path to which the source 460 # file path (i.e. "log_file_path") should be copied. 461 # return_data If this is set to ${1}, this keyword will return the SOL 462 # data to the caller as a unicode string. 463 464 ${log_file_path}= Create OS Console File Path ${log_file_path} 465 # Find the pid of the active system console logging session (if any). 466 ${search_string}= Create OS Console Command String 467 ${cmd_buf}= Catenate echo $(ps -ef | egrep '${search_string}' 468 ... | egrep -v grep | cut -c10-14) 469 Rdpissuing ${cmd_buf} 470 ${rc} ${os_con_pid}= Run And Return Rc And Output ${cmd_buf} 471 Rdpvars os_con_pid 472 # If rc is not zero it just means that there is no OS Console process 473 # running. 474 475 ${cmd_buf}= Catenate kill -9 ${os_con_pid} 476 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpissuing ${cmd_buf} 477 ${rc} ${output}= Run Keyword If '${os_con_pid}' != '${EMPTY}' 478 ... Run And Return Rc And Output ${cmd_buf} 479 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpvars rc output 480 481 Run Keyword If '${targ_file_path}' != '${EMPTY}' 482 ... Run Keyword And Ignore Error 483 ... Copy File ${log_file_path} ${targ_file_path} 484 485 ${output}= Set Variable ${EMPTY} 486 ${loc_quiet}= Evaluate ${debug}^1 487 ${rc} ${output}= Run Keyword If '${return_data}' == '${1}' 488 ... Cmd Fnc cat ${log_file_path} quiet=${loc_quiet} print_output=${0} 489 490 [Return] ${output} 491 492Start SOL Console Logging 493 [Documentation] Start system console log to file. 494 [Arguments] ${log_file_path}=${EMPTY} ${return_data}=${1} 495 496 # This keyword will first call "Stop SOL Console Logging". Only then will 497 # it start SOL console logging. The data returned by "Stop SOL Console 498 # Logging" will in turn be returned by this keyword. 499 500 # Description of arguments: 501 # log_file_path The file path to which system console log data should be 502 # written. Note that this path is taken to be a location 503 # on the machine where this program is running rather than 504 # on the Open BMC system. 505 # return_data If this is set to ${1}, this keyword will return any SOL 506 # data to the caller as a unicode string. 507 508 ${log_file_path}= Create OS Console File Path ${log_file_path} 509 510 ${log_output}= Stop SOL Console Logging ${log_file_path} 511 ... return_data=${return_data} 512 513 # Validate by making sure we can create the file. Problems creating the 514 # file would not be noticed by the subsequent ssh command because we fork 515 # the command. 516 Create File ${log_file_path} 517 ${sub_cmd_buf}= Create OS Console Command String 518 # Routing stderr to stdout so that any startup error text will go to the 519 # output file. 520 ${cmd_buf}= Catenate ${sub_cmd_buf} > ${log_file_path} 2>&1 & 521 Rdpissuing ${cmd_buf} 522 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 523 # Because we are forking this command, we essentially will never get a 524 # non-zero return code or any output. 525 Should Be Equal ${rc} ${0} 526 527 [Return] ${log_output} 528 529Get Time Stamp 530 [Documentation] Get the current time stamp data 531 ${cur_time}= Get Current Date result_format=%Y%m%d%H%M%S%f 532 [Return] ${cur_time} 533 534 535Verify BMC State 536 [Documentation] Get the BMC state and verify if the current 537 ... BMC state is as expected. 538 [Arguments] ${expected} 539 540 ${current}= Get BMC State Deprecated 541 Should Contain ${expected} ${current} 542 543Start Journal Log 544 [Documentation] Start capturing journal log to a file in /tmp using 545 ... journalctl command. By default journal log is collected 546 ... at /tmp/journal_log else user input location. 547 ... The File is appended with datetime. 548 [Arguments] ${file_path}=/tmp/journal_log 549 550 Open Connection And Log In 551 552 ${cur_time}= Get Time Stamp 553 Set Global Variable ${LOG_TIME} ${cur_time} 554 Start Command 555 ... journalctl -f > ${file_path}-${LOG_TIME} 556 Log Journal Log Started: ${file_path}-${LOG_TIME} 557 558 559Stop Journal Log 560 [Documentation] Stop journalctl process if its running. 561 ... By default return log from /tmp/journal_log else 562 ... user input location. 563 [Arguments] ${file_path}=/tmp/journal_log 564 565 Open Connection And Log In 566 567 ${rc}= 568 ... Execute Command 569 ... ps ax | grep journalctl | grep -v grep 570 ... return_stdout=False return_rc=True 571 572 Return From Keyword If '${rc}' == '${1}' 573 ... No journal log process running 574 575 ${output} ${stderr}= 576 ... Execute Command killall journalctl 577 ... return_stderr=True 578 Should Be Empty ${stderr} 579 580 ${journal_log} ${stderr}= 581 ... Execute Command 582 ... cat ${file_path}-${LOG_TIME} 583 ... return_stderr=True 584 Should Be Empty ${stderr} 585 586 Log ${journal_log} 587 588 Execute Command rm ${file_path}-${LOG_TIME} 589 590 [Return] ${journal_log} 591 592Mac Address To Hex String 593 [Documentation] Converts MAC address into hex format. 594 ... Example 595 ... Given the following MAC: 00:01:6C:80:02:78 596 ... This keyword will return: 0x00 0x01 0x6C 0x80 0x02 0x78 597 ... Description of arguments: 598 ... i_macaddress MAC address in the following format 599 ... 00:01:6C:80:02:78 600 [Arguments] ${i_macaddress} 601 602 ${mac_hex}= Catenate 0x${i_macaddress.replace(':', ' 0x')} 603 [Return] ${mac_hex} 604 605IP Address To Hex String 606 [Documentation] Converts IP address into hex format. 607 ... Example: 608 ... Given the following IP: 10.3.164.100 609 ... This keyword will return: 0xa 0x3 0xa4 0xa0 610 ... Description of arguments: 611 ... i_ipaddress IP address in the following format 612 ... 10.10.10.10 613 [Arguments] ${i_ipaddress} 614 615 @{ip}= Split String ${i_ipaddress} . 616 ${index}= Set Variable ${0} 617 618 :FOR ${item} IN @{ip} 619 \ ${hex}= Convert To Hex ${item} prefix=0x lowercase=yes 620 \ Set List Value ${ip} ${index} ${hex} 621 \ ${index}= Set Variable ${index + 1} 622 ${ip_hex}= Catenate @{ip} 623 [Return] ${ip_hex} 624 625BMC CPU Performance Check 626 [Documentation] Minimal 10% of proc should be free in this instance 627 628 ${bmc_cpu_usage_output} ${stderr}= Execute Command ${bmc_cpu_usage_cmd} 629 ... return_stderr=True 630 Should be empty ${stderr} 631 ${bmc_cpu_percentage}= Fetch From Left ${bmc_cpu_usage_output} % 632 Should be true ${bmc_cpu_percentage} < 90 633 634BMC Mem Performance Check 635 [Documentation] Minimal 10% of memory should be free in this instance 636 637 ${bmc_mem_free_output} ${stderr}= Execute Command ${bmc_mem_free_cmd} 638 ... return_stderr=True 639 Should be empty ${stderr} 640 641 ${bmc_mem_total_output} ${stderr}= Execute Command ${bmc_mem_total_cmd} 642 ... return_stderr=True 643 Should be empty ${stderr} 644 645 ${bmc_mem_percentage}= Evaluate ${bmc_mem_free_output}*100 646 ${bmc_mem_percentage}= Evaluate 647 ... ${bmc_mem_percentage}/${bmc_mem_total_output} 648 Should be true ${bmc_mem_percentage} > 10 649 650BMC File System Usage Check 651 [Documentation] Check the file system space. None should be 100% full 652 ... except /run/initramfs/ro 653 ${bmc_fs_usage_output} ${stderr}= Execute Command 654 ... ${bmc_file_system_usage_cmd} return_stderr=True 655 Should Be Empty ${stderr} 656 Should Be True ${bmc_fs_usage_output}==0 657 658Check BMC CPU Performance 659 [Documentation] Minimal 10% of proc should be free in 3 sample 660 :FOR ${var} IN Range 1 4 661 \ BMC CPU Performance check 662 663Check BMC Mem Performance 664 [Documentation] Minimal 10% of memory should be free 665 666 :FOR ${var} IN Range 1 4 667 \ BMC Mem Performance check 668 669Check BMC File System Performance 670 [Documentation] Check for file system usage for 4 times 671 672 :FOR ${var} IN Range 1 4 673 \ BMC File System Usage check 674 675Get URL List 676 [Documentation] Return list of URLs under given URL. 677 [Arguments] ${openbmc_url} 678 # Description of argument(s): 679 # openbmc_url URL for list operation (e.g. 680 # /xyz/openbmc_project/inventory). 681 682 ${url_list}= Read Properties ${openbmc_url}/list 683 [Return] ${url_list} 684 685Get Endpoint Paths 686 [Documentation] Returns all url paths ending with given endpoint 687 ... Example: 688 ... Given the following endpoint: cpu 689 ... This keyword will return: list of all urls ending with 690 ... cpu - 691 ... /org/openbmc/inventory/system/chassis/motherboard/cpu0, 692 ... /org/openbmc/inventory/system/chassis/motherboard/cpu1 693 ... Description of arguments: 694 ... path URL path for enumeration 695 ... endpoint string for which url path ending 696 [Arguments] ${path} ${endpoint} 697 698 ${resp}= Read Properties ${path}/enumerate timeout=30 699 log Dictionary ${resp} 700 701 ${list}= Get Dictionary Keys ${resp} 702 ${resp}= Get Matches ${list} regexp=^.*[0-9a-z_].${endpoint}[0-9]*$ 703 [Return] ${resp} 704 705Check Zombie Process 706 [Documentation] Check if any defunct process exist or not on BMC 707 ${count} ${stderr} ${rc}= Execute Command ps -o stat | grep Z | wc -l 708 ... return_stderr=True return_rc=True 709 Should Be True ${count}==0 710 Should Be Empty ${stderr} 711 712Prune Journal Log 713 [Documentation] Prune archived journal logs. 714 [Arguments] ${vacuum_size}=1M 715 716 # This keyword can be used to prevent the journal 717 # log from filling up the /run filesystem. 718 # This command will retain only the latest logs 719 # of the user specified size. 720 721 Open Connection And Log In 722 ${output} ${stderr} ${rc}= 723 ... Execute Command 724 ... journalctl --vacuum-size=${vacuum_size} 725 ... return_stderr=True return_rc=True 726 727 Should Be Equal ${rc} ${0} msg=${stderr} 728 729Set BMC Power Policy 730 [Documentation] Set the given BMC power policy. 731 [arguments] ${policy} 732 733 ${valueDict}= create dictionary data=${policy} 734 Write Attribute ${HOST_SETTING} power_policy data=${valueDict} 735 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 736 Should Be Equal ${currentPolicy} ${policy} 737 738Get System Power Policy 739 [Documentation] Get the BMC power policy. 740 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 741 [Return] ${currentPolicy} 742 743Get Auto Reboot 744 [Documentation] Returns auto reboot setting. 745 ${setting}= Read Attribute ${HOST_SETTING} auto_reboot 746 [Return] ${setting} 747 748 749Set Auto Reboot 750 [Documentation] Set the given auto reboot setting. 751 [Arguments] ${setting} 752 # setting auto reboot's setting, i.e. yes or no 753 754 ${valueDict}= Set Variable ${setting} 755 ${data}= Create Dictionary data=${valueDict} 756 Write Attribute ${HOST_SETTING} auto_reboot data=${data} 757 ${current_setting}= Get Auto Reboot 758 Should Be Equal ${current_setting} ${setting} 759 760 761Set BMC Reset Reference Time 762 [Documentation] Set current boot time as a reference and increment 763 ... boot count. 764 765 ${cur_btime}= Get BMC Boot Time 766 Run Keyword If ${BOOT_TIME} == ${0} and ${BOOT_COUNT} == ${0} 767 ... Set Global Variable ${BOOT_TIME} ${cur_btime} 768 ... ELSE IF ${cur_btime} > ${BOOT_TIME} 769 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 770 ... AND 771 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 772 773Get BMC Boot Time 774 [Documentation] Get boot time from /proc/stat. 775 776 Open Connection And Log In 777 ${output} ${stderr}= 778 ... Execute Command egrep '^btime ' /proc/stat | cut -f 2 -d ' ' 779 ... return_stderr=True 780 Should Be Empty ${stderr} 781 ${btime}= Convert To Integer ${output} 782 [Return] ${btime} 783 784Execute Command On BMC 785 [Documentation] Execute given command on BMC and return output. 786 [Arguments] ${command} 787 ${stdout} ${stderr}= Execute Command ${command} return_stderr=True 788 Should Be Empty ${stderr} 789 [Return] ${stdout} 790 791 792Enable Core Dump On BMC 793 [Documentation] Enable core dump collection. 794 Open Connection And Log In 795 ${core_pattern}= Execute Command On BMC 796 ... echo '/tmp/core_%e.%p' | tee /proc/sys/kernel/core_pattern 797 Should Be Equal As Strings ${core_pattern} /tmp/core_%e.%p 798 799Get Number Of BMC Core Dump Files 800 [Documentation] Get number of core dump files on BMC. 801 Open Connection And Log In 802 ${num_of_core_dump}= Execute Command 803 ... ls /tmp/core* 2>/dev/null | wc -l 804 [Return] ${num_of_core_dump} 805 806Set Core Dump File Size Unlimited 807 [Documentation] Set core dump file size to unlimited. 808 Open Connection And Log In 809 Execute Command On BMC 810 ... ulimit -c unlimited 811 812Check For Core Dumps 813 [Documentation] Check for any core dumps exist. 814 ${output}= Get Number Of BMC Core Dump Files 815 Run Keyword If ${output} > 0 816 ... Log **Warning** BMC core dump files exist level=WARN 817 818Trigger Host Watchdog Error 819 [Documentation] Inject host watchdog error using BMC. 820 [Arguments] ${milliseconds}=1000 ${sleep_time}=5s 821 # Description of argument(s): 822 # milliseconds The time watchdog timer value in milliseconds (e.g. 1000 = 823 # 1 second). 824 # sleep_time Time delay for host watchdog error to get injected. 825 # Default is 5 seconds. 826 827 Execute Command On BMC 828 ... /usr/sbin/mapper call /org/openbmc/watchdog/host0 org.openbmc.Watchdog set i ${milliseconds} 829 Execute Command On BMC 830 ... /usr/sbin/mapper call /org/openbmc/watchdog/host0 org.openbmc.Watchdog start 831 Sleep ${sleep_time} 832 833Login To OS Host 834 [Documentation] Login to OS Host. 835 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 836 ... ${os_password}=${OS_PASSWORD} 837 # Desription of arguments: 838 # ${os_host} IP address of the OS Host. 839 # ${os_username} OS Host Login user name. 840 # ${os_password} OS Host Login passwrd. 841 842 ${os_state}= Run Keyword And Return Status Ping Host ${os_host} 843 Run Keyword If '${os_state}' == 'False' 844 ... Run Keywords Initiate Host Reboot AND 845 ... Is Host Running AND 846 ... Wait for OS ${os_host} ${os_username} ${os_password} 847 848 Open Connection ${os_host} 849 ${resp}= Login ${os_username} ${os_password} 850 [Return] ${resp} 851 852Configure Initial Settings 853 [Documentation] Restore old IP and route. 854 ... This keyword requires initial settings viz IP address, 855 ... Network Mask, default gatway and serial console IP and port 856 ... information which should be provided in command line. 857 858 [Arguments] ${host}=${OPENBMC_HOST} ${mask}=${NET_MASK} 859 ... ${gw_ip}=${GW_IP} 860 861 # Open telnet connection and ignore the error, in case telnet session is 862 # already opened by the program calling this keyword. 863 864 Run Keyword And Ignore Error Open Telnet Connection to BMC Serial Console 865 Telnet.write ifconfig eth0 ${host} netmask ${mask} 866 Telnet.write route add default gw ${gw_ip} 867 868Install Debug Tarball On BMC 869 [Documentation] Copy the debug tar file to BMC and install. 870 [Arguments] ${tarball_file_path}=${EXECDIR}/obmc-phosphor-debug-tarball-witherspoon.tar.xz 871 ... ${targ_tarball_dir_path}=/home/root/tarball/ 872 873 # Description of arguments: 874 # tarball_file_path Path of the debug tarball file. 875 # The tar file is downloaded from the build page 876 # https://openpower.xyz/job/openbmc-build/ 877 # obmc-phosphor-debug-tarball-witherspoon.tar.xz 878 # 879 # targ_tarball_dir_path The directory path where the tarball is to be 880 # installed. 881 882 OperatingSystem.File Should Exist ${tarball_file_path} 883 ... msg=${tarball_file_path} doesn't exist. 884 885 # Upload the file to BMC. 886 Import Library SCPLibrary WITH NAME scp 887 Open Connection for SCP 888 scp.Put File ${tarball_file_path} /tmp/debug-tarball.tar.xz 889 890 # Create tarball directory and install. 891 Open Connection And Log In 892 Execute Command On BMC mkdir -p ${targ_tarball_dir_path} 893 Execute Command On BMC 894 ... tar -xf /tmp/debug-tarball.tar.xz -C ${targ_tarball_dir_path} 895 896 # Create symlink to callout-test binary. 897 Execute Command On BMC 898 ... ln -s ${targ_tarball_dir_path}/bin/callout-test /usr/bin/callout-test 899 900 # Create symlink to logging-test binary. 901 Execute Command On BMC 902 ... ln -s ${targ_tarball_dir_path}/bin/logging-test /usr/bin/logging-test 903 904 # Remove the tarball file from BMC 905 Execute Command On BMC rm /tmp/debug-tarball.tar.xz 906 907 908Get BMC Boot Count 909 [Documentation] Get BMC boot count based on boot time. 910 ${cur_btime}= Get BMC Boot Time 911 912 # Set global variable BOOT_TIME to current boot time if current boot time 913 # is changed. Also increase value of global variable BOOT_COUNT by 1. 914 Run Keyword If ${cur_btime} > ${BOOT_TIME} 915 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 916 ... AND 917 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 918 [Return] ${BOOT_COUNT} 919 920Set BMC Boot Count 921 [Documentation] Set BMC boot count to given value. 922 [Arguments] ${count} 923 924 # Description of arguments: 925 # count boot count value. 926 ${cur_btime}= Get BMC Boot Time 927 928 # Set global variable BOOT_COUNT to given value. 929 Set Global Variable ${BOOT_COUNT} ${count} 930 931 # Set BOOT_TIME variable to current boot time. 932 Set Global Variable ${BOOT_COUNT} ${count} 933 934Get System LED State 935 [Documentation] Return the state of given system LED. 936 [Arguments] ${led_name} 937 938 # Description of argument(s): 939 # led_name System LED name (e.g. heartbeat, identify, beep). 940 941 ${state}= Read Attribute ${LED_PHYSICAL_URI}${led_name} State 942 [Return] ${state.rsplit('.', 1)[1]} 943 944############################################################################### 945Delete Error logs 946 [Documentation] Delete error logs. 947 948 # The REST method to delete error openbmc/openbmc#1327 949 # until then using logging restart. 950 Open Connection And Log In 951 Execute Command On BMC 952 ... systemctl restart xyz.openbmc_project.Logging.service 953 Sleep 10s reason=Wait for logging service to restart properly. 954 955############################################################################### 956Delete Error log Entry 957 [Documentation] Delete error log entry. 958 [Arguments] ${entry_path} 959 960 # Description of argument(s): 961 # entry_path Delete an error log entry. 962 # Ex. /xyz/openbmc_project/logging/entry/1 963 964 ${data}= Create Dictionary data=@{EMPTY} 965 ${resp}= Openbmc Delete Request ${entry_path} data=${data} 966 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 967 968 969Get LED State XYZ 970 [Documentation] Returns state of given LED. 971 [Arguments] ${led_name} 972 # Description of argument(s): 973 # led_name Name of LED 974 975 ${state}= Read Attribute ${LED_GROUPS_URI}${led_name} Asserted 976 [Return] ${state} 977