1*** Settings *** 2Resource ../lib/resource.txt 3Resource ../lib/rest_client.robot 4Resource ../lib/connection_client.robot 5Library DateTime 6Library Process 7Library OperatingSystem 8Library gen_print.py 9Library gen_robot_print.py 10Library gen_cmd.py 11 12*** Variables *** 13${SYSTEM_SHUTDOWN_TIME} ${5} 14${dbuscmdBase} 15... dbus-send --system --print-reply --dest=${OPENBMC_BASE_DBUS}.settings.Host 16${dbuscmdGet} 17... ${SETTINGS_URI}host0 org.freedesktop.DBus.Properties.Get 18# Enable when ready with openbmc/openbmc-test-automation#203 19#${dbuscmdString}= string:"xyz.openbmc_project.settings.Host" string: 20${dbuscmdString}= string:"org.openbmc.settings.Host" string: 21 22# Assign default value to QUIET for programs which may not define it. 23${QUIET} ${0} 24${bmc_mem_free_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f4 25${bmc_mem_total_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f2 26${bmc_cpu_usage_cmd}= top -n 1 | grep CPU: | cut -c 7-9 27${HOST_SETTING} ${SETTINGS_URI}host0 28# /run/initramfs/ro associate filesystem should be 100% full always 29${bmc_file_system_usage_cmd}= 30... df -h | grep -v /run/initramfs/ro | cut -c 52-54 | grep 100 | wc -l 31 32${BOOT_TIME} ${0} 33${BOOT_COUNT} ${0} 34${count} ${0} 35${devicetree_base} /sys/firmware/devicetree/base/model 36 37*** Keywords *** 38 39Get BMC System Model 40 [Documentation] Get the BMC model from the device tree. 41 42 Open Connection And Log In 43 ${bmc_model} ${stderr}= Execute Command 44 ... cat ${devicetree_base} | cut -d " " -f 1 return_stderr=True 45 Should Be Empty ${stderr} 46 Should Not Be Empty ${bmc_model} 47 [Return] ${bmc_model} 48 49Verify BMC System Model 50 [Documentation] Verify the BMC model with ${OPENBMC_MODEL}. 51 [Arguments] ${bmc_model} 52 53 ${tmp_bmc_model}= Fetch From Right ${OPENBMC_MODEL} / 54 ${tmp_bmc_model}= Fetch From Left ${tmp_bmc_model} . 55 ${ret}= Run Keyword And Return Status Should Contain ${bmc_model} 56 ... ${tmp_bmc_model} ignore_case=True 57 [Return] ${ret} 58 59Wait For Host To Ping 60 [Arguments] ${host} ${timeout}=${OPENBMC_REBOOT_TIMEOUT}min 61 ... ${interval}=5 sec 62 63 # host The DNS name or IP of the host to ping. 64 # timeout The amount of time after which attempts to ping cease. 65 # interval The amount of time in between attempts to ping. 66 67 Wait Until Keyword Succeeds ${timeout} ${interval} Ping Host ${host} 68 69Ping Host 70 [Arguments] ${host} 71 Should Not Be Empty ${host} msg=No host provided 72 ${RC} ${output}= Run and return RC and Output ping -c 4 ${host} 73 Log RC: ${RC}\nOutput:\n${output} 74 Should be equal ${RC} ${0} 75 76Get Boot Progress 77 [Arguments] ${quiet}=${QUIET} 78 79 ${state}= Read Attribute ${OPENBMC_BASE_URI}sensors/host/BootProgress 80 ... value quiet=${quiet} 81 [Return] ${state} 82 83Is Power On 84 ${state}= Get Power State 85 Should be equal ${state} ${1} 86 87Is Power Off 88 ${state}= Get Power State 89 Should be equal ${state} ${0} 90 91Initiate Power On 92 [Documentation] Initiates the power on and waits until the Is Power On 93 ... keyword returns that the power state has switched to on. 94 [Arguments] ${wait}=${1} 95 96 @{arglist}= Create List 97 ${args}= Create Dictionary data=@{arglist} 98 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOn 99 ... data=${args} 100 should be equal as strings ${resp.status_code} ${HTTP_OK} 101 102 # Does caller want to wait for power on status? 103 Run Keyword If '${wait}' == '${0}' Return From Keyword 104 Wait Until Keyword Succeeds 3 min 10 sec Is Power On 105 106Initiate Power Off 107 [Documentation] Initiates the power off and waits until the Is Power Off 108 ... keyword returns that the power state has switched to off. 109 @{arglist}= Create List 110 ${args}= Create Dictionary data=@{arglist} 111 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOff 112 ... data=${args} 113 should be equal as strings ${resp.status_code} ${HTTP_OK} 114 Wait Until Keyword Succeeds 1 min 10 sec Is Power Off 115 116Initiate OS Host Power Off 117 [Documentation] Initiate an OS reboot. 118 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 119 ... ${os_password}=${OS_PASSWORD} 120 121 # Description of arguments: 122 # os_host The DNS name or IP of the OS. 123 # os_username The username to be used to sign in to the OS. 124 # os_password The password to be used to sign in to the OS. 125 126 SSHLibrary.Open connection ${os_host} 127 Login ${os_username} ${os_password} 128 ${cmd_buf} Catenate shutdown 129 Start Command ${cmd_buf} 130 SSHLibrary.Close Connection 131 132Initiate OS Host Reboot 133 [Documentation] Initiate an OS reboot. 134 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 135 ... ${os_password}=${OS_PASSWORD} 136 137 # Description of arguments: 138 # os_host The DNS name or IP of the OS. 139 # os_username The username to be used to sign in to the OS. 140 # os_password The password to be used to sign in to the OS. 141 142 SSHLibrary.Open connection ${os_host} 143 Login ${os_username} ${os_password} 144 ${cmd_buf} Catenate reboot 145 Start Command ${cmd_buf} 146 SSHLibrary.Close Connection 147 148Initiate Auto Reboot 149 [Documentation] Initiate an auto reboot. 150 151 # Set the auto reboot policy. 152 Set Auto Reboot yes 153 154 SSHLibrary.Open connection ${openbmc_host} 155 Login ${openbmc_username} ${openbmc_password} 156 157 # Set the watchdog timer. Note: 5000 = milliseconds which is 5 seconds. 158 ${cmd_buf}= Catenate /usr/sbin/mapper call /org/openbmc/watchdog/host0 159 ... org.openbmc.Watchdog set i 5000 160 ${output} ${stderr} ${rc}= Execute Command ${cmd_buf} 161 ... return_stderr=True return_rc=True 162 Should Be Empty ${stderr} 163 Should be equal ${rc} ${0} 164 165 # Start the watchdog. 166 ${cmd_buf}= Catenate /usr/sbin/mapper call /org/openbmc/watchdog/host0 167 ... org.openbmc.Watchdog start 168 ${output} ${stderr} ${rc}= Execute Command ${cmd_buf} 169 ... return_stderr=True return_rc=True 170 Should Be Empty ${stderr} 171 Should be equal ${rc} ${0} 172 173Trigger Warm Reset 174 log to console "Triggering warm reset" 175 ${data}= create dictionary data=@{EMPTY} 176 ${resp}= openbmc post request 177 ... ${OPENBMC_BASE_URI}control/bmc0/action/warmReset data=${data} 178 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 179 ${session_active}= Check If warmReset is Initiated 180 Run Keyword If '${session_active}' == '${True}' 181 ... Fail msg=warm reset didn't occur 182 183 Sleep ${SYSTEM_SHUTDOWN_TIME}min 184 Check If BMC Is Up 185 186Check OS 187 [Documentation] Attempts to ping the host OS and then checks that the host 188 ... OS is up by running an SSH command. 189 190 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 191 ... ${os_password}=${OS_PASSWORD} ${quiet}=${QUIET} 192 ... ${print_string}=${EMPTY} 193 [Teardown] SSHLibrary.Close Connection 194 195 # os_host The DNS name/IP of the OS host associated with our BMC. 196 # os_username The username to be used to sign on to the OS host. 197 # os_password The password to be used to sign on to the OS host. 198 # quiet Indicates whether this keyword should write to console. 199 # print_string A string to be printed before checking the OS. 200 201 rprint ${print_string} 202 203 # Attempt to ping the OS. Store the return code to check later. 204 ${ping_rc}= Run Keyword and Return Status Ping Host ${os_host} 205 206 SSHLibrary.Open connection ${os_host} 207 208 ${status} ${msg}= Run Keyword And Ignore Error Login ${os_username} 209 ... ${os_password} 210 ${err_msg1}= Sprint Error ${msg} 211 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 212 Run Keyword If '${status}' == 'FAIL' Fail msg=${err_msg} 213 ${output} ${stderr} ${rc}= Execute Command uptime return_stderr=True 214 ... return_rc=True 215 216 ${temp_msg}= Catenate Could not execute a command on the operating 217 ... system.\n 218 ${err_msg1}= Sprint Error ${temp_msg} 219 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 220 221 # If the return code returned by "Execute Command" is non-zero, this 222 # keyword will fail. 223 Should Be Equal ${rc} ${0} msg=${err_msg} 224 # We will likewise fail if there is any stderr data. 225 Should Be Empty ${stderr} 226 227 ${temp_msg}= Set Variable Could not ping the operating system.\n 228 ${err_msg1}= Sprint Error ${temp_msg} 229 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 230 # We will likewise fail if the OS did not ping, as we could SSH but not 231 # ping 232 Should Be Equal As Strings ${ping_rc} ${TRUE} msg=${err_msg} 233 234Wait for OS 235 [Documentation] Waits for the host OS to come up via calls to "Check OS". 236 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 237 ... ${os_password}=${OS_PASSWORD} ${timeout}=${OS_WAIT_TIMEOUT} 238 ... ${quiet}=${0} 239 [Teardown] rprintn 240 241 # os_host The DNS name or IP of the OS host associated with our 242 # BMC. 243 # os_username The username to be used to sign on to the OS host. 244 # os_password The password to be used to sign on to the OS host. 245 # timeout The timeout in seconds indicating how long you're 246 # willing to wait for the OS to respond. 247 # quiet Indicates whether this keyword should write to console. 248 249 # The interval to be used between calls to "Check OS". 250 ${interval}= Set Variable 5 251 252 ${message}= Catenate Checking every ${interval} seconds for up to 253 ... ${timeout} seconds for the operating system to communicate. 254 rqprint_timen ${message} 255 256 Wait Until Keyword Succeeds ${timeout} sec ${interval} Check OS 257 ... ${os_host} ${os_username} ${os_password} 258 ... print_string=\# 259 260 rqprintn 261 262 rqprint_timen The operating system is now communicating. 263 264Get BMC State Deprecated 265 [Documentation] Returns the state of the BMC as a string. (i.e: BMC_READY) 266 [Arguments] ${quiet}=${QUIET} 267 268 @{arglist}= Create List 269 ${args}= Create Dictionary data=@{arglist} 270 ${resp}= Call Method ${OPENBMC_BASE_URI}managers/System/ getSystemState 271 ... data=${args} quiet=${quiet} 272 Should be equal as strings ${resp.status_code} ${HTTP_OK} 273 ${content}= to json ${resp.content} 274 [Return] ${content["data"]} 275 276Get Power State 277 [Documentation] Returns the power state as an integer. Either 0 or 1. 278 [Arguments] ${quiet}=${QUIET} 279 280 @{arglist}= Create List 281 ${args}= Create Dictionary data=@{arglist} 282 283 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ getPowerState 284 ... data=${args} quiet=${quiet} 285 Should be equal as strings ${resp.status_code} ${HTTP_OK} 286 ${content}= to json ${resp.content} 287 [Return] ${content["data"]} 288 289Clear BMC Record Log 290 [Documentation] Clears all the event logs on the BMC. This would be 291 ... equivalent to ipmitool sel clear. 292 @{arglist}= Create List 293 ${args}= Create Dictionary data=@{arglist} 294 ${resp}= Call Method 295 ... ${OPENBMC_BASE_URI}records/events/ clear data=${args} 296 should be equal as strings ${resp.status_code} ${HTTP_OK} 297 298Copy PNOR to BMC 299 Import Library SCPLibrary WITH NAME scp 300 Open Connection for SCP 301 Log Copying ${PNOR_IMAGE_PATH} to /tmp 302 scp.Put File ${PNOR_IMAGE_PATH} /tmp 303 304Flash PNOR 305 [Documentation] Calls flash bios update method to flash PNOR image 306 [Arguments] ${pnor_image} 307 @{arglist}= Create List ${pnor_image} 308 ${args}= Create Dictionary data=@{arglist} 309 ${resp}= Call Method ${OPENBMC_BASE_URI}control/flash/bios/ update 310 ... data=${args} 311 should be equal as strings ${resp.status_code} ${HTTP_OK} 312 Wait Until Keyword Succeeds 2 min 10 sec Is PNOR Flashing 313 314Get Flash BIOS Status 315 [Documentation] Returns the status of the flash BIOS API as a string. For 316 ... example 'Flashing', 'Flash Done', etc 317 ${data}= Read Properties ${OPENBMC_BASE_URI}control/flash/bios 318 [Return] ${data['status']} 319 320Is PNOR Flashing 321 [Documentation] Get BIOS 'Flashing' status. This indicates that PNOR 322 ... flashing has started. 323 ${status}= Get Flash BIOS Status 324 Should Contain ${status} Flashing 325 326Is PNOR Flash Done 327 [Documentation] Get BIOS 'Flash Done' status. This indicates that the 328 ... PNOR flashing has completed. 329 ${status}= Get Flash BIOS Status 330 should be equal as strings ${status} Flash Done 331 332Is System State Host Booted 333 [Documentation] Checks whether system state is HOST_BOOTED. 334 ${state}= Get BMC State Deprecated 335 should be equal as strings ${state} HOST_BOOTED 336 337Is OS Starting 338 [Documentation] Check if boot progress is OS starting. 339 ${boot_progress}= Get Boot Progress 340 Should Be Equal ${boot_progress} FW Progress, Starting OS 341 342Is OS Off 343 [Documentation] Check if boot progress is "Off". 344 ${boot_progress}= Get Boot Progress 345 Should Be Equal ${boot_progress} Off 346 347Verify Ping and REST Authentication 348 ${l_ping}= Run Keyword And Return Status 349 ... Ping Host ${OPENBMC_HOST} 350 Run Keyword If '${l_ping}' == '${False}' 351 ... Fail msg=Ping Failed 352 353 ${l_rest}= Run Keyword And Return Status 354 ... Initialize OpenBMC 355 Run Keyword If '${l_rest}' == '${False}' 356 ... Fail msg=REST Authentication Failed 357 358 # Just to make sure the SSH is working for SCP 359 Open Connection And Log In 360 ${system} ${stderr}= Execute Command hostname return_stderr=True 361 Should Be Empty ${stderr} 362 363Check If BMC is Up 364 [Documentation] Wait for Host to be online. Checks every X seconds 365 ... interval for Y minutes and fails if timed out. 366 ... Default MAX timedout is 10 min, interval 10 seconds. 367 [Arguments] ${max_timeout}=${OPENBMC_REBOOT_TIMEOUT} min 368 ... ${interval}=10 sec 369 370 Wait Until Keyword Succeeds 371 ... ${max_timeout} ${interval} Verify Ping and REST Authentication 372 373 374Check If warmReset is Initiated 375 [Documentation] Ping would be still alive, so try SSH to connect 376 ... if fails the ports are down indicating reboot 377 ... is in progress 378 379 # Warm reset adds 3 seconds delay before forcing reboot 380 # To minimize race conditions, we wait for 7 seconds 381 Sleep 7s 382 ${alive}= Run Keyword and Return Status 383 ... Open Connection And Log In 384 Return From Keyword If '${alive}' == '${False}' ${False} 385 [Return] ${True} 386 387Flush REST Sessions 388 [Documentation] Removes all the active session objects 389 Delete All Sessions 390 391Initialize DBUS cmd 392 [Documentation] Initialize dbus string with property string to extract 393 [Arguments] ${boot_property} 394 ${cmd}= Catenate ${dbuscmdBase} ${dbuscmdGet} ${dbuscmdString} 395 ${cmd}= Catenate ${cmd}${boot_property} 396 Set Global Variable ${dbuscmd} ${cmd} 397 398Create OS Console File Path 399 [Documentation] Create OS console file path name and return it. 400 [Arguments] ${log_file_path}=${EMPTY} 401 402 # Description of arguements: 403 # file_path The caller's candidate value. If this value is ${EMPTY}, this 404 # keyword will compose a file path name. Otherwise, this 405 # keyword will use the caller's file_path value. In either 406 # case, the value will be returned. 407 408 ${default_file_path}= Catenate /tmp/${OPENBMC_HOST}_os_console 409 ${log_file_path}= Set Variable If '${log_file_path}' == '${EMPTY}' 410 ... ${default_file_path} ${log_file_path} 411 412 [Return] ${log_file_path} 413 414Create OS Console Command String 415 [Documentation] Return a command string to start OS console logging. 416 417 # First make sure that the ssh_pw program is available. 418 ${cmd_buf}= Catenate which ssh_pw 2>&1 419 Rdpissuing ${cmd_buf} 420 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 421 Rdpvars rc output 422 Should Be Equal ${rc} ${0} msg=${output}\n 423 424 ${cmd_buf}= Catenate ssh_pw ${OPENBMC_PASSWORD} -p 2200 425 ... ${OPENBMC_USERNAME}@${OPENBMC_HOST} 426 427 [Return] ${cmd_buf} 428 429Stop SOL Console Logging 430 [Documentation] Stop system console logging and return log output. 431 [Arguments] ${log_file_path}=${EMPTY} ${targ_file_path}=${EMPTY} 432 ... ${return_data}=${1} 433 434 # If there are muliple system console processes, they will all be stopped. 435 # If there is no existing log file this keyword will return an error 436 # message to that effect (and write that message to targ_file_path, if 437 # specified). 438 # NOTE: This keyword will not fail if there is no running system console 439 # process. 440 441 # Description of arguments: 442 # log_file_path The file path that was used to call "Start SOL 443 # Console Logging". See that keyword (above) for details. 444 # targ_file_path If specified, the file path to which the source 445 # file path (i.e. "log_file_path") should be copied. 446 # return_data If this is set to ${1}, this keyword will return the SOL 447 # data to the caller as a unicode string. 448 449 ${log_file_path}= Create OS Console File Path ${log_file_path} 450 # Find the pid of the active system console logging session (if any). 451 ${search_string}= Create OS Console Command String 452 ${cmd_buf}= Catenate echo $(ps -ef | egrep '${search_string}' 453 ... | egrep -v grep | cut -c10-14) 454 Rdpissuing ${cmd_buf} 455 ${rc} ${os_con_pid}= Run And Return Rc And Output ${cmd_buf} 456 Rdpvars os_con_pid 457 # If rc is not zero it just means that there is no OS Console process 458 # running. 459 460 ${cmd_buf}= Catenate kill -9 ${os_con_pid} 461 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpissuing ${cmd_buf} 462 ${rc} ${output}= Run Keyword If '${os_con_pid}' != '${EMPTY}' 463 ... Run And Return Rc And Output ${cmd_buf} 464 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpvars rc output 465 466 Run Keyword If '${targ_file_path}' != '${EMPTY}' 467 ... Run Keyword And Ignore Error 468 ... Copy File ${log_file_path} ${targ_file_path} 469 470 ${output}= Set Variable ${EMPTY} 471 ${loc_quiet}= Evaluate ${debug}^1 472 ${rc} ${output}= Run Keyword If '${return_data}' == '${1}' 473 ... Cmd Fnc cat ${log_file_path} quiet=${loc_quiet} print_output=${0} 474 475 [Return] ${output} 476 477Start SOL Console Logging 478 [Documentation] Start system console log to file. 479 [Arguments] ${log_file_path}=${EMPTY} ${return_data}=${1} 480 481 # This keyword will first call "Stop SOL Console Logging". Only then will 482 # it start SOL console logging. The data returned by "Stop SOL Console 483 # Logging" will in turn be returned by this keyword. 484 485 # Description of arguments: 486 # log_file_path The file path to which system console log data should be 487 # written. Note that this path is taken to be a location 488 # on the machine where this program is running rather than 489 # on the Open BMC system. 490 # return_data If this is set to ${1}, this keyword will return any SOL 491 # data to the caller as a unicode string. 492 493 ${log_file_path}= Create OS Console File Path ${log_file_path} 494 495 ${log_output}= Stop SOL Console Logging ${log_file_path} 496 ... return_data=${return_data} 497 498 # Validate by making sure we can create the file. Problems creating the 499 # file would not be noticed by the subsequent ssh command because we fork 500 # the command. 501 Create File ${log_file_path} 502 ${sub_cmd_buf}= Create OS Console Command String 503 # Routing stderr to stdout so that any startup error text will go to the 504 # output file. 505 ${cmd_buf}= Catenate ${sub_cmd_buf} > ${log_file_path} 2>&1 & 506 Rdpissuing ${cmd_buf} 507 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 508 # Because we are forking this command, we essentially will never get a 509 # non-zero return code or any output. 510 Should Be Equal ${rc} ${0} 511 512 [Return] ${log_output} 513 514Get Time Stamp 515 [Documentation] Get the current time stamp data 516 ${cur_time}= Get Current Date result_format=%Y%m%d%H%M%S%f 517 [Return] ${cur_time} 518 519 520Verify BMC State 521 [Documentation] Get the BMC state and verify if the current 522 ... BMC state is as expected. 523 [Arguments] ${expected} 524 525 ${current}= Get BMC State Deprecated 526 Should Contain ${expected} ${current} 527 528Start Journal Log 529 [Documentation] Start capturing journal log to a file in /tmp using 530 ... journalctl command. By default journal log is collected 531 ... at /tmp/journal_log else user input location. 532 ... The File is appended with datetime. 533 [Arguments] ${file_path}=/tmp/journal_log 534 535 Open Connection And Log In 536 537 ${cur_time}= Get Time Stamp 538 Set Global Variable ${LOG_TIME} ${cur_time} 539 Start Command 540 ... journalctl -f > ${file_path}-${LOG_TIME} 541 Log Journal Log Started: ${file_path}-${LOG_TIME} 542 543 544Stop Journal Log 545 [Documentation] Stop journalctl process if its running. 546 ... By default return log from /tmp/journal_log else 547 ... user input location. 548 [Arguments] ${file_path}=/tmp/journal_log 549 550 Open Connection And Log In 551 552 ${rc}= 553 ... Execute Command 554 ... ps ax | grep journalctl | grep -v grep 555 ... return_stdout=False return_rc=True 556 557 Return From Keyword If '${rc}' == '${1}' 558 ... No journal log process running 559 560 ${output} ${stderr}= 561 ... Execute Command killall journalctl 562 ... return_stderr=True 563 Should Be Empty ${stderr} 564 565 ${journal_log} ${stderr}= 566 ... Execute Command 567 ... cat ${file_path}-${LOG_TIME} 568 ... return_stderr=True 569 Should Be Empty ${stderr} 570 571 Log ${journal_log} 572 573 Execute Command rm ${file_path}-${LOG_TIME} 574 575 [Return] ${journal_log} 576 577Mac Address To Hex String 578 [Documentation] Converts MAC address into hex format. 579 ... Example 580 ... Given the following MAC: 00:01:6C:80:02:78 581 ... This keyword will return: 0x00 0x01 0x6C 0x80 0x02 0x78 582 ... Description of arguments: 583 ... i_macaddress MAC address in the following format 584 ... 00:01:6C:80:02:78 585 [Arguments] ${i_macaddress} 586 587 ${mac_hex}= Catenate 0x${i_macaddress.replace(':', ' 0x')} 588 [Return] ${mac_hex} 589 590IP Address To Hex String 591 [Documentation] Converts IP address into hex format. 592 ... Example: 593 ... Given the following IP: 10.3.164.100 594 ... This keyword will return: 0xa 0x3 0xa4 0xa0 595 ... Description of arguments: 596 ... i_ipaddress IP address in the following format 597 ... 10.10.10.10 598 [Arguments] ${i_ipaddress} 599 600 @{ip}= Split String ${i_ipaddress} . 601 ${index}= Set Variable ${0} 602 603 :FOR ${item} IN @{ip} 604 \ ${hex}= Convert To Hex ${item} prefix=0x lowercase=yes 605 \ Set List Value ${ip} ${index} ${hex} 606 \ ${index}= Set Variable ${index + 1} 607 ${ip_hex}= Catenate @{ip} 608 [Return] ${ip_hex} 609 610BMC CPU Performance Check 611 [Documentation] Minimal 10% of proc should be free in this instance 612 613 ${bmc_cpu_usage_output} ${stderr}= Execute Command ${bmc_cpu_usage_cmd} 614 ... return_stderr=True 615 Should be empty ${stderr} 616 ${bmc_cpu_percentage}= Fetch From Left ${bmc_cpu_usage_output} % 617 Should be true ${bmc_cpu_percentage} < 90 618 619BMC Mem Performance Check 620 [Documentation] Minimal 10% of memory should be free in this instance 621 622 ${bmc_mem_free_output} ${stderr}= Execute Command ${bmc_mem_free_cmd} 623 ... return_stderr=True 624 Should be empty ${stderr} 625 626 ${bmc_mem_total_output} ${stderr}= Execute Command ${bmc_mem_total_cmd} 627 ... return_stderr=True 628 Should be empty ${stderr} 629 630 ${bmc_mem_percentage}= Evaluate ${bmc_mem_free_output}*100 631 ${bmc_mem_percentage}= Evaluate 632 ... ${bmc_mem_percentage}/${bmc_mem_total_output} 633 Should be true ${bmc_mem_percentage} > 10 634 635BMC File System Usage Check 636 [Documentation] Check the file system space. None should be 100% full 637 ... except /run/initramfs/ro 638 ${bmc_fs_usage_output} ${stderr}= Execute Command 639 ... ${bmc_file_system_usage_cmd} return_stderr=True 640 Should Be Empty ${stderr} 641 Should Be True ${bmc_fs_usage_output}==0 642 643Check BMC CPU Performance 644 [Documentation] Minimal 10% of proc should be free in 3 sample 645 :FOR ${var} IN Range 1 4 646 \ BMC CPU Performance check 647 648Check BMC Mem Performance 649 [Documentation] Minimal 10% of memory should be free 650 651 :FOR ${var} IN Range 1 4 652 \ BMC Mem Performance check 653 654Check BMC File System Performance 655 [Documentation] Check for file system usage for 4 times 656 657 :FOR ${var} IN Range 1 4 658 \ BMC File System Usage check 659 660Get URL List 661 [Documentation] Return list of URLs under given URL. 662 [Arguments] ${openbmc_url} 663 # Description of argument(s): 664 # openbmc_url URL for list operation (e.g. 665 # /xyz/openbmc_project/inventory). 666 667 ${url_list}= Read Properties ${openbmc_url}/list 668 [Return] ${url_list} 669 670Get Endpoint Paths 671 [Documentation] Returns all url paths ending with given endpoint 672 ... Example: 673 ... Given the following endpoint: cpu 674 ... This keyword will return: list of all urls ending with 675 ... cpu - 676 ... /org/openbmc/inventory/system/chassis/motherboard/cpu0, 677 ... /org/openbmc/inventory/system/chassis/motherboard/cpu1 678 ... Description of arguments: 679 ... path URL path for enumeration 680 ... endpoint string for which url path ending 681 [Arguments] ${path} ${endpoint} 682 683 ${resp}= Read Properties ${path}/enumerate timeout=30 684 log Dictionary ${resp} 685 686 ${list}= Get Dictionary Keys ${resp} 687 ${resp}= Get Matches ${list} regexp=^.*[0-9a-z_].${endpoint}[0-9]*$ 688 [Return] ${resp} 689 690Check Zombie Process 691 [Documentation] Check if any defunct process exist or not on BMC 692 ${count} ${stderr} ${rc}= Execute Command ps -o stat | grep Z | wc -l 693 ... return_stderr=True return_rc=True 694 Should Be True ${count}==0 695 Should Be Empty ${stderr} 696 697Prune Journal Log 698 [Documentation] Prune archived journal logs. 699 [Arguments] ${vacuum_size}=1M 700 701 # This keyword can be used to prevent the journal 702 # log from filling up the /run filesystem. 703 # This command will retain only the latest logs 704 # of the user specified size. 705 706 Open Connection And Log In 707 ${output} ${stderr} ${rc}= 708 ... Execute Command 709 ... journalctl --vacuum-size=${vacuum_size} 710 ... return_stderr=True return_rc=True 711 712 Should Be Equal ${rc} ${0} msg=${stderr} 713 714Set BMC Power Policy 715 [Documentation] Set the given BMC power policy. 716 [arguments] ${policy} 717 718 ${valueDict}= create dictionary data=${policy} 719 Write Attribute ${HOST_SETTING} power_policy data=${valueDict} 720 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 721 Should Be Equal ${currentPolicy} ${policy} 722 723Get System Power Policy 724 [Documentation] Get the BMC power policy. 725 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 726 [Return] ${currentPolicy} 727 728Get Auto Reboot 729 [Documentation] Returns auto reboot setting. 730 ${setting}= Read Attribute ${HOST_SETTING} auto_reboot 731 [Return] ${setting} 732 733 734Set Auto Reboot 735 [Documentation] Set the given auto reboot setting. 736 [Arguments] ${setting} 737 # setting auto reboot's setting, i.e. yes or no 738 739 ${valueDict}= Set Variable ${setting} 740 ${data}= Create Dictionary data=${valueDict} 741 Write Attribute ${HOST_SETTING} auto_reboot data=${data} 742 ${current_setting}= Get Auto Reboot 743 Should Be Equal ${current_setting} ${setting} 744 745 746Set BMC Reset Reference Time 747 [Documentation] Set current boot time as a reference and increment 748 ... boot count. 749 750 ${cur_btime}= Get BMC Boot Time 751 Run Keyword If ${BOOT_TIME} == ${0} and ${BOOT_COUNT} == ${0} 752 ... Set Global Variable ${BOOT_TIME} ${cur_btime} 753 ... ELSE IF ${cur_btime} > ${BOOT_TIME} 754 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 755 ... AND 756 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 757 758Get BMC Boot Time 759 [Documentation] Get boot time from /proc/stat. 760 761 Open Connection And Log In 762 ${output} ${stderr}= 763 ... Execute Command egrep '^btime ' /proc/stat | cut -f 2 -d ' ' 764 ... return_stderr=True 765 Should Be Empty ${stderr} 766 ${btime}= Convert To Integer ${output} 767 [Return] ${btime} 768 769Execute Command On BMC 770 [Documentation] Execute given command on BMC and return output. 771 [Arguments] ${command} 772 ${stdout} ${stderr}= Execute Command ${command} return_stderr=True 773 Should Be Empty ${stderr} 774 [Return] ${stdout} 775 776 777Enable Core Dump On BMC 778 [Documentation] Enable core dump collection. 779 Open Connection And Log In 780 ${core_pattern}= Execute Command On BMC 781 ... echo '/tmp/core_%e.%p' | tee /proc/sys/kernel/core_pattern 782 Should Be Equal As Strings ${core_pattern} /tmp/core_%e.%p 783 784Get Number Of BMC Core Dump Files 785 [Documentation] Get number of core dump files on BMC. 786 Open Connection And Log In 787 ${num_of_core_dump}= Execute Command 788 ... ls /tmp/core* 2>/dev/null | wc -l 789 [Return] ${num_of_core_dump} 790 791Set Core Dump File Size Unlimited 792 [Documentation] Set core dump file size to unlimited. 793 Open Connection And Log In 794 Execute Command On BMC 795 ... ulimit -c unlimited 796 797Check For Core Dumps 798 [Documentation] Check for any core dumps exist. 799 ${output}= Get Number Of BMC Core Dump Files 800 Run Keyword If ${output} > 0 801 ... Log **Warning** BMC core dump files exist level=WARN 802 803Trigger Host Watchdog Error 804 [Documentation] Inject host watchdog error using BMC. 805 [Arguments] ${milliseconds}=1000 ${sleep_time}=5s 806 # Description of argument(s): 807 # milliseconds The time watchdog timer value in milliseconds (e.g. 1000 = 808 # 1 second). 809 # sleep_time Time delay for host watchdog error to get injected. 810 # Default is 5 seconds. 811 812 Execute Command On BMC 813 ... /usr/sbin/mapper call /org/openbmc/watchdog/host0 org.openbmc.Watchdog set i ${milliseconds} 814 Execute Command On BMC 815 ... /usr/sbin/mapper call /org/openbmc/watchdog/host0 org.openbmc.Watchdog start 816 Sleep ${sleep_time} 817 818Login To OS Host 819 [Documentation] Login to OS Host. 820 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 821 ... ${os_password}=${OS_PASSWORD} 822 # Desription of arguments: 823 # ${os_host} IP address of the OS Host. 824 # ${os_username} OS Host Login user name. 825 # ${os_password} OS Host Login passwrd. 826 827 ${os_state}= Run Keyword And Return Status Ping Host ${os_host} 828 Run Keyword If '${os_state}' == 'False' 829 ... Run Keywords Initiate Host Reboot AND 830 ... Is Host Running AND 831 ... Wait for OS ${os_host} ${os_username} ${os_password} 832 833 Open Connection ${os_host} 834 ${resp}= Login ${os_username} ${os_password} 835 [Return] ${resp} 836 837Configure Initial Settings 838 [Documentation] Restore old IP and route. 839 ... This keyword requires initial settings viz IP address, 840 ... Network Mask, default gatway and serial console IP and port 841 ... information which should be provided in command line. 842 843 [Arguments] ${host}=${OPENBMC_HOST} ${mask}=${NET_MASK} 844 ... ${gw_ip}=${GW_IP} 845 846 # Open telnet connection and ignore the error, in case telnet session is 847 # already opened by the program calling this keyword. 848 849 Run Keyword And Ignore Error Open Telnet Connection to BMC Serial Console 850 Telnet.write ifconfig eth0 ${host} netmask ${mask} 851 Telnet.write route add default gw ${gw_ip} 852 853Install Debug Tarball On BMC 854 [Documentation] Copy the debug tar file to BMC and install. 855 [Arguments] ${tarball_file_path}=${EXECDIR}/obmc-phosphor-debug-tarball-witherspoon.tar.xz 856 ... ${targ_tarball_dir_path}=/home/root/tarball/ 857 858 # Description of arguments: 859 # tarball_file_path Path of the debug tarball file. 860 # The tar file is downloaded from the build page 861 # https://openpower.xyz/job/openbmc-build/ 862 # obmc-phosphor-debug-tarball-witherspoon.tar.xz 863 # 864 # targ_tarball_dir_path The directory path where the tarball is to be 865 # installed. 866 867 OperatingSystem.File Should Exist ${tarball_file_path} 868 ... msg=${tarball_file_path} doesn't exist. 869 870 # Upload the file to BMC. 871 Import Library SCPLibrary WITH NAME scp 872 Open Connection for SCP 873 scp.Put File ${tarball_file_path} /tmp/debug-tarball.tar.xz 874 875 # Create tarball directory and install. 876 Open Connection And Log In 877 Execute Command On BMC mkdir -p ${targ_tarball_dir_path} 878 Execute Command On BMC 879 ... tar -xf /tmp/debug-tarball.tar.xz -C ${targ_tarball_dir_path} 880 881 # Create symlink to callout-test binary. 882 Execute Command On BMC 883 ... ln -s ${targ_tarball_dir_path}/bin/callout-test /usr/bin/callout-test 884 885 # Remove the tarball file from BMC 886 Execute Command On BMC rm /tmp/debug-tarball.tar.xz 887 888 889Get BMC Boot Count 890 [Documentation] Get BMC boot count based on boot time. 891 ${cur_btime}= Get BMC Boot Time 892 893 # Set global variable BOOT_TIME to current boot time if current boot time 894 # is changed. Also increase value of global variable BOOT_COUNT by 1. 895 Run Keyword If ${cur_btime} > ${BOOT_TIME} 896 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 897 ... AND 898 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 899 [Return] ${BOOT_COUNT} 900 901Set BMC Boot Count 902 [Documentation] Set BMC boot count to given value. 903 [Arguments] ${count} 904 905 # Description of arguments: 906 # count boot count value. 907 ${cur_btime}= Get BMC Boot Time 908 909 # Set global variable BOOT_COUNT to given value. 910 Set Global Variable ${BOOT_COUNT} ${count} 911 912 # Set BOOT_TIME variable to current boot time. 913 Set Global Variable ${BOOT_COUNT} ${count} 914 915############################################################################### 916Delete Error logs 917 [Documentation] Delete error logs. 918 919 # The REST method to delete error openbmc/openbmc#1327 920 # until then using logging restart. 921 Open Connection And Log In 922 Execute Command On BMC 923 ... systemctl restart xyz.openbmc_project.Logging.service 924 Sleep 10s reason=Wait for logging service to restart properly. 925 926############################################################################### 927Delete Error log Entry 928 [Documentation] Delete error log entry. 929 [Arguments] ${entry_path} 930 931 # Description of argument(s): 932 # entry_path Delete an error log entry. 933 # Ex. /xyz/openbmc_project/logging/entry/1 934 935 ${data}= Create Dictionary data=@{EMPTY} 936 ${resp}= Openbmc Delete Request ${entry_path} data=${data} 937 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 938