1*** Settings *** 2Resource ../lib/resource.txt 3Resource ../lib/rest_client.robot 4Resource ../lib/connection_client.robot 5Library DateTime 6Library Process 7Library OperatingSystem 8Library gen_print.py 9Library gen_robot_print.py 10 11*** Variables *** 12${SYSTEM_SHUTDOWN_TIME} ${5} 13${dbuscmdBase} 14... dbus-send --system --print-reply --dest=${OPENBMC_BASE_DBUS}.settings.Host 15${dbuscmdGet} 16... ${SETTINGS_URI}host0 org.freedesktop.DBus.Properties.Get 17# Enable when ready with openbmc/openbmc-test-automation#203 18#${dbuscmdString}= string:"xyz.openbmc_project.settings.Host" string: 19${dbuscmdString}= string:"org.openbmc.settings.Host" string: 20 21# Assign default value to QUIET for programs which may not define it. 22${QUIET} ${0} 23${bmc_mem_free_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f4 24${bmc_mem_total_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f2 25${bmc_cpu_usage_cmd}= top -n 1 | grep CPU: | cut -c 7-9 26${HOST_SETTING} ${SETTINGS_URI}host0 27# /run/initramfs/ro associate filesystem should be 100% full always 28${bmc_file_system_usage_cmd}= 29... df -h | grep -v /run/initramfs/ro | cut -c 52-54 | grep 100 | wc -l 30 31${BOOT_TIME} ${0} 32${BOOT_COUNT} ${0} 33${count} ${0} 34${devicetree_base} /sys/firmware/devicetree/base/model 35 36*** Keywords *** 37 38Get BMC System Model 39 [Documentation] Get the BMC model from the device tree. 40 41 ${bmc_model} ${stderr}= Execute Command 42 ... cat ${devicetree_base} | cut -d " " -f 1 return_stderr=True 43 Should Be Empty ${stderr} 44 Should Not Be Empty ${bmc_model} 45 [Return] ${bmc_model} 46 47Verify BMC System Model 48 [Documentation] Verify the BMC model with ${OPENBMC_MODEL}. 49 [Arguments] ${bmc_model} 50 51 ${tmp_bmc_model}= Fetch From Right ${OPENBMC_MODEL} / 52 ${tmp_bmc_model}= Fetch From Left ${tmp_bmc_model} . 53 ${ret}= Run Keyword And Return Status Should Contain ${bmc_model} 54 ... ${tmp_bmc_model} ignore_case=True 55 [Return] ${ret} 56 57Wait For Host To Ping 58 [Arguments] ${host} ${timeout}=${OPENBMC_REBOOT_TIMEOUT}min 59 ... ${interval}=5 sec 60 61 # host The DNS name or IP of the host to ping. 62 # timeout The amount of time after which attempts to ping cease. 63 # interval The amount of time in between attempts to ping. 64 65 Wait Until Keyword Succeeds ${timeout} ${interval} Ping Host ${host} 66 67Ping Host 68 [Arguments] ${host} 69 Should Not Be Empty ${host} msg=No host provided 70 ${RC} ${output}= Run and return RC and Output ping -c 4 ${host} 71 Log RC: ${RC}\nOutput:\n${output} 72 Should be equal ${RC} ${0} 73 74Get Boot Progress 75 [Arguments] ${quiet}=${QUIET} 76 77 ${state}= Read Attribute ${OPENBMC_BASE_URI}sensors/host/BootProgress 78 ... value quiet=${quiet} 79 [Return] ${state} 80 81Is Power On 82 ${state}= Get Power State 83 Should be equal ${state} ${1} 84 85Is Power Off 86 ${state}= Get Power State 87 Should be equal ${state} ${0} 88 89Initiate Power On 90 [Documentation] Initiates the power on and waits until the Is Power On 91 ... keyword returns that the power state has switched to on. 92 [Arguments] ${wait}=${1} 93 94 @{arglist}= Create List 95 ${args}= Create Dictionary data=@{arglist} 96 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOn 97 ... data=${args} 98 should be equal as strings ${resp.status_code} ${HTTP_OK} 99 100 # Does caller want to wait for power on status? 101 Run Keyword If '${wait}' == '${0}' Return From Keyword 102 Wait Until Keyword Succeeds 3 min 10 sec Is Power On 103 104Initiate Power Off 105 [Documentation] Initiates the power off and waits until the Is Power Off 106 ... keyword returns that the power state has switched to off. 107 @{arglist}= Create List 108 ${args}= Create Dictionary data=@{arglist} 109 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOff 110 ... data=${args} 111 should be equal as strings ${resp.status_code} ${HTTP_OK} 112 Wait Until Keyword Succeeds 1 min 10 sec Is Power Off 113 114Initiate OS Host Power Off 115 [Documentation] Initiate an OS reboot. 116 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 117 ... ${os_password}=${OS_PASSWORD} 118 119 # Description of arguments: 120 # os_host The DNS name or IP of the OS. 121 # os_username The username to be used to sign in to the OS. 122 # os_password The password to be used to sign in to the OS. 123 124 SSHLibrary.Open connection ${os_host} 125 Login ${os_username} ${os_password} 126 ${cmd_buf} Catenate shutdown 127 Start Command ${cmd_buf} 128 SSHLibrary.Close Connection 129 130Initiate OS Host Reboot 131 [Documentation] Initiate an OS reboot. 132 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 133 ... ${os_password}=${OS_PASSWORD} 134 135 # Description of arguments: 136 # os_host The DNS name or IP of the OS. 137 # os_username The username to be used to sign in to the OS. 138 # os_password The password to be used to sign in to the OS. 139 140 SSHLibrary.Open connection ${os_host} 141 Login ${os_username} ${os_password} 142 ${cmd_buf} Catenate reboot 143 Start Command ${cmd_buf} 144 SSHLibrary.Close Connection 145 146Initiate Auto Reboot 147 [Documentation] Initiate an auto reboot. 148 149 # Set the auto reboot policy. 150 Set Auto Reboot yes 151 152 SSHLibrary.Open connection ${openbmc_host} 153 Login ${openbmc_username} ${openbmc_password} 154 155 # Set the watchdog timer. Note: 5000 = milliseconds which is 5 seconds. 156 ${cmd_buf}= Catenate /usr/sbin/mapper call /org/openbmc/watchdog/host0 157 ... org.openbmc.Watchdog set i 5000 158 ${output} ${stderr} ${rc}= Execute Command ${cmd_buf} 159 ... return_stderr=True return_rc=True 160 Should Be Empty ${stderr} 161 Should be equal ${rc} ${0} 162 163 # Start the watchdog. 164 ${cmd_buf}= Catenate /usr/sbin/mapper call /org/openbmc/watchdog/host0 165 ... org.openbmc.Watchdog start 166 ${output} ${stderr} ${rc}= Execute Command ${cmd_buf} 167 ... return_stderr=True return_rc=True 168 Should Be Empty ${stderr} 169 Should be equal ${rc} ${0} 170 171Trigger Warm Reset 172 log to console "Triggering warm reset" 173 ${data}= create dictionary data=@{EMPTY} 174 ${resp}= openbmc post request 175 ... ${OPENBMC_BASE_URI}control/bmc0/action/warmReset data=${data} 176 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 177 ${session_active}= Check If warmReset is Initiated 178 Run Keyword If '${session_active}' == '${True}' 179 ... Fail msg=warm reset didn't occur 180 181 Sleep ${SYSTEM_SHUTDOWN_TIME}min 182 Check If BMC Is Up 183 184Check OS 185 [Documentation] Attempts to ping the host OS and then checks that the host 186 ... OS is up by running an SSH command. 187 188 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 189 ... ${os_password}=${OS_PASSWORD} ${quiet}=${QUIET} 190 ... ${print_string}=${EMPTY} 191 [Teardown] SSHLibrary.Close Connection 192 193 # os_host The DNS name/IP of the OS host associated with our BMC. 194 # os_username The username to be used to sign on to the OS host. 195 # os_password The password to be used to sign on to the OS host. 196 # quiet Indicates whether this keyword should write to console. 197 # print_string A string to be printed before checking the OS. 198 199 rprint ${print_string} 200 201 # Attempt to ping the OS. Store the return code to check later. 202 ${ping_rc}= Run Keyword and Return Status Ping Host ${os_host} 203 204 SSHLibrary.Open connection ${os_host} 205 206 ${status} ${msg}= Run Keyword And Ignore Error Login ${os_username} 207 ... ${os_password} 208 ${err_msg1}= Sprint Error ${msg} 209 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 210 Run Keyword If '${status}' == 'FAIL' Fail msg=${err_msg} 211 ${output} ${stderr} ${rc}= Execute Command uptime return_stderr=True 212 ... return_rc=True 213 214 ${temp_msg}= Catenate Could not execute a command on the operating 215 ... system.\n 216 ${err_msg1}= Sprint Error ${temp_msg} 217 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 218 219 # If the return code returned by "Execute Command" is non-zero, this 220 # keyword will fail. 221 Should Be Equal ${rc} ${0} msg=${err_msg} 222 # We will likewise fail if there is any stderr data. 223 Should Be Empty ${stderr} 224 225 ${temp_msg}= Set Variable Could not ping the operating system.\n 226 ${err_msg1}= Sprint Error ${temp_msg} 227 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 228 # We will likewise fail if the OS did not ping, as we could SSH but not 229 # ping 230 Should Be Equal As Strings ${ping_rc} ${TRUE} msg=${err_msg} 231 232Wait for OS 233 [Documentation] Waits for the host OS to come up via calls to "Check OS". 234 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 235 ... ${os_password}=${OS_PASSWORD} ${timeout}=${OS_WAIT_TIMEOUT} 236 ... ${quiet}=${0} 237 [Teardown] rprintn 238 239 # os_host The DNS name or IP of the OS host associated with our 240 # BMC. 241 # os_username The username to be used to sign on to the OS host. 242 # os_password The password to be used to sign on to the OS host. 243 # timeout The timeout in seconds indicating how long you're 244 # willing to wait for the OS to respond. 245 # quiet Indicates whether this keyword should write to console. 246 247 # The interval to be used between calls to "Check OS". 248 ${interval}= Set Variable 5 249 250 ${message}= Catenate Checking every ${interval} seconds for up to 251 ... ${timeout} seconds for the operating system to communicate. 252 rqprint_timen ${message} 253 254 Wait Until Keyword Succeeds ${timeout} sec ${interval} Check OS 255 ... ${os_host} ${os_username} ${os_password} 256 ... print_string=\# 257 258 rqprintn 259 260 rqprint_timen The operating system is now communicating. 261 262Get BMC State Deprecated 263 [Documentation] Returns the state of the BMC as a string. (i.e: BMC_READY) 264 [Arguments] ${quiet}=${QUIET} 265 266 @{arglist}= Create List 267 ${args}= Create Dictionary data=@{arglist} 268 ${resp}= Call Method ${OPENBMC_BASE_URI}managers/System/ getSystemState 269 ... data=${args} quiet=${quiet} 270 Should be equal as strings ${resp.status_code} ${HTTP_OK} 271 ${content}= to json ${resp.content} 272 [Return] ${content["data"]} 273 274Get Power State 275 [Documentation] Returns the power state as an integer. Either 0 or 1. 276 [Arguments] ${quiet}=${QUIET} 277 278 @{arglist}= Create List 279 ${args}= Create Dictionary data=@{arglist} 280 281 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ getPowerState 282 ... data=${args} quiet=${quiet} 283 Should be equal as strings ${resp.status_code} ${HTTP_OK} 284 ${content}= to json ${resp.content} 285 [Return] ${content["data"]} 286 287Clear BMC Record Log 288 [Documentation] Clears all the event logs on the BMC. This would be 289 ... equivalent to ipmitool sel clear. 290 @{arglist}= Create List 291 ${args}= Create Dictionary data=@{arglist} 292 ${resp}= Call Method 293 ... ${OPENBMC_BASE_URI}records/events/ clear data=${args} 294 should be equal as strings ${resp.status_code} ${HTTP_OK} 295 296Copy PNOR to BMC 297 Import Library SCPLibrary WITH NAME scp 298 Open Connection for SCP 299 Log Copying ${PNOR_IMAGE_PATH} to /tmp 300 scp.Put File ${PNOR_IMAGE_PATH} /tmp 301 302Flash PNOR 303 [Documentation] Calls flash bios update method to flash PNOR image 304 [Arguments] ${pnor_image} 305 @{arglist}= Create List ${pnor_image} 306 ${args}= Create Dictionary data=@{arglist} 307 ${resp}= Call Method ${OPENBMC_BASE_URI}control/flash/bios/ update 308 ... data=${args} 309 should be equal as strings ${resp.status_code} ${HTTP_OK} 310 Wait Until Keyword Succeeds 2 min 10 sec Is PNOR Flashing 311 312Get Flash BIOS Status 313 [Documentation] Returns the status of the flash BIOS API as a string. For 314 ... example 'Flashing', 'Flash Done', etc 315 ${data}= Read Properties ${OPENBMC_BASE_URI}control/flash/bios 316 [Return] ${data['status']} 317 318Is PNOR Flashing 319 [Documentation] Get BIOS 'Flashing' status. This indicates that PNOR 320 ... flashing has started. 321 ${status}= Get Flash BIOS Status 322 should be equal as strings ${status} Flashing 323 324Is PNOR Flash Done 325 [Documentation] Get BIOS 'Flash Done' status. This indicates that the 326 ... PNOR flashing has completed. 327 ${status}= Get Flash BIOS Status 328 should be equal as strings ${status} Flash Done 329 330Is System State Host Booted 331 [Documentation] Checks whether system state is HOST_BOOTED. 332 ${state}= Get BMC State Deprecated 333 should be equal as strings ${state} HOST_BOOTED 334 335Is OS Starting 336 [Documentation] Check if boot progress is OS starting. 337 ${boot_progress}= Get Boot Progress 338 Should Be Equal ${boot_progress} FW Progress, Starting OS 339 340Is OS Off 341 [Documentation] Check if boot progress is "Off". 342 ${boot_progress}= Get Boot Progress 343 Should Be Equal ${boot_progress} Off 344 345Verify Ping and REST Authentication 346 ${l_ping}= Run Keyword And Return Status 347 ... Ping Host ${OPENBMC_HOST} 348 Run Keyword If '${l_ping}' == '${False}' 349 ... Fail msg=Ping Failed 350 351 ${l_rest}= Run Keyword And Return Status 352 ... Initialize OpenBMC 353 Run Keyword If '${l_rest}' == '${False}' 354 ... Fail msg=REST Authentication Failed 355 356 # Just to make sure the SSH is working for SCP 357 Open Connection And Log In 358 ${system} ${stderr}= Execute Command hostname return_stderr=True 359 Should Be Empty ${stderr} 360 361Check If BMC is Up 362 [Documentation] Wait for Host to be online. Checks every X seconds 363 ... interval for Y minutes and fails if timed out. 364 ... Default MAX timedout is 10 min, interval 10 seconds. 365 [Arguments] ${max_timeout}=${OPENBMC_REBOOT_TIMEOUT} min 366 ... ${interval}=10 sec 367 368 Wait Until Keyword Succeeds 369 ... ${max_timeout} ${interval} Verify Ping and REST Authentication 370 371 372Check If warmReset is Initiated 373 [Documentation] Ping would be still alive, so try SSH to connect 374 ... if fails the ports are down indicating reboot 375 ... is in progress 376 377 # Warm reset adds 3 seconds delay before forcing reboot 378 # To minimize race conditions, we wait for 7 seconds 379 Sleep 7s 380 ${alive}= Run Keyword and Return Status 381 ... Open Connection And Log In 382 Return From Keyword If '${alive}' == '${False}' ${False} 383 [Return] ${True} 384 385Flush REST Sessions 386 [Documentation] Removes all the active session objects 387 Delete All Sessions 388 389Initialize DBUS cmd 390 [Documentation] Initialize dbus string with property string to extract 391 [Arguments] ${boot_property} 392 ${cmd}= Catenate ${dbuscmdBase} ${dbuscmdGet} ${dbuscmdString} 393 ${cmd}= Catenate ${cmd}${boot_property} 394 Set Global Variable ${dbuscmd} ${cmd} 395 396Create OS Console File Path 397 [Documentation] Create OS console file path name and return it. 398 [Arguments] ${log_file_path}=${EMPTY} 399 400 # Description of arguements: 401 # file_path The caller's candidate value. If this value is ${EMPTY}, this 402 # keyword will compose a file path name. Otherwise, this 403 # keyword will use the caller's file_path value. In either 404 # case, the value will be returned. 405 406 ${default_file_path}= Catenate /tmp/${OPENBMC_HOST}_os_console 407 ${log_file_path}= Set Variable If '${log_file_path}' == '${EMPTY}' 408 ... ${default_file_path} ${log_file_path} 409 410 [Return] ${log_file_path} 411 412Create OS Console Command String 413 [Documentation] Return a command string to start OS console logging. 414 415 # First make sure that the ssh_pw program is available. 416 ${cmd_buf}= Catenate which ssh_pw 2>&1 417 Rdpissuing ${cmd_buf} 418 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 419 Rdpvars rc output 420 Should Be Equal ${rc} ${0} msg=${output}\n 421 422 ${cmd_buf}= Catenate ssh_pw ${OPENBMC_PASSWORD} -p 2200 423 ... ${OPENBMC_USERNAME}@${OPENBMC_HOST} 424 425 [Return] ${cmd_buf} 426 427Stop SOL Console Logging 428 [Documentation] Stop system console logging and return log output. 429 [Arguments] ${log_file_path}=${EMPTY} ${targ_file_path}=${EMPTY} 430 431 # If there are muliple system console processes, they will all be stopped. 432 # If there is no existing log file this keyword will return an error 433 # message to that effect (and write that message to targ_file_path, if 434 # specified). 435 # NOTE: This keyword will not fail if there is no running system console 436 # process. 437 438 # Description of arguments: 439 # log_file_path The file path that was used to call "Start SOL 440 # Console Logging". See that keyword (above) for details. 441 # targ_file_path If specified, the file path to which the source 442 # file path (i.e. "log_file_path") should be copied. 443 444 ${log_file_path}= Create OS Console File Path ${log_file_path} 445 # Find the pid of the active system console logging session (if any). 446 ${search_string}= Create OS Console Command String 447 ${cmd_buf}= Catenate echo $(ps -ef | egrep '${search_string}' 448 ... | egrep -v grep | cut -c10-14) 449 Rdpissuing ${cmd_buf} 450 ${rc} ${os_con_pid}= Run And Return Rc And Output ${cmd_buf} 451 Rdpvars os_con_pid 452 # If rc is not zero it just means that there is no OS Console process 453 # running. 454 455 ${cmd_buf}= Catenate kill -9 ${os_con_pid} 456 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpissuing ${cmd_buf} 457 ${rc} ${output}= Run Keyword If '${os_con_pid}' != '${EMPTY}' 458 ... Run And Return Rc And Output ${cmd_buf} 459 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpvars rc output 460 461 ${cmd_buf}= Set Variable cat ${log_file_path} 2>&1 462 Rdpissuing ${cmd_buf} 463 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 464 Rdpvars rc 465 466 Run Keyword If '${targ_file_path}' != '${EMPTY}' 467 ... Run Keyword And Ignore Error 468 ... Copy File ${log_file_path} ${targ_file_path} 469 470 [Return] ${output} 471 472Start SOL Console Logging 473 [Documentation] Start system console log to file. 474 [Arguments] ${log_file_path}=${EMPTY} 475 476 # This keyword will first call "Stop SOL Console Logging". Only then will 477 # it start SOL console logging. The data returned by "Stop SOL Console 478 # Logging" will in turn be returned by this keyword. 479 480 # Description of arguments: 481 # log_file_path The file path to which system console log data should be 482 # written. Note that this path is taken to be a location on 483 # the machine where this program is running rather than on 484 # the Open BMC system. 485 486 ${log_file_path}= Create OS Console File Path ${log_file_path} 487 488 ${log_output}= Stop SOL Console Logging ${log_file_path} 489 490 # Validate by making sure we can create the file. Problems creating the 491 # file would not be noticed by the subsequent ssh command because we fork 492 # the command. 493 Create File ${log_file_path} 494 ${sub_cmd_buf}= Create OS Console Command String 495 # Routing stderr to stdout so that any startup error text will go to the 496 # output file. 497 ${cmd_buf}= Catenate ${sub_cmd_buf} > ${log_file_path} 2>&1 & 498 Rdpissuing ${cmd_buf} 499 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 500 # Because we are forking this command, we essentially will never get a 501 # non-zero return code or any output. 502 Should Be Equal ${rc} ${0} 503 504 [Return] ${log_output} 505 506Get Time Stamp 507 [Documentation] Get the current time stamp data 508 ${cur_time}= Get Current Date result_format=%Y%m%d%H%M%S%f 509 [Return] ${cur_time} 510 511 512Verify BMC State 513 [Documentation] Get the BMC state and verify if the current 514 ... BMC state is as expected. 515 [Arguments] ${expected} 516 517 ${current}= Get BMC State Deprecated 518 Should Contain ${expected} ${current} 519 520Start Journal Log 521 [Documentation] Start capturing journal log to a file in /tmp using 522 ... journalctl command. By default journal log is collected 523 ... at /tmp/journal_log else user input location. 524 ... The File is appended with datetime. 525 [Arguments] ${file_path}=/tmp/journal_log 526 527 Open Connection And Log In 528 529 ${cur_time}= Get Time Stamp 530 Set Global Variable ${LOG_TIME} ${cur_time} 531 Start Command 532 ... journalctl -f > ${file_path}-${LOG_TIME} 533 Log Journal Log Started: ${file_path}-${LOG_TIME} 534 535 536Stop Journal Log 537 [Documentation] Stop journalctl process if its running. 538 ... By default return log from /tmp/journal_log else 539 ... user input location. 540 [Arguments] ${file_path}=/tmp/journal_log 541 542 Open Connection And Log In 543 544 ${rc}= 545 ... Execute Command 546 ... ps ax | grep journalctl | grep -v grep 547 ... return_stdout=False return_rc=True 548 549 Return From Keyword If '${rc}' == '${1}' 550 ... No journal log process running 551 552 ${output} ${stderr}= 553 ... Execute Command killall journalctl 554 ... return_stderr=True 555 Should Be Empty ${stderr} 556 557 ${journal_log} ${stderr}= 558 ... Execute Command 559 ... cat ${file_path}-${LOG_TIME} 560 ... return_stderr=True 561 Should Be Empty ${stderr} 562 563 Log ${journal_log} 564 565 Execute Command rm ${file_path}-${LOG_TIME} 566 567 [Return] ${journal_log} 568 569Mac Address To Hex String 570 [Documentation] Converts MAC address into hex format. 571 ... Example 572 ... Given the following MAC: 00:01:6C:80:02:78 573 ... This keyword will return: 0x00 0x01 0x6C 0x80 0x02 0x78 574 ... Description of arguments: 575 ... i_macaddress MAC address in the following format 576 ... 00:01:6C:80:02:78 577 [Arguments] ${i_macaddress} 578 579 ${mac_hex}= Catenate 0x${i_macaddress.replace(':', ' 0x')} 580 [Return] ${mac_hex} 581 582IP Address To Hex String 583 [Documentation] Converts IP address into hex format. 584 ... Example: 585 ... Given the following IP: 10.3.164.100 586 ... This keyword will return: 0xa 0x3 0xa4 0xa0 587 ... Description of arguments: 588 ... i_ipaddress IP address in the following format 589 ... 10.10.10.10 590 [Arguments] ${i_ipaddress} 591 592 @{ip}= Split String ${i_ipaddress} . 593 ${index}= Set Variable ${0} 594 595 :FOR ${item} IN @{ip} 596 \ ${hex}= Convert To Hex ${item} prefix=0x lowercase=yes 597 \ Set List Value ${ip} ${index} ${hex} 598 \ ${index}= Set Variable ${index + 1} 599 ${ip_hex}= Catenate @{ip} 600 [Return] ${ip_hex} 601 602BMC CPU Performance Check 603 [Documentation] Minimal 10% of proc should be free in this instance 604 605 ${bmc_cpu_usage_output} ${stderr}= Execute Command ${bmc_cpu_usage_cmd} 606 ... return_stderr=True 607 Should be empty ${stderr} 608 ${bmc_cpu_percentage}= Fetch From Left ${bmc_cpu_usage_output} % 609 Should be true ${bmc_cpu_percentage} < 90 610 611BMC Mem Performance Check 612 [Documentation] Minimal 10% of memory should be free in this instance 613 614 ${bmc_mem_free_output} ${stderr}= Execute Command ${bmc_mem_free_cmd} 615 ... return_stderr=True 616 Should be empty ${stderr} 617 618 ${bmc_mem_total_output} ${stderr}= Execute Command ${bmc_mem_total_cmd} 619 ... return_stderr=True 620 Should be empty ${stderr} 621 622 ${bmc_mem_percentage}= Evaluate ${bmc_mem_free_output}*100 623 ${bmc_mem_percentage}= Evaluate 624 ... ${bmc_mem_percentage}/${bmc_mem_total_output} 625 Should be true ${bmc_mem_percentage} > 10 626 627BMC File System Usage Check 628 [Documentation] Check the file system space. None should be 100% full 629 ... except /run/initramfs/ro 630 ${bmc_fs_usage_output} ${stderr}= Execute Command 631 ... ${bmc_file_system_usage_cmd} return_stderr=True 632 Should Be Empty ${stderr} 633 Should Be True ${bmc_fs_usage_output}==0 634 635Check BMC CPU Performance 636 [Documentation] Minimal 10% of proc should be free in 3 sample 637 :FOR ${var} IN Range 1 4 638 \ BMC CPU Performance check 639 640Check BMC Mem Performance 641 [Documentation] Minimal 10% of memory should be free 642 643 :FOR ${var} IN Range 1 4 644 \ BMC Mem Performance check 645 646Check BMC File System Performance 647 [Documentation] Check for file system usage for 4 times 648 649 :FOR ${var} IN Range 1 4 650 \ BMC File System Usage check 651 652Get Endpoint Paths 653 [Documentation] Returns all url paths ending with given endpoint 654 ... Example: 655 ... Given the following endpoint: cpu 656 ... This keyword will return: list of all urls ending with 657 ... cpu - 658 ... /org/openbmc/inventory/system/chassis/motherboard/cpu0, 659 ... /org/openbmc/inventory/system/chassis/motherboard/cpu1 660 ... Description of arguments: 661 ... path URL path for enumeration 662 ... endpoint string for which url path ending 663 [Arguments] ${path} ${endpoint} 664 665 ${resp}= Read Properties ${path}/enumerate timeout=30 666 log Dictionary ${resp} 667 668 ${list}= Get Dictionary Keys ${resp} 669 ${resp}= Get Matches ${list} regexp=^.*[0-9a-z_].${endpoint}[0-9]*$ 670 [Return] ${resp} 671 672Check Zombie Process 673 [Documentation] Check if any defunct process exist or not on BMC 674 ${count} ${stderr} ${rc}= Execute Command ps -o stat | grep Z | wc -l 675 ... return_stderr=True return_rc=True 676 Should Be True ${count}==0 677 Should Be Empty ${stderr} 678 679Prune Journal Log 680 [Documentation] Prune archived journal logs. 681 [Arguments] ${vacuum_size}=1M 682 683 # This keyword can be used to prevent the journal 684 # log from filling up the /run filesystem. 685 # This command will retain only the latest logs 686 # of the user specified size. 687 688 Open Connection And Log In 689 ${output} ${stderr} ${rc}= 690 ... Execute Command 691 ... journalctl --vacuum-size=${vacuum_size} 692 ... return_stderr=True return_rc=True 693 694 Should Be Equal ${rc} ${0} msg=${stderr} 695 696Set BMC Power Policy 697 [Documentation] Set the given BMC power policy. 698 [arguments] ${policy} 699 700 ${valueDict}= create dictionary data=${policy} 701 Write Attribute ${HOST_SETTING} power_policy data=${valueDict} 702 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 703 Should Be Equal ${currentPolicy} ${policy} 704 705Get System Power Policy 706 [Documentation] Get the BMC power policy. 707 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 708 [Return] ${currentPolicy} 709 710Get Auto Reboot 711 [Documentation] Returns auto reboot setting. 712 ${setting}= Read Attribute ${HOST_SETTINGS} auto_reboot 713 [Return] ${setting} 714 715 716Set Auto Reboot 717 [Documentation] Set the given auto reboot setting. 718 [Arguments] ${setting} 719 # setting auto reboot's setting, i.e. yes or no 720 721 ${valueDict}= Set Variable ${setting} 722 ${data}= Create Dictionary data=${valueDict} 723 Write Attribute ${HOST_SETTINGS} auto_reboot data=${data} 724 ${current_setting}= Get Auto Reboot 725 Should Be Equal ${current_setting} ${setting} 726 727 728Set BMC Reset Reference Time 729 [Documentation] Set current boot time as a reference and increment 730 ... boot count. 731 732 ${cur_btime}= Get BMC Boot Time 733 Run Keyword If ${BOOT_TIME} == ${0} and ${BOOT_COUNT} == ${0} 734 ... Set Global Variable ${BOOT_TIME} ${cur_btime} 735 ... ELSE IF ${cur_btime} > ${BOOT_TIME} 736 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 737 ... AND 738 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 739 740Get BMC Boot Time 741 [Documentation] Get boot time from /proc/stat. 742 743 Open Connection And Log In 744 ${output} ${stderr}= 745 ... Execute Command egrep '^btime ' /proc/stat | cut -f 2 -d ' ' 746 ... return_stderr=True 747 Should Be Empty ${stderr} 748 ${btime}= Convert To Integer ${output} 749 [Return] ${btime} 750 751Execute Command On BMC 752 [Documentation] Execute given command on BMC and return output. 753 [Arguments] ${command} 754 ${stdout} ${stderr}= Execute Command ${command} return_stderr=True 755 Should Be Empty ${stderr} 756 [Return] ${stdout} 757 758 759Enable Core Dump On BMC 760 [Documentation] Enable core dump collection. 761 Open Connection And Log In 762 ${core_pattern}= Execute Command On BMC 763 ... echo '/tmp/core_%e.%p' | tee /proc/sys/kernel/core_pattern 764 Should Be Equal As Strings ${core_pattern} /tmp/core_%e.%p 765 766Get Number Of BMC Core Dump Files 767 [Documentation] Get number of core dump files on BMC. 768 Open Connection And Log In 769 ${num_of_core_dump}= Execute Command 770 ... ls /tmp/core* 2>/dev/null | wc -l 771 [Return] ${num_of_core_dump} 772 773Set Core Dump File Size Unlimited 774 [Documentation] Set core dump file size to unlimited. 775 Open Connection And Log In 776 Execute Command On BMC 777 ... ulimit -c unlimited 778 779Check For Core Dumps 780 [Documentation] Check for any core dumps exist. 781 ${output}= Get Number Of BMC Core Dump Files 782 Run Keyword If ${output} > 0 783 ... Log **Warning** BMC core dump files exist level=WARN 784 785Trigger Host Watchdog Error 786 [Documentation] Inject host watchdog error using BMC. 787 [Arguments] ${milliseconds}=1000 ${sleep_time}=5s 788 # Description of arguments: 789 # milliseconds The time watchdog timer value in milliseconds (e.g. 1000 = 1 second). 790 # sleep_time Time delay for host watchdog error to get injected. 791 # Default is 5 seconds. 792 793 Execute Command On BMC 794 ... /usr/sbin/mapper call /org/openbmc/watchdog/host0 org.openbmc.Watchdog set i ${milliseconds} 795 Execute Command On BMC 796 ... /usr/sbin/mapper call /org/openbmc/watchdog/host0 org.openbmc.Watchdog start 797 Sleep ${sleep_time} 798 799Login To OS Host 800 [Documentation] Login to OS Host. 801 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 802 ... ${os_password}=${OS_PASSWORD} 803 # Desription of arguments: 804 # ${os_host} IP address of the OS Host. 805 # ${os_username} OS Host Login user name. 806 # ${os_password} OS Host Login passwrd. 807 808 ${os_state}= Run Keyword And Return Status Ping Host ${os_host} 809 Run Keyword If '${os_state}' == 'False' 810 ... Run Keywords Initiate Host Reboot AND 811 ... Is Host Running AND 812 ... Wait for OS ${os_host} ${os_username} ${os_password} 813 814 Open Connection ${os_host} 815 ${resp}= Login ${os_username} ${os_password} 816 [Return] ${resp} 817 818Configure Initial Settings 819 [Documentation] Restore old IP and route. 820 ... This keyword requires initial settings viz IP address, 821 ... Network Mask, default gatway and serial console IP and port 822 ... information which should be provided in command line. 823 824 [Arguments] ${host}=${OPENBMC_HOST} ${mask}=${NET_MASK} ${gw_ip}=${GW_IP} 825 826 # Open telnet connection and ignore the error, in case telnet session is already 827 # opened by the program calling this keyword. 828 829 Run Keyword And Ignore Error Open Telnet Connection to BMC Serial Console 830 Telnet.write ifconfig eth0 ${host} netmask ${mask} 831 Telnet.write route add default gw ${gw_ip} 832 833Install Debug Tarball On BMC 834 [Documentation] Copy the tar file to BMC and install in "/usr/". 835 [Arguments] ${tarball_path} 836 # Description of arguments: 837 # tarball_path Absolute path of the debug tarball file. 838 # The tar file is downloaded from the build page 839 # https://openpower.xyz/job/openbmc-build/665/distro=ubuntu, 840 # target=witherspoon/artifact/images/witherspoon/ 841 # obmc-phosphor-debug-tarball-witherspoon.tar.xz 842 843 OperatingSystem.File Should Exist ${tarball_path} 844 ... msg=${tarball_path} doesn't exist. 845 Import Library SCPLibrary WITH NAME scp 846 Open Connection for SCP 847 scp.Put File ${tarball_path} /tmp/debug-tarball.tar.xz 848 Open Connection And Log In 849 Execute Command On BMC tar -xf /tmp/debug-tarball.tar.xz -C /usr/ 850 # Remove the tarball file from BMC 851 Execute Command On BMC rm /tmp/debug-tarball.tar.xz 852 853Get BMC Boot Count 854 [Documentation] Get BMC boot count based on boot time. 855 ${cur_btime}= Get BMC Boot Time 856 857 # Set global variable BOOT_TIME to current boot time if current boot time 858 # is changed. Also increase value of global variable BOOT_COUNT by 1. 859 Run Keyword If ${cur_btime} > ${BOOT_TIME} 860 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 861 ... AND 862 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 863 [Return] ${BOOT_COUNT} 864 865Set BMC Boot Count 866 [Documentation] Set BMC boot count to given value. 867 [Arguments] ${count} 868 869 # Description of arguments: 870 # count boot count value. 871 ${cur_btime}= Get BMC Boot Time 872 873 # Set global variable BOOT_COUNT to given value. 874 Set Global Variable ${BOOT_COUNT} ${count} 875 876 # Set BOOT_TIME variable to current boot time. 877 Set Global Variable ${BOOT_COUNT} ${count} 878