1*** Settings *** 2Resource ../lib/resource.txt 3Resource ../lib/rest_client.robot 4Resource ../lib/connection_client.robot 5Resource ../extended/obmc_boot_test_resource.robot 6Library DateTime 7Library Process 8Library OperatingSystem 9Library gen_print.py 10Library gen_robot_print.py 11Library gen_cmd.py 12Library gen_robot_keyword.py 13 14*** Variables *** 15${pflash_cmd} /usr/sbin/pflash -r /dev/stdout -P VERSION 16${SYSTEM_SHUTDOWN_TIME} ${5} 17${dbuscmdBase} 18... dbus-send --system --print-reply --dest=${OPENBMC_BASE_DBUS}.settings.Host 19${dbuscmdGet} 20... ${SETTINGS_URI}host0 org.freedesktop.DBus.Properties.Get 21# Enable when ready with openbmc/openbmc-test-automation#203 22#${dbuscmdString}= string:"xyz.openbmc_project.settings.Host" string: 23${dbuscmdString}= string:"org.openbmc.settings.Host" string: 24 25# Assign default value to QUIET for programs which may not define it. 26${QUIET} ${0} 27${bmc_mem_free_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f4 28${bmc_mem_total_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f2 29${bmc_cpu_usage_cmd}= top -n 1 | grep CPU: | cut -c 7-9 30${HOST_SETTING} ${SETTINGS_URI}host0 31# /run/initramfs/ro associate filesystem should be 100% full always 32${bmc_file_system_usage_cmd}= 33... df -h | grep -v /run/initramfs/ro | cut -c 52-54 | grep 100 | wc -l 34 35${BOOT_TIME} ${0} 36${BOOT_COUNT} ${0} 37${count} ${0} 38${devicetree_base} /sys/firmware/devicetree/base/model 39# MAC input from user. 40${MAC_ADDRESS} ${EMPTY} 41 42# Initialize default debug value to 0. 43${DEBUG} ${0} 44 45*** Keywords *** 46 47Verify PNOR Update 48 [Documentation] Verify that the PNOR is not corrupted. 49 # Example: 50 # FFS: Flash header not found. Code: 100 51 # Error 100 opening ffs ! 52 53 Open Connection And Log In 54 ${pnor_info}= Execute Command On BMC ${pflash_cmd} 55 Should Not Contain Any ${pnor_info} Flash header not found Error 56 57Get BMC System Model 58 [Documentation] Get the BMC model from the device tree. 59 60 Open Connection And Log In 61 ${bmc_model} ${stderr}= Execute Command 62 ... cat ${devicetree_base} | cut -d " " -f 1 return_stderr=True 63 Should Be Empty ${stderr} 64 Should Not Be Empty ${bmc_model} 65 [Return] ${bmc_model} 66 67Verify BMC System Model 68 [Documentation] Verify the BMC model with ${OPENBMC_MODEL}. 69 [Arguments] ${bmc_model} 70 71 ${tmp_bmc_model}= Fetch From Right ${OPENBMC_MODEL} / 72 ${tmp_bmc_model}= Fetch From Left ${tmp_bmc_model} . 73 ${ret}= Run Keyword And Return Status Should Contain ${bmc_model} 74 ... ${tmp_bmc_model} ignore_case=True 75 [Return] ${ret} 76 77Wait For Host To Ping 78 [Arguments] ${host} ${timeout}=${OPENBMC_REBOOT_TIMEOUT}min 79 ... ${interval}=5 sec 80 81 # host The DNS name or IP of the host to ping. 82 # timeout The amount of time after which attempts to ping cease. 83 # interval The amount of time in between attempts to ping. 84 85 Wait Until Keyword Succeeds ${timeout} ${interval} Ping Host ${host} 86 87Ping Host 88 [Arguments] ${host} 89 Should Not Be Empty ${host} msg=No host provided 90 ${RC} ${output}= Run and return RC and Output ping -c 4 ${host} 91 Log RC: ${RC}\nOutput:\n${output} 92 Should be equal ${RC} ${0} 93 94Get Boot Progress 95 [Arguments] ${quiet}=${QUIET} 96 97 ${state}= Read Attribute ${OPENBMC_BASE_URI}sensors/host/BootProgress 98 ... value quiet=${quiet} 99 [Return] ${state} 100 101Is Power On 102 ${state}= Get Power State 103 Should be equal ${state} ${1} 104 105Is Power Off 106 ${state}= Get Power State 107 Should be equal ${state} ${0} 108 109Initiate Power On 110 [Documentation] Initiates the power on and waits until the Is Power On 111 ... keyword returns that the power state has switched to on. 112 [Arguments] ${wait}=${1} 113 114 @{arglist}= Create List 115 ${args}= Create Dictionary data=@{arglist} 116 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOn 117 ... data=${args} 118 should be equal as strings ${resp.status_code} ${HTTP_OK} 119 120 # Does caller want to wait for power on status? 121 Run Keyword If '${wait}' == '${0}' Return From Keyword 122 Wait Until Keyword Succeeds 3 min 10 sec Is Power On 123 124Initiate Power Off 125 [Documentation] Initiates the power off and waits until the Is Power Off 126 ... keyword returns that the power state has switched to off. 127 @{arglist}= Create List 128 ${args}= Create Dictionary data=@{arglist} 129 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOff 130 ... data=${args} 131 should be equal as strings ${resp.status_code} ${HTTP_OK} 132 Wait Until Keyword Succeeds 1 min 10 sec Is Power Off 133 134Initiate OS Host Power Off 135 [Documentation] Initiate an OS reboot. 136 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 137 ... ${os_password}=${OS_PASSWORD} 138 139 # Description of arguments: 140 # os_host The DNS name or IP of the OS. 141 # os_username The username to be used to sign in to the OS. 142 # os_password The password to be used to sign in to the OS. 143 144 SSHLibrary.Open connection ${os_host} 145 Login ${os_username} ${os_password} 146 ${cmd_buf} Catenate shutdown 147 Start Command ${cmd_buf} 148 SSHLibrary.Close Connection 149 150Initiate OS Host Reboot 151 [Documentation] Initiate an OS reboot. 152 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 153 ... ${os_password}=${OS_PASSWORD} 154 155 # Description of arguments: 156 # os_host The DNS name or IP of the OS. 157 # os_username The username to be used to sign in to the OS. 158 # os_password The password to be used to sign in to the OS. 159 160 SSHLibrary.Open connection ${os_host} 161 Login ${os_username} ${os_password} 162 ${cmd_buf} Catenate reboot 163 Start Command ${cmd_buf} 164 SSHLibrary.Close Connection 165 166Initiate Auto Reboot 167 [Documentation] Initiate an auto reboot. 168 169 # Set the auto reboot policy. 170 Set Auto Reboot yes 171 172 SSHLibrary.Open connection ${openbmc_host} 173 Login ${openbmc_username} ${openbmc_password} 174 175 # Set the watchdog timer. Note: 5000 = milliseconds which is 5 seconds. 176 ${cmd_buf}= Catenate /usr/sbin/mapper call /org/openbmc/watchdog/host0 177 ... org.openbmc.Watchdog set i 5000 178 ${output} ${stderr} ${rc}= Execute Command ${cmd_buf} 179 ... return_stderr=True return_rc=True 180 Should Be Empty ${stderr} 181 Should be equal ${rc} ${0} 182 183 # Start the watchdog. 184 ${cmd_buf}= Catenate /usr/sbin/mapper call /org/openbmc/watchdog/host0 185 ... org.openbmc.Watchdog start 186 ${output} ${stderr} ${rc}= Execute Command ${cmd_buf} 187 ... return_stderr=True return_rc=True 188 Should Be Empty ${stderr} 189 Should be equal ${rc} ${0} 190 191Trigger Warm Reset 192 log to console "Triggering warm reset" 193 ${data}= create dictionary data=@{EMPTY} 194 ${resp}= openbmc post request 195 ... ${OPENBMC_BASE_URI}control/bmc0/action/warmReset data=${data} 196 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 197 ${session_active}= Check If warmReset is Initiated 198 Run Keyword If '${session_active}' == '${True}' 199 ... Fail msg=warm reset didn't occur 200 201 Sleep ${SYSTEM_SHUTDOWN_TIME}min 202 Check If BMC Is Up 203 204Check OS 205 [Documentation] Attempts to ping the host OS and then checks that the host 206 ... OS is up by running an SSH command. 207 208 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 209 ... ${os_password}=${OS_PASSWORD} ${quiet}=${QUIET} 210 ... ${print_string}=${EMPTY} 211 [Teardown] SSHLibrary.Close Connection 212 213 # os_host The DNS name/IP of the OS host associated with our BMC. 214 # os_username The username to be used to sign on to the OS host. 215 # os_password The password to be used to sign on to the OS host. 216 # quiet Indicates whether this keyword should write to console. 217 # print_string A string to be printed before checking the OS. 218 219 rprint ${print_string} 220 221 # Attempt to ping the OS. Store the return code to check later. 222 ${ping_rc}= Run Keyword and Return Status Ping Host ${os_host} 223 224 SSHLibrary.Open connection ${os_host} 225 226 ${status} ${msg}= Run Keyword And Ignore Error Login ${os_username} 227 ... ${os_password} 228 ${err_msg1}= Sprint Error ${msg} 229 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 230 Run Keyword If '${status}' == 'FAIL' Fail msg=${err_msg} 231 ${output} ${stderr} ${rc}= Execute Command uptime return_stderr=True 232 ... return_rc=True 233 234 ${temp_msg}= Catenate Could not execute a command on the operating 235 ... system.\n 236 ${err_msg1}= Sprint Error ${temp_msg} 237 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 238 239 # If the return code returned by "Execute Command" is non-zero, this 240 # keyword will fail. 241 Should Be Equal ${rc} ${0} msg=${err_msg} 242 # We will likewise fail if there is any stderr data. 243 Should Be Empty ${stderr} 244 245 ${temp_msg}= Set Variable Could not ping the operating system.\n 246 ${err_msg1}= Sprint Error ${temp_msg} 247 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 248 # We will likewise fail if the OS did not ping, as we could SSH but not 249 # ping 250 Should Be Equal As Strings ${ping_rc} ${TRUE} msg=${err_msg} 251 252Wait for OS 253 [Documentation] Waits for the host OS to come up via calls to "Check OS". 254 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 255 ... ${os_password}=${OS_PASSWORD} ${timeout}=${OS_WAIT_TIMEOUT} 256 ... ${quiet}=${0} 257 [Teardown] rprintn 258 259 # os_host The DNS name or IP of the OS host associated with our 260 # BMC. 261 # os_username The username to be used to sign on to the OS host. 262 # os_password The password to be used to sign on to the OS host. 263 # timeout The timeout in seconds indicating how long you're 264 # willing to wait for the OS to respond. 265 # quiet Indicates whether this keyword should write to console. 266 267 # The interval to be used between calls to "Check OS". 268 ${interval}= Set Variable 5 269 270 ${message}= Catenate Checking every ${interval} seconds for up to 271 ... ${timeout} seconds for the operating system to communicate. 272 rqprint_timen ${message} 273 274 Wait Until Keyword Succeeds ${timeout} sec ${interval} Check OS 275 ... ${os_host} ${os_username} ${os_password} 276 ... print_string=\# 277 278 rqprintn 279 280 rqprint_timen The operating system is now communicating. 281 282Get BMC State Deprecated 283 [Documentation] Returns the state of the BMC as a string. (i.e: BMC_READY) 284 [Arguments] ${quiet}=${QUIET} 285 286 @{arglist}= Create List 287 ${args}= Create Dictionary data=@{arglist} 288 ${resp}= Call Method ${OPENBMC_BASE_URI}managers/System/ getSystemState 289 ... data=${args} quiet=${quiet} 290 Should be equal as strings ${resp.status_code} ${HTTP_OK} 291 ${content}= to json ${resp.content} 292 [Return] ${content["data"]} 293 294Get Power State 295 [Documentation] Returns the power state as an integer. Either 0 or 1. 296 [Arguments] ${quiet}=${QUIET} 297 298 @{arglist}= Create List 299 ${args}= Create Dictionary data=@{arglist} 300 301 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ getPowerState 302 ... data=${args} quiet=${quiet} 303 Should be equal as strings ${resp.status_code} ${HTTP_OK} 304 ${content}= to json ${resp.content} 305 [Return] ${content["data"]} 306 307Clear BMC Record Log 308 [Documentation] Clears all the event logs on the BMC. This would be 309 ... equivalent to ipmitool sel clear. 310 @{arglist}= Create List 311 ${args}= Create Dictionary data=@{arglist} 312 ${resp}= Call Method 313 ... ${OPENBMC_BASE_URI}records/events/ clear data=${args} 314 should be equal as strings ${resp.status_code} ${HTTP_OK} 315 316Copy PNOR to BMC 317 Import Library SCPLibrary WITH NAME scp 318 Open Connection for SCP 319 Log Copying ${PNOR_IMAGE_PATH} to /tmp 320 scp.Put File ${PNOR_IMAGE_PATH} /tmp 321 322Flash PNOR 323 [Documentation] Calls flash bios update method to flash PNOR image 324 [Arguments] ${pnor_image} 325 @{arglist}= Create List ${pnor_image} 326 ${args}= Create Dictionary data=@{arglist} 327 ${resp}= Call Method ${OPENBMC_BASE_URI}control/flash/bios/ update 328 ... data=${args} 329 should be equal as strings ${resp.status_code} ${HTTP_OK} 330 Wait Until Keyword Succeeds 2 min 10 sec Is PNOR Flashing 331 332Get Flash BIOS Status 333 [Documentation] Returns the status of the flash BIOS API as a string. For 334 ... example 'Flashing', 'Flash Done', etc 335 ${data}= Read Properties ${OPENBMC_BASE_URI}control/flash/bios 336 [Return] ${data['status']} 337 338Is PNOR Flashing 339 [Documentation] Get BIOS 'Flashing' status. This indicates that PNOR 340 ... flashing has started. 341 ${status}= Get Flash BIOS Status 342 Should Contain ${status} Flashing 343 344Is PNOR Flash Done 345 [Documentation] Get BIOS 'Flash Done' status. This indicates that the 346 ... PNOR flashing has completed. 347 ${status}= Get Flash BIOS Status 348 should be equal as strings ${status} Flash Done 349 350Is System State Host Booted 351 [Documentation] Checks whether system state is HOST_BOOTED. 352 ${state}= Get BMC State Deprecated 353 should be equal as strings ${state} HOST_BOOTED 354 355Is OS Starting 356 [Documentation] Check if boot progress is OS starting. 357 ${boot_progress}= Get Boot Progress 358 Should Be Equal ${boot_progress} FW Progress, Starting OS 359 360Is OS Off 361 [Documentation] Check if boot progress is "Off". 362 ${boot_progress}= Get Boot Progress 363 Should Be Equal ${boot_progress} Off 364 365Verify Ping and REST Authentication 366 ${l_ping}= Run Keyword And Return Status 367 ... Ping Host ${OPENBMC_HOST} 368 Run Keyword If '${l_ping}' == '${False}' 369 ... Fail msg=Ping Failed 370 371 ${l_rest}= Run Keyword And Return Status 372 ... Initialize OpenBMC 373 Run Keyword If '${l_rest}' == '${False}' 374 ... Fail msg=REST Authentication Failed 375 376 # Just to make sure the SSH is working for SCP 377 Open Connection And Log In 378 ${system} ${stderr}= Execute Command hostname return_stderr=True 379 Should Be Empty ${stderr} 380 381Check If BMC is Up 382 [Documentation] Wait for Host to be online. Checks every X seconds 383 ... interval for Y minutes and fails if timed out. 384 ... Default MAX timedout is 10 min, interval 10 seconds. 385 [Arguments] ${max_timeout}=${OPENBMC_REBOOT_TIMEOUT} min 386 ... ${interval}=10 sec 387 388 Wait Until Keyword Succeeds 389 ... ${max_timeout} ${interval} Verify Ping and REST Authentication 390 391 392Check If warmReset is Initiated 393 [Documentation] Ping would be still alive, so try SSH to connect 394 ... if fails the ports are down indicating reboot 395 ... is in progress 396 397 # Warm reset adds 3 seconds delay before forcing reboot 398 # To minimize race conditions, we wait for 7 seconds 399 Sleep 7s 400 ${alive}= Run Keyword and Return Status 401 ... Open Connection And Log In 402 Return From Keyword If '${alive}' == '${False}' ${False} 403 [Return] ${True} 404 405Flush REST Sessions 406 [Documentation] Removes all the active session objects 407 Delete All Sessions 408 409Initialize DBUS cmd 410 [Documentation] Initialize dbus string with property string to extract 411 [Arguments] ${boot_property} 412 ${cmd}= Catenate ${dbuscmdBase} ${dbuscmdGet} ${dbuscmdString} 413 ${cmd}= Catenate ${cmd}${boot_property} 414 Set Global Variable ${dbuscmd} ${cmd} 415 416Create OS Console File Path 417 [Documentation] Create OS console file path name and return it. 418 [Arguments] ${log_file_path}=${EMPTY} 419 420 # Description of arguements: 421 # file_path The caller's candidate value. If this value is ${EMPTY}, this 422 # keyword will compose a file path name. Otherwise, this 423 # keyword will use the caller's file_path value. In either 424 # case, the value will be returned. 425 426 ${default_file_path}= Catenate /tmp/${OPENBMC_HOST}_os_console 427 ${log_file_path}= Set Variable If '${log_file_path}' == '${EMPTY}' 428 ... ${default_file_path} ${log_file_path} 429 430 [Return] ${log_file_path} 431 432Create OS Console Command String 433 [Documentation] Return a command string to start OS console logging. 434 435 # First make sure that the ssh_pw program is available. 436 ${cmd_buf}= Catenate which ssh_pw 2>&1 437 Rdpissuing ${cmd_buf} 438 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 439 Rdpvars rc output 440 Should Be Equal ${rc} ${0} msg=${output}\n 441 442 ${cmd_buf}= Catenate ssh_pw ${OPENBMC_PASSWORD} -p 2200 443 ... -o "StrictHostKeyChecking no" ${OPENBMC_USERNAME}@${OPENBMC_HOST} 444 445 [Return] ${cmd_buf} 446 447Stop SOL Console Logging 448 [Documentation] Stop system console logging and return log output. 449 [Arguments] ${log_file_path}=${EMPTY} ${targ_file_path}=${EMPTY} 450 ... ${return_data}=${1} 451 452 # If there are muliple system console processes, they will all be stopped. 453 # If there is no existing log file this keyword will return an error 454 # message to that effect (and write that message to targ_file_path, if 455 # specified). 456 # NOTE: This keyword will not fail if there is no running system console 457 # process. 458 459 # Description of arguments: 460 # log_file_path The file path that was used to call "Start SOL 461 # Console Logging". See that keyword (above) for details. 462 # targ_file_path If specified, the file path to which the source 463 # file path (i.e. "log_file_path") should be copied. 464 # return_data If this is set to ${1}, this keyword will return the SOL 465 # data to the caller as a unicode string. 466 467 ${log_file_path}= Create OS Console File Path ${log_file_path} 468 # Find the pid of the active system console logging session (if any). 469 ${search_string}= Create OS Console Command String 470 ${cmd_buf}= Catenate echo $(ps -ef | egrep '${search_string}' 471 ... | egrep -v grep | cut -c10-14) 472 Rdpissuing ${cmd_buf} 473 ${rc} ${os_con_pid}= Run And Return Rc And Output ${cmd_buf} 474 Rdpvars os_con_pid 475 # If rc is not zero it just means that there is no OS Console process 476 # running. 477 478 ${cmd_buf}= Catenate kill -9 ${os_con_pid} 479 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpissuing ${cmd_buf} 480 ${rc} ${output}= Run Keyword If '${os_con_pid}' != '${EMPTY}' 481 ... Run And Return Rc And Output ${cmd_buf} 482 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpvars rc output 483 484 Run Keyword If '${targ_file_path}' != '${EMPTY}' 485 ... Run Keyword And Ignore Error 486 ... Copy File ${log_file_path} ${targ_file_path} 487 488 ${output}= Set Variable ${EMPTY} 489 ${loc_quiet}= Evaluate ${debug}^1 490 ${rc} ${output}= Run Keyword If '${return_data}' == '${1}' 491 ... Cmd Fnc cat ${log_file_path} quiet=${loc_quiet} print_output=${0} 492 493 [Return] ${output} 494 495Start SOL Console Logging 496 [Documentation] Start system console log to file. 497 [Arguments] ${log_file_path}=${EMPTY} ${return_data}=${1} 498 499 # This keyword will first call "Stop SOL Console Logging". Only then will 500 # it start SOL console logging. The data returned by "Stop SOL Console 501 # Logging" will in turn be returned by this keyword. 502 503 # Description of arguments: 504 # log_file_path The file path to which system console log data should be 505 # written. Note that this path is taken to be a location 506 # on the machine where this program is running rather than 507 # on the Open BMC system. 508 # return_data If this is set to ${1}, this keyword will return any SOL 509 # data to the caller as a unicode string. 510 511 ${log_file_path}= Create OS Console File Path ${log_file_path} 512 513 ${log_output}= Stop SOL Console Logging ${log_file_path} 514 ... return_data=${return_data} 515 516 # Validate by making sure we can create the file. Problems creating the 517 # file would not be noticed by the subsequent ssh command because we fork 518 # the command. 519 Create File ${log_file_path} 520 ${sub_cmd_buf}= Create OS Console Command String 521 # Routing stderr to stdout so that any startup error text will go to the 522 # output file. 523 ${cmd_buf}= Catenate ${sub_cmd_buf} > ${log_file_path} 2>&1 & 524 Rdpissuing ${cmd_buf} 525 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 526 # Because we are forking this command, we essentially will never get a 527 # non-zero return code or any output. 528 Should Be Equal ${rc} ${0} 529 530 [Return] ${log_output} 531 532Get Time Stamp 533 [Documentation] Get the current time stamp data 534 ${cur_time}= Get Current Date result_format=%Y%m%d%H%M%S%f 535 [Return] ${cur_time} 536 537 538Verify BMC State 539 [Documentation] Get the BMC state and verify if the current 540 ... BMC state is as expected. 541 [Arguments] ${expected} 542 543 ${current}= Get BMC State Deprecated 544 Should Contain ${expected} ${current} 545 546Start Journal Log 547 [Documentation] Start capturing journal log to a file in /tmp using 548 ... journalctl command. By default journal log is collected 549 ... at /tmp/journal_log else user input location. 550 ... The File is appended with datetime. 551 [Arguments] ${file_path}=/tmp/journal_log 552 553 Open Connection And Log In 554 555 ${cur_time}= Get Time Stamp 556 Set Global Variable ${LOG_TIME} ${cur_time} 557 Start Command 558 ... journalctl -f > ${file_path}-${LOG_TIME} 559 Log Journal Log Started: ${file_path}-${LOG_TIME} 560 561 562Stop Journal Log 563 [Documentation] Stop journalctl process if its running. 564 ... By default return log from /tmp/journal_log else 565 ... user input location. 566 [Arguments] ${file_path}=/tmp/journal_log 567 568 Open Connection And Log In 569 570 ${rc}= 571 ... Execute Command 572 ... ps ax | grep journalctl | grep -v grep 573 ... return_stdout=False return_rc=True 574 575 Return From Keyword If '${rc}' == '${1}' 576 ... No journal log process running 577 578 ${output} ${stderr}= 579 ... Execute Command killall journalctl 580 ... return_stderr=True 581 Should Be Empty ${stderr} 582 583 ${journal_log} ${stderr}= 584 ... Execute Command 585 ... cat ${file_path}-${LOG_TIME} 586 ... return_stderr=True 587 Should Be Empty ${stderr} 588 589 Log ${journal_log} 590 591 Execute Command rm ${file_path}-${LOG_TIME} 592 593 [Return] ${journal_log} 594 595Mac Address To Hex String 596 [Documentation] Converts MAC address into hex format. 597 ... Example 598 ... Given the following MAC: 00:01:6C:80:02:78 599 ... This keyword will return: 0x00 0x01 0x6C 0x80 0x02 0x78 600 ... Description of arguments: 601 ... i_macaddress MAC address in the following format 602 ... 00:01:6C:80:02:78 603 [Arguments] ${i_macaddress} 604 605 ${mac_hex}= Catenate 0x${i_macaddress.replace(':', ' 0x')} 606 [Return] ${mac_hex} 607 608IP Address To Hex String 609 [Documentation] Converts IP address into hex format. 610 ... Example: 611 ... Given the following IP: 10.3.164.100 612 ... This keyword will return: 0xa 0x3 0xa4 0xa0 613 ... Description of arguments: 614 ... i_ipaddress IP address in the following format 615 ... 10.10.10.10 616 [Arguments] ${i_ipaddress} 617 618 @{ip}= Split String ${i_ipaddress} . 619 ${index}= Set Variable ${0} 620 621 :FOR ${item} IN @{ip} 622 \ ${hex}= Convert To Hex ${item} prefix=0x lowercase=yes 623 \ Set List Value ${ip} ${index} ${hex} 624 \ ${index}= Set Variable ${index + 1} 625 ${ip_hex}= Catenate @{ip} 626 [Return] ${ip_hex} 627 628BMC CPU Performance Check 629 [Documentation] Minimal 10% of proc should be free in this instance 630 631 ${bmc_cpu_usage_output} ${stderr}= Execute Command ${bmc_cpu_usage_cmd} 632 ... return_stderr=True 633 Should be empty ${stderr} 634 ${bmc_cpu_percentage}= Fetch From Left ${bmc_cpu_usage_output} % 635 Should be true ${bmc_cpu_percentage} < 90 636 637BMC Mem Performance Check 638 [Documentation] Minimal 10% of memory should be free in this instance 639 640 ${bmc_mem_free_output} ${stderr}= Execute Command ${bmc_mem_free_cmd} 641 ... return_stderr=True 642 Should be empty ${stderr} 643 644 ${bmc_mem_total_output} ${stderr}= Execute Command ${bmc_mem_total_cmd} 645 ... return_stderr=True 646 Should be empty ${stderr} 647 648 ${bmc_mem_percentage}= Evaluate ${bmc_mem_free_output}*100 649 ${bmc_mem_percentage}= Evaluate 650 ... ${bmc_mem_percentage}/${bmc_mem_total_output} 651 Should be true ${bmc_mem_percentage} > 10 652 653BMC File System Usage Check 654 [Documentation] Check the file system space. None should be 100% full 655 ... except /run/initramfs/ro 656 ${bmc_fs_usage_output} ${stderr}= Execute Command 657 ... ${bmc_file_system_usage_cmd} return_stderr=True 658 Should Be Empty ${stderr} 659 Should Be True ${bmc_fs_usage_output}==0 660 661Check BMC CPU Performance 662 [Documentation] Minimal 10% of proc should be free in 3 sample 663 :FOR ${var} IN Range 1 4 664 \ BMC CPU Performance check 665 666Check BMC Mem Performance 667 [Documentation] Minimal 10% of memory should be free 668 669 :FOR ${var} IN Range 1 4 670 \ BMC Mem Performance check 671 672Check BMC File System Performance 673 [Documentation] Check for file system usage for 4 times 674 675 :FOR ${var} IN Range 1 4 676 \ BMC File System Usage check 677 678Get URL List 679 [Documentation] Return list of URLs under given URL. 680 [Arguments] ${openbmc_url} 681 # Description of argument(s): 682 # openbmc_url URL for list operation (e.g. 683 # /xyz/openbmc_project/inventory). 684 685 ${url_list}= Read Properties ${openbmc_url}/list 686 [Return] ${url_list} 687 688Get Endpoint Paths 689 [Documentation] Returns all url paths ending with given endpoint 690 ... Example: 691 ... Given the following endpoint: cpu 692 ... This keyword will return: list of all urls ending with 693 ... cpu - 694 ... /org/openbmc/inventory/system/chassis/motherboard/cpu0, 695 ... /org/openbmc/inventory/system/chassis/motherboard/cpu1 696 ... Description of arguments: 697 ... path URL path for enumeration 698 ... endpoint string for which url path ending 699 [Arguments] ${path} ${endpoint} 700 701 ${resp}= Read Properties ${path}/enumerate timeout=30 702 log Dictionary ${resp} 703 704 ${list}= Get Dictionary Keys ${resp} 705 ${resp}= Get Matches ${list} regexp=^.*[0-9a-z_].${endpoint}[0-9]*$ 706 [Return] ${resp} 707 708Check Zombie Process 709 [Documentation] Check if any defunct process exist or not on BMC 710 ${count} ${stderr} ${rc}= Execute Command ps -o stat | grep Z | wc -l 711 ... return_stderr=True return_rc=True 712 Should Be True ${count}==0 713 Should Be Empty ${stderr} 714 715Prune Journal Log 716 [Documentation] Prune archived journal logs. 717 [Arguments] ${vacuum_size}=1M 718 719 # This keyword can be used to prevent the journal 720 # log from filling up the /run filesystem. 721 # This command will retain only the latest logs 722 # of the user specified size. 723 724 Open Connection And Log In 725 ${output} ${stderr} ${rc}= 726 ... Execute Command 727 ... journalctl --vacuum-size=${vacuum_size} 728 ... return_stderr=True return_rc=True 729 730 Should Be Equal ${rc} ${0} msg=${stderr} 731 732Set BMC Power Policy 733 [Documentation] Set the given BMC power policy. 734 [arguments] ${policy} 735 736 ${valueDict}= create dictionary data=${policy} 737 Write Attribute ${HOST_SETTING} power_policy data=${valueDict} 738 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 739 Should Be Equal ${currentPolicy} ${policy} 740 741Get System Power Policy 742 [Documentation] Get the BMC power policy. 743 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 744 [Return] ${currentPolicy} 745 746Get Auto Reboot 747 [Documentation] Returns auto reboot setting. 748 ${setting}= Read Attribute ${HOST_SETTING} auto_reboot 749 [Return] ${setting} 750 751 752Set Auto Reboot 753 [Documentation] Set the given auto reboot setting. 754 [Arguments] ${setting} 755 # setting auto reboot's setting, i.e. yes or no 756 757 ${valueDict}= Set Variable ${setting} 758 ${data}= Create Dictionary data=${valueDict} 759 Write Attribute ${HOST_SETTING} auto_reboot data=${data} 760 ${current_setting}= Get Auto Reboot 761 Should Be Equal ${current_setting} ${setting} 762 763 764Set BMC Reset Reference Time 765 [Documentation] Set current boot time as a reference and increment 766 ... boot count. 767 768 ${cur_btime}= Get BMC Boot Time 769 Run Keyword If ${BOOT_TIME} == ${0} and ${BOOT_COUNT} == ${0} 770 ... Set Global Variable ${BOOT_TIME} ${cur_btime} 771 ... ELSE IF ${cur_btime} > ${BOOT_TIME} 772 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 773 ... AND 774 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 775 776Get BMC Boot Time 777 [Documentation] Get boot time from /proc/stat. 778 779 Open Connection And Log In 780 ${output} ${stderr}= 781 ... Execute Command egrep '^btime ' /proc/stat | cut -f 2 -d ' ' 782 ... return_stderr=True 783 Should Be Empty ${stderr} 784 ${btime}= Convert To Integer ${output} 785 [Return] ${btime} 786 787Execute Command On BMC 788 [Documentation] Execute given command on BMC and return output. 789 [Arguments] ${command} 790 ${stdout} ${stderr}= Execute Command ${command} return_stderr=True 791 Should Be Empty ${stderr} 792 [Return] ${stdout} 793 794 795Enable Core Dump On BMC 796 [Documentation] Enable core dump collection. 797 Open Connection And Log In 798 ${core_pattern}= Execute Command On BMC 799 ... echo '/tmp/core_%e.%p' | tee /proc/sys/kernel/core_pattern 800 Should Be Equal As Strings ${core_pattern} /tmp/core_%e.%p 801 802Get Number Of BMC Core Dump Files 803 [Documentation] Get number of core dump files on BMC. 804 Open Connection And Log In 805 ${num_of_core_dump}= Execute Command 806 ... ls /tmp/core* 2>/dev/null | wc -l 807 [Return] ${num_of_core_dump} 808 809Set Core Dump File Size Unlimited 810 [Documentation] Set core dump file size to unlimited. 811 Open Connection And Log In 812 Execute Command On BMC 813 ... ulimit -c unlimited 814 815Check For Core Dumps 816 [Documentation] Check for any core dumps exist. 817 ${output}= Get Number Of BMC Core Dump Files 818 Run Keyword If ${output} > 0 819 ... Log **Warning** BMC core dump files exist level=WARN 820 821Trigger Host Watchdog Error 822 [Documentation] Inject host watchdog error using BMC. 823 [Arguments] ${milliseconds}=1000 ${sleep_time}=5s 824 # Description of argument(s): 825 # milliseconds The time watchdog timer value in milliseconds (e.g. 1000 = 826 # 1 second). 827 # sleep_time Time delay for host watchdog error to get injected. 828 # Default is 5 seconds. 829 830 Execute Command On BMC 831 ... /usr/sbin/mapper call /org/openbmc/watchdog/host0 org.openbmc.Watchdog set i ${milliseconds} 832 Execute Command On BMC 833 ... /usr/sbin/mapper call /org/openbmc/watchdog/host0 org.openbmc.Watchdog start 834 Sleep ${sleep_time} 835 836Login To OS Host 837 [Documentation] Login to OS Host. 838 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 839 ... ${os_password}=${OS_PASSWORD} 840 # Desription of arguments: 841 # ${os_host} IP address of the OS Host. 842 # ${os_username} OS Host Login user name. 843 # ${os_password} OS Host Login passwrd. 844 845 ${os_state}= Run Keyword And Return Status Ping Host ${os_host} 846 Run Keyword If '${os_state}' == 'False' 847 ... Run Keywords Initiate Host Reboot AND 848 ... Is Host Running AND 849 ... Wait for OS ${os_host} ${os_username} ${os_password} 850 851 Open Connection ${os_host} 852 ${resp}= Login ${os_username} ${os_password} 853 [Return] ${resp} 854 855Configure Initial Settings 856 [Documentation] Restore old IP and route. 857 ... This keyword requires initial settings viz IP address, 858 ... Network Mask, default gatway and serial console IP and port 859 ... information which should be provided in command line. 860 861 [Arguments] ${host}=${OPENBMC_HOST} ${mask}=${NET_MASK} 862 ... ${gw_ip}=${GW_IP} 863 864 # Open telnet connection and ignore the error, in case telnet session is 865 # already opened by the program calling this keyword. 866 867 Run Keyword And Ignore Error Open Telnet Connection to BMC Serial Console 868 Telnet.write ifconfig eth0 ${host} netmask ${mask} 869 Telnet.write route add default gw ${gw_ip} 870 871Install Debug Tarball On BMC 872 [Documentation] Copy the debug tar file to BMC and install. 873 [Arguments] ${tarball_file_path}=${EXECDIR}/obmc-phosphor-debug-tarball-witherspoon.tar.xz 874 ... ${targ_tarball_dir_path}=/home/root/tarball/ 875 876 # Description of arguments: 877 # tarball_file_path Path of the debug tarball file. 878 # The tar file is downloaded from the build page 879 # https://openpower.xyz/job/openbmc-build/ 880 # obmc-phosphor-debug-tarball-witherspoon.tar.xz 881 # 882 # targ_tarball_dir_path The directory path where the tarball is to be 883 # installed. 884 885 OperatingSystem.File Should Exist ${tarball_file_path} 886 ... msg=${tarball_file_path} doesn't exist. 887 888 # Upload the file to BMC. 889 Import Library SCPLibrary WITH NAME scp 890 Open Connection for SCP 891 scp.Put File ${tarball_file_path} /tmp/debug-tarball.tar.xz 892 893 # Create tarball directory and install. 894 Open Connection And Log In 895 Execute Command On BMC mkdir -p ${targ_tarball_dir_path} 896 Execute Command On BMC 897 ... tar -xf /tmp/debug-tarball.tar.xz -C ${targ_tarball_dir_path} 898 899 # Create symlink to callout-test binary. 900 Execute Command On BMC 901 ... ln -s ${targ_tarball_dir_path}/bin/callout-test /usr/bin/callout-test 902 903 # Create symlink to logging-test binary. 904 Execute Command On BMC 905 ... ln -s ${targ_tarball_dir_path}/bin/logging-test /usr/bin/logging-test 906 907 # Remove the tarball file from BMC 908 Execute Command On BMC rm /tmp/debug-tarball.tar.xz 909 910 911Get BMC Boot Count 912 [Documentation] Get BMC boot count based on boot time. 913 ${cur_btime}= Get BMC Boot Time 914 915 # Set global variable BOOT_TIME to current boot time if current boot time 916 # is changed. Also increase value of global variable BOOT_COUNT by 1. 917 Run Keyword If ${cur_btime} > ${BOOT_TIME} 918 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 919 ... AND 920 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 921 [Return] ${BOOT_COUNT} 922 923Set BMC Boot Count 924 [Documentation] Set BMC boot count to given value. 925 [Arguments] ${count} 926 927 # Description of arguments: 928 # count boot count value. 929 ${cur_btime}= Get BMC Boot Time 930 931 # Set global variable BOOT_COUNT to given value. 932 Set Global Variable ${BOOT_COUNT} ${count} 933 934 # Set BOOT_TIME variable to current boot time. 935 Set Global Variable ${BOOT_COUNT} ${count} 936 937Check And Reset MAC 938 [Documentation] Update BMC with user input MAC address. 939 [Arguments] ${mac_address}=${MAC_ADDRESS} 940 941 # Description of argument(s): 942 # mac_address The mac address (e.g. 00:01:6c:80:02:28). 943 944 Should Not Be Empty ${mac_address} 945 Open Connection And Log In 946 ${bmc_mac_addr}= Execute Command On BMC cat /sys/class/net/eth0/address 947 Run Keyword If '${mac_address}' != '${bmc_mac_addr}' 948 ... Set MAC Address 949 950Set MAC Address 951 [Documentation] Update eth0 with input MAC address. 952 [Arguments] ${mac_address}=${MAC_ADDRESS} 953 954 # Description of argument(s): 955 # mac_address The mac address (e.g. 00:01:6c:80:02:28). 956 Write fw_setenv ethaddr ${mac_address} 957 Run Key U OBMC Boot Test \ OBMC Reboot (off) 958 ${bmc_mac_addr}= Execute Command On BMC cat /sys/class/net/eth0/address 959 Should Be Equal ${bmc_mac_addr} ${mac_address} 960 961############################################################################### 962Delete Error logs 963 [Documentation] Delete error logs. 964 965 # The REST method to delete error openbmc/openbmc#1327 966 # until then using logging restart. 967 Open Connection And Log In 968 Execute Command On BMC 969 ... systemctl restart xyz.openbmc_project.Logging.service 970 Sleep 10s reason=Wait for logging service to restart properly. 971 972############################################################################### 973Delete Error log Entry 974 [Documentation] Delete error log entry. 975 [Arguments] ${entry_path} 976 977 # Description of argument(s): 978 # entry_path Delete an error log entry. 979 # Ex. /xyz/openbmc_project/logging/entry/1 980 981 ${data}= Create Dictionary data=@{EMPTY} 982 ${resp}= Openbmc Delete Request ${entry_path} data=${data} 983 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 984 985 986Get LED State XYZ 987 [Documentation] Returns state of given LED. 988 [Arguments] ${led_name} 989 # Description of argument(s): 990 # led_name Name of LED 991 992 ${state}= Read Attribute ${LED_GROUPS_URI}${led_name} Asserted 993 [Return] ${state} 994