1*** Settings *** 2Resource ../lib/resource.txt 3Resource ../lib/rest_client.robot 4Resource ../lib/connection_client.robot 5Resource ../extended/obmc_boot_test_resource.robot 6Library DateTime 7Library Process 8Library OperatingSystem 9Library gen_print.py 10Library gen_robot_print.py 11Library gen_cmd.py 12Library gen_robot_keyword.py 13 14*** Variables *** 15${SYSTEM_SHUTDOWN_TIME} ${5} 16${dbuscmdBase} 17... dbus-send --system --print-reply --dest=${OPENBMC_BASE_DBUS}.settings.Host 18${dbuscmdGet} 19... ${SETTINGS_URI}host0 org.freedesktop.DBus.Properties.Get 20# Enable when ready with openbmc/openbmc-test-automation#203 21#${dbuscmdString}= string:"xyz.openbmc_project.settings.Host" string: 22${dbuscmdString}= string:"org.openbmc.settings.Host" string: 23 24# Assign default value to QUIET for programs which may not define it. 25${QUIET} ${0} 26${bmc_mem_free_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f4 27${bmc_mem_total_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f2 28${bmc_cpu_usage_cmd}= top -n 1 | grep CPU: | cut -c 7-9 29${HOST_SETTING} ${SETTINGS_URI}host0 30# /run/initramfs/ro associate filesystem should be 100% full always 31${bmc_file_system_usage_cmd}= 32... df -h | grep -v /run/initramfs/ro | cut -c 52-54 | grep 100 | wc -l 33 34${BOOT_TIME} ${0} 35${BOOT_COUNT} ${0} 36${count} ${0} 37${devicetree_base} /sys/firmware/devicetree/base/model 38# MAC input from user. 39${MAC_ADDRESS} ${EMPTY} 40 41# Initialize default debug value to 0. 42${DEBUG} ${0} 43 44*** Keywords *** 45 46Get BMC System Model 47 [Documentation] Get the BMC model from the device tree. 48 49 Open Connection And Log In 50 ${bmc_model} ${stderr}= Execute Command 51 ... cat ${devicetree_base} | cut -d " " -f 1 return_stderr=True 52 Should Be Empty ${stderr} 53 Should Not Be Empty ${bmc_model} 54 [Return] ${bmc_model} 55 56Verify BMC System Model 57 [Documentation] Verify the BMC model with ${OPENBMC_MODEL}. 58 [Arguments] ${bmc_model} 59 60 ${tmp_bmc_model}= Fetch From Right ${OPENBMC_MODEL} / 61 ${tmp_bmc_model}= Fetch From Left ${tmp_bmc_model} . 62 ${ret}= Run Keyword And Return Status Should Contain ${bmc_model} 63 ... ${tmp_bmc_model} ignore_case=True 64 [Return] ${ret} 65 66Wait For Host To Ping 67 [Arguments] ${host} ${timeout}=${OPENBMC_REBOOT_TIMEOUT}min 68 ... ${interval}=5 sec 69 70 # host The DNS name or IP of the host to ping. 71 # timeout The amount of time after which attempts to ping cease. 72 # interval The amount of time in between attempts to ping. 73 74 Wait Until Keyword Succeeds ${timeout} ${interval} Ping Host ${host} 75 76Ping Host 77 [Arguments] ${host} 78 Should Not Be Empty ${host} msg=No host provided 79 ${RC} ${output}= Run and return RC and Output ping -c 4 ${host} 80 Log RC: ${RC}\nOutput:\n${output} 81 Should be equal ${RC} ${0} 82 83Get Boot Progress 84 [Arguments] ${quiet}=${QUIET} 85 86 ${state}= Read Attribute ${OPENBMC_BASE_URI}sensors/host/BootProgress 87 ... value quiet=${quiet} 88 [Return] ${state} 89 90Is Power On 91 ${state}= Get Power State 92 Should be equal ${state} ${1} 93 94Is Power Off 95 ${state}= Get Power State 96 Should be equal ${state} ${0} 97 98Initiate Power On 99 [Documentation] Initiates the power on and waits until the Is Power On 100 ... keyword returns that the power state has switched to on. 101 [Arguments] ${wait}=${1} 102 103 @{arglist}= Create List 104 ${args}= Create Dictionary data=@{arglist} 105 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOn 106 ... data=${args} 107 should be equal as strings ${resp.status_code} ${HTTP_OK} 108 109 # Does caller want to wait for power on status? 110 Run Keyword If '${wait}' == '${0}' Return From Keyword 111 Wait Until Keyword Succeeds 3 min 10 sec Is Power On 112 113Initiate Power Off 114 [Documentation] Initiates the power off and waits until the Is Power Off 115 ... keyword returns that the power state has switched to off. 116 @{arglist}= Create List 117 ${args}= Create Dictionary data=@{arglist} 118 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOff 119 ... data=${args} 120 should be equal as strings ${resp.status_code} ${HTTP_OK} 121 Wait Until Keyword Succeeds 1 min 10 sec Is Power Off 122 123Initiate OS Host Power Off 124 [Documentation] Initiate an OS reboot. 125 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 126 ... ${os_password}=${OS_PASSWORD} 127 128 # Description of arguments: 129 # os_host The DNS name or IP of the OS. 130 # os_username The username to be used to sign in to the OS. 131 # os_password The password to be used to sign in to the OS. 132 133 SSHLibrary.Open connection ${os_host} 134 Login ${os_username} ${os_password} 135 ${cmd_buf} Catenate shutdown 136 Start Command ${cmd_buf} 137 SSHLibrary.Close Connection 138 139Initiate OS Host Reboot 140 [Documentation] Initiate an OS reboot. 141 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 142 ... ${os_password}=${OS_PASSWORD} 143 144 # Description of arguments: 145 # os_host The DNS name or IP of the OS. 146 # os_username The username to be used to sign in to the OS. 147 # os_password The password to be used to sign in to the OS. 148 149 SSHLibrary.Open connection ${os_host} 150 Login ${os_username} ${os_password} 151 ${cmd_buf} Catenate reboot 152 Start Command ${cmd_buf} 153 SSHLibrary.Close Connection 154 155Initiate Auto Reboot 156 [Documentation] Initiate an auto reboot. 157 158 # Set the auto reboot policy. 159 Set Auto Reboot yes 160 161 SSHLibrary.Open connection ${openbmc_host} 162 Login ${openbmc_username} ${openbmc_password} 163 164 # Set the watchdog timer. Note: 5000 = milliseconds which is 5 seconds. 165 ${cmd_buf}= Catenate /usr/sbin/mapper call /org/openbmc/watchdog/host0 166 ... org.openbmc.Watchdog set i 5000 167 ${output} ${stderr} ${rc}= Execute Command ${cmd_buf} 168 ... return_stderr=True return_rc=True 169 Should Be Empty ${stderr} 170 Should be equal ${rc} ${0} 171 172 # Start the watchdog. 173 ${cmd_buf}= Catenate /usr/sbin/mapper call /org/openbmc/watchdog/host0 174 ... org.openbmc.Watchdog start 175 ${output} ${stderr} ${rc}= Execute Command ${cmd_buf} 176 ... return_stderr=True return_rc=True 177 Should Be Empty ${stderr} 178 Should be equal ${rc} ${0} 179 180Trigger Warm Reset 181 log to console "Triggering warm reset" 182 ${data}= create dictionary data=@{EMPTY} 183 ${resp}= openbmc post request 184 ... ${OPENBMC_BASE_URI}control/bmc0/action/warmReset data=${data} 185 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 186 ${session_active}= Check If warmReset is Initiated 187 Run Keyword If '${session_active}' == '${True}' 188 ... Fail msg=warm reset didn't occur 189 190 Sleep ${SYSTEM_SHUTDOWN_TIME}min 191 Check If BMC Is Up 192 193Check OS 194 [Documentation] Attempts to ping the host OS and then checks that the host 195 ... OS is up by running an SSH command. 196 197 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 198 ... ${os_password}=${OS_PASSWORD} ${quiet}=${QUIET} 199 ... ${print_string}=${EMPTY} 200 [Teardown] SSHLibrary.Close Connection 201 202 # os_host The DNS name/IP of the OS host associated with our BMC. 203 # os_username The username to be used to sign on to the OS host. 204 # os_password The password to be used to sign on to the OS host. 205 # quiet Indicates whether this keyword should write to console. 206 # print_string A string to be printed before checking the OS. 207 208 rprint ${print_string} 209 210 # Attempt to ping the OS. Store the return code to check later. 211 ${ping_rc}= Run Keyword and Return Status Ping Host ${os_host} 212 213 SSHLibrary.Open connection ${os_host} 214 215 ${status} ${msg}= Run Keyword And Ignore Error Login ${os_username} 216 ... ${os_password} 217 ${err_msg1}= Sprint Error ${msg} 218 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 219 Run Keyword If '${status}' == 'FAIL' Fail msg=${err_msg} 220 ${output} ${stderr} ${rc}= Execute Command uptime return_stderr=True 221 ... return_rc=True 222 223 ${temp_msg}= Catenate Could not execute a command on the operating 224 ... system.\n 225 ${err_msg1}= Sprint Error ${temp_msg} 226 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 227 228 # If the return code returned by "Execute Command" is non-zero, this 229 # keyword will fail. 230 Should Be Equal ${rc} ${0} msg=${err_msg} 231 # We will likewise fail if there is any stderr data. 232 Should Be Empty ${stderr} 233 234 ${temp_msg}= Set Variable Could not ping the operating system.\n 235 ${err_msg1}= Sprint Error ${temp_msg} 236 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 237 # We will likewise fail if the OS did not ping, as we could SSH but not 238 # ping 239 Should Be Equal As Strings ${ping_rc} ${TRUE} msg=${err_msg} 240 241Wait for OS 242 [Documentation] Waits for the host OS to come up via calls to "Check OS". 243 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 244 ... ${os_password}=${OS_PASSWORD} ${timeout}=${OS_WAIT_TIMEOUT} 245 ... ${quiet}=${0} 246 [Teardown] rprintn 247 248 # os_host The DNS name or IP of the OS host associated with our 249 # BMC. 250 # os_username The username to be used to sign on to the OS host. 251 # os_password The password to be used to sign on to the OS host. 252 # timeout The timeout in seconds indicating how long you're 253 # willing to wait for the OS to respond. 254 # quiet Indicates whether this keyword should write to console. 255 256 # The interval to be used between calls to "Check OS". 257 ${interval}= Set Variable 5 258 259 ${message}= Catenate Checking every ${interval} seconds for up to 260 ... ${timeout} seconds for the operating system to communicate. 261 rqprint_timen ${message} 262 263 Wait Until Keyword Succeeds ${timeout} sec ${interval} Check OS 264 ... ${os_host} ${os_username} ${os_password} 265 ... print_string=\# 266 267 rqprintn 268 269 rqprint_timen The operating system is now communicating. 270 271Get BMC State Deprecated 272 [Documentation] Returns the state of the BMC as a string. (i.e: BMC_READY) 273 [Arguments] ${quiet}=${QUIET} 274 275 @{arglist}= Create List 276 ${args}= Create Dictionary data=@{arglist} 277 ${resp}= Call Method ${OPENBMC_BASE_URI}managers/System/ getSystemState 278 ... data=${args} quiet=${quiet} 279 Should be equal as strings ${resp.status_code} ${HTTP_OK} 280 ${content}= to json ${resp.content} 281 [Return] ${content["data"]} 282 283Get Power State 284 [Documentation] Returns the power state as an integer. Either 0 or 1. 285 [Arguments] ${quiet}=${QUIET} 286 287 @{arglist}= Create List 288 ${args}= Create Dictionary data=@{arglist} 289 290 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ getPowerState 291 ... data=${args} quiet=${quiet} 292 Should be equal as strings ${resp.status_code} ${HTTP_OK} 293 ${content}= to json ${resp.content} 294 [Return] ${content["data"]} 295 296Clear BMC Record Log 297 [Documentation] Clears all the event logs on the BMC. This would be 298 ... equivalent to ipmitool sel clear. 299 @{arglist}= Create List 300 ${args}= Create Dictionary data=@{arglist} 301 ${resp}= Call Method 302 ... ${OPENBMC_BASE_URI}records/events/ clear data=${args} 303 should be equal as strings ${resp.status_code} ${HTTP_OK} 304 305Copy PNOR to BMC 306 Import Library SCPLibrary WITH NAME scp 307 Open Connection for SCP 308 Log Copying ${PNOR_IMAGE_PATH} to /tmp 309 scp.Put File ${PNOR_IMAGE_PATH} /tmp 310 311Flash PNOR 312 [Documentation] Calls flash bios update method to flash PNOR image 313 [Arguments] ${pnor_image} 314 @{arglist}= Create List ${pnor_image} 315 ${args}= Create Dictionary data=@{arglist} 316 ${resp}= Call Method ${OPENBMC_BASE_URI}control/flash/bios/ update 317 ... data=${args} 318 should be equal as strings ${resp.status_code} ${HTTP_OK} 319 Wait Until Keyword Succeeds 2 min 10 sec Is PNOR Flashing 320 321Get Flash BIOS Status 322 [Documentation] Returns the status of the flash BIOS API as a string. For 323 ... example 'Flashing', 'Flash Done', etc 324 ${data}= Read Properties ${OPENBMC_BASE_URI}control/flash/bios 325 [Return] ${data['status']} 326 327Is PNOR Flashing 328 [Documentation] Get BIOS 'Flashing' status. This indicates that PNOR 329 ... flashing has started. 330 ${status}= Get Flash BIOS Status 331 Should Contain ${status} Flashing 332 333Is PNOR Flash Done 334 [Documentation] Get BIOS 'Flash Done' status. This indicates that the 335 ... PNOR flashing has completed. 336 ${status}= Get Flash BIOS Status 337 should be equal as strings ${status} Flash Done 338 339Is System State Host Booted 340 [Documentation] Checks whether system state is HOST_BOOTED. 341 ${state}= Get BMC State Deprecated 342 should be equal as strings ${state} HOST_BOOTED 343 344Is OS Starting 345 [Documentation] Check if boot progress is OS starting. 346 ${boot_progress}= Get Boot Progress 347 Should Be Equal ${boot_progress} FW Progress, Starting OS 348 349Is OS Off 350 [Documentation] Check if boot progress is "Off". 351 ${boot_progress}= Get Boot Progress 352 Should Be Equal ${boot_progress} Off 353 354Verify Ping and REST Authentication 355 ${l_ping}= Run Keyword And Return Status 356 ... Ping Host ${OPENBMC_HOST} 357 Run Keyword If '${l_ping}' == '${False}' 358 ... Fail msg=Ping Failed 359 360 ${l_rest}= Run Keyword And Return Status 361 ... Initialize OpenBMC 362 Run Keyword If '${l_rest}' == '${False}' 363 ... Fail msg=REST Authentication Failed 364 365 # Just to make sure the SSH is working for SCP 366 Open Connection And Log In 367 ${system} ${stderr}= Execute Command hostname return_stderr=True 368 Should Be Empty ${stderr} 369 370Check If BMC is Up 371 [Documentation] Wait for Host to be online. Checks every X seconds 372 ... interval for Y minutes and fails if timed out. 373 ... Default MAX timedout is 10 min, interval 10 seconds. 374 [Arguments] ${max_timeout}=${OPENBMC_REBOOT_TIMEOUT} min 375 ... ${interval}=10 sec 376 377 Wait Until Keyword Succeeds 378 ... ${max_timeout} ${interval} Verify Ping and REST Authentication 379 380 381Check If warmReset is Initiated 382 [Documentation] Ping would be still alive, so try SSH to connect 383 ... if fails the ports are down indicating reboot 384 ... is in progress 385 386 # Warm reset adds 3 seconds delay before forcing reboot 387 # To minimize race conditions, we wait for 7 seconds 388 Sleep 7s 389 ${alive}= Run Keyword and Return Status 390 ... Open Connection And Log In 391 Return From Keyword If '${alive}' == '${False}' ${False} 392 [Return] ${True} 393 394Flush REST Sessions 395 [Documentation] Removes all the active session objects 396 Delete All Sessions 397 398Initialize DBUS cmd 399 [Documentation] Initialize dbus string with property string to extract 400 [Arguments] ${boot_property} 401 ${cmd}= Catenate ${dbuscmdBase} ${dbuscmdGet} ${dbuscmdString} 402 ${cmd}= Catenate ${cmd}${boot_property} 403 Set Global Variable ${dbuscmd} ${cmd} 404 405Create OS Console File Path 406 [Documentation] Create OS console file path name and return it. 407 [Arguments] ${log_file_path}=${EMPTY} 408 409 # Description of arguements: 410 # file_path The caller's candidate value. If this value is ${EMPTY}, this 411 # keyword will compose a file path name. Otherwise, this 412 # keyword will use the caller's file_path value. In either 413 # case, the value will be returned. 414 415 ${default_file_path}= Catenate /tmp/${OPENBMC_HOST}_os_console 416 ${log_file_path}= Set Variable If '${log_file_path}' == '${EMPTY}' 417 ... ${default_file_path} ${log_file_path} 418 419 [Return] ${log_file_path} 420 421Create OS Console Command String 422 [Documentation] Return a command string to start OS console logging. 423 424 # First make sure that the ssh_pw program is available. 425 ${cmd_buf}= Catenate which ssh_pw 2>&1 426 Rdpissuing ${cmd_buf} 427 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 428 Rdpvars rc output 429 Should Be Equal ${rc} ${0} msg=${output}\n 430 431 ${cmd_buf}= Catenate ssh_pw ${OPENBMC_PASSWORD} -p 2200 432 ... ${OPENBMC_USERNAME}@${OPENBMC_HOST} 433 434 [Return] ${cmd_buf} 435 436Stop SOL Console Logging 437 [Documentation] Stop system console logging and return log output. 438 [Arguments] ${log_file_path}=${EMPTY} ${targ_file_path}=${EMPTY} 439 ... ${return_data}=${1} 440 441 # If there are muliple system console processes, they will all be stopped. 442 # If there is no existing log file this keyword will return an error 443 # message to that effect (and write that message to targ_file_path, if 444 # specified). 445 # NOTE: This keyword will not fail if there is no running system console 446 # process. 447 448 # Description of arguments: 449 # log_file_path The file path that was used to call "Start SOL 450 # Console Logging". See that keyword (above) for details. 451 # targ_file_path If specified, the file path to which the source 452 # file path (i.e. "log_file_path") should be copied. 453 # return_data If this is set to ${1}, this keyword will return the SOL 454 # data to the caller as a unicode string. 455 456 ${log_file_path}= Create OS Console File Path ${log_file_path} 457 # Find the pid of the active system console logging session (if any). 458 ${search_string}= Create OS Console Command String 459 ${cmd_buf}= Catenate echo $(ps -ef | egrep '${search_string}' 460 ... | egrep -v grep | cut -c10-14) 461 Rdpissuing ${cmd_buf} 462 ${rc} ${os_con_pid}= Run And Return Rc And Output ${cmd_buf} 463 Rdpvars os_con_pid 464 # If rc is not zero it just means that there is no OS Console process 465 # running. 466 467 ${cmd_buf}= Catenate kill -9 ${os_con_pid} 468 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpissuing ${cmd_buf} 469 ${rc} ${output}= Run Keyword If '${os_con_pid}' != '${EMPTY}' 470 ... Run And Return Rc And Output ${cmd_buf} 471 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpvars rc output 472 473 Run Keyword If '${targ_file_path}' != '${EMPTY}' 474 ... Run Keyword And Ignore Error 475 ... Copy File ${log_file_path} ${targ_file_path} 476 477 ${output}= Set Variable ${EMPTY} 478 ${loc_quiet}= Evaluate ${debug}^1 479 ${rc} ${output}= Run Keyword If '${return_data}' == '${1}' 480 ... Cmd Fnc cat ${log_file_path} quiet=${loc_quiet} print_output=${0} 481 482 [Return] ${output} 483 484Start SOL Console Logging 485 [Documentation] Start system console log to file. 486 [Arguments] ${log_file_path}=${EMPTY} ${return_data}=${1} 487 488 # This keyword will first call "Stop SOL Console Logging". Only then will 489 # it start SOL console logging. The data returned by "Stop SOL Console 490 # Logging" will in turn be returned by this keyword. 491 492 # Description of arguments: 493 # log_file_path The file path to which system console log data should be 494 # written. Note that this path is taken to be a location 495 # on the machine where this program is running rather than 496 # on the Open BMC system. 497 # return_data If this is set to ${1}, this keyword will return any SOL 498 # data to the caller as a unicode string. 499 500 ${log_file_path}= Create OS Console File Path ${log_file_path} 501 502 ${log_output}= Stop SOL Console Logging ${log_file_path} 503 ... return_data=${return_data} 504 505 # Validate by making sure we can create the file. Problems creating the 506 # file would not be noticed by the subsequent ssh command because we fork 507 # the command. 508 Create File ${log_file_path} 509 ${sub_cmd_buf}= Create OS Console Command String 510 # Routing stderr to stdout so that any startup error text will go to the 511 # output file. 512 ${cmd_buf}= Catenate ${sub_cmd_buf} > ${log_file_path} 2>&1 & 513 Rdpissuing ${cmd_buf} 514 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 515 # Because we are forking this command, we essentially will never get a 516 # non-zero return code or any output. 517 Should Be Equal ${rc} ${0} 518 519 [Return] ${log_output} 520 521Get Time Stamp 522 [Documentation] Get the current time stamp data 523 ${cur_time}= Get Current Date result_format=%Y%m%d%H%M%S%f 524 [Return] ${cur_time} 525 526 527Verify BMC State 528 [Documentation] Get the BMC state and verify if the current 529 ... BMC state is as expected. 530 [Arguments] ${expected} 531 532 ${current}= Get BMC State Deprecated 533 Should Contain ${expected} ${current} 534 535Start Journal Log 536 [Documentation] Start capturing journal log to a file in /tmp using 537 ... journalctl command. By default journal log is collected 538 ... at /tmp/journal_log else user input location. 539 ... The File is appended with datetime. 540 [Arguments] ${file_path}=/tmp/journal_log 541 542 Open Connection And Log In 543 544 ${cur_time}= Get Time Stamp 545 Set Global Variable ${LOG_TIME} ${cur_time} 546 Start Command 547 ... journalctl -f > ${file_path}-${LOG_TIME} 548 Log Journal Log Started: ${file_path}-${LOG_TIME} 549 550 551Stop Journal Log 552 [Documentation] Stop journalctl process if its running. 553 ... By default return log from /tmp/journal_log else 554 ... user input location. 555 [Arguments] ${file_path}=/tmp/journal_log 556 557 Open Connection And Log In 558 559 ${rc}= 560 ... Execute Command 561 ... ps ax | grep journalctl | grep -v grep 562 ... return_stdout=False return_rc=True 563 564 Return From Keyword If '${rc}' == '${1}' 565 ... No journal log process running 566 567 ${output} ${stderr}= 568 ... Execute Command killall journalctl 569 ... return_stderr=True 570 Should Be Empty ${stderr} 571 572 ${journal_log} ${stderr}= 573 ... Execute Command 574 ... cat ${file_path}-${LOG_TIME} 575 ... return_stderr=True 576 Should Be Empty ${stderr} 577 578 Log ${journal_log} 579 580 Execute Command rm ${file_path}-${LOG_TIME} 581 582 [Return] ${journal_log} 583 584Mac Address To Hex String 585 [Documentation] Converts MAC address into hex format. 586 ... Example 587 ... Given the following MAC: 00:01:6C:80:02:78 588 ... This keyword will return: 0x00 0x01 0x6C 0x80 0x02 0x78 589 ... Description of arguments: 590 ... i_macaddress MAC address in the following format 591 ... 00:01:6C:80:02:78 592 [Arguments] ${i_macaddress} 593 594 ${mac_hex}= Catenate 0x${i_macaddress.replace(':', ' 0x')} 595 [Return] ${mac_hex} 596 597IP Address To Hex String 598 [Documentation] Converts IP address into hex format. 599 ... Example: 600 ... Given the following IP: 10.3.164.100 601 ... This keyword will return: 0xa 0x3 0xa4 0xa0 602 ... Description of arguments: 603 ... i_ipaddress IP address in the following format 604 ... 10.10.10.10 605 [Arguments] ${i_ipaddress} 606 607 @{ip}= Split String ${i_ipaddress} . 608 ${index}= Set Variable ${0} 609 610 :FOR ${item} IN @{ip} 611 \ ${hex}= Convert To Hex ${item} prefix=0x lowercase=yes 612 \ Set List Value ${ip} ${index} ${hex} 613 \ ${index}= Set Variable ${index + 1} 614 ${ip_hex}= Catenate @{ip} 615 [Return] ${ip_hex} 616 617BMC CPU Performance Check 618 [Documentation] Minimal 10% of proc should be free in this instance 619 620 ${bmc_cpu_usage_output} ${stderr}= Execute Command ${bmc_cpu_usage_cmd} 621 ... return_stderr=True 622 Should be empty ${stderr} 623 ${bmc_cpu_percentage}= Fetch From Left ${bmc_cpu_usage_output} % 624 Should be true ${bmc_cpu_percentage} < 90 625 626BMC Mem Performance Check 627 [Documentation] Minimal 10% of memory should be free in this instance 628 629 ${bmc_mem_free_output} ${stderr}= Execute Command ${bmc_mem_free_cmd} 630 ... return_stderr=True 631 Should be empty ${stderr} 632 633 ${bmc_mem_total_output} ${stderr}= Execute Command ${bmc_mem_total_cmd} 634 ... return_stderr=True 635 Should be empty ${stderr} 636 637 ${bmc_mem_percentage}= Evaluate ${bmc_mem_free_output}*100 638 ${bmc_mem_percentage}= Evaluate 639 ... ${bmc_mem_percentage}/${bmc_mem_total_output} 640 Should be true ${bmc_mem_percentage} > 10 641 642BMC File System Usage Check 643 [Documentation] Check the file system space. None should be 100% full 644 ... except /run/initramfs/ro 645 ${bmc_fs_usage_output} ${stderr}= Execute Command 646 ... ${bmc_file_system_usage_cmd} return_stderr=True 647 Should Be Empty ${stderr} 648 Should Be True ${bmc_fs_usage_output}==0 649 650Check BMC CPU Performance 651 [Documentation] Minimal 10% of proc should be free in 3 sample 652 :FOR ${var} IN Range 1 4 653 \ BMC CPU Performance check 654 655Check BMC Mem Performance 656 [Documentation] Minimal 10% of memory should be free 657 658 :FOR ${var} IN Range 1 4 659 \ BMC Mem Performance check 660 661Check BMC File System Performance 662 [Documentation] Check for file system usage for 4 times 663 664 :FOR ${var} IN Range 1 4 665 \ BMC File System Usage check 666 667Get URL List 668 [Documentation] Return list of URLs under given URL. 669 [Arguments] ${openbmc_url} 670 # Description of argument(s): 671 # openbmc_url URL for list operation (e.g. 672 # /xyz/openbmc_project/inventory). 673 674 ${url_list}= Read Properties ${openbmc_url}/list 675 [Return] ${url_list} 676 677Get Endpoint Paths 678 [Documentation] Returns all url paths ending with given endpoint 679 ... Example: 680 ... Given the following endpoint: cpu 681 ... This keyword will return: list of all urls ending with 682 ... cpu - 683 ... /org/openbmc/inventory/system/chassis/motherboard/cpu0, 684 ... /org/openbmc/inventory/system/chassis/motherboard/cpu1 685 ... Description of arguments: 686 ... path URL path for enumeration 687 ... endpoint string for which url path ending 688 [Arguments] ${path} ${endpoint} 689 690 ${resp}= Read Properties ${path}/enumerate timeout=30 691 log Dictionary ${resp} 692 693 ${list}= Get Dictionary Keys ${resp} 694 ${resp}= Get Matches ${list} regexp=^.*[0-9a-z_].${endpoint}[0-9]*$ 695 [Return] ${resp} 696 697Check Zombie Process 698 [Documentation] Check if any defunct process exist or not on BMC 699 ${count} ${stderr} ${rc}= Execute Command ps -o stat | grep Z | wc -l 700 ... return_stderr=True return_rc=True 701 Should Be True ${count}==0 702 Should Be Empty ${stderr} 703 704Prune Journal Log 705 [Documentation] Prune archived journal logs. 706 [Arguments] ${vacuum_size}=1M 707 708 # This keyword can be used to prevent the journal 709 # log from filling up the /run filesystem. 710 # This command will retain only the latest logs 711 # of the user specified size. 712 713 Open Connection And Log In 714 ${output} ${stderr} ${rc}= 715 ... Execute Command 716 ... journalctl --vacuum-size=${vacuum_size} 717 ... return_stderr=True return_rc=True 718 719 Should Be Equal ${rc} ${0} msg=${stderr} 720 721Set BMC Power Policy 722 [Documentation] Set the given BMC power policy. 723 [arguments] ${policy} 724 725 ${valueDict}= create dictionary data=${policy} 726 Write Attribute ${HOST_SETTING} power_policy data=${valueDict} 727 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 728 Should Be Equal ${currentPolicy} ${policy} 729 730Get System Power Policy 731 [Documentation] Get the BMC power policy. 732 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 733 [Return] ${currentPolicy} 734 735Get Auto Reboot 736 [Documentation] Returns auto reboot setting. 737 ${setting}= Read Attribute ${HOST_SETTING} auto_reboot 738 [Return] ${setting} 739 740 741Set Auto Reboot 742 [Documentation] Set the given auto reboot setting. 743 [Arguments] ${setting} 744 # setting auto reboot's setting, i.e. yes or no 745 746 ${valueDict}= Set Variable ${setting} 747 ${data}= Create Dictionary data=${valueDict} 748 Write Attribute ${HOST_SETTING} auto_reboot data=${data} 749 ${current_setting}= Get Auto Reboot 750 Should Be Equal ${current_setting} ${setting} 751 752 753Set BMC Reset Reference Time 754 [Documentation] Set current boot time as a reference and increment 755 ... boot count. 756 757 ${cur_btime}= Get BMC Boot Time 758 Run Keyword If ${BOOT_TIME} == ${0} and ${BOOT_COUNT} == ${0} 759 ... Set Global Variable ${BOOT_TIME} ${cur_btime} 760 ... ELSE IF ${cur_btime} > ${BOOT_TIME} 761 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 762 ... AND 763 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 764 765Get BMC Boot Time 766 [Documentation] Get boot time from /proc/stat. 767 768 Open Connection And Log In 769 ${output} ${stderr}= 770 ... Execute Command egrep '^btime ' /proc/stat | cut -f 2 -d ' ' 771 ... return_stderr=True 772 Should Be Empty ${stderr} 773 ${btime}= Convert To Integer ${output} 774 [Return] ${btime} 775 776Execute Command On BMC 777 [Documentation] Execute given command on BMC and return output. 778 [Arguments] ${command} 779 ${stdout} ${stderr}= Execute Command ${command} return_stderr=True 780 Should Be Empty ${stderr} 781 [Return] ${stdout} 782 783 784Enable Core Dump On BMC 785 [Documentation] Enable core dump collection. 786 Open Connection And Log In 787 ${core_pattern}= Execute Command On BMC 788 ... echo '/tmp/core_%e.%p' | tee /proc/sys/kernel/core_pattern 789 Should Be Equal As Strings ${core_pattern} /tmp/core_%e.%p 790 791Get Number Of BMC Core Dump Files 792 [Documentation] Get number of core dump files on BMC. 793 Open Connection And Log In 794 ${num_of_core_dump}= Execute Command 795 ... ls /tmp/core* 2>/dev/null | wc -l 796 [Return] ${num_of_core_dump} 797 798Set Core Dump File Size Unlimited 799 [Documentation] Set core dump file size to unlimited. 800 Open Connection And Log In 801 Execute Command On BMC 802 ... ulimit -c unlimited 803 804Check For Core Dumps 805 [Documentation] Check for any core dumps exist. 806 ${output}= Get Number Of BMC Core Dump Files 807 Run Keyword If ${output} > 0 808 ... Log **Warning** BMC core dump files exist level=WARN 809 810Trigger Host Watchdog Error 811 [Documentation] Inject host watchdog error using BMC. 812 [Arguments] ${milliseconds}=1000 ${sleep_time}=5s 813 # Description of argument(s): 814 # milliseconds The time watchdog timer value in milliseconds (e.g. 1000 = 815 # 1 second). 816 # sleep_time Time delay for host watchdog error to get injected. 817 # Default is 5 seconds. 818 819 Execute Command On BMC 820 ... /usr/sbin/mapper call /org/openbmc/watchdog/host0 org.openbmc.Watchdog set i ${milliseconds} 821 Execute Command On BMC 822 ... /usr/sbin/mapper call /org/openbmc/watchdog/host0 org.openbmc.Watchdog start 823 Sleep ${sleep_time} 824 825Login To OS Host 826 [Documentation] Login to OS Host. 827 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 828 ... ${os_password}=${OS_PASSWORD} 829 # Desription of arguments: 830 # ${os_host} IP address of the OS Host. 831 # ${os_username} OS Host Login user name. 832 # ${os_password} OS Host Login passwrd. 833 834 ${os_state}= Run Keyword And Return Status Ping Host ${os_host} 835 Run Keyword If '${os_state}' == 'False' 836 ... Run Keywords Initiate Host Reboot AND 837 ... Is Host Running AND 838 ... Wait for OS ${os_host} ${os_username} ${os_password} 839 840 Open Connection ${os_host} 841 ${resp}= Login ${os_username} ${os_password} 842 [Return] ${resp} 843 844Configure Initial Settings 845 [Documentation] Restore old IP and route. 846 ... This keyword requires initial settings viz IP address, 847 ... Network Mask, default gatway and serial console IP and port 848 ... information which should be provided in command line. 849 850 [Arguments] ${host}=${OPENBMC_HOST} ${mask}=${NET_MASK} 851 ... ${gw_ip}=${GW_IP} 852 853 # Open telnet connection and ignore the error, in case telnet session is 854 # already opened by the program calling this keyword. 855 856 Run Keyword And Ignore Error Open Telnet Connection to BMC Serial Console 857 Telnet.write ifconfig eth0 ${host} netmask ${mask} 858 Telnet.write route add default gw ${gw_ip} 859 860Install Debug Tarball On BMC 861 [Documentation] Copy the debug tar file to BMC and install. 862 [Arguments] ${tarball_file_path}=${EXECDIR}/obmc-phosphor-debug-tarball-witherspoon.tar.xz 863 ... ${targ_tarball_dir_path}=/home/root/tarball/ 864 865 # Description of arguments: 866 # tarball_file_path Path of the debug tarball file. 867 # The tar file is downloaded from the build page 868 # https://openpower.xyz/job/openbmc-build/ 869 # obmc-phosphor-debug-tarball-witherspoon.tar.xz 870 # 871 # targ_tarball_dir_path The directory path where the tarball is to be 872 # installed. 873 874 OperatingSystem.File Should Exist ${tarball_file_path} 875 ... msg=${tarball_file_path} doesn't exist. 876 877 # Upload the file to BMC. 878 Import Library SCPLibrary WITH NAME scp 879 Open Connection for SCP 880 scp.Put File ${tarball_file_path} /tmp/debug-tarball.tar.xz 881 882 # Create tarball directory and install. 883 Open Connection And Log In 884 Execute Command On BMC mkdir -p ${targ_tarball_dir_path} 885 Execute Command On BMC 886 ... tar -xf /tmp/debug-tarball.tar.xz -C ${targ_tarball_dir_path} 887 888 # Create symlink to callout-test binary. 889 Execute Command On BMC 890 ... ln -s ${targ_tarball_dir_path}/bin/callout-test /usr/bin/callout-test 891 892 # Remove the tarball file from BMC 893 Execute Command On BMC rm /tmp/debug-tarball.tar.xz 894 895 896Get BMC Boot Count 897 [Documentation] Get BMC boot count based on boot time. 898 ${cur_btime}= Get BMC Boot Time 899 900 # Set global variable BOOT_TIME to current boot time if current boot time 901 # is changed. Also increase value of global variable BOOT_COUNT by 1. 902 Run Keyword If ${cur_btime} > ${BOOT_TIME} 903 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 904 ... AND 905 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 906 [Return] ${BOOT_COUNT} 907 908Set BMC Boot Count 909 [Documentation] Set BMC boot count to given value. 910 [Arguments] ${count} 911 912 # Description of arguments: 913 # count boot count value. 914 ${cur_btime}= Get BMC Boot Time 915 916 # Set global variable BOOT_COUNT to given value. 917 Set Global Variable ${BOOT_COUNT} ${count} 918 919 # Set BOOT_TIME variable to current boot time. 920 Set Global Variable ${BOOT_COUNT} ${count} 921 922Check And Reset MAC 923 [Documentation] Update BMC with user input MAC address. 924 [Arguments] ${mac_address}=${MAC_ADDRESS} 925 926 # Description of argument(s): 927 # mac_address The mac address (e.g. 00:01:6c:80:02:28). 928 929 Should Not Be Empty ${mac_address} 930 Open Connection And Log In 931 ${bmc_mac_addr}= Execute Command On BMC cat /sys/class/net/eth0/address 932 Run Keyword If '${mac_address}' != '${bmc_mac_addr}' 933 ... Set MAC Address 934 935Set MAC Address 936 [Documentation] Update eth0 with input MAC address. 937 [Arguments] ${mac_address}=${MAC_ADDRESS} 938 939 # Description of argument(s): 940 # mac_address The mac address (e.g. 00:01:6c:80:02:28). 941 Write fw_setenv ethaddr ${mac_address} 942 Run Key U OBMC Boot Test \ OBMC Reboot (off) 943 ${bmc_mac_addr}= Execute Command On BMC cat /sys/class/net/eth0/address 944 Should Be Equal ${bmc_mac_addr} ${mac_address} 945 946############################################################################### 947Delete Error logs 948 [Documentation] Delete error logs. 949 950 # The REST method to delete error openbmc/openbmc#1327 951 # until then using logging restart. 952 Open Connection And Log In 953 Execute Command On BMC 954 ... systemctl restart xyz.openbmc_project.Logging.service 955 Sleep 10s reason=Wait for logging service to restart properly. 956 957############################################################################### 958Delete Error log Entry 959 [Documentation] Delete error log entry. 960 [Arguments] ${entry_path} 961 962 # Description of argument(s): 963 # entry_path Delete an error log entry. 964 # Ex. /xyz/openbmc_project/logging/entry/1 965 966 ${data}= Create Dictionary data=@{EMPTY} 967 ${resp}= Openbmc Delete Request ${entry_path} data=${data} 968 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 969