1*** Settings *** 2Resource ../lib/resource.txt 3Resource ../lib/rest_client.robot 4Resource ../lib/connection_client.robot 5Library String 6Library DateTime 7Library Process 8Library OperatingSystem 9Library gen_print.py 10Library gen_robot_print.py 11Library gen_cmd.py 12Library gen_robot_keyword.py 13Library bmc_ssh_utils.py 14Library utils.py 15 16*** Variables *** 17${pflash_cmd} /usr/sbin/pflash -r /dev/stdout -P VERSION 18${SYSTEM_SHUTDOWN_TIME} ${5} 19${dbuscmdBase} 20... dbus-send --system --print-reply --dest=${OPENBMC_BASE_DBUS}.settings.Host 21${dbuscmdGet} 22... ${SETTINGS_URI}host0 org.freedesktop.DBus.Properties.Get 23# Enable when ready with openbmc/openbmc-test-automation#203 24#${dbuscmdString}= string:"xyz.openbmc_project.settings.Host" string: 25${dbuscmdString}= string:"org.openbmc.settings.Host" string: 26 27# Assign default value to QUIET for programs which may not define it. 28${QUIET} ${0} 29${bmc_mem_free_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f4 30${bmc_mem_total_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f2 31${bmc_cpu_usage_cmd}= top -n 1 | grep CPU: | cut -c 7-9 32${HOST_SETTING} ${SETTINGS_URI}host0 33# /run/initramfs/ro associate filesystem should be 100% full always 34${bmc_file_system_usage_cmd}= 35... df -h | grep -v /run/initramfs/ro | cut -c 52-54 | grep 100 | wc -l 36 37${BOOT_TIME} ${0} 38${BOOT_COUNT} ${0} 39${count} ${0} 40${devicetree_base} /sys/firmware/devicetree/base/model 41 42# Initialize default debug value to 0. 43${DEBUG} ${0} 44 45# These variables are used to straddle between new and old methods of setting 46# values. 47${boot_prog_method} ${EMPTY} 48 49${power_policy_setup} ${0} 50${bmc_power_policy_method} ${EMPTY} 51@{valid_power_policy_vars} RESTORE_LAST_STATE ALWAYS_POWER_ON 52... ALWAYS_POWER_OFF 53 54*** Keywords *** 55 56Verify PNOR Update 57 [Documentation] Verify that the PNOR is not corrupted. 58 # Example: 59 # FFS: Flash header not found. Code: 100 60 # Error 100 opening ffs ! 61 62 Open Connection And Log In 63 ${pnor_info}= Execute Command On BMC ${pflash_cmd} 64 Should Not Contain Any ${pnor_info} Flash header not found Error 65 66Get BMC System Model 67 [Documentation] Get the BMC model from the device tree. 68 69 ${bmc_model} ${stderr} ${rc}= BMC Execute Command 70 ... cat ${devicetree_base} | cut -d " " -f 1 return_stderr=True 71 Should Be Empty ${stderr} 72 Should Not Be Empty ${bmc_model} 73 [Return] ${bmc_model} 74 75Verify BMC System Model 76 [Documentation] Verify the BMC model with ${OPENBMC_MODEL}. 77 [Arguments] ${bmc_model} 78 79 ${tmp_bmc_model}= Fetch From Right ${OPENBMC_MODEL} / 80 ${tmp_bmc_model}= Fetch From Left ${tmp_bmc_model} . 81 ${ret}= Run Keyword And Return Status Should Contain ${bmc_model} 82 ... ${tmp_bmc_model} ignore_case=True 83 [Return] ${ret} 84 85Wait For Host To Ping 86 [Arguments] ${host} ${timeout}=${OPENBMC_REBOOT_TIMEOUT}min 87 ... ${interval}=5 sec 88 89 # host The DNS name or IP of the host to ping. 90 # timeout The amount of time after which attempts to ping cease. 91 # interval The amount of time in between attempts to ping. 92 93 Wait Until Keyword Succeeds ${timeout} ${interval} Ping Host ${host} 94 95Ping Host 96 [Arguments] ${host} 97 Should Not Be Empty ${host} msg=No host provided 98 ${RC} ${output}= Run and return RC and Output ping -c 4 ${host} 99 Log RC: ${RC}\nOutput:\n${output} 100 Should be equal ${RC} ${0} 101 102Get Boot Progress 103 [Documentation] Get the boot progress and return it. 104 [Arguments] ${quiet}=${QUIET} 105 106 # Description of argument(s): 107 # quiet Indicates whether this keyword should run without any output to 108 # the console. 109 110 Set Boot Progress Method 111 ${state}= Run Keyword If '${boot_prog_method}' == 'New' 112 ... New Get Boot Progress quiet=${quiet} 113 ... ELSE 114 ... Old Get Boot Progress quiet=${quiet} 115 116 [Return] ${state} 117 118Set Boot Progress Method 119 [Documentation] Set the boot_prog_method to either 'Old' or 'New'. 120 121 # The boot progress data has moved from an 'org' location to an 'xyz' 122 # location. This keyword will determine whether the new method of getting 123 # the boot progress is valid and will set the global boot_prog_method 124 # variable accordingly. If boot_prog_method is already set (either by a 125 # prior call to this function or via a -v parm), this keyword will simply 126 # return. 127 128 # Note: There are interim builds that contain boot_progress in both the old 129 # and the new location values. It is nearly impossible for this keyword to 130 # determine whether the old boot_progress or the new one is active. When 131 # using such builds where the old boot_progress is active, the only recourse 132 # users will have is that they may specify -v boot_prog_method:Old to force 133 # old behavior on such builds. 134 135 Run Keyword If '${boot_prog_method}' != '${EMPTY}' Return From Keyword 136 137 ${new_status} ${new_value}= Run Keyword And Ignore Error 138 ... New Get Boot Progress 139 # If the new style read fails, the method must necessarily be "Old". 140 Run Keyword If '${new_status}' == 'PASS' 141 ... Run Keywords 142 ... Set Global Variable ${boot_prog_method} New AND 143 ... Rqpvars boot_prog_method AND 144 ... Return From Keyword 145 146 # Default method is "Old". 147 Set Global Variable ${boot_prog_method} Old 148 Rqpvars boot_prog_method 149 150Old Get Boot Progress 151 [Documentation] Get the boot progress the old way (via org location). 152 [Arguments] ${quiet}=${QUIET} 153 154 # Description of argument(s): 155 # quiet Indicates whether this keyword should run without any output to 156 # the console. 157 158 ${state}= Read Attribute ${OPENBMC_BASE_URI}sensors/host/BootProgress 159 ... value quiet=${quiet} 160 161 [Return] ${state} 162 163New Get Boot Progress 164 [Documentation] Get the boot progress the new way (via xyz location). 165 [Arguments] ${quiet}=${QUIET} 166 167 # Description of argument(s): 168 # quiet Indicates whether this keyword should run without any output to 169 # the console. 170 171 ${state}= Read Attribute ${HOST_STATE_URI} BootProgress quiet=${quiet} 172 173 [Return] ${state.rsplit('.', 1)[1]} 174 175Is Power On 176 ${state}= Get Power State 177 Should be equal ${state} ${1} 178 179Is Power Off 180 ${state}= Get Power State 181 Should be equal ${state} ${0} 182 183Initiate Power On 184 [Documentation] Initiates the power on and waits until the Is Power On 185 ... keyword returns that the power state has switched to on. 186 [Arguments] ${wait}=${1} 187 188 @{arglist}= Create List 189 ${args}= Create Dictionary data=@{arglist} 190 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOn 191 ... data=${args} 192 should be equal as strings ${resp.status_code} ${HTTP_OK} 193 194 # Does caller want to wait for power on status? 195 Run Keyword If '${wait}' == '${0}' Return From Keyword 196 Wait Until Keyword Succeeds 3 min 10 sec Is Power On 197 198Initiate Power Off 199 [Documentation] Initiates the power off and waits until the Is Power Off 200 ... keyword returns that the power state has switched to off. 201 @{arglist}= Create List 202 ${args}= Create Dictionary data=@{arglist} 203 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOff 204 ... data=${args} 205 should be equal as strings ${resp.status_code} ${HTTP_OK} 206 Wait Until Keyword Succeeds 1 min 10 sec Is Power Off 207 208Initiate OS Host Power Off 209 [Documentation] Initiate an OS reboot. 210 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 211 ... ${os_password}=${OS_PASSWORD} 212 213 # Description of arguments: 214 # os_host The DNS name or IP of the OS. 215 # os_username The username to be used to sign in to the OS. 216 # os_password The password to be used to sign in to the OS. 217 218 SSHLibrary.Open connection ${os_host} 219 Login ${os_username} ${os_password} 220 ${cmd_buf} Catenate shutdown 221 Start Command ${cmd_buf} 222 SSHLibrary.Close Connection 223 224Initiate OS Host Reboot 225 [Documentation] Initiate an OS reboot. 226 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 227 ... ${os_password}=${OS_PASSWORD} 228 229 # Description of arguments: 230 # os_host The DNS name or IP of the OS. 231 # os_username The username to be used to sign in to the OS. 232 # os_password The password to be used to sign in to the OS. 233 234 SSHLibrary.Open connection ${os_host} 235 Login ${os_username} ${os_password} 236 ${cmd_buf} Catenate reboot 237 Start Command ${cmd_buf} 238 SSHLibrary.Close Connection 239 240Initiate Auto Reboot 241 [Documentation] Initiate an auto reboot. 242 243 # Set the auto reboot policy. 244 Set Auto Reboot ${1} 245 # Set the watchdog timer. Note: 5000 = milliseconds which is 5 seconds. 246 Trigger Host Watchdog Error 5000 247 248Trigger Warm Reset 249 log to console "Triggering warm reset" 250 ${data}= create dictionary data=@{EMPTY} 251 ${resp}= openbmc post request 252 ... ${OPENBMC_BASE_URI}control/bmc0/action/warmReset data=${data} 253 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 254 ${session_active}= Check If warmReset is Initiated 255 Run Keyword If '${session_active}' == '${True}' 256 ... Fail msg=warm reset didn't occur 257 258 Sleep ${SYSTEM_SHUTDOWN_TIME}min 259 Check If BMC Is Up 260 261Check OS 262 [Documentation] Attempts to ping the host OS and then checks that the host 263 ... OS is up by running an SSH command. 264 265 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 266 ... ${os_password}=${OS_PASSWORD} ${quiet}=${QUIET} 267 ... ${print_string}=${EMPTY} 268 [Teardown] SSHLibrary.Close Connection 269 270 # os_host The DNS name/IP of the OS host associated with our BMC. 271 # os_username The username to be used to sign on to the OS host. 272 # os_password The password to be used to sign on to the OS host. 273 # quiet Indicates whether this keyword should write to console. 274 # print_string A string to be printed before checking the OS. 275 276 rprint ${print_string} 277 278 # Attempt to ping the OS. Store the return code to check later. 279 ${ping_rc}= Run Keyword and Return Status Ping Host ${os_host} 280 281 SSHLibrary.Open connection ${os_host} 282 283 ${status} ${msg}= Run Keyword And Ignore Error Login ${os_username} 284 ... ${os_password} 285 ${err_msg1}= Sprint Error ${msg} 286 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 287 Run Keyword If '${status}' == 'FAIL' Fail msg=${err_msg} 288 ${output} ${stderr} ${rc}= Execute Command uptime return_stderr=True 289 ... return_rc=True 290 291 ${temp_msg}= Catenate Could not execute a command on the operating 292 ... system.\n 293 ${err_msg1}= Sprint Error ${temp_msg} 294 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 295 296 # If the return code returned by "Execute Command" is non-zero, this 297 # keyword will fail. 298 Should Be Equal ${rc} ${0} msg=${err_msg} 299 # We will likewise fail if there is any stderr data. 300 Should Be Empty ${stderr} 301 302 ${temp_msg}= Set Variable Could not ping the operating system.\n 303 ${err_msg1}= Sprint Error ${temp_msg} 304 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 305 # We will likewise fail if the OS did not ping, as we could SSH but not 306 # ping 307 Should Be Equal As Strings ${ping_rc} ${TRUE} msg=${err_msg} 308 309Wait for OS 310 [Documentation] Waits for the host OS to come up via calls to "Check OS". 311 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 312 ... ${os_password}=${OS_PASSWORD} ${timeout}=${OS_WAIT_TIMEOUT} 313 ... ${quiet}=${0} 314 [Teardown] rprintn 315 316 # os_host The DNS name or IP of the OS host associated with our 317 # BMC. 318 # os_username The username to be used to sign on to the OS host. 319 # os_password The password to be used to sign on to the OS host. 320 # timeout The timeout in seconds indicating how long you're 321 # willing to wait for the OS to respond. 322 # quiet Indicates whether this keyword should write to console. 323 324 # The interval to be used between calls to "Check OS". 325 ${interval}= Set Variable 5 326 327 ${message}= Catenate Checking every ${interval} seconds for up to 328 ... ${timeout} seconds for the operating system to communicate. 329 rqprint_timen ${message} 330 331 Wait Until Keyword Succeeds ${timeout} sec ${interval} Check OS 332 ... ${os_host} ${os_username} ${os_password} 333 ... print_string=\# 334 335 rqprintn 336 337 rqprint_timen The operating system is now communicating. 338 339Get BMC State Deprecated 340 [Documentation] Returns the state of the BMC as a string. (i.e: BMC_READY) 341 [Arguments] ${quiet}=${QUIET} 342 343 @{arglist}= Create List 344 ${args}= Create Dictionary data=@{arglist} 345 ${resp}= Call Method ${OPENBMC_BASE_URI}managers/System/ getSystemState 346 ... data=${args} quiet=${quiet} 347 Should be equal as strings ${resp.status_code} ${HTTP_OK} 348 ${content}= to json ${resp.content} 349 [Return] ${content["data"]} 350 351Get Power State 352 [Documentation] Returns the power state as an integer. Either 0 or 1. 353 [Arguments] ${quiet}=${QUIET} 354 355 @{arglist}= Create List 356 ${args}= Create Dictionary data=@{arglist} 357 358 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ getPowerState 359 ... data=${args} quiet=${quiet} 360 Should be equal as strings ${resp.status_code} ${HTTP_OK} 361 ${content}= to json ${resp.content} 362 [Return] ${content["data"]} 363 364Clear BMC Record Log 365 [Documentation] Clears all the event logs on the BMC. This would be 366 ... equivalent to ipmitool sel clear. 367 @{arglist}= Create List 368 ${args}= Create Dictionary data=@{arglist} 369 ${resp}= Call Method 370 ... ${OPENBMC_BASE_URI}records/events/ clear data=${args} 371 should be equal as strings ${resp.status_code} ${HTTP_OK} 372 373Copy PNOR to BMC 374 Import Library SCPLibrary WITH NAME scp 375 Open Connection for SCP 376 Log Copying ${PNOR_IMAGE_PATH} to /tmp 377 scp.Put File ${PNOR_IMAGE_PATH} /tmp 378 379Flash PNOR 380 [Documentation] Calls flash bios update method to flash PNOR image 381 [Arguments] ${pnor_image} 382 @{arglist}= Create List ${pnor_image} 383 ${args}= Create Dictionary data=@{arglist} 384 ${resp}= Call Method ${OPENBMC_BASE_URI}control/flash/bios/ update 385 ... data=${args} 386 should be equal as strings ${resp.status_code} ${HTTP_OK} 387 Wait Until Keyword Succeeds 2 min 10 sec Is PNOR Flashing 388 389Get Flash BIOS Status 390 [Documentation] Returns the status of the flash BIOS API as a string. For 391 ... example 'Flashing', 'Flash Done', etc 392 ${data}= Read Properties ${OPENBMC_BASE_URI}control/flash/bios 393 [Return] ${data['status']} 394 395Is PNOR Flashing 396 [Documentation] Get BIOS 'Flashing' status. This indicates that PNOR 397 ... flashing has started. 398 ${status}= Get Flash BIOS Status 399 Should Contain ${status} Flashing 400 401Is PNOR Flash Done 402 [Documentation] Get BIOS 'Flash Done' status. This indicates that the 403 ... PNOR flashing has completed. 404 ${status}= Get Flash BIOS Status 405 should be equal as strings ${status} Flash Done 406 407Is System State Host Booted 408 [Documentation] Checks whether system state is HOST_BOOTED. 409 ${state}= Get BMC State Deprecated 410 should be equal as strings ${state} HOST_BOOTED 411 412Is OS Starting 413 [Documentation] Check if boot progress is OS starting. 414 ${boot_progress}= Get Boot Progress 415 Should Be Equal ${boot_progress} OSStart 416 417Is OS Off 418 [Documentation] Check if boot progress is "Off". 419 ${boot_progress}= Get Boot Progress 420 Should Be Equal ${boot_progress} Off 421 422Get Boot Progress To OS Starting State 423 [Documentation] Get the system to a boot progress state of 'FW Progress, 424 ... Starting OS'. 425 426 ${boot_progress}= Get Boot Progress 427 Run Keyword If '${boot_progress}' == 'OSStart' 428 ... Log Host is already in OS starting state 429 ... ELSE 430 ... Run Keywords Initiate Host PowerOff AND Initiate Host Boot 431 ... AND Wait Until Keyword Succeeds 10 min 10 sec Is OS Starting 432 433Verify Ping and REST Authentication 434 ${l_ping}= Run Keyword And Return Status 435 ... Ping Host ${OPENBMC_HOST} 436 Run Keyword If '${l_ping}' == '${False}' 437 ... Fail msg=Ping Failed 438 439 ${l_rest}= Run Keyword And Return Status 440 ... Initialize OpenBMC 441 Run Keyword If '${l_rest}' == '${False}' 442 ... Fail msg=REST Authentication Failed 443 444 # Just to make sure the SSH is working for SCP 445 Open Connection And Log In 446 ${system} ${stderr}= Execute Command hostname return_stderr=True 447 Should Be Empty ${stderr} 448 449Check If BMC is Up 450 [Documentation] Wait for Host to be online. Checks every X seconds 451 ... interval for Y minutes and fails if timed out. 452 ... Default MAX timedout is 10 min, interval 10 seconds. 453 [Arguments] ${max_timeout}=${OPENBMC_REBOOT_TIMEOUT} min 454 ... ${interval}=10 sec 455 456 Wait Until Keyword Succeeds 457 ... ${max_timeout} ${interval} Verify Ping and REST Authentication 458 459 460Check If warmReset is Initiated 461 [Documentation] Ping would be still alive, so try SSH to connect 462 ... if fails the ports are down indicating reboot 463 ... is in progress 464 465 # Warm reset adds 3 seconds delay before forcing reboot 466 # To minimize race conditions, we wait for 7 seconds 467 Sleep 7s 468 ${alive}= Run Keyword and Return Status 469 ... Open Connection And Log In 470 Return From Keyword If '${alive}' == '${False}' ${False} 471 [Return] ${True} 472 473Flush REST Sessions 474 [Documentation] Removes all the active session objects 475 Delete All Sessions 476 477Initialize DBUS cmd 478 [Documentation] Initialize dbus string with property string to extract 479 [Arguments] ${boot_property} 480 ${cmd}= Catenate ${dbuscmdBase} ${dbuscmdGet} ${dbuscmdString} 481 ${cmd}= Catenate ${cmd}${boot_property} 482 Set Global Variable ${dbuscmd} ${cmd} 483 484Create OS Console File Path 485 [Documentation] Create OS console file path name and return it. 486 [Arguments] ${log_file_path}=${EMPTY} 487 488 # Description of arguements: 489 # file_path The caller's candidate value. If this value is ${EMPTY}, this 490 # keyword will compose a file path name. Otherwise, this 491 # keyword will use the caller's file_path value. In either 492 # case, the value will be returned. 493 494 ${default_file_path}= Catenate /tmp/${OPENBMC_HOST}_os_console 495 ${log_file_path}= Set Variable If '${log_file_path}' == '${EMPTY}' 496 ... ${default_file_path} ${log_file_path} 497 498 [Return] ${log_file_path} 499 500Create OS Console Command String 501 [Documentation] Return a command string to start OS console logging. 502 503 # First make sure that the ssh_pw program is available. 504 ${cmd_buf}= Catenate which ssh_pw 2>&1 505 Rdpissuing ${cmd_buf} 506 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 507 Rdpvars rc output 508 Should Be Equal ${rc} ${0} msg=${output}\n 509 510 ${cmd_buf}= Catenate ssh_pw ${OPENBMC_PASSWORD} -p 2200 511 ... -o "StrictHostKeyChecking no" ${OPENBMC_USERNAME}@${OPENBMC_HOST} 512 513 [Return] ${cmd_buf} 514 515Get SOL Console Pid 516 [Documentation] Get the pid of the active sol conole job. 517 518 # Find the pid of the active system console logging session (if any). 519 ${search_string}= Create OS Console Command String 520 # At least in some cases, ps output does not show double quotes so we must 521 # replace them in our search string with the regexes to indicate that they 522 # are optional. 523 ${search_string}= Replace String ${search_string} " ["]? 524 ${cmd_buf}= Catenate echo $(ps -ef | egrep '${search_string}' 525 ... | egrep -v grep | cut -c10-14) 526 Rdpissuing ${cmd_buf} 527 ${rc} ${os_con_pid}= Run And Return Rc And Output ${cmd_buf} 528 Rdpvars os_con_pid 529 # If rc is not zero it just means that there is no OS Console process 530 # running. 531 532 [Return] ${os_con_pid} 533 534 535Stop SOL Console Logging 536 [Documentation] Stop system console logging and return log output. 537 [Arguments] ${log_file_path}=${EMPTY} 538 ... ${targ_file_path}=${EXECDIR}${/}logs${/} 539 ... ${return_data}=${1} 540 541 # If there are muliple system console processes, they will all be stopped. 542 # If there is no existing log file this keyword will return an error 543 # message to that effect (and write that message to targ_file_path, if 544 # specified). 545 # NOTE: This keyword will not fail if there is no running system console 546 # process. 547 548 # Description of arguments: 549 # log_file_path The file path that was used to call "Start SOL 550 # Console Logging". See that keyword (above) for details. 551 # targ_file_path If specified, the file path to which the source 552 # file path (i.e. "log_file_path") should be copied. 553 # return_data If this is set to ${1}, this keyword will return the SOL 554 # data to the caller as a unicode string. 555 556 ${log_file_path}= Create OS Console File Path ${log_file_path} 557 558 ${os_con_pid}= Get SOL Console Pid 559 560 ${cmd_buf}= Catenate kill -9 ${os_con_pid} 561 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpissuing ${cmd_buf} 562 ${rc} ${output}= Run Keyword If '${os_con_pid}' != '${EMPTY}' 563 ... Run And Return Rc And Output ${cmd_buf} 564 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpvars rc output 565 566 Run Keyword If '${targ_file_path}' != '${EMPTY}' 567 ... Run Keyword And Ignore Error 568 ... Copy File ${log_file_path} ${targ_file_path} 569 570 ${output}= Set Variable ${EMPTY} 571 ${loc_quiet}= Evaluate ${debug}^1 572 ${rc} ${output}= Run Keyword If '${return_data}' == '${1}' 573 ... Cmd Fnc cat ${log_file_path} quiet=${loc_quiet} print_output=${0} 574 575 [Return] ${output} 576 577Start SOL Console Logging 578 [Documentation] Start system console log to file. 579 [Arguments] ${log_file_path}=${EMPTY} ${return_data}=${1} 580 581 # This keyword will first call "Stop SOL Console Logging". Only then will 582 # it start SOL console logging. The data returned by "Stop SOL Console 583 # Logging" will in turn be returned by this keyword. 584 585 # Description of arguments: 586 # log_file_path The file path to which system console log data should be 587 # written. Note that this path is taken to be a location 588 # on the machine where this program is running rather than 589 # on the Open BMC system. 590 # return_data If this is set to ${1}, this keyword will return any SOL 591 # data to the caller as a unicode string. 592 593 ${log_file_path}= Create OS Console File Path ${log_file_path} 594 595 ${log_output}= Stop SOL Console Logging ${log_file_path} 596 ... return_data=${return_data} 597 598 # Validate by making sure we can create the file. Problems creating the 599 # file would not be noticed by the subsequent ssh command because we fork 600 # the command. 601 Create File ${log_file_path} 602 ${sub_cmd_buf}= Create OS Console Command String 603 # Routing stderr to stdout so that any startup error text will go to the 604 # output file. 605 ${cmd_buf}= Catenate ${sub_cmd_buf} > ${log_file_path} 2>&1 & 606 Rdpissuing ${cmd_buf} 607 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 608 # Because we are forking this command, we essentially will never get a 609 # non-zero return code or any output. 610 Should Be Equal ${rc} ${0} 611 612 Sleep 1 613 ${os_con_pid}= Get SOL Console Pid 614 615 Should Not Be Empty ${os_con_pid} 616 617 [Return] ${log_output} 618 619Get Time Stamp 620 [Documentation] Get the current time stamp data 621 ${cur_time}= Get Current Date result_format=%Y%m%d%H%M%S%f 622 [Return] ${cur_time} 623 624 625Verify BMC State 626 [Documentation] Get the BMC state and verify if the current 627 ... BMC state is as expected. 628 [Arguments] ${expected} 629 630 ${current}= Get BMC State Deprecated 631 Should Contain ${expected} ${current} 632 633Start Journal Log 634 [Documentation] Start capturing journal log to a file in /tmp using 635 ... journalctl command. By default journal log is collected 636 ... at /tmp/journal_log else user input location. 637 ... The File is appended with datetime. 638 [Arguments] ${file_path}=/tmp/journal_log 639 640 Open Connection And Log In 641 642 ${cur_time}= Get Time Stamp 643 Set Global Variable ${LOG_TIME} ${cur_time} 644 Start Command 645 ... journalctl -f > ${file_path}-${LOG_TIME} 646 Log Journal Log Started: ${file_path}-${LOG_TIME} 647 648 649Stop Journal Log 650 [Documentation] Stop journalctl process if its running. 651 ... By default return log from /tmp/journal_log else 652 ... user input location. 653 [Arguments] ${file_path}=/tmp/journal_log 654 655 Open Connection And Log In 656 657 ${rc}= 658 ... Execute Command 659 ... ps ax | grep journalctl | grep -v grep 660 ... return_stdout=False return_rc=True 661 662 Return From Keyword If '${rc}' == '${1}' 663 ... No journal log process running 664 665 ${output} ${stderr}= 666 ... Execute Command killall journalctl 667 ... return_stderr=True 668 Should Be Empty ${stderr} 669 670 ${journal_log} ${stderr}= 671 ... Execute Command 672 ... cat ${file_path}-${LOG_TIME} 673 ... return_stderr=True 674 Should Be Empty ${stderr} 675 676 Log ${journal_log} 677 678 Execute Command rm ${file_path}-${LOG_TIME} 679 680 [Return] ${journal_log} 681 682Mac Address To Hex String 683 [Documentation] Converts MAC address into hex format. 684 ... Example 685 ... Given the following MAC: 00:01:6C:80:02:78 686 ... This keyword will return: 0x00 0x01 0x6C 0x80 0x02 0x78 687 ... Description of arguments: 688 ... i_macaddress MAC address in the following format 689 ... 00:01:6C:80:02:78 690 [Arguments] ${i_macaddress} 691 692 ${mac_hex}= Catenate 0x${i_macaddress.replace(':', ' 0x')} 693 [Return] ${mac_hex} 694 695IP Address To Hex String 696 [Documentation] Converts IP address into hex format. 697 ... Example: 698 ... Given the following IP: 10.3.164.100 699 ... This keyword will return: 0xa 0x3 0xa4 0xa0 700 ... Description of arguments: 701 ... i_ipaddress IP address in the following format 702 ... 10.10.10.10 703 [Arguments] ${i_ipaddress} 704 705 @{ip}= Split String ${i_ipaddress} . 706 ${index}= Set Variable ${0} 707 708 :FOR ${item} IN @{ip} 709 \ ${hex}= Convert To Hex ${item} prefix=0x lowercase=yes 710 \ Set List Value ${ip} ${index} ${hex} 711 \ ${index}= Set Variable ${index + 1} 712 ${ip_hex}= Catenate @{ip} 713 [Return] ${ip_hex} 714 715BMC CPU Performance Check 716 [Documentation] Minimal 10% of proc should be free in this instance 717 718 ${bmc_cpu_usage_output} ${stderr}= Execute Command ${bmc_cpu_usage_cmd} 719 ... return_stderr=True 720 Should be empty ${stderr} 721 ${bmc_cpu_percentage}= Fetch From Left ${bmc_cpu_usage_output} % 722 Should be true ${bmc_cpu_percentage} < 90 723 724BMC Mem Performance Check 725 [Documentation] Minimal 10% of memory should be free in this instance 726 727 ${bmc_mem_free_output} ${stderr}= Execute Command ${bmc_mem_free_cmd} 728 ... return_stderr=True 729 Should be empty ${stderr} 730 731 ${bmc_mem_total_output} ${stderr}= Execute Command ${bmc_mem_total_cmd} 732 ... return_stderr=True 733 Should be empty ${stderr} 734 735 ${bmc_mem_percentage}= Evaluate ${bmc_mem_free_output}*100 736 ${bmc_mem_percentage}= Evaluate 737 ... ${bmc_mem_percentage}/${bmc_mem_total_output} 738 Should be true ${bmc_mem_percentage} > 10 739 740BMC File System Usage Check 741 [Documentation] Check the file system space. None should be 100% full 742 ... except /run/initramfs/ro 743 ${bmc_fs_usage_output} ${stderr}= Execute Command 744 ... ${bmc_file_system_usage_cmd} return_stderr=True 745 Should Be Empty ${stderr} 746 Should Be True ${bmc_fs_usage_output}==0 747 748Check BMC CPU Performance 749 [Documentation] Minimal 10% of proc should be free in 3 sample 750 :FOR ${var} IN Range 1 4 751 \ BMC CPU Performance check 752 753Check BMC Mem Performance 754 [Documentation] Minimal 10% of memory should be free 755 756 :FOR ${var} IN Range 1 4 757 \ BMC Mem Performance check 758 759Check BMC File System Performance 760 [Documentation] Check for file system usage for 4 times 761 762 :FOR ${var} IN Range 1 4 763 \ BMC File System Usage check 764 765Get URL List 766 [Documentation] Return list of URLs under given URL. 767 [Arguments] ${openbmc_url} 768 # Description of argument(s): 769 # openbmc_url URL for list operation (e.g. 770 # /xyz/openbmc_project/inventory). 771 772 ${url_list}= Read Properties ${openbmc_url}/list quiet=${1} 773 Sort List ${url_list} 774 [Return] ${url_list} 775 776Get Endpoint Paths 777 [Documentation] Returns all url paths ending with given endpoint 778 ... Example: 779 ... Given the following endpoint: cpu 780 ... This keyword will return: list of all urls ending with 781 ... cpu - 782 ... /org/openbmc/inventory/system/chassis/motherboard/cpu0, 783 ... /org/openbmc/inventory/system/chassis/motherboard/cpu1 784 ... Description of arguments: 785 ... path URL path for enumeration 786 ... endpoint string for which url path ending 787 [Arguments] ${path} ${endpoint} 788 789 ${resp}= Read Properties ${path}/enumerate timeout=30 790 log Dictionary ${resp} 791 792 ${list}= Get Dictionary Keys ${resp} 793 ${resp}= Get Matches ${list} regexp=^.*[0-9a-z_].${endpoint}[0-9]*$ 794 [Return] ${resp} 795 796Check Zombie Process 797 [Documentation] Check if any defunct process exist or not on BMC 798 ${count} ${stderr} ${rc}= Execute Command ps -o stat | grep Z | wc -l 799 ... return_stderr=True return_rc=True 800 Should Be True ${count}==0 801 Should Be Empty ${stderr} 802 803Prune Journal Log 804 [Documentation] Prune archived journal logs. 805 [Arguments] ${vacuum_size}=1M 806 807 # This keyword can be used to prevent the journal 808 # log from filling up the /run filesystem. 809 # This command will retain only the latest logs 810 # of the user specified size. 811 812 Open Connection And Log In 813 ${output} ${stderr} ${rc}= 814 ... Execute Command 815 ... journalctl --vacuum-size=${vacuum_size} 816 ... return_stderr=True return_rc=True 817 818 Should Be Equal ${rc} ${0} msg=${stderr} 819 820Set BMC Power Policy 821 [Documentation] Set the given BMC power policy. 822 [Arguments] ${policy} 823 824 # Note that this function will translate the old style "RESTORE_LAST_STATE" 825 # policy to the new style "xyz.openbmc_project.Control.Power.RestorePolicy. 826 # Policy.Restore" for you. 827 828 # Description of argument(s): 829 # policy Power restore policy (e.g "RESTORE_LAST_STATE", 830 # ${RESTORE_LAST_STATE}). 831 832 # Set the bmc_power_policy_method to either 'Old' or 'New'. 833 Set Power Policy Method 834 # This translation helps bridge between old and new method for calling. 835 ${policy}= Translate Power Policy Value ${policy} 836 # Run the appropriate keyword. 837 Run Key ${bmc_power_policy_method} Set Power Policy \ ${policy} 838 ${currentPolicy}= Get System Power Policy 839 Should Be Equal ${currentPolicy} ${policy} 840 841New Set Power Policy 842 [Documentation] Set the given BMC power policy (new method). 843 [Arguments] ${policy} 844 845 # Description of argument(s): 846 # policy Power restore policy (e.g. ${RESTORE_LAST_STATE}). 847 848 ${valueDict}= Create Dictionary data=${policy} 849 Write Attribute 850 ... ${POWER_RESTORE_URI} PowerRestorePolicy data=${valueDict} 851 852Old Set Power Policy 853 [Documentation] Set the given BMC power policy (old method). 854 [Arguments] ${policy} 855 856 # Description of argument(s): 857 # policy Power restore policy (e.g. "RESTORE_LAST_STATE"). 858 859 ${valueDict}= create dictionary data=${policy} 860 Write Attribute ${HOST_SETTING} power_policy data=${valueDict} 861 862Get System Power Policy 863 [Documentation] Get the BMC power policy. 864 865 # Set the bmc_power_policy_method to either 'Old' or 'New'. 866 Set Power Policy Method 867 ${cmd_buf}= Create List ${bmc_power_policy_method} Get Power Policy 868 # Run the appropriate keyword. 869 ${currentPolicy}= Run Keyword @{cmd_buf} 870 [Return] ${currentPolicy} 871 872New Get Power Policy 873 [Documentation] Get the BMC power policy (new method). 874 ${currentPolicy}= Read Attribute ${POWER_RESTORE_URI} PowerRestorePolicy 875 [Return] ${currentPolicy} 876 877Old Get Power Policy 878 [Documentation] Get the BMC power policy (old method). 879 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 880 [Return] ${currentPolicy} 881 882Get Auto Reboot 883 [Documentation] Returns auto reboot setting. 884 ${setting}= Read Attribute ${CONTROL_HOST_URI}/auto_reboot AutoReboot 885 [Return] ${setting} 886 887Set Auto Reboot 888 [Documentation] Set the given auto reboot setting. 889 [Arguments] ${setting} 890 # setting auto reboot's setting, i.e. 1 for enabling and 0 for disabling. 891 892 ${valueDict}= Set Variable ${setting} 893 ${data}= Create Dictionary data=${valueDict} 894 Write Attribute ${CONTROL_HOST_URI}/auto_reboot AutoReboot data=${data} 895 ${current_setting}= Get Auto Reboot 896 Should Be Equal As Integers ${current_setting} ${setting} 897 898 899Set BMC Reset Reference Time 900 [Documentation] Set current boot time as a reference and increment 901 ... boot count. 902 903 ${cur_btime}= Get BMC Boot Time 904 Run Keyword If ${BOOT_TIME} == ${0} and ${BOOT_COUNT} == ${0} 905 ... Set Global Variable ${BOOT_TIME} ${cur_btime} 906 ... ELSE IF ${cur_btime} > ${BOOT_TIME} 907 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 908 ... AND 909 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 910 911Get BMC Boot Time 912 [Documentation] Get boot time from /proc/stat. 913 914 Open Connection And Log In 915 ${output} ${stderr}= 916 ... Execute Command egrep '^btime ' /proc/stat | cut -f 2 -d ' ' 917 ... return_stderr=True 918 Should Be Empty ${stderr} 919 ${btime}= Convert To Integer ${output} 920 [Return] ${btime} 921 922Execute Command On BMC 923 [Documentation] Execute given command on BMC and return output. 924 [Arguments] ${command} 925 ${stdout} ${stderr}= Execute Command ${command} return_stderr=True 926 Should Be Empty ${stderr} 927 [Return] ${stdout} 928 929 930Enable Core Dump On BMC 931 [Documentation] Enable core dump collection. 932 Open Connection And Log In 933 ${core_pattern}= Execute Command On BMC 934 ... echo '/tmp/core_%e.%p' | tee /proc/sys/kernel/core_pattern 935 Should Be Equal As Strings ${core_pattern} /tmp/core_%e.%p 936 937Get Number Of BMC Core Dump Files 938 [Documentation] Get number of core dump files on BMC. 939 Open Connection And Log In 940 ${num_of_core_dump}= Execute Command 941 ... ls /tmp/core* 2>/dev/null | wc -l 942 [Return] ${num_of_core_dump} 943 944Set Core Dump File Size Unlimited 945 [Documentation] Set core dump file size to unlimited. 946 Open Connection And Log In 947 Execute Command On BMC 948 ... ulimit -c unlimited 949 950Check For Core Dumps 951 [Documentation] Check for any core dumps exist. 952 ${output}= Get Number Of BMC Core Dump Files 953 Run Keyword If ${output} > 0 954 ... Log **Warning** BMC core dump files exist level=WARN 955 956Trigger Host Watchdog Error 957 [Documentation] Inject host watchdog timeout error via REST. 958 [Arguments] ${milliseconds}=1000 ${sleep_time}=5s 959 # Description of argument(s): 960 # milliseconds The time watchdog timer value in milliseconds (e.g. 1000 = 961 # 1 second). 962 # sleep_time Time delay for host watchdog error to get injected. 963 # Default is 5 seconds. 964 965 ${data}= Create Dictionary data=${True} 966 Write Attribute /xyz/openbmc_project/watchdog/host0 Enabled data=${data} 967 968 ${data}= Create Dictionary data=${milliseconds} 969 Write Attribute /xyz/openbmc_project/watchdog/host0 TimeRemaining data=${data} 970 971 Sleep ${sleep_time} 972 973Login To OS Host 974 [Documentation] Login to OS Host. 975 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 976 ... ${os_password}=${OS_PASSWORD} 977 # Desription of arguments: 978 # ${os_host} IP address of the OS Host. 979 # ${os_username} OS Host Login user name. 980 # ${os_password} OS Host Login passwrd. 981 982 ${os_state}= Run Keyword And Return Status Ping Host ${os_host} 983 Run Keyword If '${os_state}' == 'False' 984 ... Run Keywords Initiate Host Reboot AND 985 ... Is Host Running AND 986 ... Wait for OS ${os_host} ${os_username} ${os_password} 987 988 SSHLibrary.Open Connection ${os_host} 989 ${resp}= Login ${os_username} ${os_password} 990 [Return] ${resp} 991 992Configure Initial Settings 993 [Documentation] Restore old IP and route. 994 ... This keyword requires initial settings viz IP address, 995 ... Network Mask, default gatway and serial console IP and port 996 ... information which should be provided in command line. 997 998 [Arguments] ${host}=${OPENBMC_HOST} ${mask}=${NET_MASK} 999 ... ${gw_ip}=${GW_IP} 1000 1001 # Open telnet connection and ignore the error, in case telnet session is 1002 # already opened by the program calling this keyword. 1003 1004 Run Keyword And Ignore Error Open Telnet Connection to BMC Serial Console 1005 Telnet.write ifconfig eth0 ${host} netmask ${mask} 1006 Telnet.write route add default gw ${gw_ip} 1007 1008Install Debug Tarball On BMC 1009 [Documentation] Copy the debug tar file to BMC and install. 1010 [Arguments] ${tarball_file_path}=${EXECDIR}/obmc-phosphor-debug-tarball-witherspoon.tar.xz 1011 ... ${targ_tarball_dir_path}=/home/root/tarball/ 1012 1013 # Description of arguments: 1014 # tarball_file_path Path of the debug tarball file. 1015 # The tar file is downloaded from the build page 1016 # https://openpower.xyz/job/openbmc-build/ 1017 # obmc-phosphor-debug-tarball-witherspoon.tar.xz 1018 # 1019 # targ_tarball_dir_path The directory path where the tarball is to be 1020 # installed. 1021 1022 OperatingSystem.File Should Exist ${tarball_file_path} 1023 ... msg=${tarball_file_path} doesn't exist. 1024 1025 # Upload the file to BMC. 1026 Import Library SCPLibrary WITH NAME scp 1027 Open Connection for SCP 1028 scp.Put File ${tarball_file_path} /tmp/debug-tarball.tar.xz 1029 1030 # Create tarball directory and install. 1031 Open Connection And Log In 1032 Execute Command On BMC mkdir -p ${targ_tarball_dir_path} 1033 Execute Command On BMC 1034 ... tar -xf /tmp/debug-tarball.tar.xz -C ${targ_tarball_dir_path} 1035 1036 # Create symlink to callout-test binary. 1037 Execute Command On BMC 1038 ... ln -s ${targ_tarball_dir_path}/bin/callout-test /usr/bin/callout-test 1039 1040 # Create symlink to logging-test binary. 1041 Execute Command On BMC 1042 ... ln -s ${targ_tarball_dir_path}/bin/logging-test /usr/bin/logging-test 1043 1044 # Remove the tarball file from BMC 1045 Execute Command On BMC rm /tmp/debug-tarball.tar.xz 1046 1047 1048Get BMC Boot Count 1049 [Documentation] Get BMC boot count based on boot time. 1050 ${cur_btime}= Get BMC Boot Time 1051 1052 # Set global variable BOOT_TIME to current boot time if current boot time 1053 # is changed. Also increase value of global variable BOOT_COUNT by 1. 1054 Run Keyword If ${cur_btime} > ${BOOT_TIME} 1055 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 1056 ... AND 1057 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 1058 [Return] ${BOOT_COUNT} 1059 1060Set BMC Boot Count 1061 [Documentation] Set BMC boot count to given value. 1062 [Arguments] ${count} 1063 1064 # Description of arguments: 1065 # count boot count value. 1066 ${cur_btime}= Get BMC Boot Time 1067 1068 # Set global variable BOOT_COUNT to given value. 1069 Set Global Variable ${BOOT_COUNT} ${count} 1070 1071 # Set BOOT_TIME variable to current boot time. 1072 Set Global Variable ${BOOT_COUNT} ${count} 1073 1074Get System LED State 1075 [Documentation] Return the state of given system LED. 1076 [Arguments] ${led_name} 1077 1078 # Description of argument(s): 1079 # led_name System LED name (e.g. heartbeat, identify, beep). 1080 1081 ${state}= Read Attribute ${LED_PHYSICAL_URI}${led_name} State 1082 [Return] ${state.rsplit('.', 1)[1]} 1083 1084############################################################################### 1085Delete Error Logs 1086 [Documentation] Delete error logs. 1087 1088 # Check if error logs entries exist, if not return. 1089 ${resp}= OpenBMC Get Request ${BMC_LOGGING_ENTRY}${/}list quiet=${1} 1090 Return From Keyword If ${resp.status_code} == ${HTTP_NOT_FOUND} 1091 1092 # Get the list of error logs entries and delete them all. 1093 ${elog_entries}= Get URL List ${BMC_LOGGING_ENTRY} 1094 :FOR ${entry} IN @{elog_entries} 1095 \ Delete Error Log Entry ${entry} 1096 1097############################################################################### 1098Delete Error Log Entry 1099 [Documentation] Delete error log entry. 1100 [Arguments] ${entry_path} 1101 1102 # Description of argument(s): 1103 # entry_path Delete an error log entry. 1104 # Ex. /xyz/openbmc_project/logging/entry/1 1105 1106 # Skip delete if entry URI is a callout. 1107 # Example: /xyz/openbmc_project/logging/entry/1/callout 1108 Return From Keyword If '${entry_path.rsplit('/', 1)[1]}' == 'callout' 1109 1110 ${data}= Create Dictionary data=@{EMPTY} 1111 ${resp}= Openbmc Delete Request ${entry_path} data=${data} 1112 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 1113 1114 1115############################################################################### 1116Get LED State XYZ 1117 [Documentation] Returns state of given LED. 1118 [Arguments] ${led_name} 1119 # Description of argument(s): 1120 # led_name Name of LED 1121 1122 ${state}= Read Attribute ${LED_GROUPS_URI}${led_name} Asserted 1123 [Return] ${state} 1124 1125 1126############################################################################### 1127Get BMC Version 1128 [Documentation] Returns BMC version from /etc/os-release. 1129 ... e.g. "v1.99.6-141-ge662190" 1130 1131 Open Connection And Log In 1132 ${cmd}= Set Variable grep ^VERSION_ID= /etc/os-release | cut -f 2 -d '=' 1133 ${output}= Execute Command On BMC ${cmd} 1134 [Return] ${output} 1135 1136 1137############################################################################### 1138Get Elog URL List 1139 [Documentation] Return error log entry list of URLs. 1140 1141 ${url_list}= Read Properties /xyz/openbmc_project/logging/entry/ 1142 Sort List ${url_list} 1143 [Return] ${url_list} 1144 1145 1146############################################################################### 1147