1*** Settings *** 2Resource ../lib/resource.txt 3Resource ../lib/rest_client.robot 4Resource ../lib/connection_client.robot 5Library String 6Library DateTime 7Library Process 8Library OperatingSystem 9Library gen_print.py 10Library gen_robot_print.py 11Library gen_cmd.py 12Library gen_robot_keyword.py 13Library bmc_ssh_utils.py 14Library utils.py 15 16*** Variables *** 17${pflash_cmd} /usr/sbin/pflash -r /dev/stdout -P VERSION 18${SYSTEM_SHUTDOWN_TIME} ${5} 19${dbuscmdBase} 20... dbus-send --system --print-reply --dest=${OPENBMC_BASE_DBUS}.settings.Host 21${dbuscmdGet} 22... ${SETTINGS_URI}host0 org.freedesktop.DBus.Properties.Get 23# Enable when ready with openbmc/openbmc-test-automation#203 24#${dbuscmdString}= string:"xyz.openbmc_project.settings.Host" string: 25${dbuscmdString}= string:"org.openbmc.settings.Host" string: 26 27# Assign default value to QUIET for programs which may not define it. 28${QUIET} ${0} 29${bmc_mem_free_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f4 30${bmc_mem_total_cmd}= free | tr -s ' ' | sed '/^Mem/!d' | cut -d" " -f2 31${bmc_cpu_usage_cmd}= top -n 1 | grep CPU: | cut -c 7-9 32${HOST_SETTING} ${SETTINGS_URI}host0 33# /run/initramfs/ro associate filesystem should be 100% full always 34${bmc_file_system_usage_cmd}= 35... df -h | grep -v /run/initramfs/ro | cut -c 52-54 | grep 100 | wc -l 36 37${BOOT_TIME} ${0} 38${BOOT_COUNT} ${0} 39${count} ${0} 40${devicetree_base} /sys/firmware/devicetree/base/model 41 42# Initialize default debug value to 0. 43${DEBUG} ${0} 44 45# These variables are used to straddle between new and old methods of setting 46# values. 47${boot_prog_method} ${EMPTY} 48 49${power_policy_setup} ${0} 50${bmc_power_policy_method} ${EMPTY} 51@{valid_power_policy_vars} RESTORE_LAST_STATE ALWAYS_POWER_ON 52... ALWAYS_POWER_OFF 53 54*** Keywords *** 55 56Verify PNOR Update 57 [Documentation] Verify that the PNOR is not corrupted. 58 # Example: 59 # FFS: Flash header not found. Code: 100 60 # Error 100 opening ffs ! 61 62 Open Connection And Log In 63 ${pnor_info}= Execute Command On BMC ${pflash_cmd} 64 Should Not Contain Any ${pnor_info} Flash header not found Error 65 66Get BMC System Model 67 [Documentation] Get the BMC model from the device tree. 68 69 ${bmc_model} ${stderr} ${rc}= BMC Execute Command 70 ... cat ${devicetree_base} | cut -d " " -f 1 return_stderr=True 71 Should Be Empty ${stderr} 72 Should Not Be Empty ${bmc_model} 73 [Return] ${bmc_model} 74 75Verify BMC System Model 76 [Documentation] Verify the BMC model with ${OPENBMC_MODEL}. 77 [Arguments] ${bmc_model} 78 79 ${tmp_bmc_model}= Fetch From Right ${OPENBMC_MODEL} / 80 ${tmp_bmc_model}= Fetch From Left ${tmp_bmc_model} . 81 ${ret}= Run Keyword And Return Status Should Contain ${bmc_model} 82 ... ${tmp_bmc_model} ignore_case=True 83 [Return] ${ret} 84 85Wait For Host To Ping 86 [Arguments] ${host} ${timeout}=${OPENBMC_REBOOT_TIMEOUT}min 87 ... ${interval}=5 sec 88 89 # host The DNS name or IP of the host to ping. 90 # timeout The amount of time after which attempts to ping cease. 91 # interval The amount of time in between attempts to ping. 92 93 Wait Until Keyword Succeeds ${timeout} ${interval} Ping Host ${host} 94 95Ping Host 96 [Arguments] ${host} 97 Should Not Be Empty ${host} msg=No host provided 98 ${RC} ${output}= Run and return RC and Output ping -c 4 ${host} 99 Log RC: ${RC}\nOutput:\n${output} 100 Should be equal ${RC} ${0} 101 102Get Boot Progress 103 [Documentation] Get the boot progress and return it. 104 [Arguments] ${quiet}=${QUIET} 105 106 # Description of argument(s): 107 # quiet Indicates whether this keyword should run without any output to 108 # the console. 109 110 Set Boot Progress Method 111 ${state}= Run Keyword If '${boot_prog_method}' == 'New' 112 ... New Get Boot Progress quiet=${quiet} 113 ... ELSE 114 ... Old Get Boot Progress quiet=${quiet} 115 116 [Return] ${state} 117 118Set Boot Progress Method 119 [Documentation] Set the boot_prog_method to either 'Old' or 'New'. 120 121 # The boot progress data has moved from an 'org' location to an 'xyz' 122 # location. This keyword will determine whether the new method of getting 123 # the boot progress is valid and will set the global boot_prog_method 124 # variable accordingly. If boot_prog_method is already set (either by a 125 # prior call to this function or via a -v parm), this keyword will simply 126 # return. 127 128 # Note: There are interim builds that contain boot_progress in both the old 129 # and the new location values. It is nearly impossible for this keyword to 130 # determine whether the old boot_progress or the new one is active. When 131 # using such builds where the old boot_progress is active, the only recourse 132 # users will have is that they may specify -v boot_prog_method:Old to force 133 # old behavior on such builds. 134 135 Run Keyword If '${boot_prog_method}' != '${EMPTY}' Return From Keyword 136 137 ${new_status} ${new_value}= Run Keyword And Ignore Error 138 ... New Get Boot Progress 139 # If the new style read fails, the method must necessarily be "Old". 140 Run Keyword If '${new_status}' == 'PASS' 141 ... Run Keywords 142 ... Set Global Variable ${boot_prog_method} New AND 143 ... Rqpvars boot_prog_method AND 144 ... Return From Keyword 145 146 # Default method is "Old". 147 Set Global Variable ${boot_prog_method} Old 148 Rqpvars boot_prog_method 149 150Old Get Boot Progress 151 [Documentation] Get the boot progress the old way (via org location). 152 [Arguments] ${quiet}=${QUIET} 153 154 # Description of argument(s): 155 # quiet Indicates whether this keyword should run without any output to 156 # the console. 157 158 ${state}= Read Attribute ${OPENBMC_BASE_URI}sensors/host/BootProgress 159 ... value quiet=${quiet} 160 161 [Return] ${state} 162 163New Get Boot Progress 164 [Documentation] Get the boot progress the new way (via xyz location). 165 [Arguments] ${quiet}=${QUIET} 166 167 # Description of argument(s): 168 # quiet Indicates whether this keyword should run without any output to 169 # the console. 170 171 ${state}= Read Attribute ${HOST_STATE_URI} BootProgress quiet=${quiet} 172 173 [Return] ${state.rsplit('.', 1)[1]} 174 175Is Power On 176 ${state}= Get Power State 177 Should be equal ${state} ${1} 178 179Is Power Off 180 ${state}= Get Power State 181 Should be equal ${state} ${0} 182 183Initiate Power On 184 [Documentation] Initiates the power on and waits until the Is Power On 185 ... keyword returns that the power state has switched to on. 186 [Arguments] ${wait}=${1} 187 188 @{arglist}= Create List 189 ${args}= Create Dictionary data=@{arglist} 190 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOn 191 ... data=${args} 192 should be equal as strings ${resp.status_code} ${HTTP_OK} 193 194 # Does caller want to wait for power on status? 195 Run Keyword If '${wait}' == '${0}' Return From Keyword 196 Wait Until Keyword Succeeds 3 min 10 sec Is Power On 197 198Initiate Power Off 199 [Documentation] Initiates the power off and waits until the Is Power Off 200 ... keyword returns that the power state has switched to off. 201 @{arglist}= Create List 202 ${args}= Create Dictionary data=@{arglist} 203 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ powerOff 204 ... data=${args} 205 should be equal as strings ${resp.status_code} ${HTTP_OK} 206 Wait Until Keyword Succeeds 1 min 10 sec Is Power Off 207 208Initiate OS Host Power Off 209 [Documentation] Initiate an OS reboot. 210 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 211 ... ${os_password}=${OS_PASSWORD} 212 213 # Description of arguments: 214 # os_host The DNS name or IP of the OS. 215 # os_username The username to be used to sign in to the OS. 216 # os_password The password to be used to sign in to the OS. 217 218 ${cmd_buf}= Run Keyword If '${os_username}' == 'root' 219 ... Set Variable shutdown 220 ... ELSE 221 ... Set Variable echo ${os_password} | sudo -S shutdown 222 223 ${output} ${stderr} ${rc}= OS Execute Command 224 ... ${cmd_buf} fork=${1} 225 226Initiate OS Host Reboot 227 [Documentation] Initiate an OS reboot. 228 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 229 ... ${os_password}=${OS_PASSWORD} 230 231 # Description of argument(s): 232 # os_host The host name or IP address of the OS. 233 # os_username The username to be used to sign in to the OS. 234 # os_password The password to be used to sign in to the OS. 235 236 ${cmd_buf}= Run Keyword If '${os_username}' == 'root' 237 ... Set Variable reboot 238 ... ELSE 239 ... Set Variable echo ${os_password} | sudo -S reboot 240 241 ${output} ${stderr} ${rc}= OS Execute Command 242 ... ${cmd_buf} fork=${1} 243 244Initiate Auto Reboot 245 [Documentation] Initiate an auto reboot. 246 247 # Set the auto reboot policy. 248 Set Auto Reboot ${1} 249 # Set the watchdog timer. Note: 5000 = milliseconds which is 5 seconds. 250 Trigger Host Watchdog Error 5000 251 252Trigger Warm Reset 253 log to console "Triggering warm reset" 254 ${data}= create dictionary data=@{EMPTY} 255 ${resp}= openbmc post request 256 ... ${OPENBMC_BASE_URI}control/bmc0/action/warmReset data=${data} 257 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 258 ${session_active}= Check If warmReset is Initiated 259 Run Keyword If '${session_active}' == '${True}' 260 ... Fail msg=warm reset didn't occur 261 262 Sleep ${SYSTEM_SHUTDOWN_TIME}min 263 Check If BMC Is Up 264 265Check OS 266 [Documentation] Attempts to ping the host OS and then checks that the host 267 ... OS is up by running an SSH command. 268 269 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 270 ... ${os_password}=${OS_PASSWORD} ${quiet}=${QUIET} 271 ... ${print_string}=${EMPTY} 272 [Teardown] SSHLibrary.Close Connection 273 274 # os_host The DNS name/IP of the OS host associated with our BMC. 275 # os_username The username to be used to sign on to the OS host. 276 # os_password The password to be used to sign on to the OS host. 277 # quiet Indicates whether this keyword should write to console. 278 # print_string A string to be printed before checking the OS. 279 280 rprint ${print_string} 281 282 # Attempt to ping the OS. Store the return code to check later. 283 ${ping_rc}= Run Keyword and Return Status Ping Host ${os_host} 284 285 SSHLibrary.Open connection ${os_host} 286 287 ${status} ${msg}= Run Keyword And Ignore Error Login ${os_username} 288 ... ${os_password} 289 ${err_msg1}= Sprint Error ${msg} 290 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 291 Run Keyword If '${status}' == 'FAIL' Fail msg=${err_msg} 292 ${output} ${stderr} ${rc}= Execute Command uptime return_stderr=True 293 ... return_rc=True 294 295 ${temp_msg}= Catenate Could not execute a command on the operating 296 ... system.\n 297 ${err_msg1}= Sprint Error ${temp_msg} 298 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 299 300 # If the return code returned by "Execute Command" is non-zero, this 301 # keyword will fail. 302 Should Be Equal ${rc} ${0} msg=${err_msg} 303 # We will likewise fail if there is any stderr data. 304 Should Be Empty ${stderr} 305 306 ${temp_msg}= Set Variable Could not ping the operating system.\n 307 ${err_msg1}= Sprint Error ${temp_msg} 308 ${err_msg}= Catenate SEPARATOR= \n ${err_msg1} 309 # We will likewise fail if the OS did not ping, as we could SSH but not 310 # ping 311 Should Be Equal As Strings ${ping_rc} ${TRUE} msg=${err_msg} 312 313Wait for OS 314 [Documentation] Waits for the host OS to come up via calls to "Check OS". 315 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 316 ... ${os_password}=${OS_PASSWORD} ${timeout}=${OS_WAIT_TIMEOUT} 317 ... ${quiet}=${0} 318 [Teardown] rprintn 319 320 # os_host The DNS name or IP of the OS host associated with our 321 # BMC. 322 # os_username The username to be used to sign on to the OS host. 323 # os_password The password to be used to sign on to the OS host. 324 # timeout The timeout in seconds indicating how long you're 325 # willing to wait for the OS to respond. 326 # quiet Indicates whether this keyword should write to console. 327 328 # The interval to be used between calls to "Check OS". 329 ${interval}= Set Variable 5 330 331 ${message}= Catenate Checking every ${interval} seconds for up to 332 ... ${timeout} seconds for the operating system to communicate. 333 rqprint_timen ${message} 334 335 Wait Until Keyword Succeeds ${timeout} sec ${interval} Check OS 336 ... ${os_host} ${os_username} ${os_password} 337 ... print_string=\# 338 339 rqprintn 340 341 rqprint_timen The operating system is now communicating. 342 343Get BMC State Deprecated 344 [Documentation] Returns the state of the BMC as a string. (i.e: BMC_READY) 345 [Arguments] ${quiet}=${QUIET} 346 347 @{arglist}= Create List 348 ${args}= Create Dictionary data=@{arglist} 349 ${resp}= Call Method ${OPENBMC_BASE_URI}managers/System/ getSystemState 350 ... data=${args} quiet=${quiet} 351 Should be equal as strings ${resp.status_code} ${HTTP_OK} 352 ${content}= to json ${resp.content} 353 [Return] ${content["data"]} 354 355Get Power State 356 [Documentation] Returns the power state as an integer. Either 0 or 1. 357 [Arguments] ${quiet}=${QUIET} 358 359 @{arglist}= Create List 360 ${args}= Create Dictionary data=@{arglist} 361 362 ${resp}= Call Method ${OPENBMC_BASE_URI}control/chassis0/ getPowerState 363 ... data=${args} quiet=${quiet} 364 Should be equal as strings ${resp.status_code} ${HTTP_OK} 365 ${content}= to json ${resp.content} 366 [Return] ${content["data"]} 367 368Clear BMC Record Log 369 [Documentation] Clears all the event logs on the BMC. This would be 370 ... equivalent to ipmitool sel clear. 371 @{arglist}= Create List 372 ${args}= Create Dictionary data=@{arglist} 373 ${resp}= Call Method 374 ... ${OPENBMC_BASE_URI}records/events/ clear data=${args} 375 should be equal as strings ${resp.status_code} ${HTTP_OK} 376 377Copy PNOR to BMC 378 Import Library SCPLibrary WITH NAME scp 379 Open Connection for SCP 380 Log Copying ${PNOR_IMAGE_PATH} to /tmp 381 scp.Put File ${PNOR_IMAGE_PATH} /tmp 382 383Flash PNOR 384 [Documentation] Calls flash bios update method to flash PNOR image 385 [Arguments] ${pnor_image} 386 @{arglist}= Create List ${pnor_image} 387 ${args}= Create Dictionary data=@{arglist} 388 ${resp}= Call Method ${OPENBMC_BASE_URI}control/flash/bios/ update 389 ... data=${args} 390 should be equal as strings ${resp.status_code} ${HTTP_OK} 391 Wait Until Keyword Succeeds 2 min 10 sec Is PNOR Flashing 392 393Get Flash BIOS Status 394 [Documentation] Returns the status of the flash BIOS API as a string. For 395 ... example 'Flashing', 'Flash Done', etc 396 ${data}= Read Properties ${OPENBMC_BASE_URI}control/flash/bios 397 [Return] ${data['status']} 398 399Is PNOR Flashing 400 [Documentation] Get BIOS 'Flashing' status. This indicates that PNOR 401 ... flashing has started. 402 ${status}= Get Flash BIOS Status 403 Should Contain ${status} Flashing 404 405Is PNOR Flash Done 406 [Documentation] Get BIOS 'Flash Done' status. This indicates that the 407 ... PNOR flashing has completed. 408 ${status}= Get Flash BIOS Status 409 should be equal as strings ${status} Flash Done 410 411Is System State Host Booted 412 [Documentation] Checks whether system state is HOST_BOOTED. 413 ${state}= Get BMC State Deprecated 414 should be equal as strings ${state} HOST_BOOTED 415 416Is OS Starting 417 [Documentation] Check if boot progress is OS starting. 418 ${boot_progress}= Get Boot Progress 419 Should Be Equal ${boot_progress} OSStart 420 421Is OS Off 422 [Documentation] Check if boot progress is "Off". 423 ${boot_progress}= Get Boot Progress 424 Should Be Equal ${boot_progress} Off 425 426Get Boot Progress To OS Starting State 427 [Documentation] Get the system to a boot progress state of 'FW Progress, 428 ... Starting OS'. 429 430 ${boot_progress}= Get Boot Progress 431 Run Keyword If '${boot_progress}' == 'OSStart' 432 ... Log Host is already in OS starting state 433 ... ELSE 434 ... Run Keywords Initiate Host PowerOff AND Initiate Host Boot 435 ... AND Wait Until Keyword Succeeds 10 min 10 sec Is OS Starting 436 437Verify Ping and REST Authentication 438 ${l_ping}= Run Keyword And Return Status 439 ... Ping Host ${OPENBMC_HOST} 440 Run Keyword If '${l_ping}' == '${False}' 441 ... Fail msg=Ping Failed 442 443 ${l_rest}= Run Keyword And Return Status 444 ... Initialize OpenBMC 445 Run Keyword If '${l_rest}' == '${False}' 446 ... Fail msg=REST Authentication Failed 447 448 # Just to make sure the SSH is working for SCP 449 Open Connection And Log In 450 ${system} ${stderr}= Execute Command hostname return_stderr=True 451 Should Be Empty ${stderr} 452 453Check If BMC is Up 454 [Documentation] Wait for Host to be online. Checks every X seconds 455 ... interval for Y minutes and fails if timed out. 456 ... Default MAX timedout is 10 min, interval 10 seconds. 457 [Arguments] ${max_timeout}=${OPENBMC_REBOOT_TIMEOUT} min 458 ... ${interval}=10 sec 459 460 Wait Until Keyword Succeeds 461 ... ${max_timeout} ${interval} Verify Ping and REST Authentication 462 463 464Check If warmReset is Initiated 465 [Documentation] Ping would be still alive, so try SSH to connect 466 ... if fails the ports are down indicating reboot 467 ... is in progress 468 469 # Warm reset adds 3 seconds delay before forcing reboot 470 # To minimize race conditions, we wait for 7 seconds 471 Sleep 7s 472 ${alive}= Run Keyword and Return Status 473 ... Open Connection And Log In 474 Return From Keyword If '${alive}' == '${False}' ${False} 475 [Return] ${True} 476 477Flush REST Sessions 478 [Documentation] Removes all the active session objects 479 Delete All Sessions 480 481Initialize DBUS cmd 482 [Documentation] Initialize dbus string with property string to extract 483 [Arguments] ${boot_property} 484 ${cmd}= Catenate ${dbuscmdBase} ${dbuscmdGet} ${dbuscmdString} 485 ${cmd}= Catenate ${cmd}${boot_property} 486 Set Global Variable ${dbuscmd} ${cmd} 487 488Create OS Console File Path 489 [Documentation] Create OS console file path name and return it. 490 [Arguments] ${log_file_path}=${EMPTY} 491 492 # Description of arguements: 493 # file_path The caller's candidate value. If this value is ${EMPTY}, this 494 # keyword will compose a file path name. Otherwise, this 495 # keyword will use the caller's file_path value. In either 496 # case, the value will be returned. 497 498 ${default_file_path}= Catenate /tmp/${OPENBMC_HOST}_os_console 499 ${log_file_path}= Set Variable If '${log_file_path}' == '${EMPTY}' 500 ... ${default_file_path} ${log_file_path} 501 502 [Return] ${log_file_path} 503 504Create OS Console Command String 505 [Documentation] Return a command string to start OS console logging. 506 507 ${ssh_pw_file_path}= Set Variable ssh_pw 508 509 # First make sure that the ssh_pw program is available. 510 ${cmd_buf}= Catenate which ssh_pw 2>&1 511 Rdpissuing ${cmd_buf} 512 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 513 Rdpvars rc output 514 515 ${ssh_pw_file_path}= Run Keyword If ${rc} != ${0} 516 ... Set Variable ${EXECDIR}${/}bin/ssh_pw 517 518 ${cmd_buf}= Catenate ${ssh_pw_file_path} ${OPENBMC_PASSWORD} -p 2200 519 ... -o "StrictHostKeyChecking no" ${OPENBMC_USERNAME}@${OPENBMC_HOST} 520 521 [Return] ${cmd_buf} 522 523Get SOL Console Pid 524 [Documentation] Get the pid of the active sol conole job. 525 526 # Find the pid of the active system console logging session (if any). 527 ${search_string}= Create OS Console Command String 528 # At least in some cases, ps output does not show double quotes so we must 529 # replace them in our search string with the regexes to indicate that they 530 # are optional. 531 ${search_string}= Replace String ${search_string} " ["]? 532 ${cmd_buf}= Catenate echo $(ps -ef | egrep '${search_string}' 533 ... | egrep -v grep | cut -c10-14) 534 Rdpissuing ${cmd_buf} 535 ${rc} ${os_con_pid}= Run And Return Rc And Output ${cmd_buf} 536 Rdpvars os_con_pid 537 # If rc is not zero it just means that there is no OS Console process 538 # running. 539 540 [Return] ${os_con_pid} 541 542 543Stop SOL Console Logging 544 [Documentation] Stop system console logging and return log output. 545 [Arguments] ${log_file_path}=${EMPTY} 546 ... ${targ_file_path}=${EXECDIR}${/}logs${/} 547 ... ${return_data}=${1} 548 549 # If there are muliple system console processes, they will all be stopped. 550 # If there is no existing log file this keyword will return an error 551 # message to that effect (and write that message to targ_file_path, if 552 # specified). 553 # NOTE: This keyword will not fail if there is no running system console 554 # process. 555 556 # Description of arguments: 557 # log_file_path The file path that was used to call "Start SOL 558 # Console Logging". See that keyword (above) for details. 559 # targ_file_path If specified, the file path to which the source 560 # file path (i.e. "log_file_path") should be copied. 561 # return_data If this is set to ${1}, this keyword will return the SOL 562 # data to the caller as a unicode string. 563 564 ${log_file_path}= Create OS Console File Path ${log_file_path} 565 566 ${os_con_pid}= Get SOL Console Pid 567 568 ${cmd_buf}= Catenate kill -9 ${os_con_pid} 569 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpissuing ${cmd_buf} 570 ${rc} ${output}= Run Keyword If '${os_con_pid}' != '${EMPTY}' 571 ... Run And Return Rc And Output ${cmd_buf} 572 Run Keyword If '${os_con_pid}' != '${EMPTY}' Rdpvars rc output 573 574 Run Keyword If '${targ_file_path}' != '${EMPTY}' 575 ... Run Keyword And Ignore Error 576 ... Copy File ${log_file_path} ${targ_file_path} 577 578 ${output}= Set Variable ${EMPTY} 579 ${loc_quiet}= Evaluate ${debug}^1 580 ${rc} ${output}= Run Keyword If '${return_data}' == '${1}' 581 ... Cmd Fnc cat ${log_file_path} quiet=${loc_quiet} print_output=${0} 582 583 [Return] ${output} 584 585Start SOL Console Logging 586 [Documentation] Start system console log to file. 587 [Arguments] ${log_file_path}=${EMPTY} ${return_data}=${1} 588 589 # This keyword will first call "Stop SOL Console Logging". Only then will 590 # it start SOL console logging. The data returned by "Stop SOL Console 591 # Logging" will in turn be returned by this keyword. 592 593 # Description of arguments: 594 # log_file_path The file path to which system console log data should be 595 # written. Note that this path is taken to be a location 596 # on the machine where this program is running rather than 597 # on the Open BMC system. 598 # return_data If this is set to ${1}, this keyword will return any SOL 599 # data to the caller as a unicode string. 600 601 ${log_file_path}= Create OS Console File Path ${log_file_path} 602 603 ${log_output}= Stop SOL Console Logging ${log_file_path} 604 ... return_data=${return_data} 605 606 # Validate by making sure we can create the file. Problems creating the 607 # file would not be noticed by the subsequent ssh command because we fork 608 # the command. 609 Create File ${log_file_path} 610 ${sub_cmd_buf}= Create OS Console Command String 611 # Routing stderr to stdout so that any startup error text will go to the 612 # output file. 613 # TODO: Doesn't work with tox so reverting temporarily. 614 # nohup detaches the process completely from our pty. 615 #${cmd_buf}= Catenate nohup ${sub_cmd_buf} &> ${log_file_path} & 616 ${cmd_buf}= Catenate ${sub_cmd_buf} > ${log_file_path} 2>&1 & 617 Rdpissuing ${cmd_buf} 618 ${rc} ${output}= Run And Return Rc And Output ${cmd_buf} 619 # Because we are forking this command, we essentially will never get a 620 # non-zero return code or any output. 621 Should Be Equal ${rc} ${0} 622 623 Sleep 1 624 ${os_con_pid}= Get SOL Console Pid 625 626 Should Not Be Empty ${os_con_pid} 627 628 [Return] ${log_output} 629 630Get Time Stamp 631 [Documentation] Get the current time stamp data 632 ${cur_time}= Get Current Date result_format=%Y%m%d%H%M%S%f 633 [Return] ${cur_time} 634 635 636Verify BMC State 637 [Documentation] Get the BMC state and verify if the current 638 ... BMC state is as expected. 639 [Arguments] ${expected} 640 641 ${current}= Get BMC State Deprecated 642 Should Contain ${expected} ${current} 643 644Start Journal Log 645 [Documentation] Start capturing journal log to a file in /tmp using 646 ... journalctl command. By default journal log is collected 647 ... at /tmp/journal_log else user input location. 648 ... The File is appended with datetime. 649 [Arguments] ${file_path}=/tmp/journal_log 650 651 Open Connection And Log In 652 653 ${cur_time}= Get Time Stamp 654 Set Global Variable ${LOG_TIME} ${cur_time} 655 Start Command 656 ... journalctl -f > ${file_path}-${LOG_TIME} 657 Log Journal Log Started: ${file_path}-${LOG_TIME} 658 659 660Stop Journal Log 661 [Documentation] Stop journalctl process if its running. 662 ... By default return log from /tmp/journal_log else 663 ... user input location. 664 [Arguments] ${file_path}=/tmp/journal_log 665 666 Open Connection And Log In 667 668 ${rc}= 669 ... Execute Command 670 ... ps ax | grep journalctl | grep -v grep 671 ... return_stdout=False return_rc=True 672 673 Return From Keyword If '${rc}' == '${1}' 674 ... No journal log process running 675 676 ${output} ${stderr}= 677 ... Execute Command killall journalctl 678 ... return_stderr=True 679 Should Be Empty ${stderr} 680 681 ${journal_log} ${stderr}= 682 ... Execute Command 683 ... cat ${file_path}-${LOG_TIME} 684 ... return_stderr=True 685 Should Be Empty ${stderr} 686 687 Log ${journal_log} 688 689 Execute Command rm ${file_path}-${LOG_TIME} 690 691 [Return] ${journal_log} 692 693Mac Address To Hex String 694 [Documentation] Converts MAC address into hex format. 695 ... Example 696 ... Given the following MAC: 00:01:6C:80:02:78 697 ... This keyword will return: 0x00 0x01 0x6C 0x80 0x02 0x78 698 ... Description of arguments: 699 ... i_macaddress MAC address in the following format 700 ... 00:01:6C:80:02:78 701 [Arguments] ${i_macaddress} 702 703 ${mac_hex}= Catenate 0x${i_macaddress.replace(':', ' 0x')} 704 [Return] ${mac_hex} 705 706IP Address To Hex String 707 [Documentation] Converts IP address into hex format. 708 ... Example: 709 ... Given the following IP: 10.3.164.100 710 ... This keyword will return: 0xa 0x3 0xa4 0xa0 711 ... Description of arguments: 712 ... i_ipaddress IP address in the following format 713 ... 10.10.10.10 714 [Arguments] ${i_ipaddress} 715 716 @{ip}= Split String ${i_ipaddress} . 717 ${index}= Set Variable ${0} 718 719 :FOR ${item} IN @{ip} 720 \ ${hex}= Convert To Hex ${item} prefix=0x lowercase=yes 721 \ Set List Value ${ip} ${index} ${hex} 722 \ ${index}= Set Variable ${index + 1} 723 ${ip_hex}= Catenate @{ip} 724 [Return] ${ip_hex} 725 726BMC CPU Performance Check 727 [Documentation] Minimal 10% of proc should be free in this instance 728 729 ${bmc_cpu_usage_output} ${stderr}= Execute Command ${bmc_cpu_usage_cmd} 730 ... return_stderr=True 731 Should be empty ${stderr} 732 ${bmc_cpu_percentage}= Fetch From Left ${bmc_cpu_usage_output} % 733 Should be true ${bmc_cpu_percentage} < 90 734 735BMC Mem Performance Check 736 [Documentation] Minimal 10% of memory should be free in this instance 737 738 ${bmc_mem_free_output} ${stderr}= Execute Command ${bmc_mem_free_cmd} 739 ... return_stderr=True 740 Should be empty ${stderr} 741 742 ${bmc_mem_total_output} ${stderr}= Execute Command ${bmc_mem_total_cmd} 743 ... return_stderr=True 744 Should be empty ${stderr} 745 746 ${bmc_mem_percentage}= Evaluate ${bmc_mem_free_output}*100 747 ${bmc_mem_percentage}= Evaluate 748 ... ${bmc_mem_percentage}/${bmc_mem_total_output} 749 Should be true ${bmc_mem_percentage} > 10 750 751BMC File System Usage Check 752 [Documentation] Check the file system space. None should be 100% full 753 ... except /run/initramfs/ro 754 ${bmc_fs_usage_output} ${stderr}= Execute Command 755 ... ${bmc_file_system_usage_cmd} return_stderr=True 756 Should Be Empty ${stderr} 757 Should Be True ${bmc_fs_usage_output}==0 758 759Check BMC CPU Performance 760 [Documentation] Minimal 10% of proc should be free in 3 sample 761 :FOR ${var} IN Range 1 4 762 \ BMC CPU Performance check 763 764Check BMC Mem Performance 765 [Documentation] Minimal 10% of memory should be free 766 767 :FOR ${var} IN Range 1 4 768 \ BMC Mem Performance check 769 770Check BMC File System Performance 771 [Documentation] Check for file system usage for 4 times 772 773 :FOR ${var} IN Range 1 4 774 \ BMC File System Usage check 775 776Get URL List 777 [Documentation] Return list of URLs under given URL. 778 [Arguments] ${openbmc_url} 779 # Description of argument(s): 780 # openbmc_url URL for list operation (e.g. 781 # /xyz/openbmc_project/inventory). 782 783 ${url_list}= Read Properties ${openbmc_url}/list quiet=${1} 784 Sort List ${url_list} 785 [Return] ${url_list} 786 787Get Endpoint Paths 788 [Documentation] Returns all url paths ending with given endpoint 789 ... Example: 790 ... Given the following endpoint: cpu 791 ... This keyword will return: list of all urls ending with 792 ... cpu - 793 ... /org/openbmc/inventory/system/chassis/motherboard/cpu0, 794 ... /org/openbmc/inventory/system/chassis/motherboard/cpu1 795 ... Description of arguments: 796 ... path URL path for enumeration 797 ... endpoint string for which url path ending 798 [Arguments] ${path} ${endpoint} 799 800 ${resp}= Read Properties ${path}/enumerate timeout=30 801 log Dictionary ${resp} 802 803 ${list}= Get Dictionary Keys ${resp} 804 ${resp}= Get Matches ${list} regexp=^.*[0-9a-z_].${endpoint}[0-9]*$ 805 [Return] ${resp} 806 807Check Zombie Process 808 [Documentation] Check if any defunct process exist or not on BMC 809 ${count} ${stderr} ${rc}= Execute Command ps -o stat | grep Z | wc -l 810 ... return_stderr=True return_rc=True 811 Should Be True ${count}==0 812 Should Be Empty ${stderr} 813 814Prune Journal Log 815 [Documentation] Prune archived journal logs. 816 [Arguments] ${vacuum_size}=1M 817 818 # This keyword can be used to prevent the journal 819 # log from filling up the /run filesystem. 820 # This command will retain only the latest logs 821 # of the user specified size. 822 823 Open Connection And Log In 824 ${output} ${stderr} ${rc}= 825 ... Execute Command 826 ... journalctl --vacuum-size=${vacuum_size} 827 ... return_stderr=True return_rc=True 828 829 Should Be Equal ${rc} ${0} msg=${stderr} 830 831Set BMC Power Policy 832 [Documentation] Set the given BMC power policy. 833 [Arguments] ${policy} 834 835 # Note that this function will translate the old style "RESTORE_LAST_STATE" 836 # policy to the new style "xyz.openbmc_project.Control.Power.RestorePolicy. 837 # Policy.Restore" for you. 838 839 # Description of argument(s): 840 # policy Power restore policy (e.g "RESTORE_LAST_STATE", 841 # ${RESTORE_LAST_STATE}). 842 843 # Set the bmc_power_policy_method to either 'Old' or 'New'. 844 Set Power Policy Method 845 # This translation helps bridge between old and new method for calling. 846 ${policy}= Translate Power Policy Value ${policy} 847 # Run the appropriate keyword. 848 Run Key ${bmc_power_policy_method} Set Power Policy \ ${policy} 849 ${currentPolicy}= Get System Power Policy 850 Should Be Equal ${currentPolicy} ${policy} 851 852New Set Power Policy 853 [Documentation] Set the given BMC power policy (new method). 854 [Arguments] ${policy} 855 856 # Description of argument(s): 857 # policy Power restore policy (e.g. ${RESTORE_LAST_STATE}). 858 859 ${valueDict}= Create Dictionary data=${policy} 860 Write Attribute 861 ... ${POWER_RESTORE_URI} PowerRestorePolicy data=${valueDict} 862 863Old Set Power Policy 864 [Documentation] Set the given BMC power policy (old method). 865 [Arguments] ${policy} 866 867 # Description of argument(s): 868 # policy Power restore policy (e.g. "RESTORE_LAST_STATE"). 869 870 ${valueDict}= create dictionary data=${policy} 871 Write Attribute ${HOST_SETTING} power_policy data=${valueDict} 872 873Get System Power Policy 874 [Documentation] Get the BMC power policy. 875 876 # Set the bmc_power_policy_method to either 'Old' or 'New'. 877 Set Power Policy Method 878 ${cmd_buf}= Create List ${bmc_power_policy_method} Get Power Policy 879 # Run the appropriate keyword. 880 ${currentPolicy}= Run Keyword @{cmd_buf} 881 [Return] ${currentPolicy} 882 883New Get Power Policy 884 [Documentation] Get the BMC power policy (new method). 885 ${currentPolicy}= Read Attribute ${POWER_RESTORE_URI} PowerRestorePolicy 886 [Return] ${currentPolicy} 887 888Old Get Power Policy 889 [Documentation] Get the BMC power policy (old method). 890 ${currentPolicy}= Read Attribute ${HOST_SETTING} power_policy 891 [Return] ${currentPolicy} 892 893Get Auto Reboot 894 [Documentation] Returns auto reboot setting. 895 ${setting}= Read Attribute ${CONTROL_HOST_URI}/auto_reboot AutoReboot 896 [Return] ${setting} 897 898Set Auto Reboot 899 [Documentation] Set the given auto reboot setting. 900 [Arguments] ${setting} 901 # setting auto reboot's setting, i.e. 1 for enabling and 0 for disabling. 902 903 ${valueDict}= Set Variable ${setting} 904 ${data}= Create Dictionary data=${valueDict} 905 Write Attribute ${CONTROL_HOST_URI}/auto_reboot AutoReboot data=${data} 906 ${current_setting}= Get Auto Reboot 907 Should Be Equal As Integers ${current_setting} ${setting} 908 909 910Set BMC Reset Reference Time 911 [Documentation] Set current boot time as a reference and increment 912 ... boot count. 913 914 ${cur_btime}= Get BMC Boot Time 915 Run Keyword If ${BOOT_TIME} == ${0} and ${BOOT_COUNT} == ${0} 916 ... Set Global Variable ${BOOT_TIME} ${cur_btime} 917 ... ELSE IF ${cur_btime} > ${BOOT_TIME} 918 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 919 ... AND 920 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 921 922Get BMC Boot Time 923 [Documentation] Get boot time from /proc/stat. 924 925 Open Connection And Log In 926 ${output} ${stderr}= 927 ... Execute Command egrep '^btime ' /proc/stat | cut -f 2 -d ' ' 928 ... return_stderr=True 929 Should Be Empty ${stderr} 930 ${btime}= Convert To Integer ${output} 931 [Return] ${btime} 932 933Execute Command On BMC 934 [Documentation] Execute given command on BMC and return output. 935 [Arguments] ${command} 936 ${stdout} ${stderr}= Execute Command ${command} return_stderr=True 937 Should Be Empty ${stderr} 938 [Return] ${stdout} 939 940 941Enable Core Dump On BMC 942 [Documentation] Enable core dump collection. 943 Open Connection And Log In 944 ${core_pattern}= Execute Command On BMC 945 ... echo '/tmp/core_%e.%p' | tee /proc/sys/kernel/core_pattern 946 Should Be Equal As Strings ${core_pattern} /tmp/core_%e.%p 947 948Get Number Of BMC Core Dump Files 949 [Documentation] Get number of core dump files on BMC. 950 Open Connection And Log In 951 ${num_of_core_dump}= Execute Command 952 ... ls /tmp/core* 2>/dev/null | wc -l 953 [Return] ${num_of_core_dump} 954 955Set Core Dump File Size Unlimited 956 [Documentation] Set core dump file size to unlimited. 957 Open Connection And Log In 958 Execute Command On BMC 959 ... ulimit -c unlimited 960 961Check For Core Dumps 962 [Documentation] Check for any core dumps exist. 963 ${output}= Get Number Of BMC Core Dump Files 964 Run Keyword If ${output} > 0 965 ... Log **Warning** BMC core dump files exist level=WARN 966 967Trigger Host Watchdog Error 968 [Documentation] Inject host watchdog timeout error via REST. 969 [Arguments] ${milliseconds}=1000 ${sleep_time}=5s 970 # Description of argument(s): 971 # milliseconds The time watchdog timer value in milliseconds (e.g. 1000 = 972 # 1 second). 973 # sleep_time Time delay for host watchdog error to get injected. 974 # Default is 5 seconds. 975 976 ${data}= Create Dictionary data=${True} 977 Write Attribute /xyz/openbmc_project/watchdog/host0 Enabled data=${data} 978 979 ${data}= Create Dictionary data=${milliseconds} 980 Write Attribute /xyz/openbmc_project/watchdog/host0 TimeRemaining data=${data} 981 982 Sleep ${sleep_time} 983 984Login To OS Host 985 [Documentation] Login to OS Host. 986 [Arguments] ${os_host}=${OS_HOST} ${os_username}=${OS_USERNAME} 987 ... ${os_password}=${OS_PASSWORD} 988 # Desription of arguments: 989 # ${os_host} IP address of the OS Host. 990 # ${os_username} OS Host Login user name. 991 # ${os_password} OS Host Login passwrd. 992 993 ${os_state}= Run Keyword And Return Status Ping Host ${os_host} 994 Run Keyword If '${os_state}' == 'False' 995 ... Run Keywords Initiate Host Reboot AND 996 ... Is Host Running AND 997 ... Wait for OS ${os_host} ${os_username} ${os_password} 998 999 SSHLibrary.Open Connection ${os_host} 1000 ${resp}= Login ${os_username} ${os_password} 1001 [Return] ${resp} 1002 1003Configure Initial Settings 1004 [Documentation] Restore old IP and route. 1005 ... This keyword requires initial settings viz IP address, 1006 ... Network Mask, default gatway and serial console IP and port 1007 ... information which should be provided in command line. 1008 1009 [Arguments] ${host}=${OPENBMC_HOST} ${mask}=${NET_MASK} 1010 ... ${gw_ip}=${GW_IP} 1011 1012 # Open telnet connection and ignore the error, in case telnet session is 1013 # already opened by the program calling this keyword. 1014 1015 Run Keyword And Ignore Error Open Telnet Connection to BMC Serial Console 1016 Telnet.write ifconfig eth0 ${host} netmask ${mask} 1017 Telnet.write route add default gw ${gw_ip} 1018 1019Install Debug Tarball On BMC 1020 [Documentation] Copy the debug tar file to BMC and install. 1021 [Arguments] ${tarball_file_path}=${EXECDIR}/obmc-phosphor-debug-tarball-witherspoon.tar.xz 1022 ... ${targ_tarball_dir_path}=/home/root/tarball/ 1023 1024 # Description of arguments: 1025 # tarball_file_path Path of the debug tarball file. 1026 # The tar file is downloaded from the build page 1027 # https://openpower.xyz/job/openbmc-build/ 1028 # obmc-phosphor-debug-tarball-witherspoon.tar.xz 1029 # 1030 # targ_tarball_dir_path The directory path where the tarball is to be 1031 # installed. 1032 1033 OperatingSystem.File Should Exist ${tarball_file_path} 1034 ... msg=${tarball_file_path} doesn't exist. 1035 1036 # Upload the file to BMC. 1037 Import Library SCPLibrary WITH NAME scp 1038 Open Connection for SCP 1039 scp.Put File ${tarball_file_path} /tmp/debug-tarball.tar.xz 1040 1041 # Create tarball directory and install. 1042 Open Connection And Log In 1043 Execute Command On BMC mkdir -p ${targ_tarball_dir_path} 1044 Execute Command On BMC 1045 ... tar -xf /tmp/debug-tarball.tar.xz -C ${targ_tarball_dir_path} 1046 1047 # Create symlink to callout-test binary. 1048 Execute Command On BMC 1049 ... ln -s ${targ_tarball_dir_path}/bin/callout-test /usr/bin/callout-test 1050 1051 # Create symlink to logging-test binary. 1052 Execute Command On BMC 1053 ... ln -s ${targ_tarball_dir_path}/bin/logging-test /usr/bin/logging-test 1054 1055 # Remove the tarball file from BMC 1056 Execute Command On BMC rm /tmp/debug-tarball.tar.xz 1057 1058 1059Get BMC Boot Count 1060 [Documentation] Get BMC boot count based on boot time. 1061 ${cur_btime}= Get BMC Boot Time 1062 1063 # Set global variable BOOT_TIME to current boot time if current boot time 1064 # is changed. Also increase value of global variable BOOT_COUNT by 1. 1065 Run Keyword If ${cur_btime} > ${BOOT_TIME} 1066 ... Run Keywords Set Global Variable ${BOOT_TIME} ${cur_btime} 1067 ... AND 1068 ... Set Global Variable ${BOOT_COUNT} ${BOOT_COUNT + 1} 1069 [Return] ${BOOT_COUNT} 1070 1071Set BMC Boot Count 1072 [Documentation] Set BMC boot count to given value. 1073 [Arguments] ${count} 1074 1075 # Description of arguments: 1076 # count boot count value. 1077 ${cur_btime}= Get BMC Boot Time 1078 1079 # Set global variable BOOT_COUNT to given value. 1080 Set Global Variable ${BOOT_COUNT} ${count} 1081 1082 # Set BOOT_TIME variable to current boot time. 1083 Set Global Variable ${BOOT_COUNT} ${count} 1084 1085Get System LED State 1086 [Documentation] Return the state of given system LED. 1087 [Arguments] ${led_name} 1088 1089 # Description of argument(s): 1090 # led_name System LED name (e.g. heartbeat, identify, beep). 1091 1092 ${state}= Read Attribute ${LED_PHYSICAL_URI}${led_name} State 1093 [Return] ${state.rsplit('.', 1)[1]} 1094 1095############################################################################### 1096Delete Error Logs 1097 [Documentation] Delete error logs. 1098 1099 # Check if error logs entries exist, if not return. 1100 ${resp}= OpenBMC Get Request ${BMC_LOGGING_ENTRY}${/}list quiet=${1} 1101 Return From Keyword If ${resp.status_code} == ${HTTP_NOT_FOUND} 1102 1103 # Get the list of error logs entries and delete them all. 1104 ${elog_entries}= Get URL List ${BMC_LOGGING_ENTRY} 1105 :FOR ${entry} IN @{elog_entries} 1106 \ Delete Error Log Entry ${entry} 1107 1108############################################################################### 1109Delete Error Log Entry 1110 [Documentation] Delete error log entry. 1111 [Arguments] ${entry_path} 1112 1113 # Description of argument(s): 1114 # entry_path Delete an error log entry. 1115 # Ex. /xyz/openbmc_project/logging/entry/1 1116 1117 # Skip delete if entry URI is a callout. 1118 # Example: /xyz/openbmc_project/logging/entry/1/callout 1119 Return From Keyword If '${entry_path.rsplit('/', 1)[1]}' == 'callout' 1120 1121 ${data}= Create Dictionary data=@{EMPTY} 1122 ${resp}= Openbmc Delete Request ${entry_path} data=${data} 1123 Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 1124 1125 1126############################################################################### 1127Get LED State XYZ 1128 [Documentation] Returns state of given LED. 1129 [Arguments] ${led_name} 1130 # Description of argument(s): 1131 # led_name Name of LED 1132 1133 ${state}= Read Attribute ${LED_GROUPS_URI}${led_name} Asserted 1134 [Return] ${state} 1135 1136 1137############################################################################### 1138Get BMC Version 1139 [Documentation] Returns BMC version from /etc/os-release. 1140 ... e.g. "v1.99.6-141-ge662190" 1141 1142 Open Connection And Log In 1143 ${cmd}= Set Variable grep ^VERSION_ID= /etc/os-release | cut -f 2 -d '=' 1144 ${output}= Execute Command On BMC ${cmd} 1145 [Return] ${output} 1146 1147 1148############################################################################### 1149Get Elog URL List 1150 [Documentation] Return error log entry list of URLs. 1151 1152 ${url_list}= Read Properties /xyz/openbmc_project/logging/entry/ 1153 Sort List ${url_list} 1154 [Return] ${url_list} 1155 1156 1157############################################################################### 1158