1*** Settings *** 2Documentation Methods to execute commands on BMC and collect 3... data to a list of FFDC files 4 5Resource openbmc_ffdc_utils.robot 6Resource rest_client.robot 7Resource utils.robot 8Resource list_utils.robot 9Library SSHLibrary 10Library OperatingSystem 11Library Collections 12Library String 13Library gen_print.py 14Library gen_robot_keyword.py 15 16*** Keywords *** 17 18################################################################ 19# Method : Call FFDC Methods # 20# Execute the user define keywords from the FFDC List # 21# Unlike any other keywords this will call into the # 22# list of keywords defined in the FFDC list at one go # 23################################################################ 24 25Call FFDC Methods 26 [Documentation] Call into FFDC Keyword index list. 27 [Arguments] ${ffdc_function_list}=${EMPTY} 28 29 # Description of argument(s): 30 # ffdc_function_list A colon-delimited list naming the kinds of FFDC that 31 # are to be collected 32 # (e.g. "FFDC Generic Report:BMC Specific Files"). 33 # Acceptable values can be found in the description 34 # field of FFDC_METHOD_CALL in 35 # lib/openbmc_ffdc_list.py. Those values can be 36 # obtained via a call to 'Get FFDC Method Desc' (also 37 # from lib/openbmc_ffdc_list.py). 38 39 @{entries}= Get FFDC Method Index 40 # Example entries: 41 # entries: 42 # entries[0]: BMC LOGS 43 44 @{ffdc_file_list}= Create List 45 :FOR ${index} IN @{entries} 46 \ ${ffdc_file_sub_list}= Method Call Keyword List ${index} 47 ... ${ffdc_function_list} 48 \ ${ffdc_file_list}= Smart Combine Lists ${ffdc_file_list} 49 ... ${ffdc_file_sub_list} 50 51 Run Key U SSHLibrary.Close All Connections 52 53 [Return] ${ffdc_file_list} 54 55Method Call Keyword List 56 [Documentation] Process FFDC request and return a list of generated files. 57 [Arguments] ${index} ${ffdc_function_list}=${EMPTY} 58 59 # Description of argument(s): 60 # index The index into the FFDC_METHOD_CALL dictionary (e.g. 61 # 'BMC LOGS'). 62 # ffdc_function_list See ffdc_function_list description in 63 # "Call FFDC Methods" (above). 64 65 @{method_list}= Get FFDC Method Call ${index} 66 # Example method_list: 67 # method_list: 68 # method_list[0]: 69 # method_list[0][0]: FFDC Generic Report 70 # method_list[0][1]: BMC FFDC Manifest 71 # method_list[1]: 72 # method_list[1][0]: Get Request FFDC 73 # method_list[1][1]: BMC FFDC Get Requests 74 # (etc.) 75 76 # If function list is empty assign default (i.e. a list of all allowable 77 # values). In either case, convert ffdc_function_list from a string to 78 # a list. 79 @{ffdc_function_list}= 80 ... Run Keyword If '${ffdc_function_list}' == '${EMPTY}' 81 ... Get FFDC Method Desc ${index} 82 ... ELSE 83 ... Split String ${ffdc_function_list} separator=: 84 85 @{ffdc_file_list}= Create List 86 :FOR ${method} IN @{method_list} 87 \ ${ffdc_file_sub_list}= Execute Keyword Method ${method[0]} 88 ... ${method[1]} @{ffdc_function_list} 89 \ ${ffdc_file_list}= Smart Combine Lists ${ffdc_file_list} 90 ... ${ffdc_file_sub_list} 91 92 [Return] ${ffdc_file_list} 93 94 95Execute Keyword Method 96 [Documentation] Call into BMC method keywords. Don't let one 97 ... failure skip the remaining. Get whatever data 98 ... it could gather at worse case scenario. 99 [Arguments] ${description} ${keyword_name} @{ffdc_function_list} 100 101 # Description of argument(s): 102 # description The description of the FFDC to be collected. This 103 # would be any value returned by 104 # 'Get FFDC Method Desc' (e.g. "FFDC Generic Report"). 105 # keyword_name The name of the keyword to call to collect the FFDC 106 # data (again, see FFDC_METHOD_CALL). 107 # ffdc_function_list See ffdc_function_list description in 108 # "Call FFDC Methods" (above). The only difference is 109 # in this case, it should be a list rather than a 110 # colon-delimited value. 111 112 @{ffdc_file_list}= Create List 113 114 ${index}= Get Index From List ${ffdc_function_list} ${description} 115 Run Keyword If '${index}' == '${-1}' Return from Keyword 116 ... ${ffdc_file_list} 117 118 ${status} ${ffdc_file_list}= Run Key ${keyword_name} ignore=1 119 [Return] ${ffdc_file_list} 120 121################################################################ 122# Method : BMC FFDC Manifest # 123# Execute command on BMC and write to ffdc_report.txt # 124################################################################ 125 126BMC FFDC Manifest 127 [Documentation] Run the ssh commands from FFDC_BMC_CMD and return a list 128 ... of generated files. 129 130 @{ffdc_file_list}= Create List ${FFDC_FILE_PATH} 131 @{entries}= Get FFDC Cmd Index 132 :FOR ${index} IN @{entries} 133 \ Iterate BMC Command List Pairs ${index} 134 135 [Return] ${ffdc_file_list} 136 137 138Iterate BMC Command List Pairs 139 [Documentation] Feed in key pair list from dictionary to execute 140 [Arguments] ${key_index} 141 142 @{cmd_list}= Get ffdc bmc cmd ${key_index} 143 Set Suite Variable ${ENTRY_INDEX} ${key_index} 144 :FOR ${cmd} IN @{cmd_list} 145 \ Execute Command and Write FFDC ${cmd[0]} ${cmd[1]} 146 147 148Execute Command and Write FFDC 149 [Documentation] Run a command on the BMC or OS, write the output to the 150 ... specified file and return a list of generated files. 151 [Arguments] ${key_index} ${cmd} ${logpath}=${FFDC_FILE_PATH} 152 ... ${target}=BMC 153 154 Run Keyword If '${logpath}' == '${FFDC_FILE_PATH}' 155 ... Write Cmd Output to FFDC File ${key_index} ${cmd} 156 157 @{ffdc_file_list}= Create List ${log_path} 158 159 ${cmd_buf}= Catenate ${target} Execute Command \ ${cmd} \ ignore_err=${1} 160 ${status} ${ret_values}= Run Key ${cmd_buf} ignore=${1} 161 162 ${stdout}= Set Variable @{ret_values}[0] 163 ${stderr}= Set Variable @{ret_values}[1] 164 165 # Write stdout on success and stderr/stdout to the file on failure. 166 Run Keyword If $stderr == '${EMPTY}' 167 ... Write Data To File ${stdout}${\n} ${logpath} 168 ... ELSE Write Data To File 169 ... ERROR output:${\n}${stderr}${\n}Output:${\n}${stdout}${\n} 170 ... ${logpath} 171 172 [Return] ${ffdc_file_list} 173 174 175################################################################ 176# Method : BMC FFDC Files # 177# Execute command on BMC and write to individual file # 178# based on the file name pre-defined in the list # 179################################################################ 180 181BMC FFDC Files 182 [Documentation] Run the commands from FFDC_BMC_FILE and return a list of 183 ... generated files. 184 185 @{entries}= Get FFDC File Index 186 # Example of entries: 187 # entries: 188 # entries[0]: BMC FILES 189 190 @{ffdc_file_list}= Create List 191 :FOR ${index} IN @{entries} 192 \ ${ffdc_file_sub_list}= Create File and Write Data ${index} 193 \ ${ffdc_file_list}= Smart Combine Lists ${ffdc_file_list} 194 ... ${ffdc_file_sub_list} 195 196 [Return] ${ffdc_file_list} 197 198 199Create File and Write Data 200 [Documentation] Run commands from FFDC_BMC_FILE to create FFDC files and 201 ... return a list of generated files. 202 [Arguments] ${key_index} 203 204 # Description of argument(s): 205 # key_index The index into the FFDC_BMC_FILE dictionary. 206 207 @{ffdc_file_list}= Create List 208 @{cmd_list}= Get FFDC BMC File ${key_index} 209 :FOR ${cmd} IN @{cmd_list} 210 \ ${logpath}= Catenate SEPARATOR= ${LOG_PREFIX} ${cmd[0]}.txt 211 \ ${ffdc_file_sub_list}= Execute Command and Write FFDC ${cmd[0]} 212 ... ${cmd[1]} ${logpath} 213 \ ${ffdc_file_list}= Smart Combine Lists ${ffdc_file_list} 214 ... ${ffdc_file_sub_list} 215 216 [Return] ${ffdc_file_list} 217 218 219################################################################ 220# Method : Log Test Case Status # 221# Creates test result history footprint for reference # 222################################################################ 223 224Log Test Case Status 225 [Documentation] Test case execution result history. 226 ... Create once and append to this file 227 ... logs/test_history.txt 228 ... Format Date:Test suite:Test case:Status 229 ... 20160909214053719992:Test Warmreset:Test WarmReset via REST:FAIL 230 231 ${FFDC_DIR_PATH_STYLE}= Get Variable Value ${FFDC_DIR_PATH_STYLE} 232 ... ${EMPTY} 233 ${FFDC_DIR_PATH}= Get Variable Value ${FFDC_DIR_PATH} ${EMPTY} 234 235 Run Keyword If '${FFDC_DIR_PATH}' == '${EMPTY}' Set FFDC Defaults 236 237 Run Keyword If '${FFDC_DIR_PATH_STYLE}' == '${1}' Run Keywords 238 ... Set Global Variable ${FFDC_LOG_PATH} ${FFDC_DIR_PATH} AND 239 ... Set Global Variable ${TEST_HISTORY} ${FFDC_DIR_PATH}test_history.txt 240 241 Create Directory ${FFDC_LOG_PATH} 242 243 ${exist}= Run Keyword and Return Status 244 ... OperatingSystem.File Should Exist ${TEST_HISTORY} 245 246 Run Keyword If '${exist}' == '${False}' 247 ... Create File ${TEST_HISTORY} 248 249 Rpvars TEST_HISTORY 250 251 ${cur_time}= Get Current Time Stamp 252 253 Append To File ${TEST_HISTORY} 254 ... ${cur_time}:${SUITE_NAME}:${TEST_NAME}:${TEST_STATUS}${\n} 255 256 257Log FFDC Get Requests 258 [Documentation] Run the get requests associated with the key and return a 259 ... list of generated files. 260 [Arguments] ${key_index} 261 262 # Note: Output will be in JSON pretty_print format. 263 264 # Description of argument(s): 265 # key_index The key to the FFDC_GET_REQUEST dictionary that contains the 266 # get requests that are to be run. 267 268 @{ffdc_file_list}= Create List 269 @{cmd_list}= Get FFDC Get Request ${key_index} 270 :FOR ${cmd} IN @{cmd_list} 271 \ ${logpath}= Catenate SEPARATOR= ${LOG_PREFIX} ${cmd[0]}.txt 272 \ ${resp}= OpenBMC Get Request ${cmd[1]} quiet=${1} 273 \ ${status}= Run Keyword and Return Status 274 ... Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 275 \ Run Keyword If '${status}' == '${False}' Continue For Loop 276 \ ${jsondata}= to json ${resp.content} pretty_print=True 277 \ Write Data To File ${\n}${jsondata}${\n} ${logpath} 278 \ Append To List ${ffdc_file_list} ${logpath} 279 280 [Return] ${ffdc_file_list} 281 282BMC FFDC Get Requests 283 [Documentation] Iterate over get request list and return a list of 284 ... generated files. 285 286 @{ffdc_file_list}= Create List 287 288 @{entries}= Get ffdc get request index 289 # Example of entries: 290 # entries: 291 # entries[0]: GET REQUESTS 292 :FOR ${index} IN @{entries} 293 \ ${ffdc_file_sub_list}= Log FFDC Get Requests ${index} 294 \ ${ffdc_file_list}= Smart Combine Lists ${ffdc_file_list} 295 ... ${ffdc_file_sub_list} 296 297 [Return] ${ffdc_file_list} 298 299Log OS All distros FFDC 300 [Documentation] Run commands from FFDC_OS_ALL_DISTROS_FILE to create FFDC 301 ... files and return a list of generated files. 302 [Arguments] ${key_index} 303 304 # Description of argument(s): 305 # key_index The index into the FFDC_OS_ALL_DISTROS_FILE dictionary. 306 307 @{ffdc_file_list}= Create List 308 309 @{cmd_list}= Get FFDC OS All Distros Call ${key_index} 310 :FOR ${cmd} IN @{cmd_list} 311 \ ${logpath}= Catenate SEPARATOR= ${LOG_PREFIX} ${cmd[0]}.txt 312 \ ${ffdc_file_sub_list}= Execute Command and Write FFDC ${cmd[0]} 313 ... ${cmd[1]} ${logpath} target=OS 314 \ ${ffdc_file_list}= Smart Combine Lists ${ffdc_file_list} 315 ... ${ffdc_file_sub_list} 316 317 [Return] ${ffdc_file_list} 318 319 320Log OS SPECIFIC DISTRO FFDC 321 [Documentation] Run commands from the FFDC_OS_<distro>_FILE to create FFDC 322 ... files and return a list of generated files. 323 [Arguments] ${key_index} ${linux_distro} 324 325 # Description of argument(s): 326 # key_index The index into the FFDC_OS_<distro>_FILE dictionary. 327 # linux_distro Your OS's linux distro (e.g. "UBUNTU", "RHEL", etc). 328 329 @{ffdc_file_list}= Create List 330 331 @{cmd_list}= Get FFDC OS Distro Call ${key_index} ${linux_distro} 332 :FOR ${cmd} IN @{cmd_list} 333 \ ${logpath}= Catenate SEPARATOR= ${LOG_PREFIX} ${cmd[0]}.txt 334 \ ${ffdc_file_sub_list}= Execute Command and Write FFDC ${cmd[0]} 335 ... ${cmd[1]} ${logpath} target=OS 336 \ ${ffdc_file_list}= Smart Combine Lists ${ffdc_file_list} 337 ... ${ffdc_file_sub_list} 338 339 [Return] ${ffdc_file_list} 340 341 342OS FFDC Files 343 [Documentation] Run the commands from FFDC_OS_ALL_DISTROS_FILE to create 344 ... FFDC files and return a list of generated files. 345 [Arguments] ${OS_HOST}=${OS_HOST} ${OS_USERNAME}=${OS_USERNAME} 346 ... ${OS_PASSWORD}=${OS_PASSWORD} 347 348 @{ffdc_file_list}= Create List 349 350 Run Keyword If '${OS_HOST}' == '${EMPTY}' Run Keywords 351 ... Print Timen No OS Host provided so no OS FFDC will be done. AND 352 ... Return From Keyword ${ffdc_file_list} 353 354 ${match_state}= Create Dictionary os_ping=^1$ os_login=^1$ 355 ... os_run_cmd=^1$ 356 ${status} ${ret_values}= Run Keyword and Ignore Error Check State 357 ... ${match_state} quiet=0 358 359 Run Keyword If '${status}' == 'FAIL' Run Keywords 360 ... Print Error The OS is not communicating so no OS FFDC will be done.\n 361 ... AND 362 ... Return From Keyword ${ffdc_file_list} 363 364 ${stdout} ${stderr} ${rc}= OS Execute Command 365 ... . /etc/os-release; echo $ID ignore_err=${0} 366 Set Global Variable ${linux_distro} ${stdout} 367 368 Rpvars linux_distro 369 370 @{entries}= Get FFDC OS All Distros Index 371 :FOR ${index} IN @{entries} 372 \ ${ffdc_file_sub_list}= Log OS All distros FFDC ${index} 373 \ ${ffdc_file_list}= Smart Combine Lists ${ffdc_file_list} 374 ... ${ffdc_file_sub_list} 375 376 Return From Keyword If 377 ... '${linux_distro}' == '${EMPTY}' or '${linux_distro}' == 'None' 378 ... ${ffdc_file_list} 379 380 @{entries}= Get ffdc os distro index ${linux_distro} 381 :FOR ${index} IN @{entries} 382 \ ${ffdc_file_sub_list}= Log OS SPECIFIC DISTRO FFDC ${index} 383 ... ${linux_distro} 384 \ ${ffdc_file_list}= Smart Combine Lists ${ffdc_file_list} 385 ... ${ffdc_file_sub_list} 386 387 [Return] ${ffdc_file_list} 388 389 390System Inventory Files 391 [Documentation] Copy systest os_inventory files and return a list of 392 ... generated files.. 393 # The os_inventory files are the result of running 394 # systest/htx_hardbootme_test. If these files exist 395 # they are copied to the FFDC directory. 396 # Global variable ffdc_dir_path is the path name of the 397 # directory they are copied to. 398 399 @{ffdc_file_list}= Create List 400 401 ${globex}= Set Variable os_inventory_*.json 402 403 @{file_list}= OperatingSystem.List Files In Directory . ${globex} 404 405 Copy Files ${globex} ${ffdc_dir_path} 406 407 : FOR ${file_name} IN @{file_list} 408 \ Append To List ${ffdc_file_list} ${ffdc_dir_path}${file_name} 409 410 Run Keyword and Ignore Error Remove Files ${globex} 411 412 [Return] ${ffdc_file_list} 413 414 415############################################################################## 416SCP Coredump Files 417 [Documentation] Copy core dump files from BMC to local system and return a 418 ... list of generated file names. 419 420 @{ffdc_file_list}= Create List 421 422 # Check if core dump exist in the /tmp 423 ${core_files} ${stderr} ${rc}= BMC Execute Command ls /tmp/core_* 424 ... ignore_err=${1} 425 Run Keyword If '${rc}' != '${0}' Return From Keyword ${ffdc_file_list} 426 427 @{core_list}= Split String ${core_files} 428 # Copy the core files 429 Run Key U Open Connection for SCP 430 :FOR ${index} IN @{core_list} 431 \ ${ffdc_file_path}= Catenate ${LOG_PREFIX}${index.lstrip("/tmp/")} 432 \ ${status}= Run Keyword and Return Status 433 ... scp.Get File ${index} ${ffdc_file_path} 434 \ Run Keyword If '${status}' == '${False}' Continue For Loop 435 \ Append To List ${ffdc_file_list} ${ffdc_file_path} 436 # Remove the file from remote to avoid re-copying on next FFDC call 437 \ BMC Execute Command rm ${index} ignore_err=${1} 438 # I can't find a way to do this: scp.Close Connection 439 440 [Return] ${ffdc_file_list} 441 442 443############################################################################## 444Collect eSEL Log 445 [Documentation] Collect eSEL log from logging entry and convert eSEL data 446 ... to elog formatted string text file. 447 [Arguments] ${log_prefix_path}=${LOG_PREFIX} 448 449 @{ffdc_file_list}= Create List 450 451 ${resp}= OpenBMC Get Request ${BMC_LOGGING_ENTRY}/enumerate quiet=${1} 452 ${status}= Run Keyword And Return Status 453 ... Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 454 Return From Keyword If '${status}' == '${False}' ${ffdc_file_list} 455 456 ${content}= To Json ${resp.content} 457 # Grab the list of entries from logging/entry/ 458 # The data shown below is the result of the "Get Dictionary Keys". 459 # Example: 460 # /xyz/openbmc_project/logging/entry/1 461 # /xyz/openbmc_project/logging/entry/2 462 ${esel_list}= Get Dictionary Keys ${content['data']} 463 464 ${logpath}= Catenate SEPARATOR= ${log_prefix_path} esel 465 Create File ${logpath} 466 # Fetch data from /xyz/openbmc_project/logging/entry/1/attr/AdditionalData 467 # "ESEL=00 00 df 00 00 00 00 20 00 04 12 35 6f aa 00 00 " 468 # Sample eSEL entry: 469 # "/xyz/openbmc_project/logging/entry/1": { 470 # "Timestamp": 1487744317025, 471 # "AdditionalData": [ 472 # "ESEL=00 00 df 00 00 00 00 20 00 04 12 35 6f aa 00 00 " 473 # ], 474 # "Message": "org.open_power.Error.Host.Event.Event", 475 # "Id": 1, 476 # "Severity": "xyz.openbmc_project.Logging.Entry.Level.Emergency" 477 # } 478 479 :FOR ${entry_path} IN @{esel_list} 480 # Skip reading attribute if entry URI is a callout. 481 # Example: /xyz/openbmc_project/logging/entry/1/callout 482 \ Continue For Loop If '${entry_path.rsplit('/', 1)[1]}' == 'callout' 483 \ ${esel_data}= Read Attribute ${entry_path} AdditionalData quiet=${1} 484 \ ${status}= Run Keyword And Return Status 485 ... Should Contain Match ${esel_data} ESEL* 486 \ Continue For Loop If ${status} == ${False} 487 \ ${index}= Get Esel Index ${esel_data} 488 \ Write Data To File "${esel_data[${index}]}" ${logpath} 489 \ Write Data To File ${\n} ${logpath} 490 491 ${out}= Run which eSEL.pl 492 ${status}= Run Keyword And Return Status 493 ... Should Contain ${out} eSEL.pl 494 Return From Keyword If '${status}' == '${False}' ${ffdc_file_list} 495 496 Convert eSEL To Elog Format ${logpath} 497 498 Append To List ${ffdc_file_list} ${logpath} 499 500 [Return] ${ffdc_file_list} 501 502 503############################################################################## 504Convert eSEL To Elog Format 505 [Documentation] Execute parser tool on the eSEL data file to generate 506 ... formatted error log. 507 [Arguments] ${esel_file_path} 508 # Description of arguments: 509 # esel_file_path Absolute path of the eSEL data (e.g. 510 # /tmp/w55.170404.154820.esel). 511 512 # Note: The only way to get eSEL.pl to put the output in a particular 513 # directory is to cd to that directory. 514 ${cmd_buf}= Catenate cd $(dirname ${esel_file_path}) ; eSEL.pl -l 515 ... ${esel_file_path} -p decode_obmc_data 516 Run ${cmd_buf} 517 518############################################################################## 519