1*** Settings *** 2Documentation Methods to execute commands on BMC and collect 3... data to a list of FFDC files 4 5Resource openbmc_ffdc_utils.robot 6Resource rest_client.robot 7Resource utils.robot 8Resource list_utils.robot 9Library SSHLibrary 10Library OperatingSystem 11Library Collections 12Library String 13Library gen_print.py 14Library gen_robot_keyword.py 15 16*** Keywords *** 17 18# Method : Call FFDC Methods # 19# Execute the user define keywords from the FFDC List # 20# Unlike any other keywords this will call into the # 21# list of keywords defined in the FFDC list at one go # 22 23Call FFDC Methods 24 [Documentation] Call into FFDC Keyword index list. 25 [Arguments] ${ffdc_function_list}=${EMPTY} 26 27 # Description of argument(s): 28 # ffdc_function_list A colon-delimited list naming the kinds of FFDC that 29 # are to be collected 30 # (e.g. "FFDC Generic Report:BMC Specific Files"). 31 # Acceptable values can be found in the description 32 # field of FFDC_METHOD_CALL in 33 # lib/openbmc_ffdc_list.py. Those values can be 34 # obtained via a call to 'Get FFDC Method Desc' (also 35 # from lib/openbmc_ffdc_list.py). 36 37 @{entries}= Get FFDC Method Index 38 # Example entries: 39 # entries: 40 # entries[0]: BMC LOGS 41 42 @{ffdc_file_list}= Create List 43 :FOR ${index} IN @{entries} 44 \ ${ffdc_file_sub_list}= Method Call Keyword List ${index} 45 ... ${ffdc_function_list} 46 \ ${ffdc_file_list}= Smart Combine Lists ${ffdc_file_list} 47 ... ${ffdc_file_sub_list} 48 49 Run Key U SSHLibrary.Close All Connections 50 51 [Return] ${ffdc_file_list} 52 53Method Call Keyword List 54 [Documentation] Process FFDC request and return a list of generated files. 55 [Arguments] ${index} ${ffdc_function_list}=${EMPTY} 56 57 # Description of argument(s): 58 # index The index into the FFDC_METHOD_CALL dictionary (e.g. 59 # 'BMC LOGS'). 60 # ffdc_function_list See ffdc_function_list description in 61 # "Call FFDC Methods" (above). 62 63 @{method_list}= Get FFDC Method Call ${index} 64 # Example method_list: 65 # method_list: 66 # method_list[0]: 67 # method_list[0][0]: FFDC Generic Report 68 # method_list[0][1]: BMC FFDC Manifest 69 # method_list[1]: 70 # method_list[1][0]: Get Request FFDC 71 # method_list[1][1]: BMC FFDC Get Requests 72 # (etc.) 73 74 # If function list is empty assign default (i.e. a list of all allowable 75 # values). In either case, convert ffdc_function_list from a string to 76 # a list. 77 @{ffdc_function_list}= 78 ... Run Keyword If '${ffdc_function_list}' == '${EMPTY}' 79 ... Get FFDC Method Desc ${index} 80 ... ELSE 81 ... Split String ${ffdc_function_list} separator=: 82 83 @{ffdc_file_list}= Create List 84 :FOR ${method} IN @{method_list} 85 \ ${ffdc_file_sub_list}= Execute Keyword Method ${method[0]} 86 ... ${method[1]} @{ffdc_function_list} 87 \ ${ffdc_file_list}= Smart Combine Lists ${ffdc_file_list} 88 ... ${ffdc_file_sub_list} 89 90 [Return] ${ffdc_file_list} 91 92 93Execute Keyword Method 94 [Documentation] Call into BMC method keywords. Don't let one 95 ... failure skip the remaining. Get whatever data 96 ... it could gather at worse case scenario. 97 [Arguments] ${description} ${keyword_name} @{ffdc_function_list} 98 99 # Description of argument(s): 100 # description The description of the FFDC to be collected. This 101 # would be any value returned by 102 # 'Get FFDC Method Desc' (e.g. "FFDC Generic Report"). 103 # keyword_name The name of the keyword to call to collect the FFDC 104 # data (again, see FFDC_METHOD_CALL). 105 # ffdc_function_list See ffdc_function_list description in 106 # "Call FFDC Methods" (above). The only difference is 107 # in this case, it should be a list rather than a 108 # colon-delimited value. 109 110 @{ffdc_file_list}= Create List 111 112 ${index}= Get Index From List ${ffdc_function_list} ${description} 113 Run Keyword If '${index}' == '${-1}' Return from Keyword 114 ... ${ffdc_file_list} 115 116 ${status} ${ffdc_file_list}= Run Key ${keyword_name} ignore=1 117 [Return] ${ffdc_file_list} 118 119# Method : BMC FFDC Manifest # 120# Execute command on BMC and write to ffdc_report.txt # 121 122BMC FFDC Manifest 123 [Documentation] Run the ssh commands from FFDC_BMC_CMD and return a list 124 ... of generated files. 125 126 @{ffdc_file_list}= Create List ${FFDC_FILE_PATH} 127 @{entries}= Get FFDC Cmd Index 128 :FOR ${index} IN @{entries} 129 \ Iterate BMC Command List Pairs ${index} 130 131 [Return] ${ffdc_file_list} 132 133 134Iterate BMC Command List Pairs 135 [Documentation] Feed in key pair list from dictionary to execute 136 [Arguments] ${key_index} 137 138 @{cmd_list}= Get ffdc bmc cmd ${key_index} 139 Set Suite Variable ${ENTRY_INDEX} ${key_index} 140 :FOR ${cmd} IN @{cmd_list} 141 \ Execute Command and Write FFDC ${cmd[0]} ${cmd[1]} 142 143 144Execute Command and Write FFDC 145 [Documentation] Run a command on the BMC or OS, write the output to the 146 ... specified file and return a list of generated files. 147 [Arguments] ${key_index} ${cmd} ${logpath}=${FFDC_FILE_PATH} 148 ... ${target}=BMC 149 150 Run Keyword If '${logpath}' == '${FFDC_FILE_PATH}' 151 ... Write Cmd Output to FFDC File ${key_index} ${cmd} 152 153 @{ffdc_file_list}= Create List ${log_path} 154 155 ${cmd_buf}= Catenate ${target} Execute Command \ ${cmd} \ ignore_err=${1} 156 ${status} ${ret_values}= Run Key ${cmd_buf} ignore=${1} 157 158 ${stdout}= Set Variable @{ret_values}[0] 159 ${stderr}= Set Variable @{ret_values}[1] 160 161 # Write stdout on success and stderr/stdout to the file on failure. 162 Run Keyword If $stderr == '${EMPTY}' 163 ... Write Data To File ${stdout}${\n} ${logpath} 164 ... ELSE Write Data To File 165 ... ERROR output:${\n}${stderr}${\n}Output:${\n}${stdout}${\n} 166 ... ${logpath} 167 168 [Return] ${ffdc_file_list} 169 170 171# Method : BMC FFDC Files # 172# Execute command on BMC and write to individual file # 173# based on the file name pre-defined in the list # 174 175BMC FFDC Files 176 [Documentation] Run the commands from FFDC_BMC_FILE and return a list of 177 ... generated files. 178 179 @{entries}= Get FFDC File Index 180 # Example of entries: 181 # entries: 182 # entries[0]: BMC FILES 183 184 @{ffdc_file_list}= Create List 185 :FOR ${index} IN @{entries} 186 \ ${ffdc_file_sub_list}= Create File and Write Data ${index} 187 \ ${ffdc_file_list}= Smart Combine Lists ${ffdc_file_list} 188 ... ${ffdc_file_sub_list} 189 190 [Return] ${ffdc_file_list} 191 192 193Create File and Write Data 194 [Documentation] Run commands from FFDC_BMC_FILE to create FFDC files and 195 ... return a list of generated files. 196 [Arguments] ${key_index} 197 198 # Description of argument(s): 199 # key_index The index into the FFDC_BMC_FILE dictionary. 200 201 @{ffdc_file_list}= Create List 202 @{cmd_list}= Get FFDC BMC File ${key_index} 203 :FOR ${cmd} IN @{cmd_list} 204 \ ${logpath}= Catenate SEPARATOR= ${LOG_PREFIX} ${cmd[0]}.txt 205 \ ${ffdc_file_sub_list}= Execute Command and Write FFDC ${cmd[0]} 206 ... ${cmd[1]} ${logpath} 207 \ ${ffdc_file_list}= Smart Combine Lists ${ffdc_file_list} 208 ... ${ffdc_file_sub_list} 209 210 [Return] ${ffdc_file_list} 211 212 213# Method : Log Test Case Status # 214# Creates test result history footprint for reference # 215 216Log Test Case Status 217 [Documentation] Test case execution result history. 218 ... Create once and append to this file 219 ... logs/test_history.txt 220 ... Format Date:Test suite:Test case:Status 221 ... 20160909214053719992:Test Warmreset:Test WarmReset via REST:FAIL 222 223 ${FFDC_DIR_PATH_STYLE}= Get Variable Value ${FFDC_DIR_PATH_STYLE} 224 ... ${EMPTY} 225 ${FFDC_DIR_PATH}= Get Variable Value ${FFDC_DIR_PATH} ${EMPTY} 226 227 Run Keyword If '${FFDC_DIR_PATH}' == '${EMPTY}' Set FFDC Defaults 228 229 Run Keyword If '${FFDC_DIR_PATH_STYLE}' == '${1}' Run Keywords 230 ... Set Global Variable ${FFDC_LOG_PATH} ${FFDC_DIR_PATH} AND 231 ... Set Global Variable ${TEST_HISTORY} ${FFDC_DIR_PATH}test_history.txt 232 233 Create Directory ${FFDC_LOG_PATH} 234 235 ${exist}= Run Keyword and Return Status 236 ... OperatingSystem.File Should Exist ${TEST_HISTORY} 237 238 Run Keyword If '${exist}' == '${False}' 239 ... Create File ${TEST_HISTORY} 240 241 Rpvars TEST_HISTORY 242 243 ${cur_time}= Get Current Time Stamp 244 245 Append To File ${TEST_HISTORY} 246 ... ${cur_time}:${SUITE_NAME}:${TEST_NAME}:${TEST_STATUS}${\n} 247 248 249Log FFDC Get Requests 250 [Documentation] Run the get requests associated with the key and return a 251 ... list of generated files. 252 [Arguments] ${key_index} 253 254 # Note: Output will be in JSON pretty_print format. 255 256 # Description of argument(s): 257 # key_index The key to the FFDC_GET_REQUEST dictionary that contains the 258 # get requests that are to be run. 259 260 @{ffdc_file_list}= Create List 261 @{cmd_list}= Get FFDC Get Request ${key_index} 262 :FOR ${cmd} IN @{cmd_list} 263 \ ${logpath}= Catenate SEPARATOR= ${LOG_PREFIX} ${cmd[0]}.txt 264 \ ${resp}= OpenBMC Get Request ${cmd[1]} quiet=${1} 265 \ ${status}= Run Keyword and Return Status 266 ... Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 267 \ Run Keyword If '${status}' == '${False}' Continue For Loop 268 \ ${jsondata}= to json ${resp.content} pretty_print=True 269 \ Write Data To File ${\n}${jsondata}${\n} ${logpath} 270 \ Append To List ${ffdc_file_list} ${logpath} 271 272 [Return] ${ffdc_file_list} 273 274BMC FFDC Get Requests 275 [Documentation] Iterate over get request list and return a list of 276 ... generated files. 277 278 @{ffdc_file_list}= Create List 279 280 @{entries}= Get ffdc get request index 281 # Example of entries: 282 # entries: 283 # entries[0]: GET REQUESTS 284 :FOR ${index} IN @{entries} 285 \ ${ffdc_file_sub_list}= Log FFDC Get Requests ${index} 286 \ ${ffdc_file_list}= Smart Combine Lists ${ffdc_file_list} 287 ... ${ffdc_file_sub_list} 288 289 [Return] ${ffdc_file_list} 290 291Log OS All distros FFDC 292 [Documentation] Run commands from FFDC_OS_ALL_DISTROS_FILE to create FFDC 293 ... files and return a list of generated files. 294 [Arguments] ${key_index} 295 296 # Description of argument(s): 297 # key_index The index into the FFDC_OS_ALL_DISTROS_FILE dictionary. 298 299 @{ffdc_file_list}= Create List 300 301 @{cmd_list}= Get FFDC OS All Distros Call ${key_index} 302 :FOR ${cmd} IN @{cmd_list} 303 \ ${logpath}= Catenate SEPARATOR= ${LOG_PREFIX} ${cmd[0]}.txt 304 \ ${ffdc_file_sub_list}= Execute Command and Write FFDC ${cmd[0]} 305 ... ${cmd[1]} ${logpath} target=OS 306 \ ${ffdc_file_list}= Smart Combine Lists ${ffdc_file_list} 307 ... ${ffdc_file_sub_list} 308 309 [Return] ${ffdc_file_list} 310 311 312Log OS SPECIFIC DISTRO FFDC 313 [Documentation] Run commands from the FFDC_OS_<distro>_FILE to create FFDC 314 ... files and return a list of generated files. 315 [Arguments] ${key_index} ${linux_distro} 316 317 # Description of argument(s): 318 # key_index The index into the FFDC_OS_<distro>_FILE dictionary. 319 # linux_distro Your OS's linux distro (e.g. "UBUNTU", "RHEL", etc). 320 321 @{ffdc_file_list}= Create List 322 323 @{cmd_list}= Get FFDC OS Distro Call ${key_index} ${linux_distro} 324 :FOR ${cmd} IN @{cmd_list} 325 \ ${logpath}= Catenate SEPARATOR= ${LOG_PREFIX} ${cmd[0]}.txt 326 \ ${ffdc_file_sub_list}= Execute Command and Write FFDC ${cmd[0]} 327 ... ${cmd[1]} ${logpath} target=OS 328 \ ${ffdc_file_list}= Smart Combine Lists ${ffdc_file_list} 329 ... ${ffdc_file_sub_list} 330 331 [Return] ${ffdc_file_list} 332 333 334OS FFDC Files 335 [Documentation] Run the commands from FFDC_OS_ALL_DISTROS_FILE to create 336 ... FFDC files and return a list of generated files. 337 [Arguments] ${OS_HOST}=${OS_HOST} ${OS_USERNAME}=${OS_USERNAME} 338 ... ${OS_PASSWORD}=${OS_PASSWORD} 339 340 @{ffdc_file_list}= Create List 341 342 Run Keyword If '${OS_HOST}' == '${EMPTY}' Run Keywords 343 ... Print Timen No OS Host provided so no OS FFDC will be done. AND 344 ... Return From Keyword ${ffdc_file_list} 345 346 ${match_state}= Create Dictionary os_ping=^1$ os_login=^1$ 347 ... os_run_cmd=^1$ 348 ${status} ${ret_values}= Run Keyword and Ignore Error Check State 349 ... ${match_state} quiet=0 350 351 Run Keyword If '${status}' == 'FAIL' Run Keywords 352 ... Print Error The OS is not communicating so no OS FFDC will be done.\n 353 ... AND 354 ... Return From Keyword ${ffdc_file_list} 355 356 ${stdout} ${stderr} ${rc}= OS Execute Command 357 ... . /etc/os-release; echo $ID ignore_err=${0} 358 Set Global Variable ${linux_distro} ${stdout} 359 360 Rpvars linux_distro 361 362 @{entries}= Get FFDC OS All Distros Index 363 :FOR ${index} IN @{entries} 364 \ ${ffdc_file_sub_list}= Log OS All distros FFDC ${index} 365 \ ${ffdc_file_list}= Smart Combine Lists ${ffdc_file_list} 366 ... ${ffdc_file_sub_list} 367 368 Return From Keyword If 369 ... '${linux_distro}' == '${EMPTY}' or '${linux_distro}' == 'None' 370 ... ${ffdc_file_list} 371 372 @{entries}= Get ffdc os distro index ${linux_distro} 373 :FOR ${index} IN @{entries} 374 \ ${ffdc_file_sub_list}= Log OS SPECIFIC DISTRO FFDC ${index} 375 ... ${linux_distro} 376 \ ${ffdc_file_list}= Smart Combine Lists ${ffdc_file_list} 377 ... ${ffdc_file_sub_list} 378 379 [Return] ${ffdc_file_list} 380 381 382System Inventory Files 383 [Documentation] Copy systest os_inventory files and return a list of 384 ... generated files.. 385 # The os_inventory files are the result of running 386 # systest/htx_hardbootme_test. If these files exist 387 # they are copied to the FFDC directory. 388 # Global variable ffdc_dir_path is the path name of the 389 # directory they are copied to. 390 391 @{ffdc_file_list}= Create List 392 393 ${globex}= Set Variable os_inventory_*.json 394 395 @{file_list}= OperatingSystem.List Files In Directory . ${globex} 396 397 Copy Files ${globex} ${ffdc_dir_path} 398 399 : FOR ${file_name} IN @{file_list} 400 \ Append To List ${ffdc_file_list} ${ffdc_dir_path}${file_name} 401 402 Run Keyword and Ignore Error Remove Files ${globex} 403 404 [Return] ${ffdc_file_list} 405 406 407SCP Coredump Files 408 [Documentation] Copy core dump files from BMC to local system and return a 409 ... list of generated file names. 410 411 @{ffdc_file_list}= Create List 412 413 # Check if core dump exist in the /tmp 414 ${core_files} ${stderr} ${rc}= BMC Execute Command ls /tmp/core_* 415 ... ignore_err=${1} 416 Run Keyword If '${rc}' != '${0}' Return From Keyword ${ffdc_file_list} 417 418 @{core_list}= Split String ${core_files} 419 # Copy the core files 420 Run Key U Open Connection for SCP 421 :FOR ${index} IN @{core_list} 422 \ ${ffdc_file_path}= Catenate ${LOG_PREFIX}${index.lstrip("/tmp/")} 423 \ ${status}= Run Keyword and Return Status 424 ... scp.Get File ${index} ${ffdc_file_path} 425 \ Run Keyword If '${status}' == '${False}' Continue For Loop 426 \ Append To List ${ffdc_file_list} ${ffdc_file_path} 427 # Remove the file from remote to avoid re-copying on next FFDC call 428 \ BMC Execute Command rm ${index} ignore_err=${1} 429 # I can't find a way to do this: scp.Close Connection 430 431 [Return] ${ffdc_file_list} 432 433 434Collect eSEL Log 435 [Documentation] Collect eSEL log from logging entry and convert eSEL data 436 ... to elog formatted string text file. 437 [Arguments] ${log_prefix_path}=${LOG_PREFIX} 438 439 @{ffdc_file_list}= Create List 440 441 ${resp}= OpenBMC Get Request ${BMC_LOGGING_ENTRY}/enumerate quiet=${1} 442 ${status}= Run Keyword And Return Status 443 ... Should Be Equal As Strings ${resp.status_code} ${HTTP_OK} 444 Return From Keyword If '${status}' == '${False}' ${ffdc_file_list} 445 446 ${content}= To Json ${resp.content} 447 # Grab the list of entries from logging/entry/ 448 # The data shown below is the result of the "Get Dictionary Keys". 449 # Example: 450 # /xyz/openbmc_project/logging/entry/1 451 # /xyz/openbmc_project/logging/entry/2 452 ${esel_list}= Get Dictionary Keys ${content['data']} 453 454 ${logpath}= Catenate SEPARATOR= ${log_prefix_path} esel 455 Create File ${logpath} 456 # Fetch data from /xyz/openbmc_project/logging/entry/1/attr/AdditionalData 457 # "ESEL=00 00 df 00 00 00 00 20 00 04 12 35 6f aa 00 00 " 458 # Sample eSEL entry: 459 # "/xyz/openbmc_project/logging/entry/1": { 460 # "Timestamp": 1487744317025, 461 # "AdditionalData": [ 462 # "ESEL=00 00 df 00 00 00 00 20 00 04 12 35 6f aa 00 00 " 463 # ], 464 # "Message": "org.open_power.Error.Host.Event.Event", 465 # "Id": 1, 466 # "Severity": "xyz.openbmc_project.Logging.Entry.Level.Emergency" 467 # } 468 469 :FOR ${entry_path} IN @{esel_list} 470 # Skip reading attribute if entry URI is a callout. 471 # Example: /xyz/openbmc_project/logging/entry/1/callout 472 \ Continue For Loop If '${entry_path.rsplit('/', 1)[1]}' == 'callout' 473 \ ${esel_data}= Read Attribute ${entry_path} AdditionalData quiet=${1} 474 \ ${status}= Run Keyword And Return Status 475 ... Should Contain Match ${esel_data} ESEL* 476 \ Continue For Loop If ${status} == ${False} 477 \ ${index}= Get Esel Index ${esel_data} 478 \ Write Data To File "${esel_data[${index}]}" ${logpath} 479 \ Write Data To File ${\n} ${logpath} 480 481 ${out}= Run which eSEL.pl 482 ${status}= Run Keyword And Return Status 483 ... Should Contain ${out} eSEL.pl 484 Return From Keyword If '${status}' == '${False}' ${ffdc_file_list} 485 486 Convert eSEL To Elog Format ${logpath} 487 488 Append To List ${ffdc_file_list} ${logpath} 489 490 [Return] ${ffdc_file_list} 491 492 493Convert eSEL To Elog Format 494 [Documentation] Execute parser tool on the eSEL data file to generate 495 ... formatted error log. 496 [Arguments] ${esel_file_path} 497 # Description of arguments: 498 # esel_file_path Absolute path of the eSEL data (e.g. 499 # /tmp/w55.170404.154820.esel). 500 501 # Note: The only way to get eSEL.pl to put the output in a particular 502 # directory is to cd to that directory. 503 ${cmd_buf}= Catenate cd $(dirname ${esel_file_path}) ; eSEL.pl -l 504 ... ${esel_file_path} -p decode_obmc_data 505 Run ${cmd_buf} 506 507