openbmc-test-automation/lib/openbmc_ffdc_methods.robot

*** Settings ***
Documentation      Methods to execute commands on BMC and collect
...                data to a list of FFDC files

Resource           openbmc_ffdc_utils.robot
Resource           rest_client.robot
Resource           utils.robot
Resource           list_utils.robot
Library            SSHLibrary
Library            OperatingSystem
Library            Collections
Library            String
Library            gen_print.py
Library            gen_robot_keyword.py
Library            dump_utils.py

*** Keywords ***

# Method : Call FFDC Methods                                   #
#          Execute the user define keywords from the FFDC List #
#          Unlike any other keywords this will call into the   #
#          list of keywords defined in the FFDC list at one go #

Call FFDC Methods
    [Documentation]   Call into FFDC Keyword index list.
    [Arguments]  ${ffdc_function_list}=${EMPTY}

    # Description of argument(s):
    # ffdc_function_list  A colon-delimited list naming the kinds of FFDC that
    #                     are to be collected
    #                     (e.g. "FFDC Generic Report:BMC Specific Files").
    #                     Acceptable values can be found in the description
    #                     field of FFDC_METHOD_CALL in
    #                     lib/openbmc_ffdc_list.py.  Those values can be
    #                     obtained via a call to 'Get FFDC Method Desc' (also
    #                     from lib/openbmc_ffdc_list.py).

    @{entries}=  Get FFDC Method Index
    # Example entries:
    # entries:
    #   entries[0]:  BMC LOGS

    @{ffdc_file_list}=  Create List
    :FOR  ${index}  IN  @{entries}
    \    ${ffdc_file_sub_list}=  Method Call Keyword List  ${index}
    ...      ${ffdc_function_list}
    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
    ...      ${ffdc_file_sub_list}

    Run Key U  SSHLibrary.Close All Connections

    [Return]  ${ffdc_file_list}

Method Call Keyword List
    [Documentation]  Process FFDC request and return a list of generated files.
    [Arguments]  ${index}  ${ffdc_function_list}=${EMPTY}

    # Description of argument(s):
    # index               The index into the FFDC_METHOD_CALL dictionary (e.g.
    #                     'BMC LOGS').
    # ffdc_function_list  See ffdc_function_list description in
    #                     "Call FFDC Methods" (above).

    @{method_list}=  Get FFDC Method Call  ${index}
    # Example method_list:
    # method_list:
    #   method_list[0]:
    #     method_list[0][0]: FFDC Generic Report
    #     method_list[0][1]: BMC FFDC Manifest
    #   method_list[1]:
    #     method_list[1][0]: Get Request FFDC
    #     method_list[1][1]: BMC FFDC Get Requests
    # (etc.)

    # If function list is empty assign default (i.e. a list of all allowable
    # values).  In either case, convert ffdc_function_list from a string to
    # a list.
    @{ffdc_function_list}=
    ...  Run Keyword If  '${ffdc_function_list}' == '${EMPTY}'
    ...    Get FFDC Method Desc  ${index}
    ...  ELSE
    ...    Split String  ${ffdc_function_list}  separator=:

    @{ffdc_file_list}=  Create List
    :FOR  ${method}  IN  @{method_list}
    \    ${ffdc_file_sub_list}=  Execute Keyword Method  ${method[0]}
    ...      ${method[1]}  @{ffdc_function_list}
    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
    ...      ${ffdc_file_sub_list}

    [Return]  ${ffdc_file_list}


Execute Keyword Method
    [Documentation]  Call into BMC method keywords. Don't let one
    ...              failure skip the remaining. Get whatever data
    ...              it could gather at worse case scenario.
    [Arguments]  ${description}  ${keyword_name}  @{ffdc_function_list}

    # Description of argument(s):
    # description         The description of the FFDC to be collected.  This
    #                     would be any value returned by
    #                     'Get FFDC Method Desc' (e.g. "FFDC Generic Report").
    # keyword_name        The name of the keyword to call to collect the FFDC
    #                     data (again, see FFDC_METHOD_CALL).
    # ffdc_function_list  See ffdc_function_list description in
    #                     "Call FFDC Methods" (above).  The only difference is
    #                     in this case, it should be a list rather than a
    #                     colon-delimited value.

    @{ffdc_file_list}=  Create List

    ${index}=  Get Index From List  ${ffdc_function_list}  ${description}
    Run Keyword If  '${index}' == '${-1}'  Return from Keyword
    ...  ${ffdc_file_list}

    ${status}  ${ffdc_file_list}=  Run Key  ${keyword_name}  ignore=1
    [Return]  ${ffdc_file_list}

# Method : BMC FFDC Manifest                                   #
#          Execute command on BMC and write to ffdc_report.txt #

BMC FFDC Manifest
    [Documentation]  Run the ssh commands from FFDC_BMC_CMD and return a list
    ...              of generated files.

    @{ffdc_file_list}=  Create List  ${FFDC_FILE_PATH}
    @{entries}=  Get FFDC Cmd Index
    :FOR  ${index}  IN  @{entries}
    \    Iterate BMC Command List Pairs  ${index}

    [Return]  ${ffdc_file_list}


Iterate BMC Command List Pairs
    [Documentation]    Feed in key pair list from dictionary to execute
    [Arguments]        ${key_index}

    @{cmd_list}=      Get ffdc bmc cmd    ${key_index}
    Set Suite Variable   ${ENTRY_INDEX}   ${key_index}
    :FOR  ${cmd}  IN  @{cmd_list}
    \    Execute Command and Write FFDC    ${cmd[0]}  ${cmd[1]}


Execute Command and Write FFDC
    [Documentation]  Run a command on the BMC or OS, write the output to the
    ...              specified file and return a list of generated files.
    [Arguments]  ${key_index}  ${cmd}  ${logpath}=${FFDC_FILE_PATH}
    ...          ${target}=BMC

    Run Keyword If  '${logpath}' == '${FFDC_FILE_PATH}'
    ...    Write Cmd Output to FFDC File  ${key_index}  ${cmd}

    @{ffdc_file_list}=  Create List  ${log_path}

    ${cmd_buf}=  Catenate  ${target} Execute Command \ ${cmd} \ ignore_err=${1}
    ${status}  ${ret_values}=  Run Key  ${cmd_buf}  ignore=${1}

    ${stdout}=  Set Variable  @{ret_values}[0]
    ${stderr}=  Set Variable  @{ret_values}[1]

    # Write stdout on success and stderr/stdout to the file on failure.
    Run Keyword If  $stderr == '${EMPTY}'
    ...    Write Data To File  ${stdout}${\n}  ${logpath}
    ...  ELSE  Write Data To File
    ...    ERROR output:${\n}${stderr}${\n}Output:${\n}${stdout}${\n}
    ...    ${logpath}

    [Return]  ${ffdc_file_list}


# Method : BMC FFDC Files                                      #
#          Execute command on BMC and write to individual file #
#          based on the file name pre-defined in the list      #

BMC FFDC Files
    [Documentation]  Run the commands from FFDC_BMC_FILE and return a list of
    ...              generated files.

    @{entries}=  Get FFDC File Index
    # Example of entries:
    # entries:
    #   entries[0]: BMC FILES

    @{ffdc_file_list}=  Create List
    :FOR  ${index}  IN  @{entries}
    \    ${ffdc_file_sub_list}=  Create File and Write Data  ${index}
    \     ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
    ...       ${ffdc_file_sub_list}

    [Return]  ${ffdc_file_list}


Create File and Write Data
    [Documentation]  Run commands from FFDC_BMC_FILE to create FFDC files and
    ...              return a list of generated files.
    [Arguments]  ${key_index}

    # Description of argument(s):
    # key_index  The index into the FFDC_BMC_FILE dictionary.

    @{ffdc_file_list}=  Create List
    @{cmd_list}=  Get FFDC BMC File  ${key_index}
    :FOR  ${cmd}  IN  @{cmd_list}
    \    ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
    \    ${ffdc_file_sub_list}=  Execute Command and Write FFDC  ${cmd[0]}
    ...      ${cmd[1]}  ${logpath}
    \     ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
    ...       ${ffdc_file_sub_list}

    [Return]  ${ffdc_file_list}


# Method : Log Test Case Status                                #
#          Creates test result history footprint for reference #

Log Test Case Status
    [Documentation]  Test case execution result history.
    ...  Create once and append to this file
    ...  logs/test_history.txt
    ...  Format   Date:Test suite:Test case:Status
    ...  20160909214053719992:Test Warmreset:Test WarmReset via REST:FAIL

    ${FFDC_DIR_PATH_STYLE}=  Get Variable Value  ${FFDC_DIR_PATH_STYLE}
    ...  ${EMPTY}
    ${FFDC_DIR_PATH}=  Get Variable Value  ${FFDC_DIR_PATH}  ${EMPTY}

    Run Keyword If  '${FFDC_DIR_PATH}' == '${EMPTY}'  Set FFDC Defaults

    Run Keyword If  '${FFDC_DIR_PATH_STYLE}' == '${1}'  Run Keywords
    ...  Set Global Variable  ${FFDC_LOG_PATH}  ${FFDC_DIR_PATH}  AND
    ...  Set Global Variable  ${TEST_HISTORY}  ${FFDC_DIR_PATH}test_history.txt

    Create Directory   ${FFDC_LOG_PATH}

    ${exist}=   Run Keyword and Return Status
    ...   OperatingSystem.File Should Exist   ${TEST_HISTORY}

    Run Keyword If  '${exist}' == '${False}'
    ...   Create File  ${TEST_HISTORY}

    Rpvars  TEST_HISTORY

    ${cur_time}=      Get Current Time Stamp

    Append To File    ${TEST_HISTORY}
    ...   ${cur_time}:${SUITE_NAME}:${TEST_NAME}:${TEST_STATUS}${\n}


Log FFDC Get Requests
    [Documentation]  Run the get requests associated with the key and return a
    ...              list of generated files.
    [Arguments]  ${key_index}

    # Note: Output will be in JSON pretty_print format.

    # Description of argument(s):
    # key_index  The key to the FFDC_GET_REQUEST dictionary that contains the
    #            get requests that are to be run.

    @{ffdc_file_list}=  Create List
    @{cmd_list}=  Get FFDC Get Request  ${key_index}
    :FOR  ${cmd}  IN  @{cmd_list}
    \    ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
    \    ${resp}=  OpenBMC Get Request  ${cmd[1]}  quiet=${1}
    \    ${status}=  Run Keyword and Return Status
    ...  Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
    \    Run Keyword If  '${status}' == '${False}'  Continue For Loop
    \    ${jsondata}=  to json  ${resp.content}  pretty_print=True
    \    Write Data To File  ${\n}${jsondata}${\n}  ${logpath}
    \    Append To List  ${ffdc_file_list}  ${logpath}

    [Return]  ${ffdc_file_list}

BMC FFDC Get Requests
    [Documentation]  Iterate over get request list and return a list of
    ...              generated files.

    @{ffdc_file_list}=  Create List

    @{entries}=  Get ffdc get request index
    # Example of entries:
    # entries:
    #  entries[0]:  GET REQUESTS
    :FOR  ${index}  IN  @{entries}
    \    ${ffdc_file_sub_list}=  Log FFDC Get Requests  ${index}
    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
    ...  ${ffdc_file_sub_list}

    [Return]  ${ffdc_file_list}

Log OS All distros FFDC
    [Documentation]  Run commands from FFDC_OS_ALL_DISTROS_FILE to create FFDC
    ...              files and return a list of generated files.
    [Arguments]  ${key_index}

    # Description of argument(s):
    # key_index  The index into the FFDC_OS_ALL_DISTROS_FILE dictionary.

    @{ffdc_file_list}=  Create List

    @{cmd_list}=  Get FFDC OS All Distros Call  ${key_index}
    :FOR  ${cmd}  IN  @{cmd_list}
    \    ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
    \    ${ffdc_file_sub_list}=  Execute Command and Write FFDC  ${cmd[0]}
    ...      ${cmd[1]}  ${logpath}  target=OS
    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
    ...      ${ffdc_file_sub_list}

    [Return]  ${ffdc_file_list}


Log OS SPECIFIC DISTRO FFDC
    [Documentation]  Run commands from the FFDC_OS_<distro>_FILE to create FFDC
    ...              files and return a list of generated files.
    [Arguments]  ${key_index}  ${linux_distro}

    # Description of argument(s):
    # key_index  The index into the FFDC_OS_<distro>_FILE dictionary.
    # linux_distro  Your OS's linux distro (e.g. "UBUNTU", "RHEL", etc).

    @{ffdc_file_list}=  Create List

    @{cmd_list}=  Get FFDC OS Distro Call  ${key_index}  ${linux_distro}
    :FOR  ${cmd}  IN  @{cmd_list}
    \    ${logpath}=  Catenate  SEPARATOR=  ${LOG_PREFIX}  ${cmd[0]}.txt
    \    ${ffdc_file_sub_list}=  Execute Command and Write FFDC  ${cmd[0]}
    ...      ${cmd[1]}  ${logpath}  target=OS
    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
    ...      ${ffdc_file_sub_list}

    [Return]  ${ffdc_file_list}


OS FFDC Files
    [Documentation]  Run the commands from FFDC_OS_ALL_DISTROS_FILE to create
    ...              FFDC files and return a list of generated files.
    [Arguments]  ${OS_HOST}=${OS_HOST}  ${OS_USERNAME}=${OS_USERNAME}
    ...  ${OS_PASSWORD}=${OS_PASSWORD}

    @{ffdc_file_list}=  Create List

    Run Keyword If  '${OS_HOST}' == '${EMPTY}'  Run Keywords
    ...  Print Timen  No OS Host provided so no OS FFDC will be done.  AND
    ...  Return From Keyword  ${ffdc_file_list}

    ${match_state}=  Create Dictionary  os_ping=^1$  os_login=^1$
    ...  os_run_cmd=^1$
    ${status}  ${ret_values}=  Run Keyword and Ignore Error  Check State
    ...  ${match_state}  quiet=0

    Run Keyword If  '${status}' == 'FAIL'  Run Keywords
    ...  Print Timen  The OS is not communicating so no OS FFDC will be done.\n
    ...  AND
    ...  Return From Keyword  ${ffdc_file_list}

    ${stdout}  ${stderr}  ${rc}=  OS Execute Command
    ...  . /etc/os-release; echo $ID  ignore_err=${0}
    Set Global Variable  ${linux_distro}  ${stdout}

    Rpvars  linux_distro

    @{entries}=  Get FFDC OS All Distros Index
    :FOR  ${index}  IN  @{entries}
    \    ${ffdc_file_sub_list}=  Log OS All distros FFDC  ${index}
    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
    ...      ${ffdc_file_sub_list}

    Return From Keyword If
    ...  '${linux_distro}' == '${EMPTY}' or '${linux_distro}' == 'None'
    ...  ${ffdc_file_list}

    @{entries}=  Get ffdc os distro index  ${linux_distro}
    :FOR  ${index}  IN  @{entries}
    \    ${ffdc_file_sub_list}=  Log OS SPECIFIC DISTRO FFDC  ${index}
    ...      ${linux_distro}
    \    ${ffdc_file_list}=  Smart Combine Lists  ${ffdc_file_list}
    ...      ${ffdc_file_sub_list}

    [Return]  ${ffdc_file_list}


System Inventory Files
    [Documentation]  Copy systest os_inventory files and return a list of
    ...              generated files..
    # The os_inventory files are the result of running
    # systest/htx_hardbootme_test.  If these files exist
    # they are copied to the FFDC directory.
    # Global variable ffdc_dir_path is the path name of the
    # directory they are copied to.

    @{ffdc_file_list}=  Create List

    ${globex}=  Set Variable  os_inventory_*.json

    @{file_list}=  OperatingSystem.List Files In Directory  .  ${globex}

    Copy Files  ${globex}  ${ffdc_dir_path}

    : FOR  ${file_name}  IN  @{file_list}
    \    Append To List  ${ffdc_file_list}  ${ffdc_dir_path}${file_name}

    Run Keyword and Ignore Error  Remove Files  ${globex}

    [Return]  ${ffdc_file_list}


SCP Coredump Files
    [Documentation]  Copy core dump files from BMC to local system and return a
    ...              list of generated file names.

    @{ffdc_file_list}=  Create List

    # Check if core dump exist in the /tmp
    ${core_files}  ${stderr}  ${rc}=  BMC Execute Command  ls /tmp/core_*
    ...  ignore_err=${1}
    Run Keyword If  '${rc}' != '${0}'  Return From Keyword  ${ffdc_file_list}

    @{core_list}=  Split String  ${core_files}
    # Copy the core files
    Run Key U  Open Connection for SCP
    :FOR  ${index}  IN  @{core_list}
    \    ${ffdc_file_path}=  Catenate  ${LOG_PREFIX}${index.lstrip("/tmp/")}
    \    ${status}=  Run Keyword and Return Status
    ...  scp.Get File  ${index}  ${ffdc_file_path}
    \    Run Keyword If  '${status}' == '${False}'  Continue For Loop
    \    Append To List  ${ffdc_file_list}  ${ffdc_file_path}
    # Remove the file from remote to avoid re-copying on next FFDC call
    \    BMC Execute Command  rm ${index}  ignore_err=${1}
    # I can't find a way to do this: scp.Close Connection

    [Return]  ${ffdc_file_list}

SCP Dump Files
    [Documentation]  Copy all dump files from BMC to local system.

    # Check if dumps exist
    ${ffdc_file_list}=  Scp Dumps  ${FFDC_DIR_PATH}  ${FFDC_PREFIX}

    [Return]  ${ffdc_file_list}

Collect Dump Log
    [Documentation]  Collect dumps from dump entry.
    [Arguments]  ${log_prefix_path}=${LOG_PREFIX}

    ${data}=  Read Properties  ${DUMP_ENTRY_URI}/enumerate  quiet=${1}

    # Grab the list of entries from dump/entry/
    # The data shown below is the result of the "Get Dictionary Keys".
    # Example:
    # /xyz/openbmc_project/dump/entry/1
    # /xyz/openbmc_project/dump/entry/2

    ${dump_list}=  Get Dictionary Keys  ${data}

Collect eSEL Log
    [Documentation]  Collect eSEL log from logging entry and convert eSEL data
    ...              to elog formatted string text file.
    [Arguments]  ${log_prefix_path}=${LOG_PREFIX}

    @{ffdc_file_list}=  Create List

    ${resp}=  OpenBMC Get Request  ${BMC_LOGGING_ENTRY}/enumerate  quiet=${1}
    ${status}=  Run Keyword And Return Status
    ...  Should Be Equal As Strings  ${resp.status_code}  ${HTTP_OK}
    Return From Keyword If  '${status}' == '${False}'  ${ffdc_file_list}

    ${content}=  To Json  ${resp.content}
    # Grab the list of entries from logging/entry/
    # The data shown below is the result of the "Get Dictionary Keys".
    # Example:
    # /xyz/openbmc_project/logging/entry/1
    # /xyz/openbmc_project/logging/entry/2
    ${esel_list}=  Get Dictionary Keys  ${content['data']}

    ${logpath}=  Catenate  SEPARATOR=  ${log_prefix_path}  esel
    Create File  ${logpath}
    # Fetch data from /xyz/openbmc_project/logging/entry/1/attr/AdditionalData
    #  "ESEL=00 00 df 00 00 00 00 20 00 04 12 35 6f aa 00 00 "
    # Sample eSEL entry:
    #  "/xyz/openbmc_project/logging/entry/1": {
    #    "Timestamp": 1487744317025,
    #    "AdditionalData": [
    #        "ESEL=00 00 df 00 00 00 00 20 00 04 12 35 6f aa 00 00 "
    #    ],
    #    "Message": "org.open_power.Error.Host.Event.Event",
    #    "Id": 1,
    #    "Severity": "xyz.openbmc_project.Logging.Entry.Level.Emergency"
    # }

    :FOR  ${entry_path}  IN  @{esel_list}
    # Skip reading attribute if entry URI is a callout.
    # Example: /xyz/openbmc_project/logging/entry/1/callout
    \  Continue For Loop If  '${entry_path.rsplit('/', 1)[1]}' == 'callout'
    \  ${esel_data}=  Read Attribute  ${entry_path}  AdditionalData  quiet=${1}
    \  ${status}=  Run Keyword And Return Status
    ...  Should Contain Match  ${esel_data}  ESEL*
    \  Continue For Loop If  ${status} == ${False}
    \  ${index}=  Get Esel Index  ${esel_data}
    \  Write Data To File  "${esel_data[${index}]}"  ${logpath}
    \  Write Data To File  ${\n}  ${logpath}

    ${out}=  Run  which eSEL.pl
    ${status}=  Run Keyword And Return Status
    ...  Should Contain  ${out}  eSEL.pl
    Return From Keyword If  '${status}' == '${False}'  ${ffdc_file_list}

    Convert eSEL To Elog Format  ${logpath}

    Append To List  ${ffdc_file_list}  ${logpath}
    Append To List  ${ffdc_file_list}  ${logpath}.txt

    [Return]  ${ffdc_file_list}


Convert eSEL To Elog Format
    [Documentation]  Execute parser tool on the eSEL data file to generate
    ...              formatted error log.
    [Arguments]  ${esel_file_path}
    # Description of arguments:
    # esel_file_path  Absolute path of the eSEL data (e.g.
    #                 /tmp/w55.170404.154820.esel).

    # Note: The only way to get eSEL.pl to put the output in a particular
    # directory is to cd to that directory.
    ${cmd_buf}=  Catenate  cd $(dirname ${esel_file_path}) ; eSEL.pl -l
    ...  ${esel_file_path} -p decode_obmc_data
    Run  ${cmd_buf}