1*** Settings ***
2
3Documentation  Stress the system using HTX exerciser.
4
5# Test Parameters:
6# OPENBMC_HOST        The BMC host name or IP address.
7# OS_HOST             The OS host name or IP Address.
8# OS_USERNAME         The OS login userid (usually root).
9# OS_PASSWORD         The password for the OS login.
10# HTX_DURATION        Duration of HTX run, for example, 8 hours, or
11#                     30 minutes.
12# HTX_LOOP            The number of times to loop HTX.
13# HTX_INTERVAL        The time delay between consecutive checks of HTX
14#                     status, for example, 30s.
15#                     In summary: Run HTX for $HTX_DURATION, looping
16#                     $HTX_LOOP times checking every $HTX_INTERVAL.
17# HTX_KEEP_RUNNING    If set to 1, this indicates that the HTX is to
18#                     continue running after an error.
19# CHECK_INVENTORY     If set to 0 or False, OS inventory checking before
20#                     and after each HTX run will be disabled.  This
21#                     parameter is optional.
22# PREV_INV_FILE_PATH  The file path and name of an initial previous
23#                     inventory snapshot file in JSON format.  Inventory
24#                     snapshots taken before and after each HTX run will
25#                     be compared to this file.
26#                     This parameter is optional.  If not specified, an
27#                     initial inventory snapshot will be taken before
28#                     HTX startup.
29# INV_IGNORE_LIST     A comma-delimited list of colon-delimited pairs that
30#                     indicate what to ignore if there are inventory
31#                     differences.  For example, "processor:size,network:speed"
32#                     If differences are found during inventory checking
33#                     and those items are in this string, the
34#                     differences will be ignored.  This parameter is
35#                     optional.  If not specified the default value is
36#                     "processor:size".
37
38Resource        ../syslib/utils_os.robot
39Library         ../syslib/utils_keywords.py
40
41Suite Setup     Run Key  Start SOL Console Logging
42Test Setup      Pre Test Case Execution
43Test Teardown   Post Test Case Execution
44
45
46*** Variables ****
47
48${stack_mode}                skip
49${json_initial_file_path}    ${EXECDIR}/os_inventory_initial.json
50${json_final_file_path}      ${EXECDIR}/os_inventory_final.json
51${json_diff_file_path}       ${EXECDIR}/os_inventory_diff.json
52${last_inventory_file_path}  ${EMPTY}
53${CHECK_INVENTORY}           True
54${INV_IGNORE_LIST}           processor:size
55${PREV_INV_FILE_PATH}        ${EMPTY}
56
57
58*** Test Cases ***
59
60Hard Bootme Test
61    [Documentation]  Stress the system using HTX exerciser.
62    [Tags]  Hard_Bootme_Test
63
64    # Set last inventory file to PREV_INV_FILE_PATH otherwise set it
65    # to ${EMPTY}.
66    ${last_inventory_file_path}=  Get Variable Value  ${PREV_INV_FILE_PATH}
67    ...  ${EMPTY}
68
69    Rprintn
70    Rpvars  HTX_DURATION  HTX_INTERVAL  CHECK_INVENTORY  INV_IGNORE_LIST
71    ...  PREV_INV_FILE_PATH
72
73    Run Keyword If  '${last_inventory_file_path}' != '${EMPTY}'
74    ...  OperatingSystem.File Should Exist  ${last_inventory_file_path}
75
76    Set Suite Variable  ${last_inventory_file_path}  children=true
77    Set Suite Variable  ${INV_IGNORE_LIST}  children=true
78
79    Repeat Keyword  ${HTX_LOOP} times  Run HTX Exerciser
80
81
82*** Keywords ***
83
84Run HTX Exerciser
85    [Documentation]  Run HTX exerciser.
86    # Test Flow:
87    # - Power on.
88    # - Establish SSH connection session.
89    # - Do inventory collection, compare with
90    #   previous inventory run if applicable.
91    # - Create HTX mdt profile.
92    # - Run HTX exerciser.
93    # - Check HTX status for errors.
94    # - Do inventory collection, compare with
95    #   previous inventory run.
96    # - Power off.
97
98    Boot To OS
99
100    # Post Power off and on, the OS SSH session needs to be established.
101    Login To OS
102
103    Run Keyword If  '${CHECK_INVENTORY}' == 'True'
104    ...  Do Inventory And Compare  ${json_initial_file_path}
105    ...  ${last_inventory_file_path}
106
107    Run Keyword If  '${HTX_MDT_PROFILE}' == 'mdt.bu'
108    ...  Create Default MDT Profile
109
110    Run MDT Profile
111
112    Loop HTX Health Check
113
114    Shutdown HTX Exerciser
115
116    Run Keyword If  '${CHECK_INVENTORY}' == 'True'
117    ...  Do Inventory And Compare  ${json_final_file_path}
118    ...  ${last_inventory_file_path}
119
120    Power Off Host
121
122    # Close all SSH and REST active sessions.
123    Close All Connections
124    Flush REST Sessions
125
126    Rprint Timen  HTX Test ran for: ${HTX_DURATION}
127
128
129Do Inventory And Compare
130    [Documentation]  Do inventory and compare.
131    [Arguments]  ${inventory_file_path}  ${last_inventory_file_path}
132    # Description of argument(s):
133    # inventory_file_path        The file to receive the inventory snapshot.
134    # last_inventory_file_path   The previous inventory to compare with.
135
136    Create JSON Inventory File  ${inventory_file_path}
137    Run Keyword If  '${last_inventory_file_path}' != '${EMPTY}'
138    ...  Compare Json Inventory Files  ${inventory_file_path}
139    ...  ${last_inventory_file_path}
140    ${last_inventory_file_path}=   Set Variable  ${inventory_file_path}
141    Set Suite Variable  ${last_inventory_file_path}  children=true
142
143
144Compare Json Inventory Files
145    [Documentation]  Compare JSON inventory files.
146    [Arguments]  ${file1}  ${file2}
147    # Description of argument(s):
148    # file1   A file that has an inventory snapshot in JSON format.
149    # file2   A file that has an inventory snapshot, to compare with file1.
150
151    ${diff_rc}=  JSON_Inv_File_Diff_Check  ${file1}
152     ...  ${file2}  ${json_diff_file_path}  ${INV_IGNORE_LIST}
153    Run Keyword If  '${diff_rc}' != '${0}'
154    ...  Report Inventory Mismatch  ${diff_rc}  ${json_diff_file_path}
155
156
157Report Inventory Mismatch
158    [Documentation]  Report inventory mismatch.
159    [Arguments]  ${diff_rc}  ${json_diff_file_path}
160    # Description of argument(s):
161    # diff_rc              The failing return code from the difference check.
162    # json_diff_file_path  The file that has the latest inventory snapshot.
163
164    Log To Console  Significant difference in inventory found, rc=${diff_rc}
165    Log To Console  Differences are listed in file:  no_newline=true
166    Log To Console  ${json_diff_file_path}
167    Log To Console  File Contents:
168    Wait Until Created  ${json_diff_file_path}
169    ${file_contents}=  OperatingSystem.Get File  ${json_diff_file_path}
170    Log  ${file_contents}  level=WARN
171    Fail  Significant difference in inventory found, rc=${diff_rc}
172
173
174Loop HTX Health Check
175    [Documentation]  Run until HTX exerciser fails.
176    Repeat Keyword  ${HTX_DURATION}
177    ...  Run Keywords  Check HTX Run Status
178    ...  AND  Sleep  ${HTX_INTERVAL}
179
180
181Post Test Case Execution
182    [Documentation]  Do the post test teardown.
183    # 1. Shut down HTX exerciser if test Failed.
184    # 2. Capture FFDC on test failure.
185    # 3. Close all open SSH connections.
186
187    # Keep HTX running if user set HTX_KEEP_RUNNING to 1.
188    Run Keyword If
189    ...  '${TEST_STATUS}' == 'FAIL' and ${HTX_KEEP_RUNNING} == ${0}
190    ...  Shutdown HTX Exerciser
191
192    ${keyword_buf}=  Catenate  Stop SOL Console Logging
193    ...  \ targ_file_path=${EXECDIR}${/}logs${/}SOL.log
194    Run Key  ${keyword_buf}
195
196    FFDC On Test Case Fail
197    Close All Connections
198