1*** Settings ***
2Documentation       Utility for error injection scenarios through HOST & BMC.
3Resource            ../../lib/rest_client.robot
4Resource            ../../lib/utils.robot
5Resource            ../../lib/common_utils.robot
6Variables           ../../lib/ras/variables.py
7Library             ../../lib/bmc_ssh_utils.py
8Library             ../../lib/gen_print.py
9Library             ../../lib/gen_robot_print.py
10
11Library             OperatingSystem
12
13*** Keywords ***
14
15Getscom Operations On OS
16    [Documentation]  Executes getscom command on OS with the given
17    ...              input command.
18    [Arguments]      ${input_cmd}
19    # Description of arguments:
20    # input_cmd      -l|--list-chips
21    #                -c|--chip <chip-id> <addr>
22
23    ${output}  ${stderr}  ${rc}=  OS Execute Command  getscom ${input_cmd}
24    [Return]  ${output}
25
26Gard Operations On OS
27    [Documentation]  Executes opal-gard command on OS with the given
28    ...              input command.
29    [Arguments]      ${input_cmd}
30    # Description of arguments:
31    # input_cmd      list/clear all/show <gard_record_id>
32
33    ${output}  ${stderr}  ${rc}=  OS Execute Command  opal-gard ${input_cmd}
34    [Return]  ${output}
35
36Putscom Operations On OS
37    [Documentation]  Executes putscom command on OS with the given
38    ...              input arguments.
39    [Arguments]      ${proc_chip_id}  ${fru}  ${address}
40    # Description of arguments:
41    # proc_chip_id        Processor ID (e.g '0', '8').
42    # fru            FRU value (e.g. 2011400).
43    # address        Chip address (e.g 4000000000000000).
44
45    ${cmd}=  Catenate  putscom -c 0x${proc_chip_id} 0x${fru} 0x${address}
46    Start Command  ${cmd}
47
48Get ProcChipId From OS
49    [Documentation]  Get processor chip ID values based on the input.
50    [Arguments]      ${chip_type}  ${master_proc_chip}
51    # Description of arguments:
52    # chip_type         The chip type (Processor/Centaur).
53    # master_proc_chip  Processor chip type ('True' or 'False').
54
55    ${cmd}=  Catenate  -l | grep -i ${chip_type} | cut -c1-8
56    ${proc_chip_id}=  Getscom Operations On OS  ${cmd}
57    # Example output:
58    # getscom -l | grep processor | cut -c1-8
59    # 00000008     - False
60    # 00000000     - True
61
62    ${proc_ids}=  Split String  ${proc_chip_id}
63    ${proc_id}=  Run Keyword If  '${master_proc_chip}' == 'True'
64    \  ...  Get From List  ${proc_ids}  1
65    \  ...  ELSE  Get From List  ${proc_ids}  0
66
67    # Example output:
68    # 00000008
69    [Return]  ${proc_id}
70
71Get Core IDs From OS
72    [Documentation]  Get Core IDs corresponding to the input processor chip ID.
73    [Arguments]      ${proc_chip_id}
74    # Description of argument(s):
75    # proc_chip_id        Processor ID (e.g '0', '8').
76
77    ${cmd}=  Catenate  set -o pipefail ; ${probe_cpu_file_path}
78    ...    | grep -i 'CHIP ID: ${proc_chip_id}' | cut -c21-22
79    ${output}  ${stderr}  ${rc}=  OS Execute Command  ${cmd}
80    ${core_ids}=  Split String  ${output}
81    # Example output:
82    # ['2', '3', '4', '5', '6']
83    [Return]  ${core_ids}
84
85FIR Address Translation Through HOST
86    [Documentation]  Do FIR address translation through host for given FIR,
87    ...              core value & target type.
88    [Arguments]  ${fir}  ${core_id}  ${target_type}
89    # Description of argument(s):
90    # fir          FIR (Fault isolation register) value (e.g. 2011400).
91    # core_id      Core ID (e.g. 9).
92    # target_type  Target type (e.g. 'EQ', 'EX', 'C').
93
94    ${cmd}=  Catenate  set -o pipefail ; ${addr_translation_file_path} ${fir}
95    ...  ${core_id} | grep -i ${target_type}
96    ${output}  ${stderr}  ${rc}=  OS Execute Command  ${cmd}
97    ${translated_addr}=  Split String  ${output}  :${SPACE}0x
98    # Example output:
99    # 0x10010c00
100    [Return]  ${translated_addr[1]}
101
102Inject Error Through HOST
103    [Documentation]  Inject checkstop on multiple targets like
104    ...              CPU/CME/OCC/NPU/CAPP/MCA etc. through HOST.
105    ...              Test sequence:
106    ...              1. Boot To HOST.
107    ...              2. Clear any existing gard records.
108    ...              3. Inject Error on processor.
109    [Arguments]      ${fir}  ${chip_address}  ${threshold_limit}
110    ...  ${master_proc_chip}=True
111    # Description of argument(s):
112    # fir                 FIR (Fault isolation register) value (e.g. 2011400).
113    # chip_address        chip address (e.g 2000000000000000).
114    # threshold_limit     Threshold limit (e.g 1, 5, 32).
115    # master_proc_chip    Processor chip type (True' or 'False').
116
117    Delete Error Logs
118    Login To OS Host
119    Set Auto Reboot  1
120    Gard Operations On OS  clear all
121
122    # Fetch processor chip IDs.
123    ${proc_chip_id}=  Get ProcChipId From OS  Processor  ${master_proc_chip}
124
125    ${threshold_limit}=  Convert To Integer  ${threshold_limit}
126    :FOR  ${count}  IN RANGE  ${threshold_limit}
127    \  Run Keyword  Putscom Operations On OS  ${proc_chip_id}  ${fir}
128    ...  ${chip_address}
129    # Adding delay after each error injection.
130    \  Sleep  10s
131    # Adding delay to get error log after error injection.
132    Sleep  120s
133
134Code Update Unrecoverable Error Inject
135    [Documentation]  Inject UE MCACALFIR checkstop on processor through
136    ...   host during PNOR code update.
137
138    Inject Error Through HOST  05010800  4000000000000000  1
139
140Disable CPU States Through HOST
141    [Documentation]  Disable CPU states through host.
142
143    # Fetch number of states present for cpu0.
144    ${cmd}=  Catenate  ls /sys/devices/system/cpu/cpu0/cpuidle|grep state|wc -l
145    ${output}  ${stderr}  ${rc}=  OS Execute Command  ${cmd}
146    ${no_of_states}=  Convert To Integer  ${output}
147
148    # Disable state for all cpus.
149    FOR  ${count}  IN RANGE  ${no_of_states}
150        ${cmd}=  Catenate  SEPARATOR=  for file_path in /sys/devices/system/cpu/
151        ...  cpu*/cpuidle/state${count}/disable; do echo 1 > $file_path; done
152        ${output}  ${stderr}  ${rc}=  Run Keyword  OS Execute Command  ${cmd}
153    END
154
155Is Opal-PRD Service Enabled
156    [Documentation]  Check if Opal-PRD service is running & return either
157    ...              'enabled' or 'disabled'.
158
159    ${cmd}=  Catenate  systemctl list-unit-files | grep opal-prd
160    ${output}  ${stderr}  ${rc}=  OS Execute Command  ${cmd}
161    ${opal_prd_state}=  Split String  ${output}
162
163    # Example output from prior command:
164    # opal-prd.service enabled
165    [Return]  ${opal_prd_state[1]}
166
167Enable Opal-PRD Service On HOST
168    [Documentation]  Enable Opal-PRD service on host.
169
170    OS Execute Command  service opal-prd start
171    ${opal_prd_state}=  Is Opal-PRD Service Enabled
172    Should Contain  ${opal_prd_state}  enabled
173
174BMC Putscom
175    [Documentation]  Executes putscom command through BMC.
176
177    [Arguments]      ${proc_chip_id}  ${fru}  ${chip_address}
178
179    # Description of argument(s):
180    # proc_chip_id        Processor ID (e.g '0', '8').
181    # fru                 FRU (field replaceable unit) (e.g. '2011400').
182    # chip_address        Chip address (e.g. '4000000000000000').
183
184    ${cmd}=  Catenate  pdbg -d p9w -p${proc_chip_id} putscom 0x${fru} 0x${chip_address}
185
186    BMC Execute Command  ${cmd}
187
188Inject Error Through BMC
189    [Documentation]  Inject checkstop on multiple targets like
190    ...              CPU/CME/OCC/NPU/CAPP/MCA etc. through BMC.
191    ...              Test sequence:
192    ...              1. Boot To HOST.
193    ...              2. Clear any existing gard records.
194    ...              3. Inject Error on processor.
195    [Arguments]      ${fir}  ${chip_address}  ${threshold_limit}
196    ...  ${master_proc_chip}=True
197    # Description of argument(s):
198    # fir                 FIR (Fault isolation register) value (e.g. '2011400').
199    # chip_address        Chip address (e.g. '2000000000000000').
200    # threshold_limit     Recoverable error threshold limit (e.g. '1', '5', '32').
201
202    Delete Error Logs
203    Login To OS Host
204    Set Auto Reboot  1
205
206    Gard Operations On OS  clear all
207
208    ${threshold_limit}=  Convert To Integer  ${threshold_limit}
209    :FOR  ${count}  IN RANGE  ${threshold_limit}
210    \  BMC Putscom  0  ${fir}
211    ...  ${chip_address}
212    # Adding delay after each error injection.
213    \  Sleep  10s
214    # Adding delay to get error log after error injection.
215    Sleep  120s
216
217
218Inject Error Through BMC At HOST Boot
219    [Documentation]  Inject error on multiple targets like
220    ...              CPU/CME/OCC/NPU/CAPP/MCA etc. through BMC at HOST Boot.
221    ...              Test sequence:
222    ...              1. Boot To HOST.
223    ...              2. Clear any existing gard records.
224    ...              3. Power off HOST and Boot.
225    ...              4. Inject Error on processor through BMC.
226    [Arguments]      ${fir}  ${chip_address}
227    # Description of argument(s):
228    # fir                 FIR (Fault isolation register) value (e.g. '2011400').
229    # chip_address        Chip address (e.g. '2000000000000000').
230
231    Delete Error Logs
232
233    REST Power On  stack_mode=skip
234
235    Gard Operations On OS  clear all
236
237    REST Power Off
238    Set Auto Reboot  1
239    Initiate Host Boot  wait=${0}
240
241    Start SOL Console Logging   ${EXECDIR}/esol.log
242
243    Wait Until Keyword Succeeds  5 min  5 sec
244    ...  Shell Cmd  grep 'ISTEP *14' ${EXECDIR}/esol.log  quiet=1
245    ...  print_output=0  show_err=0  ignore_err=0
246
247    BMC Putscom  0  ${fir}  ${chip_address}
248    # Adding delay to get error log after error injection.
249    Sleep  10s
250
251    Stop SOL Console Logging
252