1*** Settings ***
2Documentation  Utilities for fan tests.
3
4Library        ../lib/bmc_ssh_utils.py
5Resource       ../lib/state_manager.robot
6Resource       ../lib/openbmc_ffdc_utils.robot
7Variables      ../data/variables.py
8
9*** Variables ***
10
11# Fan state values.
12${fan_functional}      ${1}
13${fan_nonfunctional}   ${0}
14
15# Criteria for a fan at maximum speed.
16${max_speed}=  ${10400}
17
18
19*** Keywords ***
20
21Is Water Cooled
22    [Documentation]  Return 1 if system is water cooled, 0 othersise.
23
24    ${water_cooled}=  Read Attribute
25    ...  ${HOST_INVENTORY_URI}system/chassis  WaterCooled
26    [Return]  ${water_cooled}
27
28
29Get Fan Names
30    [Documentation]  Get the names of the fans marked present in inventory.
31    [Arguments]  ${fan_names}
32    # This keyword populates the fan_names list with the names of
33    # fans present in inventory e.g. fan0, fan2, fan3.
34
35    # Description of Argument(s):
36    # fan_names   The list of fan names to which new fan names are to be
37    #             added to.  This list is returned to the caller.
38
39    ${fan_uris}=  Get Endpoint Paths  ${HOST_INVENTORY_URI}system  fan
40    : FOR  ${fan_uri}  IN  @{fan_uris}
41    \  ${fan_properties}=  Read Properties  ${fan_uri}
42    \  ${fan_present}=  Get Variable Value  ${fan_properties['Present']}  0
43    \  ${fan_functional}=  Get Variable Value
44    ...  ${fan_properties['Functional']}  0
45    \  Continue For Loop If  ${fan_present} == 0 or ${fan_functional} == 0
46    \  ${remaining_uri}  ${fan_name}=  Split Path  ${fan_uri}
47    \  Append To List  ${fan_names}  ${fan_name}
48
49    [Return]  ${fan_names}
50
51
52Verify System Error Indication Due To Fans
53    [Documentation]  Verify enclosure LEDs are on and there's an error log.
54
55    # Both enclosure LEDs should now be On.
56    Verify Front And Rear LED State  On
57
58    # An error log should now exist.
59    Error Logs Should Exist
60
61
62Verify Front And Rear LED State
63    [Documentation]  Check state of the front and rear enclsure fault LEDs.
64    [Arguments]  ${state}
65    # Both LEDs should be in the specified state.  If not fail the test case.
66
67    # Description of Argument(s):
68    # state    The state to check for, either 'Off' or 'On'.
69
70    ${front_fault}=  Get System LED State  front_fault
71    ${rear_fault}=  Get System LED State  rear_fault
72
73    Run Keyword If
74    ...  '${front_fault}' != '${state}' or '${rear_fault}' != '${state}'
75    ...  Fail  msg=Expecting both enclosure LEDs to be ${state}.
76
77
78Set Fan State
79    [Documentation]  Set the fan state, either functional or non-functional.
80    [Arguments]  ${fan_name}  ${fan_state}
81
82    # Description of Argument(s):
83    # fan_name     The name of the fan, e.g. "fan2".
84    # fan_state    The state to set, 1 for functional, 2 for non-functional.
85
86    ${valueDict}=  Create Dictionary  data=${fan_state}
87    Write Attribute
88    ...  ${HOST_INVENTORY_URI}system/chassis/motherboard/${fan_name}
89    ...  Functional  data=${valueDict}
90
91
92Set Fan Target Speed
93    [Documentation]  Set the target speed of a fan.
94    [Arguments]  ${fan_name}  ${fan_speed}
95
96    # Description of argument(s):
97    # fan_name    The name of the fan (e.g. "fan0").
98    # fan_speed   The target speed to set (e.g. "9000").
99
100    ${valueDict}=  Create Dictionary  data=${fan_speed}
101    Write Attribute  ${SENSORS_URI}fan_tach/${fan_name}_0
102    ...  Target  data=${valueDict}
103
104
105Get Target Speed Of Fans
106    [Documentation]  Return the maximum target speed of the system fans.
107
108    ${max_target}=  Set Variable  0
109    ${paths}=  Get Endpoint Paths  ${SENSORS_URI}fan_tach/  0
110    :FOR  ${path}  IN  @{paths}
111    \  ${response}=  OpenBMC Get Request  ${path}
112    \  ${json}=  To JSON  ${response.content}
113    \  ${target_speed}=  Set Variable  ${json["data"]["Target"]}
114    \  ${max_target}=  Run Keyword If  ${target_speed} > ${max_target}
115    ...  Set Variable  ${target_speed}  ELSE  Set Variable  ${max_target}
116    [Return]  ${max_target}
117
118
119Get Target And Blade Speeds
120    [Documentation]  Return the fan target speed setting, the speed of the
121    ...  fan's clockwise blade, and the speed of the counter-clockwise blade.
122    # Each fan unit has two counter-rotating fan blades
123    # One blade is expected to be moving but the other blade may not be
124    # moving whenever the fan unit is transitioning to a new target speed.
125    [Arguments]  ${fan_name}
126
127    # Description of argument(s):
128    # fan_name       The name of a fan (e.g. "fan0")
129
130    # Get the fan target speed and the clockwise blade speed.
131    ${path}=  Catenate  ${SENSORS_URI}fan_tach/${fan_name}_0
132    ${response}=  OpenBMC Get Request  ${path}
133    ${json}=  To JSON  ${response.content}
134    ${fan_clockwise_speed}=  Set Variable  ${json["data"]["Value"]}
135    ${target_speed}=  Set Variable  ${json["data"]["Target"]}
136
137    # Get the counter-clockwise blade speed.
138    ${path}=  Catenate  ${SENSORS_URI}fan_tach/${fan_name}_1
139    ${response}=  OpenBMC Get Request  ${path}
140    ${json}=  To JSON  ${response.content}
141    ${fan_counterclockwise_speed}=  Set Variable  ${json["data"]["Value"]}
142
143    [Return]  ${target_speed}  ${fan_clockwise_speed}
144    ...  ${fan_counterclockwise_speed}
145
146
147Get Fan Target And Speed
148    [Documentation]  Return the fan target speed setting and the
149    ...  speed of the fastest blade.
150    [Arguments]  ${fan_name}
151
152    # Description of argument(s):
153    # fan_name       The name of a fan (e.g. "fan0")
154
155    ${target_speed}  ${clockwise_speed}  ${counterclockwise_speed}=
156    ...  Get Target And Blade Speeds  ${fan_name}
157    ${blade_speed}=  Run Keyword If
158    ...  ${clockwise_speed} > ${counterclockwise_speed}
159    ...  Set Variable  ${clockwise_speed}  ELSE
160    ...  Set Variable  ${counterclockwise_speed}
161    [Return]  ${target_speed}  ${blade_speed}
162
163
164Set Fan Daemon State
165    [Documentation]  Set the state of the fan control service.
166    [Arguments]  ${state}
167
168    # Description of argument(s):
169    # state     The desired state of the service, usually
170    #           "start", "stop", or "restart".
171
172    ${cmd}=  Catenate  systemctl  ${state}  phosphor-fan-control@0.service
173    ${stdout}  ${stderr}  ${rc}=  BMC Execute Command  ${cmd}
174
175
176Verify Minimum Number Of Fans With Cooling Type
177    [Documentation]  Verify minimum number of fans.
178    [Arguments]  ${num_fans}  ${water_cooled}
179
180    # Description of argument(s):
181    # num_fans       The number of fans present in the system.
182    # water_cooled   The value 1 if the system is water cooled,
183    #                0 if air cooled.
184
185    # For a water cooled system.
186    ${min_fans_water}=  Set Variable  2
187
188    # For an air cooled system.
189    ${min_fans_air}=  Set Variable  3
190
191    Printn
192    Rpvars  num_fans  water_cooled
193
194    # If water cooled must have at least min_fans_water fans, otherwise
195    # issue Fatal Error and terminate testing.
196    Run Keyword If  ${water_cooled} == 1 and ${num_fans} < ${min_fans_water}
197    ...  Fatal Error
198    ...  msg=Water cooled but less than ${min_fans_water} fans present.
199
200    # If air cooled must have at least min_fans_air fans.
201    Run Keyword If  ${water_cooled} == 0 and ${num_fans} < ${min_fans_air}
202    ...  Fatal Error
203    ...  msg=Air cooled but less than ${min_fans_air} fans present.
204
205
206Verify Fan Monitors With State
207    [Documentation]  Verify fan monitor daemons in the system state.
208    [Arguments]  ${power_state}
209    # The number of monitoring daemons is dependent upon the system
210    # power state.  If power is off there should be 0, if power
211    # is on there should be several.
212
213    # Description of argument(s):
214    # power_state   Power staet of the system, either "On" or "Off"
215
216    ${cmd}=  Catenate  systemctl list-units | grep phosphor-fan | wc -l
217    ${num_fan_daemons}  ${stderr}  ${rc}=  BMC Execute Command  ${cmd}
218
219    Rpvars  power_state  num_fan_daemons
220
221    # Fail if system is On and there are no fan monitors.
222    Run Keyword If  '${power_state}' == 'On' and ${num_fan_daemons} == 0
223    ...  Fail  msg=No phosphor-fan monitors found at power on.
224
225    # Fail if system is Off and the fan monitors are present.
226    Run Keyword If  '${power_state}' == 'Off' and ${num_fan_daemons} != 0
227    ...  Fail  msg=Phosphor-fan monitors found at power off.
228
229
230Get Fan Count And Names
231    [Documentation]  Return the number of fans and the fan names.
232
233    # The @{fan_names} list holds the names of the fans in the system.
234    @{fan_names}  Create List
235    ${fan_names}=  Get Fan Names  ${fan_names}
236
237    ${number_of_fans}=  Get Length  ${fan_names}
238
239    [Return]  ${number_of_fans}  ${fan_names}
240
241
242
243Reset Fans
244    [Documentation]  Set the fans to functional state.
245    # Set state of fans to functional by writing 1 to the Functional
246    # attribute of each fan in the @{fan_names} list.  If @{fan_names}
247    # is empty nothing is done.
248    [Arguments]  ${fan_names}
249
250    # Description of Argument(s):
251    # fan_names    A list containing the names of the fans (e.g. fan0
252    #              fan2 fan3).
253
254    :FOR  ${fan_name}  IN  @{fan_names}
255    \  Set Fan State  ${fan_name}  ${fan_functional}
256
257
258Verify Fan Speed
259    [Documentation]  Verify fans are running at or near target speed.
260    [Arguments]  ${tolerance}  ${fan_names}
261
262    # Description of argument(s):
263    # tolerance   The speed tolerance criteria.
264    #             A tolerance value of .15 means that the fan's speed
265    #             should be within 15% of its set target speed.
266    #             Fans may be accelerating to meet a new target, so
267    #             allow .10 extra.
268    # fan_names   A list containing the names of the fans (e.g. fan0 fan1).
269
270    # Compare the fan's speed with its target speed.
271    :FOR  ${fan_name}  IN  @{fan_names}
272    \  ${target_speed}  ${fan_speed}=  Get Fan Target And Speed  ${fan_name}
273    \  Rpvars  fan_name  target_speed  fan_speed
274    \  # Calculate tolerance, which is a % of the target speed.
275    \  ${tolerance_value}=  Evaluate  ${tolerance}*${target_speed}
276    \  # Calculate upper and lower speed limits.
277    \  ${max_limit}=  Evaluate   ${target_speed}+${tolerance_value}
278    \  ${min_limit}=  Evaluate   ${target_speed}-${tolerance_value}
279    \  Run Keyword If
280    ...  ${fan_speed} < ${min_limit} or ${fan_speed} > ${max_limit}
281    ...  Fail  msg=${fan_name} speed of ${fan_speed} is out of range.
282
283
284Verify Direct Fan Control
285    [Documentation]  Verify direct control of fans.
286    [Arguments]  ${max_speed}  ${min_speed}
287    ...  ${minutes_to_stabilize}  ${number_of_fans}  ${fan_names}
288
289    # Overview:
290    # Turn off BMC's fan control daemon, then test to confirm
291    # that fans can be controlled manually.
292    # The app that takes data from sysfs and updates dbus is named hwmon.
293    # Verify hwmon functionality by comparing with what's on dbus
294    # (/xyz/openbmc_project/sensors/fan_tach/fan0_0, fan0_1, etc.)
295    # with what's in the BMC's file system at
296    # /sys/class/hwmon/hwmon9/fan*_input.
297
298    # Description of argument(s):
299    # max_speed               Integer value of maximum fan speed.
300    # min_speed               Integer value of minimum speed.
301    # minutes_to_stabilize    Time to wait for fan daemons to
302    #                         stabilize fan operation after
303    #                         tests (e.g. "4").
304    # number_of_fans          The number of fans in the system.
305    # fan_names               A list containing the names of the
306    #                         fans (e.g. fan0 fan1).
307
308    # Login to BMC and disable the fan daemon. Disabling the daemon sets
309    # manual mode.
310    Open Connection And Log In
311    Set Fan Daemon State  stop
312
313    # For each fan, set a new target speed and wait for the fan to
314    # accelerate.  Then check that the fan is running near that speed.
315    :FOR  ${fan_name}  IN  @{fan_names}
316    \  Set Fan Target Speed  ${fan_name}  ${max_speed}
317    \  Run Key U  Sleep \ 60s
318    \  ${target_speed}  ${cw_speed}  ${ccw_speed}=
319    ...  Get Target And Blade Speeds  ${fan_name}
320    \  Rpvars  fan_name  target_speed  cw_speed  ccw_speed
321    \  Run Keyword If
322    ...  ${cw_speed} < ${min_speed} or ${ccw_speed} < ${min_speed}
323    ...  Fail  msg=${fan_name} failed manual speed test.
324
325    # Check the fan speeds in the BMC file system.
326
327    # Get the location of the fan hwmon.
328    ${controller_path}  ${stderr}  ${rc}=  BMC Execute Command
329    ...  grep -ir max31785a /sys/class/hwmon/hwmon* | grep name
330    # E.g., controller_path=/sys/class/hwmon/hwmon10/name:max31785a.
331
332    ${hwmon_path}  ${file_name}=  Split Path  ${controller_path}
333    # E.g.,  /sys/class/hwmon/hwmon10  or  /sys/class/hwmon/hwmon9.
334
335    Rpvars  controller_path  hwmon_path
336
337    # Run the BMC command which gets fan speeds from the file system.
338    ${cmd}=  Catenate  cat ${hwmon_path}/fan*_input
339    ${stdout}  ${stderr}  ${rc}=
340    ...  BMC Execute Command  ${cmd}
341
342    # Convert output to integer values.
343    ${speeds}=  Evaluate  map(int, $stdout.split(${\n}))
344    Rpvars  speeds
345    # Count the number of speeds > ${min_speed}.
346    ${count}=  Set Variable  ${0}
347    :FOR  ${speed}  IN  @{speeds}
348    \  ${count}=  Run Keyword If  ${speed} > ${min_speed}
349    ...  Evaluate  ${count}+1  ELSE  Set Variable  ${count}
350    # Because each fan has two rotating fan blades, the count should be
351    # equual to 2*${number_of_fans}.  On water-cooled systems some
352    # speeds may be reported by hwmon as 0.  That is expected,
353    # and the number_of_fans reported in the system will be less.
354    ${fail_test}=  Evaluate  (2*${number_of_fans})-${count}
355
356    # Re-enable the fan daemon
357    Set Fan Daemon State  restart
358
359    Run Keyword If  ${fail_test} > ${0}  Fail
360    ...  msg=hwmon did not properly report fan speeds.
361
362    # Wait for the daemon to take control and gracefully set fan speeds
363    # back to normal.
364    ${msg}=  Catenate  Waiting ${minutes_to_stabilize} minutes
365    ...  for fan daemon to stabilize fans.
366    Print Timen  ${msg}
367    Run Key U  Sleep \ ${minutes_to_stabilize}m
368
369
370Verify Fan Speed Increase
371    [Documentation]  Verify that the speed of working fans increase when
372    ...  one fan is marked as disabled.
373    #  A non-functional fan should cause an error log and
374    #  an enclosure LED will light.  The other fans should speed up.
375    [Arguments]  ${fan_names}
376
377    # Description of argument(s):
378    # fan_names   A list containing the names of the fans (e.g. fan0 fan1).
379
380    # Allow system_response_time before checking if there was a
381    # response by the system to an applied fault.
382    ${system_response_time}=  Set Variable  60s
383
384    # Choose a fan to test with, e.g., fan0.
385    ${test_fan_name}=  Get From List  ${fan_names}  0
386
387    ${initial_speed}=  Get Target Speed Of Fans
388    Rpvars  test_fan_name  initial_speed
389
390    # If initial speed is not already at maximum, set expect_increase.
391    # This flag is used later to determine if speed checking is
392    # to be done or not.
393    ${expect_increase}=  Run Keyword If
394    ...  ${initial_speed} < ${max_speed}
395    ...  Set Variable  1  ELSE  Set Variable  0
396
397    Set Fan State  ${test_fan_name}  ${fan_nonfunctional}
398
399    Run Key U  Sleep \ ${system_response_time}
400
401    # A heavily loaded system may have powered-off.
402    ${host_state}=  Get Host State
403    Rpvars  host_state
404    Run Keyword If  'Running' != '${host_state}'  Pass Execution
405    ...  msg=System shutdown so skipping remainder of test.
406
407    ${new_fan_speed}=  Get Target Speed Of Fans
408    Rpvars  expect_increase  initial_speed  new_fan_speed
409
410    # Fail if current fan speed did not increase past the initial
411    # speed, but do this check only if not at maximum speed to begin with.
412    Run Keyword If
413    ...  ${expect_increase} == 1 and ${new_fan_speed} < ${initial_speed}
414    ...  Fail  msg=Remaining fans did not increase speed with loss of one fan.
415
416
417
418Verify System Shutdown Due To Fans
419    [Documentation]  Shut down when not enough fans.
420    [Arguments]  ${fan_names}
421
422    # Description of argument(s):
423    # fan_names   A list containing the names of the fans (e.g. fan0 fan1).
424
425    ${wait_after_poweroff}=  Set Variable  15s
426
427    # Set fans to be non-functional.
428    :FOR  ${fan_name}  IN  @{fan_names}
429    \  Set Fan State  ${fan_name}  ${fan_nonfunctional}
430
431    # System should notice the non-functional fans and power-off.
432    # The Wait For PowerOff keyword will time-out and report
433    # an error if power off does not happen within a reasonable time.
434    Wait For PowerOff
435