1*** Settings *** 2Documentation Utilities for fan tests. 3 4Library ../lib/bmc_ssh_utils.py 5Resource ../lib/state_manager.robot 6Resource ../lib/openbmc_ffdc_utils.robot 7Variables ../data/variables.py 8 9*** Variables *** 10 11# Fan state values. 12${fan_functional} ${1} 13${fan_nonfunctional} ${0} 14 15# Criteria for a fan at maximum speed. 16${max_speed}= ${10400} 17 18 19*** Keywords *** 20 21Is Water Cooled 22 [Documentation] Return 1 if system is water cooled, 0 othersise. 23 24 ${water_cooled}= Read Attribute 25 ... ${HOST_INVENTORY_URI}system/chassis WaterCooled 26 RETURN ${water_cooled} 27 28 29Get Fan Names 30 [Documentation] Get the names of the fans marked present in inventory. 31 [Arguments] ${fan_names} 32 # This keyword populates the fan_names list with the names of 33 # fans present in inventory e.g. fan0, fan2, fan3. 34 35 # Description of Argument(s): 36 # fan_names The list of fan names to which new fan names are to be 37 # added to. This list is returned to the caller. 38 39 ${fan_uris}= Get Endpoint Paths ${HOST_INVENTORY_URI}system fan 40 FOR ${fan_uri} IN @{fan_uris} 41 ${fan_properties}= Read Properties ${fan_uri} 42 ${fan_present}= Get Variable Value ${fan_properties['Present']} 0 43 ${fan_functional}= Get Variable Value 44 ... ${fan_properties['Functional']} 0 45 Continue For Loop If ${fan_present} == 0 or ${fan_functional} == 0 46 ${remaining_uri} ${fan_name}= Split Path ${fan_uri} 47 Append To List ${fan_names} ${fan_name} 48 END 49 50 RETURN ${fan_names} 51 52 53Verify System Error Indication Due To Fans 54 [Documentation] Verify enclosure LEDs are on and there's an error log. 55 56 # Both enclosure LEDs should now be On. 57 Verify Front And Rear LED State On 58 59 # An error log should now exist. 60 Error Logs Should Exist 61 62 63Verify Front And Rear LED State 64 [Documentation] Check state of the front and rear enclsure fault LEDs. 65 [Arguments] ${state} 66 # Both LEDs should be in the specified state. If not fail the test case. 67 68 # Description of Argument(s): 69 # state The state to check for, either 'Off' or 'On'. 70 71 ${front_fault}= Get System LED State front_fault 72 ${rear_fault}= Get System LED State rear_fault 73 74 IF '${front_fault}' != '${state}' or '${rear_fault}' != '${state}' 75 ... Fail msg=Expecting both enclosure LEDs to be ${state}. 76 77Set Fan State 78 [Documentation] Set the fan state, either functional or non-functional. 79 [Arguments] ${fan_name} ${fan_state} 80 81 # Description of Argument(s): 82 # fan_name The name of the fan, e.g. "fan2". 83 # fan_state The state to set, 1 for functional, 2 for non-functional. 84 85 ${valueDict}= Create Dictionary data=${fan_state} 86 Write Attribute 87 ... ${HOST_INVENTORY_URI}system/chassis/motherboard/${fan_name} 88 ... Functional data=${valueDict} 89 90 91Set Fan Target Speed 92 [Documentation] Set the target speed of a fan. 93 [Arguments] ${fan_name} ${fan_speed} 94 95 # Description of argument(s): 96 # fan_name The name of the fan (e.g. "fan0"). 97 # fan_speed The target speed to set (e.g. "9000"). 98 99 ${valueDict}= Create Dictionary data=${fan_speed} 100 Write Attribute ${SENSORS_URI}fan_tach/${fan_name}_0 101 ... Target data=${valueDict} 102 103 104Get Target Speed Of Fans 105 [Documentation] Return the maximum target speed of the system fans. 106 107 ${max_target}= Set Variable 0 108 ${paths}= Get Endpoint Paths ${SENSORS_URI}fan_tach/ 0 109 FOR ${path} IN @{paths} 110 ${response}= OpenBMC Get Request ${path} 111 ${target_speed}= Set Variable ${response.json()["data"]["Target"]} 112 ${max_target}= Set Variable If ${target_speed} > ${max_target} 113 ... ${target_speed} ${max_target} 114 END 115 RETURN ${max_target} 116 117 118Get Target And Blade Speeds 119 [Documentation] Return the fan target speed setting, the speed of the 120 ... fan's clockwise blade, and the speed of the counter-clockwise blade. 121 # Each fan unit has two counter-rotating fan blades 122 # One blade is expected to be moving but the other blade may not be 123 # moving whenever the fan unit is transitioning to a new target speed. 124 [Arguments] ${fan_name} 125 126 # Description of argument(s): 127 # fan_name The name of a fan (e.g. "fan0") 128 129 # Get the fan target speed and the clockwise blade speed. 130 ${path}= Catenate ${SENSORS_URI}fan_tach/${fan_name}_0 131 ${response}= OpenBMC Get Request ${path} 132 ${fan_clockwise_speed}= Set Variable ${response.json()["data"]["Value"]} 133 ${target_speed}= Set Variable ${response.json["data"]["Target"]} 134 135 # Get the counter-clockwise blade speed. 136 ${path}= Catenate ${SENSORS_URI}fan_tach/${fan_name}_1 137 ${response}= OpenBMC Get Request ${path} 138 ${fan_counterclockwise_speed}= Set Variable ${response.json()["data"]["Value"]} 139 140 RETURN ${target_speed} ${fan_clockwise_speed} 141 ... ${fan_counterclockwise_speed} 142 143 144Get Fan Target And Speed 145 [Documentation] Return the fan target speed setting and the 146 ... speed of the fastest blade. 147 [Arguments] ${fan_name} 148 149 # Description of argument(s): 150 # fan_name The name of a fan (e.g. "fan0") 151 152 ${target_speed} ${clockwise_speed} ${counterclockwise_speed}= 153 ... Get Target And Blade Speeds ${fan_name} 154 ${blade_speed}= Set Variable If ${clockwise_speed} > ${counterclockwise_speed} 155 ... ${clockwise_speed} ${counterclockwise_speed} 156 RETURN ${target_speed} ${blade_speed} 157 158 159Set Fan Daemon State 160 [Documentation] Set the state of the fan control service. 161 [Arguments] ${state} 162 163 # Description of argument(s): 164 # state The desired state of the service, usually 165 # "start", "stop", or "restart". 166 167 ${cmd}= Catenate systemctl ${state} phosphor-fan-control@0.service 168 ${stdout} ${stderr} ${rc}= BMC Execute Command ${cmd} 169 170 171Verify Minimum Number Of Fans With Cooling Type 172 [Documentation] Verify minimum number of fans. 173 [Arguments] ${num_fans} ${water_cooled} 174 175 # Description of argument(s): 176 # num_fans The number of fans present in the system. 177 # water_cooled The value 1 if the system is water cooled, 178 # 0 if air cooled. 179 180 # For a water cooled system. 181 ${min_fans_water}= Set Variable 2 182 183 # For an air cooled system. 184 ${min_fans_air}= Set Variable 3 185 186 Printn 187 Rpvars num_fans water_cooled 188 189 # If water cooled must have at least min_fans_water fans, otherwise 190 # issue Fatal Error and terminate testing. 191 IF ${water_cooled} == 1 and ${num_fans} < ${min_fans_water} 192 ... Fatal Error msg=Water cooled but less than ${min_fans_water} fans present. 193 194 # If air cooled must have at least min_fans_air fans. 195 IF ${water_cooled} == 0 and ${num_fans} < ${min_fans_air} 196 ... Fatal Error msg=Air cooled but less than ${min_fans_air} fans present. 197 198 199Verify Fan Monitors With State 200 [Documentation] Verify fan monitor daemons in the system state. 201 [Arguments] ${power_state} 202 # The number of monitoring daemons is dependent upon the system 203 # power state. If power is off there should be 0, if power 204 # is on there should be several. 205 206 # Description of argument(s): 207 # power_state Power staet of the system, either "On" or "Off" 208 209 ${cmd}= Catenate systemctl list-units | grep phosphor-fan | wc -l 210 ${num_fan_daemons} ${stderr} ${rc}= BMC Execute Command ${cmd} 211 212 Rpvars power_state num_fan_daemons 213 214 # Fail if system is On and there are no fan monitors. 215 IF '${power_state}' == 'On' and ${num_fan_daemons} == 0 216 ... Fail msg=No phosphor-fan monitors found at power on. 217 218 # Fail if system is Off and the fan monitors are present. 219 IF '${power_state}' == 'Off' and ${num_fan_daemons} != 0 220 ... Fail msg=Phosphor-fan monitors found at power off. 221 222Get Fan Count And Names 223 [Documentation] Return the number of fans and the fan names. 224 225 # The @{fan_names} list holds the names of the fans in the system. 226 @{fan_names} Create List 227 ${fan_names}= Get Fan Names ${fan_names} 228 229 ${number_of_fans}= Get Length ${fan_names} 230 231 RETURN ${number_of_fans} ${fan_names} 232 233 234 235Reset Fans 236 [Documentation] Set the fans to functional state. 237 # Set state of fans to functional by writing 1 to the Functional 238 # attribute of each fan in the @{fan_names} list. If @{fan_names} 239 # is empty nothing is done. 240 [Arguments] ${fan_names} 241 242 # Description of Argument(s): 243 # fan_names A list containing the names of the fans (e.g. fan0 244 # fan2 fan3). 245 246 FOR ${fan_name} IN @{fan_names} 247 Set Fan State ${fan_name} ${fan_functional} 248 END 249 250Verify Fan Speed 251 [Documentation] Verify fans are running at or near target speed. 252 [Arguments] ${tolerance} ${fan_names} 253 254 # Description of argument(s): 255 # tolerance The speed tolerance criteria. 256 # A tolerance value of .15 means that the fan's speed 257 # should be within 15% of its set target speed. 258 # Fans may be accelerating to meet a new target, so 259 # allow .10 extra. 260 # fan_names A list containing the names of the fans (e.g. fan0 fan1). 261 262 # Compare the fan's speed with its target speed. 263 FOR ${fan_name} IN @{fan_names} 264 ${target_speed} ${fan_speed}= Get Fan Target And Speed ${fan_name} 265 Rpvars fan_name target_speed fan_speed 266 # Calculate tolerance, which is a % of the target speed. 267 ${tolerance_value}= Evaluate ${tolerance}*${target_speed} 268 # Calculate upper and lower speed limits. 269 ${max_limit}= Evaluate ${target_speed}+${tolerance_value} 270 ${min_limit}= Evaluate ${target_speed}-${tolerance_value} 271 IF ${fan_speed} < ${min_limit} or ${fan_speed} > ${max_limit} 272 ... Fail msg=${fan_name} speed of ${fan_speed} is out of range. 273 END 274 275Verify Direct Fan Control 276 [Documentation] Verify direct control of fans. 277 [Arguments] ${max_speed} ${min_speed} 278 ... ${minutes_to_stabilize} ${number_of_fans} ${fan_names} 279 280 # Overview: 281 # Turn off BMC's fan control daemon, then test to confirm 282 # that fans can be controlled manually. 283 # The app that takes data from sysfs and updates dbus is named hwmon. 284 # Verify hwmon functionality by comparing with what's on dbus 285 # (/xyz/openbmc_project/sensors/fan_tach/fan0_0, fan0_1, etc.) 286 # with what's in the BMC's file system at 287 # /sys/class/hwmon/hwmon9/fan*_input. 288 289 # Description of argument(s): 290 # max_speed Integer value of maximum fan speed. 291 # min_speed Integer value of minimum speed. 292 # minutes_to_stabilize Time to wait for fan daemons to 293 # stabilize fan operation after 294 # tests (e.g. "4"). 295 # number_of_fans The number of fans in the system. 296 # fan_names A list containing the names of the 297 # fans (e.g. fan0 fan1). 298 299 # Login to BMC and disable the fan daemon. Disabling the daemon sets 300 # manual mode. 301 Open Connection And Log In 302 Set Fan Daemon State stop 303 304 # For each fan, set a new target speed and wait for the fan to 305 # accelerate. Then check that the fan is running near that speed. 306 FOR ${fan_name} IN @{fan_names} 307 Set Fan Target Speed ${fan_name} ${max_speed} 308 Run Key U Sleep \ 60s 309 ${target_speed} ${cw_speed} ${ccw_speed}= 310 ... Get Target And Blade Speeds ${fan_name} 311 Rpvars fan_name target_speed cw_speed ccw_speed 312 IF ${cw_speed} < ${min_speed} or ${ccw_speed} < ${min_speed} 313 ... Fail msg=${fan_name} failed manual speed test. 314 END 315 316 # Check the fan speeds in the BMC file system. 317 318 # Get the location of the fan hwmon. 319 ${controller_path} ${stderr} ${rc}= BMC Execute Command 320 ... grep -ir max31785a /sys/class/hwmon/hwmon* | grep name 321 # E.g., controller_path=/sys/class/hwmon/hwmon10/name:max31785a. 322 323 ${hwmon_path} ${file_name}= Split Path ${controller_path} 324 # E.g., /sys/class/hwmon/hwmon10 or /sys/class/hwmon/hwmon9. 325 326 Rpvars controller_path hwmon_path 327 328 # Run the BMC command which gets fan speeds from the file system. 329 ${cmd}= Catenate cat ${hwmon_path}/fan*_input 330 ${stdout} ${stderr} ${rc}= 331 ... BMC Execute Command ${cmd} 332 333 # Convert output to integer values. 334 ${speeds}= Evaluate map(int, $stdout.split(${\n})) 335 Rpvars speeds 336 # Count the number of speeds > ${min_speed}. 337 ${count}= Set Variable ${0} 338 FOR ${speed} IN @{speeds} 339 ${count}= Set Variable If ${speed} > ${min_speed} 340 ... Evaluate ${count}+1 ELSE Set Variable ${count} 341 # Because each fan has two rotating fan blades, the count should be 342 # equual to 2*${number_of_fans}. On water-cooled systems some 343 # speeds may be reported by hwmon as 0. That is expected, 344 # and the number_of_fans reported in the system will be less. 345 END 346 ${fail_test}= Evaluate (2*${number_of_fans})-${count} 347 348 # Re-enable the fan daemon 349 Set Fan Daemon State restart 350 351 IF ${fail_test} > ${0} Fail msg=hwmon did not properly report fan speeds. 352 353 # Wait for the daemon to take control and gracefully set fan speeds 354 # back to normal. 355 ${msg}= Catenate Waiting ${minutes_to_stabilize} minutes 356 ... for fan daemon to stabilize fans. 357 Print Timen ${msg} 358 Run Key U Sleep \ ${minutes_to_stabilize}m 359 360 361Verify Fan Speed Increase 362 [Documentation] Verify that the speed of working fans increase when 363 ... one fan is marked as disabled. 364 # A non-functional fan should cause an error log and 365 # an enclosure LED will light. The other fans should speed up. 366 [Arguments] ${fan_names} 367 368 # Description of argument(s): 369 # fan_names A list containing the names of the fans (e.g. fan0 fan1). 370 371 # Allow system_response_time before checking if there was a 372 # response by the system to an applied fault. 373 ${system_response_time}= Set Variable 60s 374 375 # Choose a fan to test with, e.g., fan0. 376 ${test_fan_name}= Get From List ${fan_names} 0 377 378 ${initial_speed}= Get Target Speed Of Fans 379 Rpvars test_fan_name initial_speed 380 381 # If initial speed is not already at maximum, set expect_increase. 382 # This flag is used later to determine if speed checking is 383 # to be done or not. 384 ${expect_increase}= Set Variable If ${initial_speed} < ${max_speed} 1 0 385 386 Set Fan State ${test_fan_name} ${fan_nonfunctional} 387 388 Run Key U Sleep \ ${system_response_time} 389 390 # A heavily loaded system may have powered-off. 391 ${host_state}= Get Host State 392 Rpvars host_state 393 IF 'Running' != '${host_state}' Pass Execution 394 ... msg=System shutdown so skipping remainder of test. 395 396 ${new_fan_speed}= Get Target Speed Of Fans 397 Rpvars expect_increase initial_speed new_fan_speed 398 399 # Fail if current fan speed did not increase past the initial 400 # speed, but do this check only if not at maximum speed to begin with. 401 402 IF ${expect_increase} == 1 and ${new_fan_speed} < ${initial_speed} 403 ... Fail msg=Remaining fans did not increase speed with loss of one fan. 404 405 406Verify System Shutdown Due To Fans 407 [Documentation] Shut down when not enough fans. 408 [Arguments] ${fan_names} 409 410 # Description of argument(s): 411 # fan_names A list containing the names of the fans (e.g. fan0 fan1). 412 413 ${wait_after_poweroff}= Set Variable 15s 414 415 # Set fans to be non-functional. 416 FOR ${fan_name} IN @{fan_names} 417 Set Fan State ${fan_name} ${fan_nonfunctional} 418 END 419 420 # System should notice the non-functional fans and power-off. 421 # The Wait For PowerOff keyword will time-out and report 422 # an error if power off does not happen within a reasonable time. 423 Wait For PowerOff 424