1 #include "Thresholds.hpp"
2
3 #include "Utils.hpp"
4 #include "VariantVisitors.hpp"
5 #include "sensor.hpp"
6
7 #include <boost/algorithm/string/replace.hpp>
8 #include <boost/asio/error.hpp>
9 #include <boost/asio/steady_timer.hpp>
10 #include <boost/container/flat_map.hpp>
11 #include <sdbusplus/asio/connection.hpp>
12 #include <sdbusplus/asio/object_server.hpp>
13 #include <sdbusplus/exception.hpp>
14 #include <sdbusplus/message.hpp>
15
16 #include <array>
17 #include <chrono>
18 #include <cstddef>
19 #include <cstdint>
20 #include <iostream>
21 #include <limits>
22 #include <memory>
23 #include <string>
24 #include <tuple>
25 #include <utility>
26 #include <variant>
27 #include <vector>
28
29 static constexpr bool debug = false;
30 namespace thresholds
31 {
findThresholdLevel(uint8_t sev)32 Level findThresholdLevel(uint8_t sev)
33 {
34 for (const ThresholdDefinition& prop : thresProp)
35 {
36 if (prop.sevOrder == sev)
37 {
38 return prop.level;
39 }
40 }
41 return Level::ERROR;
42 }
43
findThresholdDirection(const std::string & direct)44 Direction findThresholdDirection(const std::string& direct)
45 {
46 if (direct == "greater than")
47 {
48 return Direction::HIGH;
49 }
50 if (direct == "less than")
51 {
52 return Direction::LOW;
53 }
54 return Direction::ERROR;
55 }
56
parseThresholdsFromConfig(const SensorData & sensorData,std::vector<thresholds::Threshold> & thresholdVector,const std::string * matchLabel,const int * sensorIndex)57 bool parseThresholdsFromConfig(
58 const SensorData& sensorData,
59 std::vector<thresholds::Threshold>& thresholdVector,
60 const std::string* matchLabel, const int* sensorIndex)
61 {
62 for (const auto& [intf, cfg] : sensorData)
63 {
64 if (intf.find("Thresholds") == std::string::npos)
65 {
66 continue;
67 }
68 if (matchLabel != nullptr)
69 {
70 auto labelFind = cfg.find("Label");
71 if (labelFind == cfg.end())
72 {
73 continue;
74 }
75 if (std::visit(VariantToStringVisitor(), labelFind->second) !=
76 *matchLabel)
77 {
78 continue;
79 }
80 }
81
82 if (sensorIndex != nullptr)
83 {
84 auto indexFind = cfg.find("Index");
85
86 // If we're checking for index 1, a missing Index is OK.
87 if ((indexFind == cfg.end()) && (*sensorIndex != 1))
88 {
89 continue;
90 }
91
92 if ((indexFind != cfg.end()) &&
93 (std::visit(VariantToIntVisitor(), indexFind->second) !=
94 *sensorIndex))
95 {
96 continue;
97 }
98 }
99
100 double hysteresis = std::numeric_limits<double>::quiet_NaN();
101 auto hysteresisFind = cfg.find("Hysteresis");
102 if (hysteresisFind != cfg.end())
103 {
104 hysteresis = std::visit(VariantToDoubleVisitor(),
105 hysteresisFind->second);
106 }
107
108 auto directionFind = cfg.find("Direction");
109 auto severityFind = cfg.find("Severity");
110 auto valueFind = cfg.find("Value");
111 if (valueFind == cfg.end() || severityFind == cfg.end() ||
112 directionFind == cfg.end())
113 {
114 std::cerr << "Malformed threshold on configuration interface "
115 << intf << "\n";
116 return false;
117 }
118 unsigned int severity = std::visit(VariantToUnsignedIntVisitor(),
119 severityFind->second);
120
121 std::string directions = std::visit(VariantToStringVisitor(),
122 directionFind->second);
123
124 Level level = findThresholdLevel(severity);
125 Direction direction = findThresholdDirection(directions);
126
127 if ((level == Level::ERROR) || (direction == Direction::ERROR))
128 {
129 continue;
130 }
131 double val = std::visit(VariantToDoubleVisitor(), valueFind->second);
132
133 thresholdVector.emplace_back(level, direction, val, hysteresis);
134 }
135 return true;
136 }
137
persistThreshold(const std::string & path,const std::string & baseInterface,const thresholds::Threshold & threshold,std::shared_ptr<sdbusplus::asio::connection> & conn,size_t thresholdCount,const std::string & labelMatch)138 void persistThreshold(const std::string& path, const std::string& baseInterface,
139 const thresholds::Threshold& threshold,
140 std::shared_ptr<sdbusplus::asio::connection>& conn,
141 size_t thresholdCount, const std::string& labelMatch)
142 {
143 for (size_t ii = 0; ii < thresholdCount; ii++)
144 {
145 std::string thresholdInterface = baseInterface + ".Thresholds" +
146 std::to_string(ii);
147 conn->async_method_call(
148 [&, path, threshold, thresholdInterface,
149 labelMatch](const boost::system::error_code& ec,
150 const SensorBaseConfigMap& result) {
151 if (ec)
152 {
153 return; // threshold not supported
154 }
155
156 if (!labelMatch.empty())
157 {
158 auto labelFind = result.find("Label");
159 if (labelFind == result.end())
160 {
161 std::cerr << "No label in threshold configuration\n";
162 return;
163 }
164 std::string label = std::visit(VariantToStringVisitor(),
165 labelFind->second);
166 if (label != labelMatch)
167 {
168 return;
169 }
170 }
171
172 auto directionFind = result.find("Direction");
173 auto severityFind = result.find("Severity");
174 auto valueFind = result.find("Value");
175 if (valueFind == result.end() || severityFind == result.end() ||
176 directionFind == result.end())
177 {
178 std::cerr << "Malformed threshold in configuration\n";
179 return;
180 }
181 unsigned int severity = std::visit(VariantToUnsignedIntVisitor(),
182 severityFind->second);
183
184 std::string dir = std::visit(VariantToStringVisitor(),
185 directionFind->second);
186 if ((findThresholdLevel(severity) != threshold.level) ||
187 (findThresholdDirection(dir) != threshold.direction))
188 {
189 return; // not the droid we're looking for
190 }
191
192 std::variant<double> value(threshold.value);
193 conn->async_method_call(
194 [](const boost::system::error_code& ec) {
195 if (ec)
196 {
197 std::cerr << "Error setting threshold " << ec << "\n";
198 }
199 },
200 entityManagerName, path, "org.freedesktop.DBus.Properties",
201 "Set", thresholdInterface, "Value", value);
202 },
203 entityManagerName, path, "org.freedesktop.DBus.Properties",
204 "GetAll", thresholdInterface);
205 }
206 }
207
updateThresholds(Sensor * sensor)208 void updateThresholds(Sensor* sensor)
209 {
210 for (const auto& threshold : sensor->thresholds)
211 {
212 std::shared_ptr<sdbusplus::asio::dbus_interface> interface =
213 sensor->getThresholdInterface(threshold.level);
214
215 if (!interface)
216 {
217 continue;
218 }
219
220 std::string property = Sensor::propertyLevel(threshold.level,
221 threshold.direction);
222 if (property.empty())
223 {
224 continue;
225 }
226 interface->set_property(property, threshold.value);
227 }
228 }
229
230 // Debugging counters
231 static int cHiTrue = 0;
232 static int cHiFalse = 0;
233 static int cHiMidstate = 0;
234 static int cLoTrue = 0;
235 static int cLoFalse = 0;
236 static int cLoMidstate = 0;
237 static int cDebugThrottle = 0;
238 static constexpr int assertLogCount = 10;
239
240 struct ChangeParam
241 {
ChangeParamthresholds::ChangeParam242 ChangeParam(Threshold whichThreshold, bool status, double value) :
243 threshold(whichThreshold), asserted(status), assertValue(value)
244 {}
245
246 Threshold threshold;
247 bool asserted;
248 double assertValue;
249 };
250
checkThresholds(Sensor * sensor,double value)251 static std::vector<ChangeParam> checkThresholds(Sensor* sensor, double value)
252 {
253 std::vector<ChangeParam> thresholdChanges;
254 if (sensor->thresholds.empty())
255 {
256 return thresholdChanges;
257 }
258
259 for (auto& threshold : sensor->thresholds)
260 {
261 // Use "Schmitt trigger" logic to avoid threshold trigger spam,
262 // if value is noisy while hovering very close to a threshold.
263 // When a threshold is crossed, indicate true immediately,
264 // but require more distance to be crossed the other direction,
265 // before resetting the indicator back to false.
266 if (threshold.direction == thresholds::Direction::HIGH)
267 {
268 if (value >= threshold.value)
269 {
270 thresholdChanges.emplace_back(threshold, true, value);
271 if (++cHiTrue < assertLogCount)
272 {
273 std::cerr << "Sensor " << sensor->name << " high threshold "
274 << threshold.value << " assert: value " << value
275 << " raw data " << sensor->rawValue << "\n";
276 }
277 }
278 else if (value < (threshold.value - threshold.hysteresis))
279 {
280 thresholdChanges.emplace_back(threshold, false, value);
281 ++cHiFalse;
282 }
283 else
284 {
285 ++cHiMidstate;
286 }
287 }
288 else if (threshold.direction == thresholds::Direction::LOW)
289 {
290 if (value <= threshold.value)
291 {
292 thresholdChanges.emplace_back(threshold, true, value);
293 if (++cLoTrue < assertLogCount)
294 {
295 std::cerr << "Sensor " << sensor->name << " low threshold "
296 << threshold.value << " assert: value "
297 << sensor->value << " raw data "
298 << sensor->rawValue << "\n";
299 }
300 }
301 else if (value > (threshold.value + threshold.hysteresis))
302 {
303 thresholdChanges.emplace_back(threshold, false, value);
304 ++cLoFalse;
305 }
306 else
307 {
308 ++cLoMidstate;
309 }
310 }
311 else
312 {
313 std::cerr << "Error determining threshold direction\n";
314 }
315 }
316
317 // Throttle debug output, so that it does not continuously spam
318 ++cDebugThrottle;
319 if (cDebugThrottle >= 1000)
320 {
321 cDebugThrottle = 0;
322 if constexpr (debug)
323 {
324 std::cerr << "checkThresholds: High T=" << cHiTrue
325 << " F=" << cHiFalse << " M=" << cHiMidstate
326 << ", Low T=" << cLoTrue << " F=" << cLoFalse
327 << " M=" << cLoMidstate << "\n";
328 }
329 }
330
331 return thresholdChanges;
332 }
333
startTimer(const std::weak_ptr<Sensor> & weakSensor,const Threshold & threshold,bool assert,double assertValue)334 void ThresholdTimer::startTimer(const std::weak_ptr<Sensor>& weakSensor,
335 const Threshold& threshold, bool assert,
336 double assertValue)
337 {
338 struct TimerUsed timerUsed = {};
339 constexpr const size_t waitTime = 5;
340 TimerPair* pair = nullptr;
341
342 for (TimerPair& timer : timers)
343 {
344 if (!timer.first.used)
345 {
346 pair = &timer;
347 break;
348 }
349 }
350 if (pair == nullptr)
351 {
352 pair = &timers.emplace_back(timerUsed, boost::asio::steady_timer(io));
353 }
354
355 pair->first.used = true;
356 pair->first.level = threshold.level;
357 pair->first.direction = threshold.direction;
358 pair->first.assert = assert;
359 pair->second.expires_after(std::chrono::seconds(waitTime));
360 pair->second.async_wait([weakSensor, pair, threshold, assert,
361 assertValue](boost::system::error_code ec) {
362 auto sensorPtr = weakSensor.lock();
363 if (!sensorPtr)
364 {
365 return; // owner sensor has been destructed
366 }
367 // pair is valid as long as sensor is valid
368 pair->first.used = false;
369
370 if (ec == boost::asio::error::operation_aborted)
371 {
372 return; // we're being canceled
373 }
374 if (ec)
375 {
376 std::cerr << "timer error: " << ec.message() << "\n";
377 return;
378 }
379 if (sensorPtr->readingStateGood())
380 {
381 assertThresholds(sensorPtr.get(), assertValue, threshold.level,
382 threshold.direction, assert);
383 }
384 });
385 }
386
checkThresholds(Sensor * sensor)387 bool checkThresholds(Sensor* sensor)
388 {
389 bool status = true;
390 std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value);
391 for (const auto& change : changes)
392 {
393 assertThresholds(sensor, change.assertValue, change.threshold.level,
394 change.threshold.direction, change.asserted);
395 if (change.threshold.level == thresholds::Level::CRITICAL &&
396 change.asserted)
397 {
398 status = false;
399 }
400 }
401
402 return status;
403 }
404
checkThresholdsPowerDelay(const std::weak_ptr<Sensor> & weakSensor,ThresholdTimer & thresholdTimer)405 void checkThresholdsPowerDelay(const std::weak_ptr<Sensor>& weakSensor,
406 ThresholdTimer& thresholdTimer)
407 {
408 auto sensorPtr = weakSensor.lock();
409 if (!sensorPtr)
410 {
411 return; // sensor is destructed, should never be here
412 }
413
414 Sensor* sensor = sensorPtr.get();
415 std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value);
416 for (const auto& change : changes)
417 {
418 // When CPU is powered off, some volatges are expected to
419 // go below low thresholds. Filter these events with thresholdTimer.
420 // 1. always delay the assertion of low events to see if they are
421 // caused by power off event.
422 // 2. conditional delay the de-assertion of low events if there is
423 // an existing timer for assertion.
424 // 3. no delays for de-assert of low events if there is an existing
425 // de-assert for low event. This means 2nd de-assert would happen
426 // first and when timer expires for the previous one, no additional
427 // signal will be logged.
428 // 4. no delays for all high events.
429 if (change.threshold.direction == thresholds::Direction::LOW)
430 {
431 if (change.asserted || thresholdTimer.hasActiveTimer(
432 change.threshold, !change.asserted))
433 {
434 thresholdTimer.startTimer(weakSensor, change.threshold,
435 change.asserted, change.assertValue);
436 continue;
437 }
438 }
439 assertThresholds(sensor, change.assertValue, change.threshold.level,
440 change.threshold.direction, change.asserted);
441 }
442 }
443
assertThresholds(Sensor * sensor,double assertValue,thresholds::Level level,thresholds::Direction direction,bool assert)444 void assertThresholds(Sensor* sensor, double assertValue,
445 thresholds::Level level, thresholds::Direction direction,
446 bool assert)
447 {
448 std::shared_ptr<sdbusplus::asio::dbus_interface> interface =
449 sensor->getThresholdInterface(level);
450
451 if (!interface)
452 {
453 std::cout << "trying to set uninitialized interface\n";
454 return;
455 }
456
457 std::string property = Sensor::propertyAlarm(level, direction);
458 if (property.empty())
459 {
460 std::cout << "Alarm property is empty \n";
461 return;
462 }
463 if (interface->set_property<bool, true>(property, assert))
464 {
465 try
466 {
467 // msg.get_path() is interface->get_object_path()
468 sdbusplus::message_t msg =
469 interface->new_signal("ThresholdAsserted");
470
471 msg.append(sensor->name, interface->get_interface_name(), property,
472 assert, assertValue);
473 msg.signal_send();
474 }
475 catch (const sdbusplus::exception_t& e)
476 {
477 std::cerr
478 << "Failed to send thresholdAsserted signal with assertValue\n";
479 }
480 }
481 }
482
parseThresholdsFromAttr(std::vector<thresholds::Threshold> & thresholdVector,const std::string & inputPath,const double & scaleFactor,const double & offset,const double & hysteresis)483 bool parseThresholdsFromAttr(
484 std::vector<thresholds::Threshold>& thresholdVector,
485 const std::string& inputPath, const double& scaleFactor,
486 const double& offset, const double& hysteresis)
487 {
488 const boost::container::flat_map<
489 std::string, std::vector<std::tuple<const char*, thresholds::Level,
490 thresholds::Direction, double>>>
491 map = {
492 {"average",
493 {
494 std::make_tuple("average_min", Level::WARNING, Direction::LOW,
495 0.0),
496 std::make_tuple("average_max", Level::WARNING, Direction::HIGH,
497 0.0),
498 }},
499 {"input",
500 {
501 std::make_tuple("min", Level::WARNING, Direction::LOW, 0.0),
502 std::make_tuple("max", Level::WARNING, Direction::HIGH, 0.0),
503 std::make_tuple("lcrit", Level::CRITICAL, Direction::LOW, 0.0),
504 std::make_tuple("crit", Level::CRITICAL, Direction::HIGH,
505 offset),
506 }},
507 };
508
509 if (auto fileParts = splitFileName(inputPath))
510 {
511 auto& [type, nr, item] = *fileParts;
512 if (map.count(item) != 0)
513 {
514 for (const auto& t : map.at(item))
515 {
516 const auto& [suffix, level, direction, offset] = t;
517 auto attrPath = boost::replace_all_copy(inputPath, item,
518 suffix);
519 if (auto val = readFile(attrPath, scaleFactor))
520 {
521 *val += offset;
522 if (debug)
523 {
524 std::cout << "Threshold: " << attrPath << ": " << *val
525 << "\n";
526 }
527 thresholdVector.emplace_back(level, direction, *val,
528 hysteresis);
529 }
530 }
531 }
532 }
533 return true;
534 }
535
getInterface(const Level thresholdLevel)536 std::string getInterface(const Level thresholdLevel)
537 {
538 for (const ThresholdDefinition& thresh : thresProp)
539 {
540 if (thresh.level == thresholdLevel)
541 {
542 return std::string("xyz.openbmc_project.Sensor.Threshold.") +
543 thresh.levelName;
544 }
545 }
546 return "";
547 }
548 } // namespace thresholds
549