xref: /openbmc/dbus-sensors/src/Thresholds.cpp (revision 45f2702d)
1 #include <Thresholds.hpp>
2 #include <VariantVisitors.hpp>
3 #include <boost/algorithm/string/replace.hpp>
4 #include <boost/container/flat_map.hpp>
5 #include <sensor.hpp>
6 
7 #include <array>
8 #include <cmath>
9 #include <fstream>
10 #include <iostream>
11 #include <memory>
12 #include <stdexcept>
13 #include <string>
14 #include <tuple>
15 #include <utility>
16 #include <variant>
17 #include <vector>
18 
19 static constexpr bool debug = false;
20 static constexpr int thresLevel = 2;
21 namespace thresholds
22 {
23 Level findThresholdLevel(uint8_t sev, const std::string& direct)
24 {
25     for (Sensor::ThresholdProperty prop : Sensor::thresProp)
26     {
27         if ((prop.sevOrder == sev) && (prop.dirOrder == direct))
28         {
29             return prop.level;
30         }
31     }
32     return Level::ERROR;
33 }
34 
35 Direction findThresholdDirection(uint8_t sev, const std::string& direct)
36 {
37     for (Sensor::ThresholdProperty prop : Sensor::thresProp)
38     {
39         if ((prop.sevOrder == sev) && (prop.dirOrder == direct))
40         {
41             return prop.direction;
42         }
43     }
44     return Direction::ERROR;
45 }
46 
47 bool parseThresholdsFromConfig(
48     const SensorData& sensorData,
49     std::vector<thresholds::Threshold>& thresholdVector,
50     const std::string* matchLabel, const int* sensorIndex)
51 {
52     for (const auto& item : sensorData)
53     {
54         if (item.first.find("Thresholds") == std::string::npos)
55         {
56             continue;
57         }
58         if (matchLabel != nullptr)
59         {
60             auto labelFind = item.second.find("Label");
61             if (labelFind == item.second.end())
62             {
63                 continue;
64             }
65             if (std::visit(VariantToStringVisitor(), labelFind->second) !=
66                 *matchLabel)
67             {
68                 continue;
69             }
70         }
71 
72         if (sensorIndex != nullptr)
73         {
74             auto indexFind = item.second.find("Index");
75 
76             // If we're checking for index 1, a missing Index is OK.
77             if ((indexFind == item.second.end()) && (*sensorIndex != 1))
78             {
79                 continue;
80             }
81 
82             if ((indexFind != item.second.end()) &&
83                 (std::visit(VariantToIntVisitor(), indexFind->second) !=
84                  *sensorIndex))
85             {
86                 continue;
87             }
88         }
89 
90         double hysteresis = std::numeric_limits<double>::quiet_NaN();
91         auto hysteresisFind = item.second.find("Hysteresis");
92         if (hysteresisFind != item.second.end())
93         {
94             hysteresis =
95                 std::visit(VariantToDoubleVisitor(), hysteresisFind->second);
96         }
97 
98         auto directionFind = item.second.find("Direction");
99         auto severityFind = item.second.find("Severity");
100         auto valueFind = item.second.find("Value");
101         if (valueFind == item.second.end() ||
102             severityFind == item.second.end() ||
103             directionFind == item.second.end())
104         {
105             std::cerr << "Malformed threshold on configuration interface "
106                       << item.first << "\n";
107             return false;
108         }
109         uint8_t severity =
110             std::visit(VariantToUnsignedIntVisitor(), severityFind->second);
111 
112         std::string directions =
113             std::visit(VariantToStringVisitor(), directionFind->second);
114 
115         Level level = findThresholdLevel(severity, directions);
116         Direction direction = findThresholdDirection(severity, directions);
117 
118         if ((static_cast<int>(level) == thresLevel) ||
119             (static_cast<int>(direction) == thresLevel))
120         {
121             continue;
122         }
123         double val = std::visit(VariantToDoubleVisitor(), valueFind->second);
124 
125         thresholdVector.emplace_back(level, direction, val, hysteresis);
126     }
127     return true;
128 }
129 
130 void persistThreshold(const std::string& path, const std::string& baseInterface,
131                       const thresholds::Threshold& threshold,
132                       std::shared_ptr<sdbusplus::asio::connection>& conn,
133                       size_t thresholdCount, const std::string& labelMatch)
134 {
135     for (size_t ii = 0; ii < thresholdCount; ii++)
136     {
137         std::string thresholdInterface =
138             baseInterface + ".Thresholds" + std::to_string(ii);
139         conn->async_method_call(
140             [&, path, threshold, thresholdInterface, labelMatch](
141                 const boost::system::error_code& ec,
142                 const boost::container::flat_map<std::string, BasicVariantType>&
143                     result) {
144                 if (ec)
145                 {
146                     return; // threshold not supported
147                 }
148 
149                 if (!labelMatch.empty())
150                 {
151                     auto labelFind = result.find("Label");
152                     if (labelFind == result.end())
153                     {
154                         std::cerr << "No label in threshold configuration\n";
155                         return;
156                     }
157                     std::string label =
158                         std::visit(VariantToStringVisitor(), labelFind->second);
159                     if (label != labelMatch)
160                     {
161                         return;
162                     }
163                 }
164 
165                 auto directionFind = result.find("Direction");
166                 auto severityFind = result.find("Severity");
167                 auto valueFind = result.find("Value");
168                 if (valueFind == result.end() || severityFind == result.end() ||
169                     directionFind == result.end())
170                 {
171                     std::cerr << "Malformed threshold in configuration\n";
172                     return;
173                 }
174                 unsigned int level = std::visit(VariantToUnsignedIntVisitor(),
175                                                 severityFind->second);
176 
177                 std::string dir =
178                     std::visit(VariantToStringVisitor(), directionFind->second);
179                 if (((findThresholdLevel(level, dir)) != threshold.level) ||
180                     ((findThresholdDirection(level, dir)) !=
181                      threshold.direction))
182                 {
183                     return; // not the droid we're looking for
184                 }
185 
186                 std::variant<double> value(threshold.value);
187                 conn->async_method_call(
188                     [](const boost::system::error_code& ec) {
189                         if (ec)
190                         {
191                             std::cerr << "Error setting threshold " << ec
192                                       << "\n";
193                         }
194                     },
195                     entityManagerName, path, "org.freedesktop.DBus.Properties",
196                     "Set", thresholdInterface, "Value", value);
197             },
198             entityManagerName, path, "org.freedesktop.DBus.Properties",
199             "GetAll", thresholdInterface);
200     }
201 }
202 
203 void updateThresholds(Sensor* sensor)
204 {
205     if (sensor->thresholds.empty())
206     {
207         return;
208     }
209 
210     for (const auto& threshold : sensor->thresholds)
211     {
212         std::shared_ptr<sdbusplus::asio::dbus_interface> interface;
213         if (threshold.level == thresholds::Level::CRITICAL)
214         {
215             interface = sensor->thresholdInterfaceCritical;
216         }
217         else if (threshold.level == thresholds::Level::WARNING)
218         {
219             interface = sensor->thresholdInterfaceWarning;
220         }
221         else
222         {
223             continue;
224         }
225         if (!interface)
226         {
227             continue;
228         }
229 
230         std::string property =
231             sensor->propertyLevel(threshold.level, threshold.direction);
232         if (property.empty())
233         {
234             continue;
235         }
236         interface->set_property(property, threshold.value);
237     }
238 }
239 
240 // Debugging counters
241 static int cHiTrue = 0;
242 static int cHiFalse = 0;
243 static int cHiMidstate = 0;
244 static int cLoTrue = 0;
245 static int cLoFalse = 0;
246 static int cLoMidstate = 0;
247 static int cDebugThrottle = 0;
248 static constexpr int assertLogCount = 10;
249 
250 struct ChangeParam
251 {
252     ChangeParam(Threshold whichThreshold, bool status, double value) :
253         threshold(whichThreshold), asserted(status), assertValue(value)
254     {}
255 
256     Threshold threshold;
257     bool asserted;
258     double assertValue;
259 };
260 
261 static std::vector<ChangeParam> checkThresholds(Sensor* sensor, double value)
262 {
263     std::vector<ChangeParam> thresholdChanges;
264     if (sensor->thresholds.empty())
265     {
266         return thresholdChanges;
267     }
268 
269     for (auto& threshold : sensor->thresholds)
270     {
271         // Use "Schmitt trigger" logic to avoid threshold trigger spam,
272         // if value is noisy while hovering very close to a threshold.
273         // When a threshold is crossed, indicate true immediately,
274         // but require more distance to be crossed the other direction,
275         // before resetting the indicator back to false.
276         if (threshold.direction == thresholds::Direction::HIGH)
277         {
278             if (value >= threshold.value)
279             {
280                 thresholdChanges.emplace_back(threshold, true, value);
281                 if (++cHiTrue < assertLogCount)
282                 {
283                     std::cerr << "Sensor " << sensor->name << " high threshold "
284                               << threshold.value << " assert: value " << value
285                               << " raw data " << sensor->rawValue << "\n";
286                 }
287             }
288             else if (value < (threshold.value - threshold.hysteresis))
289             {
290                 thresholdChanges.emplace_back(threshold, false, value);
291                 ++cHiFalse;
292             }
293             else
294             {
295                 ++cHiMidstate;
296             }
297         }
298         else if (threshold.direction == thresholds::Direction::LOW)
299         {
300             if (value <= threshold.value)
301             {
302                 thresholdChanges.emplace_back(threshold, true, value);
303                 if (++cLoTrue < assertLogCount)
304                 {
305                     std::cerr << "Sensor " << sensor->name << " low threshold "
306                               << threshold.value << " assert: value "
307                               << sensor->value << " raw data "
308                               << sensor->rawValue << "\n";
309                 }
310             }
311             else if (value > (threshold.value + threshold.hysteresis))
312             {
313                 thresholdChanges.emplace_back(threshold, false, value);
314                 ++cLoFalse;
315             }
316             else
317             {
318                 ++cLoMidstate;
319             }
320         }
321         else
322         {
323             std::cerr << "Error determining threshold direction\n";
324         }
325     }
326 
327     // Throttle debug output, so that it does not continuously spam
328     ++cDebugThrottle;
329     if (cDebugThrottle >= 1000)
330     {
331         cDebugThrottle = 0;
332         if constexpr (debug)
333         {
334             std::cerr << "checkThresholds: High T=" << cHiTrue
335                       << " F=" << cHiFalse << " M=" << cHiMidstate
336                       << ", Low T=" << cLoTrue << " F=" << cLoFalse
337                       << " M=" << cLoMidstate << "\n";
338         }
339     }
340 
341     return thresholdChanges;
342 }
343 
344 void ThresholdTimer::startTimer(const std::weak_ptr<Sensor>& weakSensor,
345                                 const Threshold& threshold, bool assert,
346                                 double assertValue)
347 {
348     struct TimerUsed timerUsed = {};
349     constexpr const size_t waitTime = 5;
350     TimerPair* pair = nullptr;
351 
352     for (TimerPair& timer : timers)
353     {
354         if (!timer.first.used)
355         {
356             pair = &timer;
357             break;
358         }
359     }
360     if (pair == nullptr)
361     {
362         pair = &timers.emplace_back(timerUsed, boost::asio::deadline_timer(io));
363     }
364 
365     pair->first.used = true;
366     pair->first.level = threshold.level;
367     pair->first.direction = threshold.direction;
368     pair->first.assert = assert;
369     pair->second.expires_from_now(boost::posix_time::seconds(waitTime));
370     pair->second.async_wait([weakSensor, pair, threshold, assert,
371                              assertValue](boost::system::error_code ec) {
372         auto sensorPtr = weakSensor.lock();
373         if (!sensorPtr)
374         {
375             return; // owner sensor has been destructed
376         }
377         // pair is valid as long as sensor is valid
378         pair->first.used = false;
379 
380         if (ec == boost::asio::error::operation_aborted)
381         {
382             return; // we're being canceled
383         }
384         if (ec)
385         {
386             std::cerr << "timer error: " << ec.message() << "\n";
387             return;
388         }
389         if (sensorPtr->readingStateGood())
390         {
391             assertThresholds(sensorPtr.get(), assertValue, threshold.level,
392                              threshold.direction, assert);
393         }
394     });
395 }
396 
397 bool checkThresholds(Sensor* sensor)
398 {
399     bool status = true;
400     std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value);
401     for (const auto& change : changes)
402     {
403         assertThresholds(sensor, change.assertValue, change.threshold.level,
404                          change.threshold.direction, change.asserted);
405         if (change.threshold.level == thresholds::Level::CRITICAL &&
406             change.asserted)
407         {
408             status = false;
409         }
410     }
411 
412     return status;
413 }
414 
415 void checkThresholdsPowerDelay(const std::weak_ptr<Sensor>& weakSensor,
416                                ThresholdTimer& thresholdTimer)
417 {
418     auto sensorPtr = weakSensor.lock();
419     if (!sensorPtr)
420     {
421         return; // sensor is destructed, should never be here
422     }
423 
424     Sensor* sensor = sensorPtr.get();
425     std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value);
426     for (const auto& change : changes)
427     {
428         // When CPU is powered off, some volatges are expected to
429         // go below low thresholds. Filter these events with thresholdTimer.
430         // 1. always delay the assertion of low events to see if they are
431         //   caused by power off event.
432         // 2. conditional delay the de-assertion of low events if there is
433         //   an existing timer for assertion.
434         // 3. no delays for de-assert of low events if there is an existing
435         //   de-assert for low event. This means 2nd de-assert would happen
436         //   first and when timer expires for the previous one, no additional
437         //   signal will be logged.
438         // 4. no delays for all high events.
439         if (change.threshold.direction == thresholds::Direction::LOW)
440         {
441             if (change.asserted || thresholdTimer.hasActiveTimer(
442                                        change.threshold, !change.asserted))
443             {
444                 thresholdTimer.startTimer(weakSensor, change.threshold,
445                                           change.asserted, change.assertValue);
446                 continue;
447             }
448         }
449         assertThresholds(sensor, change.assertValue, change.threshold.level,
450                          change.threshold.direction, change.asserted);
451     }
452 }
453 
454 void assertThresholds(Sensor* sensor, double assertValue,
455                       thresholds::Level level, thresholds::Direction direction,
456                       bool assert)
457 {
458     std::shared_ptr<sdbusplus::asio::dbus_interface> interface;
459     if (level == thresholds::Level::WARNING)
460     {
461         interface = sensor->thresholdInterfaceWarning;
462     }
463     else if (level == thresholds::Level::CRITICAL)
464     {
465         interface = sensor->thresholdInterfaceCritical;
466     }
467     else
468     {
469         std::cerr << "Unknown threshold, level " << static_cast<int>(level)
470                   << "direction " << static_cast<int>(direction) << "\n";
471         return;
472     }
473     if (!interface)
474     {
475         std::cout << "trying to set uninitialized interface\n";
476         return;
477     }
478 
479     std::string property = sensor->propertyAlarm(level, direction);
480     if (property.empty())
481     {
482         std::cout << "Alarm property is empty \n";
483         return;
484     }
485     if (interface->set_property<bool, true>(property, assert))
486     {
487         try
488         {
489             // msg.get_path() is interface->get_object_path()
490             sdbusplus::message::message msg =
491                 interface->new_signal("ThresholdAsserted");
492 
493             msg.append(sensor->name, interface->get_interface_name(), property,
494                        assert, assertValue);
495             msg.signal_send();
496         }
497         catch (const sdbusplus::exception::exception& e)
498         {
499             std::cerr
500                 << "Failed to send thresholdAsserted signal with assertValue\n";
501         }
502     }
503 }
504 
505 bool parseThresholdsFromAttr(
506     std::vector<thresholds::Threshold>& thresholdVector,
507     const std::string& inputPath, const double& scaleFactor,
508     const double& offset)
509 {
510     const boost::container::flat_map<
511         std::string, std::vector<std::tuple<const char*, thresholds::Level,
512                                             thresholds::Direction, double>>>
513         map = {
514             {"average",
515              {
516                  std::make_tuple("average_min", Level::WARNING, Direction::LOW,
517                                  0.0),
518                  std::make_tuple("average_max", Level::WARNING, Direction::HIGH,
519                                  0.0),
520              }},
521             {"input",
522              {
523                  std::make_tuple("min", Level::WARNING, Direction::LOW, 0.0),
524                  std::make_tuple("max", Level::WARNING, Direction::HIGH, 0.0),
525                  std::make_tuple("lcrit", Level::CRITICAL, Direction::LOW, 0.0),
526                  std::make_tuple("crit", Level::CRITICAL, Direction::HIGH,
527                                  offset),
528              }},
529         };
530 
531     if (auto fileParts = splitFileName(inputPath))
532     {
533         auto& [type, nr, item] = *fileParts;
534         if (map.count(item) != 0)
535         {
536             for (const auto& t : map.at(item))
537             {
538                 auto& [suffix, level, direction, offset] = t;
539                 auto attrPath =
540                     boost::replace_all_copy(inputPath, item, suffix);
541                 if (auto val = readFile(attrPath, scaleFactor))
542                 {
543                     *val += offset;
544                     if (debug)
545                     {
546                         std::cout << "Threshold: " << attrPath << ": " << *val
547                                   << "\n";
548                     }
549                     thresholdVector.emplace_back(level, direction, *val);
550                 }
551             }
552         }
553     }
554     return true;
555 }
556 
557 bool hasCriticalInterface(
558     const std::vector<thresholds::Threshold>& thresholdVector)
559 {
560     for (auto& threshold : thresholdVector)
561     {
562         if (threshold.level == Level::CRITICAL)
563         {
564             return true;
565         }
566     }
567     return false;
568 }
569 
570 bool hasWarningInterface(
571     const std::vector<thresholds::Threshold>& thresholdVector)
572 {
573     for (auto& threshold : thresholdVector)
574     {
575         if (threshold.level == Level::WARNING)
576         {
577             return true;
578         }
579     }
580     return false;
581 }
582 } // namespace thresholds
583