xref: /openbmc/dbus-sensors/src/Thresholds.cpp (revision c8fed20b)
1 #include <Thresholds.hpp>
2 #include <VariantVisitors.hpp>
3 #include <boost/algorithm/string/replace.hpp>
4 #include <boost/container/flat_map.hpp>
5 #include <sensor.hpp>
6 
7 #include <array>
8 #include <cmath>
9 #include <fstream>
10 #include <iostream>
11 #include <memory>
12 #include <stdexcept>
13 #include <string>
14 #include <tuple>
15 #include <utility>
16 #include <variant>
17 #include <vector>
18 
19 static constexpr bool debug = false;
20 namespace thresholds
21 {
22 Level findThresholdLevel(uint8_t sev)
23 {
24     for (const ThresholdDefinition& prop : thresProp)
25     {
26         if (prop.sevOrder == sev)
27         {
28             return prop.level;
29         }
30     }
31     return Level::ERROR;
32 }
33 
34 Direction findThresholdDirection(const std::string& direct)
35 {
36     if (direct == "greater than")
37     {
38         return Direction::HIGH;
39     }
40     if (direct == "less than")
41     {
42         return Direction::LOW;
43     }
44     return Direction::ERROR;
45 }
46 
47 bool isValidLevel(Level lev)
48 {
49     for (const ThresholdDefinition& prop : thresProp)
50     {
51         if (prop.level == lev)
52         {
53             return true;
54         }
55     }
56     return false;
57 }
58 
59 bool parseThresholdsFromConfig(
60     const SensorData& sensorData,
61     std::vector<thresholds::Threshold>& thresholdVector,
62     const std::string* matchLabel, const int* sensorIndex)
63 {
64     for (const auto& item : sensorData)
65     {
66         if (item.first.find("Thresholds") == std::string::npos)
67         {
68             continue;
69         }
70         if (matchLabel != nullptr)
71         {
72             auto labelFind = item.second.find("Label");
73             if (labelFind == item.second.end())
74             {
75                 continue;
76             }
77             if (std::visit(VariantToStringVisitor(), labelFind->second) !=
78                 *matchLabel)
79             {
80                 continue;
81             }
82         }
83 
84         if (sensorIndex != nullptr)
85         {
86             auto indexFind = item.second.find("Index");
87 
88             // If we're checking for index 1, a missing Index is OK.
89             if ((indexFind == item.second.end()) && (*sensorIndex != 1))
90             {
91                 continue;
92             }
93 
94             if ((indexFind != item.second.end()) &&
95                 (std::visit(VariantToIntVisitor(), indexFind->second) !=
96                  *sensorIndex))
97             {
98                 continue;
99             }
100         }
101 
102         double hysteresis = std::numeric_limits<double>::quiet_NaN();
103         auto hysteresisFind = item.second.find("Hysteresis");
104         if (hysteresisFind != item.second.end())
105         {
106             hysteresis =
107                 std::visit(VariantToDoubleVisitor(), hysteresisFind->second);
108         }
109 
110         auto directionFind = item.second.find("Direction");
111         auto severityFind = item.second.find("Severity");
112         auto valueFind = item.second.find("Value");
113         if (valueFind == item.second.end() ||
114             severityFind == item.second.end() ||
115             directionFind == item.second.end())
116         {
117             std::cerr << "Malformed threshold on configuration interface "
118                       << item.first << "\n";
119             return false;
120         }
121         unsigned int severity =
122             std::visit(VariantToUnsignedIntVisitor(), severityFind->second);
123 
124         std::string directions =
125             std::visit(VariantToStringVisitor(), directionFind->second);
126 
127         Level level = findThresholdLevel(severity);
128         Direction direction = findThresholdDirection(directions);
129 
130         if ((level == Level::ERROR) || (direction == Direction::ERROR))
131         {
132             continue;
133         }
134         double val = std::visit(VariantToDoubleVisitor(), valueFind->second);
135 
136         thresholdVector.emplace_back(level, direction, val, hysteresis);
137     }
138     return true;
139 }
140 
141 void persistThreshold(const std::string& path, const std::string& baseInterface,
142                       const thresholds::Threshold& threshold,
143                       std::shared_ptr<sdbusplus::asio::connection>& conn,
144                       size_t thresholdCount, const std::string& labelMatch)
145 {
146     for (size_t ii = 0; ii < thresholdCount; ii++)
147     {
148         std::string thresholdInterface =
149             baseInterface + ".Thresholds" + std::to_string(ii);
150         conn->async_method_call(
151             [&, path, threshold, thresholdInterface, labelMatch](
152                 const boost::system::error_code& ec,
153                 const boost::container::flat_map<std::string, BasicVariantType>&
154                     result) {
155                 if (ec)
156                 {
157                     return; // threshold not supported
158                 }
159 
160                 if (!labelMatch.empty())
161                 {
162                     auto labelFind = result.find("Label");
163                     if (labelFind == result.end())
164                     {
165                         std::cerr << "No label in threshold configuration\n";
166                         return;
167                     }
168                     std::string label =
169                         std::visit(VariantToStringVisitor(), labelFind->second);
170                     if (label != labelMatch)
171                     {
172                         return;
173                     }
174                 }
175 
176                 auto directionFind = result.find("Direction");
177                 auto severityFind = result.find("Severity");
178                 auto valueFind = result.find("Value");
179                 if (valueFind == result.end() || severityFind == result.end() ||
180                     directionFind == result.end())
181                 {
182                     std::cerr << "Malformed threshold in configuration\n";
183                     return;
184                 }
185                 unsigned int severity = std::visit(
186                     VariantToUnsignedIntVisitor(), severityFind->second);
187 
188                 std::string dir =
189                     std::visit(VariantToStringVisitor(), directionFind->second);
190                 if ((findThresholdLevel(severity) != threshold.level) ||
191                     (findThresholdDirection(dir) != threshold.direction))
192                 {
193                     return; // not the droid we're looking for
194                 }
195 
196                 std::variant<double> value(threshold.value);
197                 conn->async_method_call(
198                     [](const boost::system::error_code& ec) {
199                         if (ec)
200                         {
201                             std::cerr << "Error setting threshold " << ec
202                                       << "\n";
203                         }
204                     },
205                     entityManagerName, path, "org.freedesktop.DBus.Properties",
206                     "Set", thresholdInterface, "Value", value);
207             },
208             entityManagerName, path, "org.freedesktop.DBus.Properties",
209             "GetAll", thresholdInterface);
210     }
211 }
212 
213 void updateThresholds(Sensor* sensor)
214 {
215     for (const auto& threshold : sensor->thresholds)
216     {
217         if (!isValidLevel(threshold.level))
218         {
219             continue;
220         }
221         std::shared_ptr<sdbusplus::asio::dbus_interface> interface =
222             sensor->getThresholdInterface(threshold.level);
223 
224         if (!interface)
225         {
226             continue;
227         }
228 
229         std::string property =
230             sensor->propertyLevel(threshold.level, threshold.direction);
231         if (property.empty())
232         {
233             continue;
234         }
235         interface->set_property(property, threshold.value);
236     }
237 }
238 
239 // Debugging counters
240 static int cHiTrue = 0;
241 static int cHiFalse = 0;
242 static int cHiMidstate = 0;
243 static int cLoTrue = 0;
244 static int cLoFalse = 0;
245 static int cLoMidstate = 0;
246 static int cDebugThrottle = 0;
247 static constexpr int assertLogCount = 10;
248 
249 struct ChangeParam
250 {
251     ChangeParam(Threshold whichThreshold, bool status, double value) :
252         threshold(whichThreshold), asserted(status), assertValue(value)
253     {}
254 
255     Threshold threshold;
256     bool asserted;
257     double assertValue;
258 };
259 
260 static std::vector<ChangeParam> checkThresholds(Sensor* sensor, double value)
261 {
262     std::vector<ChangeParam> thresholdChanges;
263     if (sensor->thresholds.empty())
264     {
265         return thresholdChanges;
266     }
267 
268     for (auto& threshold : sensor->thresholds)
269     {
270         // Use "Schmitt trigger" logic to avoid threshold trigger spam,
271         // if value is noisy while hovering very close to a threshold.
272         // When a threshold is crossed, indicate true immediately,
273         // but require more distance to be crossed the other direction,
274         // before resetting the indicator back to false.
275         if (threshold.direction == thresholds::Direction::HIGH)
276         {
277             if (value >= threshold.value)
278             {
279                 thresholdChanges.emplace_back(threshold, true, value);
280                 if (++cHiTrue < assertLogCount)
281                 {
282                     std::cerr << "Sensor " << sensor->name << " high threshold "
283                               << threshold.value << " assert: value " << value
284                               << " raw data " << sensor->rawValue << "\n";
285                 }
286             }
287             else if (value < (threshold.value - threshold.hysteresis))
288             {
289                 thresholdChanges.emplace_back(threshold, false, value);
290                 ++cHiFalse;
291             }
292             else
293             {
294                 ++cHiMidstate;
295             }
296         }
297         else if (threshold.direction == thresholds::Direction::LOW)
298         {
299             if (value <= threshold.value)
300             {
301                 thresholdChanges.emplace_back(threshold, true, value);
302                 if (++cLoTrue < assertLogCount)
303                 {
304                     std::cerr << "Sensor " << sensor->name << " low threshold "
305                               << threshold.value << " assert: value "
306                               << sensor->value << " raw data "
307                               << sensor->rawValue << "\n";
308                 }
309             }
310             else if (value > (threshold.value + threshold.hysteresis))
311             {
312                 thresholdChanges.emplace_back(threshold, false, value);
313                 ++cLoFalse;
314             }
315             else
316             {
317                 ++cLoMidstate;
318             }
319         }
320         else
321         {
322             std::cerr << "Error determining threshold direction\n";
323         }
324     }
325 
326     // Throttle debug output, so that it does not continuously spam
327     ++cDebugThrottle;
328     if (cDebugThrottle >= 1000)
329     {
330         cDebugThrottle = 0;
331         if constexpr (debug)
332         {
333             std::cerr << "checkThresholds: High T=" << cHiTrue
334                       << " F=" << cHiFalse << " M=" << cHiMidstate
335                       << ", Low T=" << cLoTrue << " F=" << cLoFalse
336                       << " M=" << cLoMidstate << "\n";
337         }
338     }
339 
340     return thresholdChanges;
341 }
342 
343 void ThresholdTimer::startTimer(const std::weak_ptr<Sensor>& weakSensor,
344                                 const Threshold& threshold, bool assert,
345                                 double assertValue)
346 {
347     struct TimerUsed timerUsed = {};
348     constexpr const size_t waitTime = 5;
349     TimerPair* pair = nullptr;
350 
351     for (TimerPair& timer : timers)
352     {
353         if (!timer.first.used)
354         {
355             pair = &timer;
356             break;
357         }
358     }
359     if (pair == nullptr)
360     {
361         pair = &timers.emplace_back(timerUsed, boost::asio::deadline_timer(io));
362     }
363 
364     pair->first.used = true;
365     pair->first.level = threshold.level;
366     pair->first.direction = threshold.direction;
367     pair->first.assert = assert;
368     pair->second.expires_from_now(boost::posix_time::seconds(waitTime));
369     pair->second.async_wait([weakSensor, pair, threshold, assert,
370                              assertValue](boost::system::error_code ec) {
371         auto sensorPtr = weakSensor.lock();
372         if (!sensorPtr)
373         {
374             return; // owner sensor has been destructed
375         }
376         // pair is valid as long as sensor is valid
377         pair->first.used = false;
378 
379         if (ec == boost::asio::error::operation_aborted)
380         {
381             return; // we're being canceled
382         }
383         if (ec)
384         {
385             std::cerr << "timer error: " << ec.message() << "\n";
386             return;
387         }
388         if (sensorPtr->readingStateGood())
389         {
390             assertThresholds(sensorPtr.get(), assertValue, threshold.level,
391                              threshold.direction, assert);
392         }
393     });
394 }
395 
396 bool checkThresholds(Sensor* sensor)
397 {
398     bool status = true;
399     std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value);
400     for (const auto& change : changes)
401     {
402         assertThresholds(sensor, change.assertValue, change.threshold.level,
403                          change.threshold.direction, change.asserted);
404         if (change.threshold.level == thresholds::Level::CRITICAL &&
405             change.asserted)
406         {
407             status = false;
408         }
409     }
410 
411     return status;
412 }
413 
414 void checkThresholdsPowerDelay(const std::weak_ptr<Sensor>& weakSensor,
415                                ThresholdTimer& thresholdTimer)
416 {
417     auto sensorPtr = weakSensor.lock();
418     if (!sensorPtr)
419     {
420         return; // sensor is destructed, should never be here
421     }
422 
423     Sensor* sensor = sensorPtr.get();
424     std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value);
425     for (const auto& change : changes)
426     {
427         // When CPU is powered off, some volatges are expected to
428         // go below low thresholds. Filter these events with thresholdTimer.
429         // 1. always delay the assertion of low events to see if they are
430         //   caused by power off event.
431         // 2. conditional delay the de-assertion of low events if there is
432         //   an existing timer for assertion.
433         // 3. no delays for de-assert of low events if there is an existing
434         //   de-assert for low event. This means 2nd de-assert would happen
435         //   first and when timer expires for the previous one, no additional
436         //   signal will be logged.
437         // 4. no delays for all high events.
438         if (change.threshold.direction == thresholds::Direction::LOW)
439         {
440             if (change.asserted || thresholdTimer.hasActiveTimer(
441                                        change.threshold, !change.asserted))
442             {
443                 thresholdTimer.startTimer(weakSensor, change.threshold,
444                                           change.asserted, change.assertValue);
445                 continue;
446             }
447         }
448         assertThresholds(sensor, change.assertValue, change.threshold.level,
449                          change.threshold.direction, change.asserted);
450     }
451 }
452 
453 void assertThresholds(Sensor* sensor, double assertValue,
454                       thresholds::Level level, thresholds::Direction direction,
455                       bool assert)
456 {
457     if (!isValidLevel(level))
458     {
459         return;
460     }
461 
462     std::shared_ptr<sdbusplus::asio::dbus_interface> interface =
463         sensor->getThresholdInterface(level);
464 
465     if (!interface)
466     {
467         std::cout << "trying to set uninitialized interface\n";
468         return;
469     }
470 
471     std::string property = sensor->propertyAlarm(level, direction);
472     if (property.empty())
473     {
474         std::cout << "Alarm property is empty \n";
475         return;
476     }
477     if (interface->set_property<bool, true>(property, assert))
478     {
479         try
480         {
481             // msg.get_path() is interface->get_object_path()
482             sdbusplus::message::message msg =
483                 interface->new_signal("ThresholdAsserted");
484 
485             msg.append(sensor->name, interface->get_interface_name(), property,
486                        assert, assertValue);
487             msg.signal_send();
488         }
489         catch (const sdbusplus::exception::exception& e)
490         {
491             std::cerr
492                 << "Failed to send thresholdAsserted signal with assertValue\n";
493         }
494     }
495 }
496 
497 bool parseThresholdsFromAttr(
498     std::vector<thresholds::Threshold>& thresholdVector,
499     const std::string& inputPath, const double& scaleFactor,
500     const double& offset)
501 {
502     const boost::container::flat_map<
503         std::string, std::vector<std::tuple<const char*, thresholds::Level,
504                                             thresholds::Direction, double>>>
505         map = {
506             {"average",
507              {
508                  std::make_tuple("average_min", Level::WARNING, Direction::LOW,
509                                  0.0),
510                  std::make_tuple("average_max", Level::WARNING, Direction::HIGH,
511                                  0.0),
512              }},
513             {"input",
514              {
515                  std::make_tuple("min", Level::WARNING, Direction::LOW, 0.0),
516                  std::make_tuple("max", Level::WARNING, Direction::HIGH, 0.0),
517                  std::make_tuple("lcrit", Level::CRITICAL, Direction::LOW, 0.0),
518                  std::make_tuple("crit", Level::CRITICAL, Direction::HIGH,
519                                  offset),
520              }},
521         };
522 
523     if (auto fileParts = splitFileName(inputPath))
524     {
525         auto& [type, nr, item] = *fileParts;
526         if (map.count(item) != 0)
527         {
528             for (const auto& t : map.at(item))
529             {
530                 auto& [suffix, level, direction, offset] = t;
531                 auto attrPath =
532                     boost::replace_all_copy(inputPath, item, suffix);
533                 if (auto val = readFile(attrPath, scaleFactor))
534                 {
535                     *val += offset;
536                     if (debug)
537                     {
538                         std::cout << "Threshold: " << attrPath << ": " << *val
539                                   << "\n";
540                     }
541                     thresholdVector.emplace_back(level, direction, *val);
542                 }
543             }
544         }
545     }
546     return true;
547 }
548 
549 std::string getInterface(const Level thresholdLevel)
550 {
551     for (const ThresholdDefinition& thresh : thresProp)
552     {
553         if (thresh.level == thresholdLevel)
554         {
555             return std::string("xyz.openbmc_project.Sensor.Threshold.") +
556                    thresh.levelName;
557         }
558     }
559     return "";
560 }
561 } // namespace thresholds
562