xref: /openbmc/dbus-sensors/src/Thresholds.cpp (revision 58fa1a5a)
1 #include <Thresholds.hpp>
2 #include <VariantVisitors.hpp>
3 #include <boost/algorithm/string/replace.hpp>
4 #include <boost/container/flat_map.hpp>
5 #include <sensor.hpp>
6 
7 #include <array>
8 #include <cmath>
9 #include <fstream>
10 #include <iostream>
11 #include <memory>
12 #include <stdexcept>
13 #include <string>
14 #include <tuple>
15 #include <utility>
16 #include <variant>
17 #include <vector>
18 
19 static constexpr bool debug = false;
20 namespace thresholds
21 {
22 Level findThresholdLevel(uint8_t sev)
23 {
24     for (const ThresholdDefinition& prop : thresProp)
25     {
26         if (prop.sevOrder == sev)
27         {
28             return prop.level;
29         }
30     }
31     return Level::ERROR;
32 }
33 
34 Direction findThresholdDirection(const std::string& direct)
35 {
36     if (direct == "greater than")
37     {
38         return Direction::HIGH;
39     }
40     if (direct == "less than")
41     {
42         return Direction::LOW;
43     }
44     return Direction::ERROR;
45 }
46 
47 bool parseThresholdsFromConfig(
48     const SensorData& sensorData,
49     std::vector<thresholds::Threshold>& thresholdVector,
50     const std::string* matchLabel, const int* sensorIndex)
51 {
52     for (const auto& item : sensorData)
53     {
54         if (item.first.find("Thresholds") == std::string::npos)
55         {
56             continue;
57         }
58         if (matchLabel != nullptr)
59         {
60             auto labelFind = item.second.find("Label");
61             if (labelFind == item.second.end())
62             {
63                 continue;
64             }
65             if (std::visit(VariantToStringVisitor(), labelFind->second) !=
66                 *matchLabel)
67             {
68                 continue;
69             }
70         }
71 
72         if (sensorIndex != nullptr)
73         {
74             auto indexFind = item.second.find("Index");
75 
76             // If we're checking for index 1, a missing Index is OK.
77             if ((indexFind == item.second.end()) && (*sensorIndex != 1))
78             {
79                 continue;
80             }
81 
82             if ((indexFind != item.second.end()) &&
83                 (std::visit(VariantToIntVisitor(), indexFind->second) !=
84                  *sensorIndex))
85             {
86                 continue;
87             }
88         }
89 
90         double hysteresis = std::numeric_limits<double>::quiet_NaN();
91         auto hysteresisFind = item.second.find("Hysteresis");
92         if (hysteresisFind != item.second.end())
93         {
94             hysteresis =
95                 std::visit(VariantToDoubleVisitor(), hysteresisFind->second);
96         }
97 
98         auto directionFind = item.second.find("Direction");
99         auto severityFind = item.second.find("Severity");
100         auto valueFind = item.second.find("Value");
101         if (valueFind == item.second.end() ||
102             severityFind == item.second.end() ||
103             directionFind == item.second.end())
104         {
105             std::cerr << "Malformed threshold on configuration interface "
106                       << item.first << "\n";
107             return false;
108         }
109         unsigned int severity =
110             std::visit(VariantToUnsignedIntVisitor(), severityFind->second);
111 
112         std::string directions =
113             std::visit(VariantToStringVisitor(), directionFind->second);
114 
115         Level level = findThresholdLevel(severity);
116         Direction direction = findThresholdDirection(directions);
117 
118         if ((level == Level::ERROR) || (direction == Direction::ERROR))
119         {
120             continue;
121         }
122         double val = std::visit(VariantToDoubleVisitor(), valueFind->second);
123 
124         thresholdVector.emplace_back(level, direction, val, hysteresis);
125     }
126     return true;
127 }
128 
129 void persistThreshold(const std::string& path, const std::string& baseInterface,
130                       const thresholds::Threshold& threshold,
131                       std::shared_ptr<sdbusplus::asio::connection>& conn,
132                       size_t thresholdCount, const std::string& labelMatch)
133 {
134     for (size_t ii = 0; ii < thresholdCount; ii++)
135     {
136         std::string thresholdInterface =
137             baseInterface + ".Thresholds" + std::to_string(ii);
138         conn->async_method_call(
139             [&, path, threshold, thresholdInterface, labelMatch](
140                 const boost::system::error_code& ec,
141                 const boost::container::flat_map<std::string, BasicVariantType>&
142                     result) {
143                 if (ec)
144                 {
145                     return; // threshold not supported
146                 }
147 
148                 if (!labelMatch.empty())
149                 {
150                     auto labelFind = result.find("Label");
151                     if (labelFind == result.end())
152                     {
153                         std::cerr << "No label in threshold configuration\n";
154                         return;
155                     }
156                     std::string label =
157                         std::visit(VariantToStringVisitor(), labelFind->second);
158                     if (label != labelMatch)
159                     {
160                         return;
161                     }
162                 }
163 
164                 auto directionFind = result.find("Direction");
165                 auto severityFind = result.find("Severity");
166                 auto valueFind = result.find("Value");
167                 if (valueFind == result.end() || severityFind == result.end() ||
168                     directionFind == result.end())
169                 {
170                     std::cerr << "Malformed threshold in configuration\n";
171                     return;
172                 }
173                 unsigned int severity = std::visit(
174                     VariantToUnsignedIntVisitor(), severityFind->second);
175 
176                 std::string dir =
177                     std::visit(VariantToStringVisitor(), directionFind->second);
178                 if ((findThresholdLevel(severity) != threshold.level) ||
179                     (findThresholdDirection(dir) != threshold.direction))
180                 {
181                     return; // not the droid we're looking for
182                 }
183 
184                 std::variant<double> value(threshold.value);
185                 conn->async_method_call(
186                     [](const boost::system::error_code& ec) {
187                         if (ec)
188                         {
189                             std::cerr << "Error setting threshold " << ec
190                                       << "\n";
191                         }
192                     },
193                     entityManagerName, path, "org.freedesktop.DBus.Properties",
194                     "Set", thresholdInterface, "Value", value);
195             },
196             entityManagerName, path, "org.freedesktop.DBus.Properties",
197             "GetAll", thresholdInterface);
198     }
199 }
200 
201 void updateThresholds(Sensor* sensor)
202 {
203     for (const auto& threshold : sensor->thresholds)
204     {
205         std::shared_ptr<sdbusplus::asio::dbus_interface> interface =
206             sensor->getThresholdInterface(threshold.level);
207 
208         if (!interface)
209         {
210             continue;
211         }
212 
213         std::string property =
214             sensor->propertyLevel(threshold.level, threshold.direction);
215         if (property.empty())
216         {
217             continue;
218         }
219         interface->set_property(property, threshold.value);
220     }
221 }
222 
223 // Debugging counters
224 static int cHiTrue = 0;
225 static int cHiFalse = 0;
226 static int cHiMidstate = 0;
227 static int cLoTrue = 0;
228 static int cLoFalse = 0;
229 static int cLoMidstate = 0;
230 static int cDebugThrottle = 0;
231 static constexpr int assertLogCount = 10;
232 
233 struct ChangeParam
234 {
235     ChangeParam(Threshold whichThreshold, bool status, double value) :
236         threshold(whichThreshold), asserted(status), assertValue(value)
237     {}
238 
239     Threshold threshold;
240     bool asserted;
241     double assertValue;
242 };
243 
244 static std::vector<ChangeParam> checkThresholds(Sensor* sensor, double value)
245 {
246     std::vector<ChangeParam> thresholdChanges;
247     if (sensor->thresholds.empty())
248     {
249         return thresholdChanges;
250     }
251 
252     for (auto& threshold : sensor->thresholds)
253     {
254         // Use "Schmitt trigger" logic to avoid threshold trigger spam,
255         // if value is noisy while hovering very close to a threshold.
256         // When a threshold is crossed, indicate true immediately,
257         // but require more distance to be crossed the other direction,
258         // before resetting the indicator back to false.
259         if (threshold.direction == thresholds::Direction::HIGH)
260         {
261             if (value >= threshold.value)
262             {
263                 thresholdChanges.emplace_back(threshold, true, value);
264                 if (++cHiTrue < assertLogCount)
265                 {
266                     std::cerr << "Sensor " << sensor->name << " high threshold "
267                               << threshold.value << " assert: value " << value
268                               << " raw data " << sensor->rawValue << "\n";
269                 }
270             }
271             else if (value < (threshold.value - threshold.hysteresis))
272             {
273                 thresholdChanges.emplace_back(threshold, false, value);
274                 ++cHiFalse;
275             }
276             else
277             {
278                 ++cHiMidstate;
279             }
280         }
281         else if (threshold.direction == thresholds::Direction::LOW)
282         {
283             if (value <= threshold.value)
284             {
285                 thresholdChanges.emplace_back(threshold, true, value);
286                 if (++cLoTrue < assertLogCount)
287                 {
288                     std::cerr << "Sensor " << sensor->name << " low threshold "
289                               << threshold.value << " assert: value "
290                               << sensor->value << " raw data "
291                               << sensor->rawValue << "\n";
292                 }
293             }
294             else if (value > (threshold.value + threshold.hysteresis))
295             {
296                 thresholdChanges.emplace_back(threshold, false, value);
297                 ++cLoFalse;
298             }
299             else
300             {
301                 ++cLoMidstate;
302             }
303         }
304         else
305         {
306             std::cerr << "Error determining threshold direction\n";
307         }
308     }
309 
310     // Throttle debug output, so that it does not continuously spam
311     ++cDebugThrottle;
312     if (cDebugThrottle >= 1000)
313     {
314         cDebugThrottle = 0;
315         if constexpr (debug)
316         {
317             std::cerr << "checkThresholds: High T=" << cHiTrue
318                       << " F=" << cHiFalse << " M=" << cHiMidstate
319                       << ", Low T=" << cLoTrue << " F=" << cLoFalse
320                       << " M=" << cLoMidstate << "\n";
321         }
322     }
323 
324     return thresholdChanges;
325 }
326 
327 void ThresholdTimer::startTimer(const std::weak_ptr<Sensor>& weakSensor,
328                                 const Threshold& threshold, bool assert,
329                                 double assertValue)
330 {
331     struct TimerUsed timerUsed = {};
332     constexpr const size_t waitTime = 5;
333     TimerPair* pair = nullptr;
334 
335     for (TimerPair& timer : timers)
336     {
337         if (!timer.first.used)
338         {
339             pair = &timer;
340             break;
341         }
342     }
343     if (pair == nullptr)
344     {
345         pair = &timers.emplace_back(timerUsed, boost::asio::deadline_timer(io));
346     }
347 
348     pair->first.used = true;
349     pair->first.level = threshold.level;
350     pair->first.direction = threshold.direction;
351     pair->first.assert = assert;
352     pair->second.expires_from_now(boost::posix_time::seconds(waitTime));
353     pair->second.async_wait([weakSensor, pair, threshold, assert,
354                              assertValue](boost::system::error_code ec) {
355         auto sensorPtr = weakSensor.lock();
356         if (!sensorPtr)
357         {
358             return; // owner sensor has been destructed
359         }
360         // pair is valid as long as sensor is valid
361         pair->first.used = false;
362 
363         if (ec == boost::asio::error::operation_aborted)
364         {
365             return; // we're being canceled
366         }
367         if (ec)
368         {
369             std::cerr << "timer error: " << ec.message() << "\n";
370             return;
371         }
372         if (sensorPtr->readingStateGood())
373         {
374             assertThresholds(sensorPtr.get(), assertValue, threshold.level,
375                              threshold.direction, assert);
376         }
377     });
378 }
379 
380 bool checkThresholds(Sensor* sensor)
381 {
382     bool status = true;
383     std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value);
384     for (const auto& change : changes)
385     {
386         assertThresholds(sensor, change.assertValue, change.threshold.level,
387                          change.threshold.direction, change.asserted);
388         if (change.threshold.level == thresholds::Level::CRITICAL &&
389             change.asserted)
390         {
391             status = false;
392         }
393     }
394 
395     return status;
396 }
397 
398 void checkThresholdsPowerDelay(const std::weak_ptr<Sensor>& weakSensor,
399                                ThresholdTimer& thresholdTimer)
400 {
401     auto sensorPtr = weakSensor.lock();
402     if (!sensorPtr)
403     {
404         return; // sensor is destructed, should never be here
405     }
406 
407     Sensor* sensor = sensorPtr.get();
408     std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value);
409     for (const auto& change : changes)
410     {
411         // When CPU is powered off, some volatges are expected to
412         // go below low thresholds. Filter these events with thresholdTimer.
413         // 1. always delay the assertion of low events to see if they are
414         //   caused by power off event.
415         // 2. conditional delay the de-assertion of low events if there is
416         //   an existing timer for assertion.
417         // 3. no delays for de-assert of low events if there is an existing
418         //   de-assert for low event. This means 2nd de-assert would happen
419         //   first and when timer expires for the previous one, no additional
420         //   signal will be logged.
421         // 4. no delays for all high events.
422         if (change.threshold.direction == thresholds::Direction::LOW)
423         {
424             if (change.asserted || thresholdTimer.hasActiveTimer(
425                                        change.threshold, !change.asserted))
426             {
427                 thresholdTimer.startTimer(weakSensor, change.threshold,
428                                           change.asserted, change.assertValue);
429                 continue;
430             }
431         }
432         assertThresholds(sensor, change.assertValue, change.threshold.level,
433                          change.threshold.direction, change.asserted);
434     }
435 }
436 
437 void assertThresholds(Sensor* sensor, double assertValue,
438                       thresholds::Level level, thresholds::Direction direction,
439                       bool assert)
440 {
441     std::shared_ptr<sdbusplus::asio::dbus_interface> interface =
442         sensor->getThresholdInterface(level);
443 
444     if (!interface)
445     {
446         std::cout << "trying to set uninitialized interface\n";
447         return;
448     }
449 
450     std::string property = sensor->propertyAlarm(level, direction);
451     if (property.empty())
452     {
453         std::cout << "Alarm property is empty \n";
454         return;
455     }
456     if (interface->set_property<bool, true>(property, assert))
457     {
458         try
459         {
460             // msg.get_path() is interface->get_object_path()
461             sdbusplus::message::message msg =
462                 interface->new_signal("ThresholdAsserted");
463 
464             msg.append(sensor->name, interface->get_interface_name(), property,
465                        assert, assertValue);
466             msg.signal_send();
467         }
468         catch (const sdbusplus::exception::exception& e)
469         {
470             std::cerr
471                 << "Failed to send thresholdAsserted signal with assertValue\n";
472         }
473     }
474 }
475 
476 bool parseThresholdsFromAttr(
477     std::vector<thresholds::Threshold>& thresholdVector,
478     const std::string& inputPath, const double& scaleFactor,
479     const double& offset)
480 {
481     const boost::container::flat_map<
482         std::string, std::vector<std::tuple<const char*, thresholds::Level,
483                                             thresholds::Direction, double>>>
484         map = {
485             {"average",
486              {
487                  std::make_tuple("average_min", Level::WARNING, Direction::LOW,
488                                  0.0),
489                  std::make_tuple("average_max", Level::WARNING, Direction::HIGH,
490                                  0.0),
491              }},
492             {"input",
493              {
494                  std::make_tuple("min", Level::WARNING, Direction::LOW, 0.0),
495                  std::make_tuple("max", Level::WARNING, Direction::HIGH, 0.0),
496                  std::make_tuple("lcrit", Level::CRITICAL, Direction::LOW, 0.0),
497                  std::make_tuple("crit", Level::CRITICAL, Direction::HIGH,
498                                  offset),
499              }},
500         };
501 
502     if (auto fileParts = splitFileName(inputPath))
503     {
504         auto& [type, nr, item] = *fileParts;
505         if (map.count(item) != 0)
506         {
507             for (const auto& t : map.at(item))
508             {
509                 auto& [suffix, level, direction, offset] = t;
510                 auto attrPath =
511                     boost::replace_all_copy(inputPath, item, suffix);
512                 if (auto val = readFile(attrPath, scaleFactor))
513                 {
514                     *val += offset;
515                     if (debug)
516                     {
517                         std::cout << "Threshold: " << attrPath << ": " << *val
518                                   << "\n";
519                     }
520                     thresholdVector.emplace_back(level, direction, *val);
521                 }
522             }
523         }
524     }
525     return true;
526 }
527 
528 std::string getInterface(const Level thresholdLevel)
529 {
530     for (const ThresholdDefinition& thresh : thresProp)
531     {
532         if (thresh.level == thresholdLevel)
533         {
534             return std::string("xyz.openbmc_project.Sensor.Threshold.") +
535                    thresh.levelName;
536         }
537     }
538     return "";
539 }
540 } // namespace thresholds
541