xref: /openbmc/dbus-sensors/src/Thresholds.cpp (revision 255da6b4)
1 #include <Thresholds.hpp>
2 #include <VariantVisitors.hpp>
3 #include <boost/algorithm/string/replace.hpp>
4 #include <boost/container/flat_map.hpp>
5 #include <sensor.hpp>
6 
7 #include <array>
8 #include <cmath>
9 #include <fstream>
10 #include <iostream>
11 #include <memory>
12 #include <stdexcept>
13 #include <string>
14 #include <tuple>
15 #include <utility>
16 #include <variant>
17 #include <vector>
18 
19 static constexpr bool debug = false;
20 namespace thresholds
21 {
22 Level findThresholdLevel(uint8_t sev)
23 {
24     for (const ThresholdDefinition& prop : thresProp)
25     {
26         if (prop.sevOrder == sev)
27         {
28             return prop.level;
29         }
30     }
31     return Level::ERROR;
32 }
33 
34 Direction findThresholdDirection(const std::string& direct)
35 {
36     if (direct == "greater than")
37     {
38         return Direction::HIGH;
39     }
40     if (direct == "less than")
41     {
42         return Direction::LOW;
43     }
44     return Direction::ERROR;
45 }
46 
47 bool parseThresholdsFromConfig(
48     const SensorData& sensorData,
49     std::vector<thresholds::Threshold>& thresholdVector,
50     const std::string* matchLabel, const int* sensorIndex)
51 {
52     for (const auto& item : sensorData)
53     {
54         if (item.first.find("Thresholds") == std::string::npos)
55         {
56             continue;
57         }
58         if (matchLabel != nullptr)
59         {
60             auto labelFind = item.second.find("Label");
61             if (labelFind == item.second.end())
62             {
63                 continue;
64             }
65             if (std::visit(VariantToStringVisitor(), labelFind->second) !=
66                 *matchLabel)
67             {
68                 continue;
69             }
70         }
71 
72         if (sensorIndex != nullptr)
73         {
74             auto indexFind = item.second.find("Index");
75 
76             // If we're checking for index 1, a missing Index is OK.
77             if ((indexFind == item.second.end()) && (*sensorIndex != 1))
78             {
79                 continue;
80             }
81 
82             if ((indexFind != item.second.end()) &&
83                 (std::visit(VariantToIntVisitor(), indexFind->second) !=
84                  *sensorIndex))
85             {
86                 continue;
87             }
88         }
89 
90         double hysteresis = std::numeric_limits<double>::quiet_NaN();
91         auto hysteresisFind = item.second.find("Hysteresis");
92         if (hysteresisFind != item.second.end())
93         {
94             hysteresis =
95                 std::visit(VariantToDoubleVisitor(), hysteresisFind->second);
96         }
97 
98         auto directionFind = item.second.find("Direction");
99         auto severityFind = item.second.find("Severity");
100         auto valueFind = item.second.find("Value");
101         if (valueFind == item.second.end() ||
102             severityFind == item.second.end() ||
103             directionFind == item.second.end())
104         {
105             std::cerr << "Malformed threshold on configuration interface "
106                       << item.first << "\n";
107             return false;
108         }
109         unsigned int severity =
110             std::visit(VariantToUnsignedIntVisitor(), severityFind->second);
111 
112         std::string directions =
113             std::visit(VariantToStringVisitor(), directionFind->second);
114 
115         Level level = findThresholdLevel(severity);
116         Direction direction = findThresholdDirection(directions);
117 
118         if ((level == Level::ERROR) || (direction == Direction::ERROR))
119         {
120             continue;
121         }
122         double val = std::visit(VariantToDoubleVisitor(), valueFind->second);
123 
124         thresholdVector.emplace_back(level, direction, val, hysteresis);
125     }
126     return true;
127 }
128 
129 void persistThreshold(const std::string& path, const std::string& baseInterface,
130                       const thresholds::Threshold& threshold,
131                       std::shared_ptr<sdbusplus::asio::connection>& conn,
132                       size_t thresholdCount, const std::string& labelMatch)
133 {
134     for (size_t ii = 0; ii < thresholdCount; ii++)
135     {
136         std::string thresholdInterface =
137             baseInterface + ".Thresholds" + std::to_string(ii);
138         conn->async_method_call(
139             [&, path, threshold, thresholdInterface,
140              labelMatch](const boost::system::error_code& ec,
141                          const SensorBaseConfigMap& result) {
142             if (ec)
143             {
144                 return; // threshold not supported
145             }
146 
147             if (!labelMatch.empty())
148             {
149                 auto labelFind = result.find("Label");
150                 if (labelFind == result.end())
151                 {
152                     std::cerr << "No label in threshold configuration\n";
153                     return;
154                 }
155                 std::string label =
156                     std::visit(VariantToStringVisitor(), labelFind->second);
157                 if (label != labelMatch)
158                 {
159                     return;
160                 }
161             }
162 
163             auto directionFind = result.find("Direction");
164             auto severityFind = result.find("Severity");
165             auto valueFind = result.find("Value");
166             if (valueFind == result.end() || severityFind == result.end() ||
167                 directionFind == result.end())
168             {
169                 std::cerr << "Malformed threshold in configuration\n";
170                 return;
171             }
172             unsigned int severity =
173                 std::visit(VariantToUnsignedIntVisitor(), severityFind->second);
174 
175             std::string dir =
176                 std::visit(VariantToStringVisitor(), directionFind->second);
177             if ((findThresholdLevel(severity) != threshold.level) ||
178                 (findThresholdDirection(dir) != threshold.direction))
179             {
180                 return; // not the droid we're looking for
181             }
182 
183             std::variant<double> value(threshold.value);
184             conn->async_method_call(
185                 [](const boost::system::error_code& ec) {
186                 if (ec)
187                 {
188                     std::cerr << "Error setting threshold " << ec << "\n";
189                 }
190                 },
191                 entityManagerName, path, "org.freedesktop.DBus.Properties",
192                 "Set", thresholdInterface, "Value", value);
193             },
194             entityManagerName, path, "org.freedesktop.DBus.Properties",
195             "GetAll", thresholdInterface);
196     }
197 }
198 
199 void updateThresholds(Sensor* sensor)
200 {
201     for (const auto& threshold : sensor->thresholds)
202     {
203         std::shared_ptr<sdbusplus::asio::dbus_interface> interface =
204             sensor->getThresholdInterface(threshold.level);
205 
206         if (!interface)
207         {
208             continue;
209         }
210 
211         std::string property =
212             Sensor::propertyLevel(threshold.level, threshold.direction);
213         if (property.empty())
214         {
215             continue;
216         }
217         interface->set_property(property, threshold.value);
218     }
219 }
220 
221 // Debugging counters
222 static int cHiTrue = 0;
223 static int cHiFalse = 0;
224 static int cHiMidstate = 0;
225 static int cLoTrue = 0;
226 static int cLoFalse = 0;
227 static int cLoMidstate = 0;
228 static int cDebugThrottle = 0;
229 static constexpr int assertLogCount = 10;
230 
231 struct ChangeParam
232 {
233     ChangeParam(Threshold whichThreshold, bool status, double value) :
234         threshold(whichThreshold), asserted(status), assertValue(value)
235     {}
236 
237     Threshold threshold;
238     bool asserted;
239     double assertValue;
240 };
241 
242 static std::vector<ChangeParam> checkThresholds(Sensor* sensor, double value)
243 {
244     std::vector<ChangeParam> thresholdChanges;
245     if (sensor->thresholds.empty())
246     {
247         return thresholdChanges;
248     }
249 
250     for (auto& threshold : sensor->thresholds)
251     {
252         // Use "Schmitt trigger" logic to avoid threshold trigger spam,
253         // if value is noisy while hovering very close to a threshold.
254         // When a threshold is crossed, indicate true immediately,
255         // but require more distance to be crossed the other direction,
256         // before resetting the indicator back to false.
257         if (threshold.direction == thresholds::Direction::HIGH)
258         {
259             if (value >= threshold.value)
260             {
261                 thresholdChanges.emplace_back(threshold, true, value);
262                 if (++cHiTrue < assertLogCount)
263                 {
264                     std::cerr << "Sensor " << sensor->name << " high threshold "
265                               << threshold.value << " assert: value " << value
266                               << " raw data " << sensor->rawValue << "\n";
267                 }
268             }
269             else if (value < (threshold.value - threshold.hysteresis))
270             {
271                 thresholdChanges.emplace_back(threshold, false, value);
272                 ++cHiFalse;
273             }
274             else
275             {
276                 ++cHiMidstate;
277             }
278         }
279         else if (threshold.direction == thresholds::Direction::LOW)
280         {
281             if (value <= threshold.value)
282             {
283                 thresholdChanges.emplace_back(threshold, true, value);
284                 if (++cLoTrue < assertLogCount)
285                 {
286                     std::cerr << "Sensor " << sensor->name << " low threshold "
287                               << threshold.value << " assert: value "
288                               << sensor->value << " raw data "
289                               << sensor->rawValue << "\n";
290                 }
291             }
292             else if (value > (threshold.value + threshold.hysteresis))
293             {
294                 thresholdChanges.emplace_back(threshold, false, value);
295                 ++cLoFalse;
296             }
297             else
298             {
299                 ++cLoMidstate;
300             }
301         }
302         else
303         {
304             std::cerr << "Error determining threshold direction\n";
305         }
306     }
307 
308     // Throttle debug output, so that it does not continuously spam
309     ++cDebugThrottle;
310     if (cDebugThrottle >= 1000)
311     {
312         cDebugThrottle = 0;
313         if constexpr (debug)
314         {
315             std::cerr << "checkThresholds: High T=" << cHiTrue
316                       << " F=" << cHiFalse << " M=" << cHiMidstate
317                       << ", Low T=" << cLoTrue << " F=" << cLoFalse
318                       << " M=" << cLoMidstate << "\n";
319         }
320     }
321 
322     return thresholdChanges;
323 }
324 
325 void ThresholdTimer::startTimer(const std::weak_ptr<Sensor>& weakSensor,
326                                 const Threshold& threshold, bool assert,
327                                 double assertValue)
328 {
329     struct TimerUsed timerUsed = {};
330     constexpr const size_t waitTime = 5;
331     TimerPair* pair = nullptr;
332 
333     for (TimerPair& timer : timers)
334     {
335         if (!timer.first.used)
336         {
337             pair = &timer;
338             break;
339         }
340     }
341     if (pair == nullptr)
342     {
343         pair = &timers.emplace_back(timerUsed, boost::asio::deadline_timer(io));
344     }
345 
346     pair->first.used = true;
347     pair->first.level = threshold.level;
348     pair->first.direction = threshold.direction;
349     pair->first.assert = assert;
350     pair->second.expires_from_now(boost::posix_time::seconds(waitTime));
351     pair->second.async_wait([weakSensor, pair, threshold, assert,
352                              assertValue](boost::system::error_code ec) {
353         auto sensorPtr = weakSensor.lock();
354         if (!sensorPtr)
355         {
356             return; // owner sensor has been destructed
357         }
358         // pair is valid as long as sensor is valid
359         pair->first.used = false;
360 
361         if (ec == boost::asio::error::operation_aborted)
362         {
363             return; // we're being canceled
364         }
365         if (ec)
366         {
367             std::cerr << "timer error: " << ec.message() << "\n";
368             return;
369         }
370         if (sensorPtr->readingStateGood())
371         {
372             assertThresholds(sensorPtr.get(), assertValue, threshold.level,
373                              threshold.direction, assert);
374         }
375     });
376 }
377 
378 bool checkThresholds(Sensor* sensor)
379 {
380     bool status = true;
381     std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value);
382     for (const auto& change : changes)
383     {
384         assertThresholds(sensor, change.assertValue, change.threshold.level,
385                          change.threshold.direction, change.asserted);
386         if (change.threshold.level == thresholds::Level::CRITICAL &&
387             change.asserted)
388         {
389             status = false;
390         }
391     }
392 
393     return status;
394 }
395 
396 void checkThresholdsPowerDelay(const std::weak_ptr<Sensor>& weakSensor,
397                                ThresholdTimer& thresholdTimer)
398 {
399     auto sensorPtr = weakSensor.lock();
400     if (!sensorPtr)
401     {
402         return; // sensor is destructed, should never be here
403     }
404 
405     Sensor* sensor = sensorPtr.get();
406     std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value);
407     for (const auto& change : changes)
408     {
409         // When CPU is powered off, some volatges are expected to
410         // go below low thresholds. Filter these events with thresholdTimer.
411         // 1. always delay the assertion of low events to see if they are
412         //   caused by power off event.
413         // 2. conditional delay the de-assertion of low events if there is
414         //   an existing timer for assertion.
415         // 3. no delays for de-assert of low events if there is an existing
416         //   de-assert for low event. This means 2nd de-assert would happen
417         //   first and when timer expires for the previous one, no additional
418         //   signal will be logged.
419         // 4. no delays for all high events.
420         if (change.threshold.direction == thresholds::Direction::LOW)
421         {
422             if (change.asserted || thresholdTimer.hasActiveTimer(
423                                        change.threshold, !change.asserted))
424             {
425                 thresholdTimer.startTimer(weakSensor, change.threshold,
426                                           change.asserted, change.assertValue);
427                 continue;
428             }
429         }
430         assertThresholds(sensor, change.assertValue, change.threshold.level,
431                          change.threshold.direction, change.asserted);
432     }
433 }
434 
435 void assertThresholds(Sensor* sensor, double assertValue,
436                       thresholds::Level level, thresholds::Direction direction,
437                       bool assert)
438 {
439     std::shared_ptr<sdbusplus::asio::dbus_interface> interface =
440         sensor->getThresholdInterface(level);
441 
442     if (!interface)
443     {
444         std::cout << "trying to set uninitialized interface\n";
445         return;
446     }
447 
448     std::string property = Sensor::propertyAlarm(level, direction);
449     if (property.empty())
450     {
451         std::cout << "Alarm property is empty \n";
452         return;
453     }
454     if (interface->set_property<bool, true>(property, assert))
455     {
456         try
457         {
458             // msg.get_path() is interface->get_object_path()
459             sdbusplus::message_t msg =
460                 interface->new_signal("ThresholdAsserted");
461 
462             msg.append(sensor->name, interface->get_interface_name(), property,
463                        assert, assertValue);
464             msg.signal_send();
465         }
466         catch (const sdbusplus::exception_t& e)
467         {
468             std::cerr
469                 << "Failed to send thresholdAsserted signal with assertValue\n";
470         }
471     }
472 }
473 
474 bool parseThresholdsFromAttr(
475     std::vector<thresholds::Threshold>& thresholdVector,
476     const std::string& inputPath, const double& scaleFactor,
477     const double& offset)
478 {
479     const boost::container::flat_map<
480         std::string, std::vector<std::tuple<const char*, thresholds::Level,
481                                             thresholds::Direction, double>>>
482         map = {
483             {"average",
484              {
485                  std::make_tuple("average_min", Level::WARNING, Direction::LOW,
486                                  0.0),
487                  std::make_tuple("average_max", Level::WARNING, Direction::HIGH,
488                                  0.0),
489              }},
490             {"input",
491              {
492                  std::make_tuple("min", Level::WARNING, Direction::LOW, 0.0),
493                  std::make_tuple("max", Level::WARNING, Direction::HIGH, 0.0),
494                  std::make_tuple("lcrit", Level::CRITICAL, Direction::LOW, 0.0),
495                  std::make_tuple("crit", Level::CRITICAL, Direction::HIGH,
496                                  offset),
497              }},
498         };
499 
500     if (auto fileParts = splitFileName(inputPath))
501     {
502         auto& [type, nr, item] = *fileParts;
503         if (map.count(item) != 0)
504         {
505             for (const auto& t : map.at(item))
506             {
507                 const auto& [suffix, level, direction, offset] = t;
508                 auto attrPath =
509                     boost::replace_all_copy(inputPath, item, suffix);
510                 if (auto val = readFile(attrPath, scaleFactor))
511                 {
512                     *val += offset;
513                     if (debug)
514                     {
515                         std::cout << "Threshold: " << attrPath << ": " << *val
516                                   << "\n";
517                     }
518                     thresholdVector.emplace_back(level, direction, *val);
519                 }
520             }
521         }
522     }
523     return true;
524 }
525 
526 std::string getInterface(const Level thresholdLevel)
527 {
528     for (const ThresholdDefinition& thresh : thresProp)
529     {
530         if (thresh.level == thresholdLevel)
531         {
532             return std::string("xyz.openbmc_project.Sensor.Threshold.") +
533                    thresh.levelName;
534         }
535     }
536     return "";
537 }
538 } // namespace thresholds
539