1 #include "Thresholds.hpp" 2 3 #include "Utils.hpp" 4 #include "VariantVisitors.hpp" 5 #include "sensor.hpp" 6 7 #include <boost/algorithm/string/replace.hpp> 8 #include <boost/asio/error.hpp> 9 #include <boost/asio/steady_timer.hpp> 10 #include <boost/container/flat_map.hpp> 11 #include <sdbusplus/asio/connection.hpp> 12 #include <sdbusplus/asio/object_server.hpp> 13 #include <sdbusplus/exception.hpp> 14 #include <sdbusplus/message.hpp> 15 16 #include <array> 17 #include <chrono> 18 #include <cstddef> 19 #include <cstdint> 20 #include <iostream> 21 #include <limits> 22 #include <memory> 23 #include <string> 24 #include <tuple> 25 #include <utility> 26 #include <variant> 27 #include <vector> 28 29 static constexpr bool debug = false; 30 namespace thresholds 31 { 32 Level findThresholdLevel(uint8_t sev) 33 { 34 for (const ThresholdDefinition& prop : thresProp) 35 { 36 if (prop.sevOrder == sev) 37 { 38 return prop.level; 39 } 40 } 41 return Level::ERROR; 42 } 43 44 Direction findThresholdDirection(const std::string& direct) 45 { 46 if (direct == "greater than") 47 { 48 return Direction::HIGH; 49 } 50 if (direct == "less than") 51 { 52 return Direction::LOW; 53 } 54 return Direction::ERROR; 55 } 56 57 bool parseThresholdsFromConfig( 58 const SensorData& sensorData, 59 std::vector<thresholds::Threshold>& thresholdVector, 60 const std::string* matchLabel, const int* sensorIndex) 61 { 62 for (const auto& [intf, cfg] : sensorData) 63 { 64 if (intf.find("Thresholds") == std::string::npos) 65 { 66 continue; 67 } 68 if (matchLabel != nullptr) 69 { 70 auto labelFind = cfg.find("Label"); 71 if (labelFind == cfg.end()) 72 { 73 continue; 74 } 75 if (std::visit(VariantToStringVisitor(), labelFind->second) != 76 *matchLabel) 77 { 78 continue; 79 } 80 } 81 82 if (sensorIndex != nullptr) 83 { 84 auto indexFind = cfg.find("Index"); 85 86 // If we're checking for index 1, a missing Index is OK. 87 if ((indexFind == cfg.end()) && (*sensorIndex != 1)) 88 { 89 continue; 90 } 91 92 if ((indexFind != cfg.end()) && 93 (std::visit(VariantToIntVisitor(), indexFind->second) != 94 *sensorIndex)) 95 { 96 continue; 97 } 98 } 99 100 double hysteresis = std::numeric_limits<double>::quiet_NaN(); 101 auto hysteresisFind = cfg.find("Hysteresis"); 102 if (hysteresisFind != cfg.end()) 103 { 104 hysteresis = 105 std::visit(VariantToDoubleVisitor(), hysteresisFind->second); 106 } 107 108 auto directionFind = cfg.find("Direction"); 109 auto severityFind = cfg.find("Severity"); 110 auto valueFind = cfg.find("Value"); 111 if (valueFind == cfg.end() || severityFind == cfg.end() || 112 directionFind == cfg.end()) 113 { 114 std::cerr << "Malformed threshold on configuration interface " 115 << intf << "\n"; 116 return false; 117 } 118 unsigned int severity = 119 std::visit(VariantToUnsignedIntVisitor(), severityFind->second); 120 121 std::string directions = 122 std::visit(VariantToStringVisitor(), directionFind->second); 123 124 Level level = findThresholdLevel(severity); 125 Direction direction = findThresholdDirection(directions); 126 127 if ((level == Level::ERROR) || (direction == Direction::ERROR)) 128 { 129 continue; 130 } 131 double val = std::visit(VariantToDoubleVisitor(), valueFind->second); 132 133 thresholdVector.emplace_back(level, direction, val, hysteresis); 134 } 135 return true; 136 } 137 138 void persistThreshold(const std::string& path, const std::string& baseInterface, 139 const thresholds::Threshold& threshold, 140 std::shared_ptr<sdbusplus::asio::connection>& conn, 141 size_t thresholdCount, const std::string& labelMatch) 142 { 143 for (size_t ii = 0; ii < thresholdCount; ii++) 144 { 145 std::string thresholdInterface = 146 baseInterface + ".Thresholds" + std::to_string(ii); 147 conn->async_method_call( 148 [&, path, threshold, thresholdInterface, 149 labelMatch](const boost::system::error_code& ec, 150 const SensorBaseConfigMap& result) { 151 if (ec) 152 { 153 return; // threshold not supported 154 } 155 156 if (!labelMatch.empty()) 157 { 158 auto labelFind = result.find("Label"); 159 if (labelFind == result.end()) 160 { 161 std::cerr << "No label in threshold configuration\n"; 162 return; 163 } 164 std::string label = 165 std::visit(VariantToStringVisitor(), labelFind->second); 166 if (label != labelMatch) 167 { 168 return; 169 } 170 } 171 172 auto directionFind = result.find("Direction"); 173 auto severityFind = result.find("Severity"); 174 auto valueFind = result.find("Value"); 175 if (valueFind == result.end() || severityFind == result.end() || 176 directionFind == result.end()) 177 { 178 std::cerr << "Malformed threshold in configuration\n"; 179 return; 180 } 181 unsigned int severity = std::visit( 182 VariantToUnsignedIntVisitor(), severityFind->second); 183 184 std::string dir = 185 std::visit(VariantToStringVisitor(), directionFind->second); 186 if ((findThresholdLevel(severity) != threshold.level) || 187 (findThresholdDirection(dir) != threshold.direction)) 188 { 189 return; // not the droid we're looking for 190 } 191 192 std::variant<double> value(threshold.value); 193 conn->async_method_call( 194 [](const boost::system::error_code& ec) { 195 if (ec) 196 { 197 std::cerr 198 << "Error setting threshold " << ec << "\n"; 199 } 200 }, 201 entityManagerName, path, "org.freedesktop.DBus.Properties", 202 "Set", thresholdInterface, "Value", value); 203 }, 204 entityManagerName, path, "org.freedesktop.DBus.Properties", 205 "GetAll", thresholdInterface); 206 } 207 } 208 209 void updateThresholds(Sensor* sensor) 210 { 211 for (const auto& threshold : sensor->thresholds) 212 { 213 std::shared_ptr<sdbusplus::asio::dbus_interface> interface = 214 sensor->getThresholdInterface(threshold.level); 215 216 if (!interface) 217 { 218 continue; 219 } 220 221 std::string property = 222 Sensor::propertyLevel(threshold.level, threshold.direction); 223 if (property.empty()) 224 { 225 continue; 226 } 227 interface->set_property(property, threshold.value); 228 } 229 } 230 231 // Debugging counters 232 static int cHiTrue = 0; 233 static int cHiFalse = 0; 234 static int cHiMidstate = 0; 235 static int cLoTrue = 0; 236 static int cLoFalse = 0; 237 static int cLoMidstate = 0; 238 static int cDebugThrottle = 0; 239 static constexpr int assertLogCount = 10; 240 241 struct ChangeParam 242 { 243 ChangeParam(Threshold whichThreshold, bool status, double value) : 244 threshold(whichThreshold), asserted(status), assertValue(value) 245 {} 246 247 Threshold threshold; 248 bool asserted; 249 double assertValue; 250 }; 251 252 static std::vector<ChangeParam> checkThresholds(Sensor* sensor, double value) 253 { 254 std::vector<ChangeParam> thresholdChanges; 255 if (sensor->thresholds.empty()) 256 { 257 return thresholdChanges; 258 } 259 260 for (auto& threshold : sensor->thresholds) 261 { 262 // Use "Schmitt trigger" logic to avoid threshold trigger spam, 263 // if value is noisy while hovering very close to a threshold. 264 // When a threshold is crossed, indicate true immediately, 265 // but require more distance to be crossed the other direction, 266 // before resetting the indicator back to false. 267 if (threshold.direction == thresholds::Direction::HIGH) 268 { 269 if (value >= threshold.value) 270 { 271 thresholdChanges.emplace_back(threshold, true, value); 272 if (++cHiTrue < assertLogCount) 273 { 274 std::cerr << "Sensor " << sensor->name << " high threshold " 275 << threshold.value << " assert: value " << value 276 << " raw data " << sensor->rawValue << "\n"; 277 } 278 } 279 else if (value < (threshold.value - threshold.hysteresis)) 280 { 281 thresholdChanges.emplace_back(threshold, false, value); 282 ++cHiFalse; 283 } 284 else 285 { 286 ++cHiMidstate; 287 } 288 } 289 else if (threshold.direction == thresholds::Direction::LOW) 290 { 291 if (value <= threshold.value) 292 { 293 thresholdChanges.emplace_back(threshold, true, value); 294 if (++cLoTrue < assertLogCount) 295 { 296 std::cerr 297 << "Sensor " << sensor->name << " low threshold " 298 << threshold.value << " assert: value " << sensor->value 299 << " raw data " << sensor->rawValue << "\n"; 300 } 301 } 302 else if (value > (threshold.value + threshold.hysteresis)) 303 { 304 thresholdChanges.emplace_back(threshold, false, value); 305 ++cLoFalse; 306 } 307 else 308 { 309 ++cLoMidstate; 310 } 311 } 312 else 313 { 314 std::cerr << "Error determining threshold direction\n"; 315 } 316 } 317 318 // Throttle debug output, so that it does not continuously spam 319 ++cDebugThrottle; 320 if (cDebugThrottle >= 1000) 321 { 322 cDebugThrottle = 0; 323 if constexpr (debug) 324 { 325 std::cerr << "checkThresholds: High T=" << cHiTrue 326 << " F=" << cHiFalse << " M=" << cHiMidstate 327 << ", Low T=" << cLoTrue << " F=" << cLoFalse 328 << " M=" << cLoMidstate << "\n"; 329 } 330 } 331 332 return thresholdChanges; 333 } 334 335 void ThresholdTimer::startTimer(const std::weak_ptr<Sensor>& weakSensor, 336 const Threshold& threshold, bool assert, 337 double assertValue) 338 { 339 struct TimerUsed timerUsed = {}; 340 constexpr const size_t waitTime = 5; 341 TimerPair* pair = nullptr; 342 343 for (TimerPair& timer : timers) 344 { 345 if (!timer.first.used) 346 { 347 pair = &timer; 348 break; 349 } 350 } 351 if (pair == nullptr) 352 { 353 pair = &timers.emplace_back(timerUsed, boost::asio::steady_timer(io)); 354 } 355 356 pair->first.used = true; 357 pair->first.level = threshold.level; 358 pair->first.direction = threshold.direction; 359 pair->first.assert = assert; 360 pair->second.expires_after(std::chrono::seconds(waitTime)); 361 pair->second.async_wait([weakSensor, pair, threshold, assert, 362 assertValue](boost::system::error_code ec) { 363 auto sensorPtr = weakSensor.lock(); 364 if (!sensorPtr) 365 { 366 return; // owner sensor has been destructed 367 } 368 // pair is valid as long as sensor is valid 369 pair->first.used = false; 370 371 if (ec == boost::asio::error::operation_aborted) 372 { 373 return; // we're being canceled 374 } 375 if (ec) 376 { 377 std::cerr << "timer error: " << ec.message() << "\n"; 378 return; 379 } 380 if (sensorPtr->readingStateGood()) 381 { 382 assertThresholds(sensorPtr.get(), assertValue, threshold.level, 383 threshold.direction, assert); 384 } 385 }); 386 } 387 388 bool checkThresholds(Sensor* sensor) 389 { 390 bool status = true; 391 std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value); 392 for (const auto& change : changes) 393 { 394 assertThresholds(sensor, change.assertValue, change.threshold.level, 395 change.threshold.direction, change.asserted); 396 if (change.threshold.level == thresholds::Level::CRITICAL && 397 change.asserted) 398 { 399 status = false; 400 } 401 } 402 403 return status; 404 } 405 406 void checkThresholdsPowerDelay(const std::weak_ptr<Sensor>& weakSensor, 407 ThresholdTimer& thresholdTimer) 408 { 409 auto sensorPtr = weakSensor.lock(); 410 if (!sensorPtr) 411 { 412 return; // sensor is destructed, should never be here 413 } 414 415 Sensor* sensor = sensorPtr.get(); 416 std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value); 417 for (const auto& change : changes) 418 { 419 // When CPU is powered off, some volatges are expected to 420 // go below low thresholds. Filter these events with thresholdTimer. 421 // 1. always delay the assertion of low events to see if they are 422 // caused by power off event. 423 // 2. conditional delay the de-assertion of low events if there is 424 // an existing timer for assertion. 425 // 3. no delays for de-assert of low events if there is an existing 426 // de-assert for low event. This means 2nd de-assert would happen 427 // first and when timer expires for the previous one, no additional 428 // signal will be logged. 429 // 4. no delays for all high events. 430 if (change.threshold.direction == thresholds::Direction::LOW) 431 { 432 if (change.asserted || thresholdTimer.hasActiveTimer( 433 change.threshold, !change.asserted)) 434 { 435 thresholdTimer.startTimer(weakSensor, change.threshold, 436 change.asserted, change.assertValue); 437 continue; 438 } 439 } 440 assertThresholds(sensor, change.assertValue, change.threshold.level, 441 change.threshold.direction, change.asserted); 442 } 443 } 444 445 void assertThresholds(Sensor* sensor, double assertValue, 446 thresholds::Level level, thresholds::Direction direction, 447 bool assert) 448 { 449 std::shared_ptr<sdbusplus::asio::dbus_interface> interface = 450 sensor->getThresholdInterface(level); 451 452 if (!interface) 453 { 454 std::cout << "trying to set uninitialized interface\n"; 455 return; 456 } 457 458 std::string property = Sensor::propertyAlarm(level, direction); 459 if (property.empty()) 460 { 461 std::cout << "Alarm property is empty \n"; 462 return; 463 } 464 if (interface->set_property<bool, true>(property, assert)) 465 { 466 try 467 { 468 // msg.get_path() is interface->get_object_path() 469 sdbusplus::message_t msg = 470 interface->new_signal("ThresholdAsserted"); 471 472 msg.append(sensor->name, interface->get_interface_name(), property, 473 assert, assertValue); 474 msg.signal_send(); 475 } 476 catch (const sdbusplus::exception_t& e) 477 { 478 std::cerr 479 << "Failed to send thresholdAsserted signal with assertValue\n"; 480 } 481 } 482 } 483 484 bool parseThresholdsFromAttr( 485 std::vector<thresholds::Threshold>& thresholdVector, 486 const std::string& inputPath, const double& scaleFactor, 487 const double& offset, const double& hysteresis) 488 { 489 const boost::container::flat_map< 490 std::string, std::vector<std::tuple<const char*, thresholds::Level, 491 thresholds::Direction, double>>> 492 map = { 493 {"average", 494 { 495 std::make_tuple("average_min", Level::WARNING, Direction::LOW, 496 0.0), 497 std::make_tuple("average_max", Level::WARNING, Direction::HIGH, 498 0.0), 499 }}, 500 {"input", 501 { 502 std::make_tuple("min", Level::WARNING, Direction::LOW, 0.0), 503 std::make_tuple("max", Level::WARNING, Direction::HIGH, 0.0), 504 std::make_tuple("lcrit", Level::CRITICAL, Direction::LOW, 0.0), 505 std::make_tuple("crit", Level::CRITICAL, Direction::HIGH, 506 offset), 507 }}, 508 }; 509 510 if (auto fileParts = splitFileName(inputPath)) 511 { 512 auto& [type, nr, item] = *fileParts; 513 if (map.count(item) != 0) 514 { 515 for (const auto& t : map.at(item)) 516 { 517 const auto& [suffix, level, direction, offset] = t; 518 auto attrPath = 519 boost::replace_all_copy(inputPath, item, suffix); 520 if (auto val = readFile(attrPath, scaleFactor)) 521 { 522 *val += offset; 523 if (debug) 524 { 525 std::cout << "Threshold: " << attrPath << ": " << *val 526 << "\n"; 527 } 528 thresholdVector.emplace_back(level, direction, *val, 529 hysteresis); 530 } 531 } 532 } 533 } 534 return true; 535 } 536 537 std::string getInterface(const Level thresholdLevel) 538 { 539 for (const ThresholdDefinition& thresh : thresProp) 540 { 541 if (thresh.level == thresholdLevel) 542 { 543 return std::string("xyz.openbmc_project.Sensor.Threshold.") + 544 thresh.levelName; 545 } 546 } 547 return ""; 548 } 549 } // namespace thresholds 550