1 #include <Thresholds.hpp> 2 #include <VariantVisitors.hpp> 3 #include <boost/algorithm/string/replace.hpp> 4 #include <boost/container/flat_map.hpp> 5 #include <sensor.hpp> 6 7 #include <array> 8 #include <cmath> 9 #include <fstream> 10 #include <iostream> 11 #include <memory> 12 #include <stdexcept> 13 #include <string> 14 #include <tuple> 15 #include <utility> 16 #include <variant> 17 #include <vector> 18 19 static constexpr bool debug = false; 20 static constexpr int thresLevel = 2; 21 namespace thresholds 22 { 23 Level findThresholdLevel(uint8_t sev, const std::string& direct) 24 { 25 for (Sensor::ThresholdProperty prop : Sensor::thresProp) 26 { 27 if ((prop.sevOrder == sev) && (prop.dirOrder == direct)) 28 { 29 return prop.level; 30 } 31 } 32 return Level::ERROR; 33 } 34 35 Direction findThresholdDirection(uint8_t sev, const std::string& direct) 36 { 37 for (Sensor::ThresholdProperty prop : Sensor::thresProp) 38 { 39 if ((prop.sevOrder == sev) && (prop.dirOrder == direct)) 40 { 41 return prop.direction; 42 } 43 } 44 return Direction::ERROR; 45 } 46 47 bool parseThresholdsFromConfig( 48 const SensorData& sensorData, 49 std::vector<thresholds::Threshold>& thresholdVector, 50 const std::string* matchLabel, const int* sensorIndex) 51 { 52 for (const auto& item : sensorData) 53 { 54 if (item.first.find("Thresholds") == std::string::npos) 55 { 56 continue; 57 } 58 if (matchLabel != nullptr) 59 { 60 auto labelFind = item.second.find("Label"); 61 if (labelFind == item.second.end()) 62 { 63 continue; 64 } 65 if (std::visit(VariantToStringVisitor(), labelFind->second) != 66 *matchLabel) 67 { 68 continue; 69 } 70 } 71 72 if (sensorIndex != nullptr) 73 { 74 auto indexFind = item.second.find("Index"); 75 76 // If we're checking for index 1, a missing Index is OK. 77 if ((indexFind == item.second.end()) && (*sensorIndex != 1)) 78 { 79 continue; 80 } 81 82 if ((indexFind != item.second.end()) && 83 (std::visit(VariantToIntVisitor(), indexFind->second) != 84 *sensorIndex)) 85 { 86 continue; 87 } 88 } 89 90 double hysteresis = std::numeric_limits<double>::quiet_NaN(); 91 auto hysteresisFind = item.second.find("Hysteresis"); 92 if (hysteresisFind != item.second.end()) 93 { 94 hysteresis = 95 std::visit(VariantToDoubleVisitor(), hysteresisFind->second); 96 } 97 98 auto directionFind = item.second.find("Direction"); 99 auto severityFind = item.second.find("Severity"); 100 auto valueFind = item.second.find("Value"); 101 if (valueFind == item.second.end() || 102 severityFind == item.second.end() || 103 directionFind == item.second.end()) 104 { 105 std::cerr << "Malformed threshold on configuration interface " 106 << item.first << "\n"; 107 return false; 108 } 109 uint8_t severity = 110 std::visit(VariantToUnsignedIntVisitor(), severityFind->second); 111 112 std::string directions = 113 std::visit(VariantToStringVisitor(), directionFind->second); 114 115 Level level = findThresholdLevel(severity, directions); 116 Direction direction = findThresholdDirection(severity, directions); 117 118 if ((static_cast<int>(level) == thresLevel) || 119 (static_cast<int>(direction) == thresLevel)) 120 { 121 continue; 122 } 123 double val = std::visit(VariantToDoubleVisitor(), valueFind->second); 124 125 thresholdVector.emplace_back(level, direction, val, hysteresis); 126 } 127 return true; 128 } 129 130 void persistThreshold(const std::string& path, const std::string& baseInterface, 131 const thresholds::Threshold& threshold, 132 std::shared_ptr<sdbusplus::asio::connection>& conn, 133 size_t thresholdCount, const std::string& labelMatch) 134 { 135 for (size_t ii = 0; ii < thresholdCount; ii++) 136 { 137 std::string thresholdInterface = 138 baseInterface + ".Thresholds" + std::to_string(ii); 139 conn->async_method_call( 140 [&, path, threshold, thresholdInterface, labelMatch]( 141 const boost::system::error_code& ec, 142 const boost::container::flat_map<std::string, BasicVariantType>& 143 result) { 144 if (ec) 145 { 146 return; // threshold not supported 147 } 148 149 if (!labelMatch.empty()) 150 { 151 auto labelFind = result.find("Label"); 152 if (labelFind == result.end()) 153 { 154 std::cerr << "No label in threshold configuration\n"; 155 return; 156 } 157 std::string label = 158 std::visit(VariantToStringVisitor(), labelFind->second); 159 if (label != labelMatch) 160 { 161 return; 162 } 163 } 164 165 auto directionFind = result.find("Direction"); 166 auto severityFind = result.find("Severity"); 167 auto valueFind = result.find("Value"); 168 if (valueFind == result.end() || severityFind == result.end() || 169 directionFind == result.end()) 170 { 171 std::cerr << "Malformed threshold in configuration\n"; 172 return; 173 } 174 unsigned int level = std::visit(VariantToUnsignedIntVisitor(), 175 severityFind->second); 176 177 std::string dir = 178 std::visit(VariantToStringVisitor(), directionFind->second); 179 if (((findThresholdLevel(level, dir)) != threshold.level) || 180 ((findThresholdDirection(level, dir)) != 181 threshold.direction)) 182 { 183 return; // not the droid we're looking for 184 } 185 186 std::variant<double> value(threshold.value); 187 conn->async_method_call( 188 [](const boost::system::error_code& ec) { 189 if (ec) 190 { 191 std::cerr << "Error setting threshold " << ec 192 << "\n"; 193 } 194 }, 195 entityManagerName, path, "org.freedesktop.DBus.Properties", 196 "Set", thresholdInterface, "Value", value); 197 }, 198 entityManagerName, path, "org.freedesktop.DBus.Properties", 199 "GetAll", thresholdInterface); 200 } 201 } 202 203 void updateThresholds(Sensor* sensor) 204 { 205 if (sensor->thresholds.empty()) 206 { 207 return; 208 } 209 210 for (const auto& threshold : sensor->thresholds) 211 { 212 std::shared_ptr<sdbusplus::asio::dbus_interface> interface; 213 if (threshold.level == thresholds::Level::CRITICAL) 214 { 215 interface = sensor->thresholdInterfaceCritical; 216 } 217 else if (threshold.level == thresholds::Level::WARNING) 218 { 219 interface = sensor->thresholdInterfaceWarning; 220 } 221 else 222 { 223 continue; 224 } 225 if (!interface) 226 { 227 continue; 228 } 229 230 std::string property = 231 sensor->propertyLevel(threshold.level, threshold.direction); 232 if (property.empty()) 233 { 234 continue; 235 } 236 interface->set_property(property, threshold.value); 237 } 238 } 239 240 // Debugging counters 241 static int cHiTrue = 0; 242 static int cHiFalse = 0; 243 static int cHiMidstate = 0; 244 static int cLoTrue = 0; 245 static int cLoFalse = 0; 246 static int cLoMidstate = 0; 247 static int cDebugThrottle = 0; 248 static constexpr int assertLogCount = 10; 249 250 struct ChangeParam 251 { 252 ChangeParam(Threshold whichThreshold, bool status, double value) : 253 threshold(whichThreshold), asserted(status), assertValue(value) 254 {} 255 256 Threshold threshold; 257 bool asserted; 258 double assertValue; 259 }; 260 261 static std::vector<ChangeParam> checkThresholds(Sensor* sensor, double value) 262 { 263 std::vector<ChangeParam> thresholdChanges; 264 if (sensor->thresholds.empty()) 265 { 266 return thresholdChanges; 267 } 268 269 for (auto& threshold : sensor->thresholds) 270 { 271 // Use "Schmitt trigger" logic to avoid threshold trigger spam, 272 // if value is noisy while hovering very close to a threshold. 273 // When a threshold is crossed, indicate true immediately, 274 // but require more distance to be crossed the other direction, 275 // before resetting the indicator back to false. 276 if (threshold.direction == thresholds::Direction::HIGH) 277 { 278 if (value >= threshold.value) 279 { 280 thresholdChanges.emplace_back(threshold, true, value); 281 if (++cHiTrue < assertLogCount) 282 { 283 std::cerr << "Sensor " << sensor->name << " high threshold " 284 << threshold.value << " assert: value " << value 285 << " raw data " << sensor->rawValue << "\n"; 286 } 287 } 288 else if (value < (threshold.value - threshold.hysteresis)) 289 { 290 thresholdChanges.emplace_back(threshold, false, value); 291 ++cHiFalse; 292 } 293 else 294 { 295 ++cHiMidstate; 296 } 297 } 298 else if (threshold.direction == thresholds::Direction::LOW) 299 { 300 if (value <= threshold.value) 301 { 302 thresholdChanges.emplace_back(threshold, true, value); 303 if (++cLoTrue < assertLogCount) 304 { 305 std::cerr << "Sensor " << sensor->name << " low threshold " 306 << threshold.value << " assert: value " 307 << sensor->value << " raw data " 308 << sensor->rawValue << "\n"; 309 } 310 } 311 else if (value > (threshold.value + threshold.hysteresis)) 312 { 313 thresholdChanges.emplace_back(threshold, false, value); 314 ++cLoFalse; 315 } 316 else 317 { 318 ++cLoMidstate; 319 } 320 } 321 else 322 { 323 std::cerr << "Error determining threshold direction\n"; 324 } 325 } 326 327 // Throttle debug output, so that it does not continuously spam 328 ++cDebugThrottle; 329 if (cDebugThrottle >= 1000) 330 { 331 cDebugThrottle = 0; 332 if constexpr (debug) 333 { 334 std::cerr << "checkThresholds: High T=" << cHiTrue 335 << " F=" << cHiFalse << " M=" << cHiMidstate 336 << ", Low T=" << cLoTrue << " F=" << cLoFalse 337 << " M=" << cLoMidstate << "\n"; 338 } 339 } 340 341 return thresholdChanges; 342 } 343 344 void ThresholdTimer::startTimer(const std::weak_ptr<Sensor>& weakSensor, 345 const Threshold& threshold, bool assert, 346 double assertValue) 347 { 348 struct TimerUsed timerUsed = {}; 349 constexpr const size_t waitTime = 5; 350 TimerPair* pair = nullptr; 351 352 for (TimerPair& timer : timers) 353 { 354 if (!timer.first.used) 355 { 356 pair = &timer; 357 break; 358 } 359 } 360 if (pair == nullptr) 361 { 362 pair = &timers.emplace_back(timerUsed, boost::asio::deadline_timer(io)); 363 } 364 365 pair->first.used = true; 366 pair->first.level = threshold.level; 367 pair->first.direction = threshold.direction; 368 pair->first.assert = assert; 369 pair->second.expires_from_now(boost::posix_time::seconds(waitTime)); 370 pair->second.async_wait([weakSensor, pair, threshold, assert, 371 assertValue](boost::system::error_code ec) { 372 auto sensorPtr = weakSensor.lock(); 373 if (!sensorPtr) 374 { 375 return; // owner sensor has been destructed 376 } 377 // pair is valid as long as sensor is valid 378 pair->first.used = false; 379 380 if (ec == boost::asio::error::operation_aborted) 381 { 382 return; // we're being canceled 383 } 384 if (ec) 385 { 386 std::cerr << "timer error: " << ec.message() << "\n"; 387 return; 388 } 389 if (sensorPtr->readingStateGood()) 390 { 391 assertThresholds(sensorPtr.get(), assertValue, threshold.level, 392 threshold.direction, assert); 393 } 394 }); 395 } 396 397 bool checkThresholds(Sensor* sensor) 398 { 399 bool status = true; 400 std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value); 401 for (const auto& change : changes) 402 { 403 assertThresholds(sensor, change.assertValue, change.threshold.level, 404 change.threshold.direction, change.asserted); 405 if (change.threshold.level == thresholds::Level::CRITICAL && 406 change.asserted) 407 { 408 status = false; 409 } 410 } 411 412 return status; 413 } 414 415 void checkThresholdsPowerDelay(const std::weak_ptr<Sensor>& weakSensor, 416 ThresholdTimer& thresholdTimer) 417 { 418 auto sensorPtr = weakSensor.lock(); 419 if (!sensorPtr) 420 { 421 return; // sensor is destructed, should never be here 422 } 423 424 Sensor* sensor = sensorPtr.get(); 425 std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value); 426 for (const auto& change : changes) 427 { 428 // When CPU is powered off, some volatges are expected to 429 // go below low thresholds. Filter these events with thresholdTimer. 430 // 1. always delay the assertion of low events to see if they are 431 // caused by power off event. 432 // 2. conditional delay the de-assertion of low events if there is 433 // an existing timer for assertion. 434 // 3. no delays for de-assert of low events if there is an existing 435 // de-assert for low event. This means 2nd de-assert would happen 436 // first and when timer expires for the previous one, no additional 437 // signal will be logged. 438 // 4. no delays for all high events. 439 if (change.threshold.direction == thresholds::Direction::LOW) 440 { 441 if (change.asserted || thresholdTimer.hasActiveTimer( 442 change.threshold, !change.asserted)) 443 { 444 thresholdTimer.startTimer(weakSensor, change.threshold, 445 change.asserted, change.assertValue); 446 continue; 447 } 448 } 449 assertThresholds(sensor, change.assertValue, change.threshold.level, 450 change.threshold.direction, change.asserted); 451 } 452 } 453 454 void assertThresholds(Sensor* sensor, double assertValue, 455 thresholds::Level level, thresholds::Direction direction, 456 bool assert) 457 { 458 std::shared_ptr<sdbusplus::asio::dbus_interface> interface; 459 if (level == thresholds::Level::WARNING) 460 { 461 interface = sensor->thresholdInterfaceWarning; 462 } 463 else if (level == thresholds::Level::CRITICAL) 464 { 465 interface = sensor->thresholdInterfaceCritical; 466 } 467 else 468 { 469 std::cerr << "Unknown threshold, level " << static_cast<int>(level) 470 << "direction " << static_cast<int>(direction) << "\n"; 471 return; 472 } 473 if (!interface) 474 { 475 std::cout << "trying to set uninitialized interface\n"; 476 return; 477 } 478 479 std::string property = sensor->propertyAlarm(level, direction); 480 if (property.empty()) 481 { 482 std::cout << "Alarm property is empty \n"; 483 return; 484 } 485 if (interface->set_property<bool, true>(property, assert)) 486 { 487 try 488 { 489 // msg.get_path() is interface->get_object_path() 490 sdbusplus::message::message msg = 491 interface->new_signal("ThresholdAsserted"); 492 493 msg.append(sensor->name, interface->get_interface_name(), property, 494 assert, assertValue); 495 msg.signal_send(); 496 } 497 catch (const sdbusplus::exception::exception& e) 498 { 499 std::cerr 500 << "Failed to send thresholdAsserted signal with assertValue\n"; 501 } 502 } 503 } 504 505 bool parseThresholdsFromAttr( 506 std::vector<thresholds::Threshold>& thresholdVector, 507 const std::string& inputPath, const double& scaleFactor, 508 const double& offset) 509 { 510 const boost::container::flat_map< 511 std::string, std::vector<std::tuple<const char*, thresholds::Level, 512 thresholds::Direction, double>>> 513 map = { 514 {"average", 515 { 516 std::make_tuple("average_min", Level::WARNING, Direction::LOW, 517 0.0), 518 std::make_tuple("average_max", Level::WARNING, Direction::HIGH, 519 0.0), 520 }}, 521 {"input", 522 { 523 std::make_tuple("min", Level::WARNING, Direction::LOW, 0.0), 524 std::make_tuple("max", Level::WARNING, Direction::HIGH, 0.0), 525 std::make_tuple("lcrit", Level::CRITICAL, Direction::LOW, 0.0), 526 std::make_tuple("crit", Level::CRITICAL, Direction::HIGH, 527 offset), 528 }}, 529 }; 530 531 if (auto fileParts = splitFileName(inputPath)) 532 { 533 auto& [type, nr, item] = *fileParts; 534 if (map.count(item) != 0) 535 { 536 for (const auto& t : map.at(item)) 537 { 538 auto& [suffix, level, direction, offset] = t; 539 auto attrPath = 540 boost::replace_all_copy(inputPath, item, suffix); 541 if (auto val = readFile(attrPath, scaleFactor)) 542 { 543 *val += offset; 544 if (debug) 545 { 546 std::cout << "Threshold: " << attrPath << ": " << *val 547 << "\n"; 548 } 549 thresholdVector.emplace_back(level, direction, *val); 550 } 551 } 552 } 553 } 554 return true; 555 } 556 557 bool hasCriticalInterface( 558 const std::vector<thresholds::Threshold>& thresholdVector) 559 { 560 for (auto& threshold : thresholdVector) 561 { 562 if (threshold.level == Level::CRITICAL) 563 { 564 return true; 565 } 566 } 567 return false; 568 } 569 570 bool hasWarningInterface( 571 const std::vector<thresholds::Threshold>& thresholdVector) 572 { 573 for (auto& threshold : thresholdVector) 574 { 575 if (threshold.level == Level::WARNING) 576 { 577 return true; 578 } 579 } 580 return false; 581 } 582 } // namespace thresholds 583