1 #include "Thresholds.hpp" 2 3 #include "VariantVisitors.hpp" 4 #include "sensor.hpp" 5 6 #include <boost/algorithm/string/replace.hpp> 7 #include <boost/container/flat_map.hpp> 8 9 #include <array> 10 #include <cmath> 11 #include <fstream> 12 #include <iostream> 13 #include <memory> 14 #include <stdexcept> 15 #include <string> 16 #include <tuple> 17 #include <utility> 18 #include <variant> 19 #include <vector> 20 21 static constexpr bool debug = false; 22 namespace thresholds 23 { 24 Level findThresholdLevel(uint8_t sev) 25 { 26 for (const ThresholdDefinition& prop : thresProp) 27 { 28 if (prop.sevOrder == sev) 29 { 30 return prop.level; 31 } 32 } 33 return Level::ERROR; 34 } 35 36 Direction findThresholdDirection(const std::string& direct) 37 { 38 if (direct == "greater than") 39 { 40 return Direction::HIGH; 41 } 42 if (direct == "less than") 43 { 44 return Direction::LOW; 45 } 46 return Direction::ERROR; 47 } 48 49 bool parseThresholdsFromConfig( 50 const SensorData& sensorData, 51 std::vector<thresholds::Threshold>& thresholdVector, 52 const std::string* matchLabel, const int* sensorIndex) 53 { 54 for (const auto& [intf, cfg] : sensorData) 55 { 56 if (intf.find("Thresholds") == std::string::npos) 57 { 58 continue; 59 } 60 if (matchLabel != nullptr) 61 { 62 auto labelFind = cfg.find("Label"); 63 if (labelFind == cfg.end()) 64 { 65 continue; 66 } 67 if (std::visit(VariantToStringVisitor(), labelFind->second) != 68 *matchLabel) 69 { 70 continue; 71 } 72 } 73 74 if (sensorIndex != nullptr) 75 { 76 auto indexFind = cfg.find("Index"); 77 78 // If we're checking for index 1, a missing Index is OK. 79 if ((indexFind == cfg.end()) && (*sensorIndex != 1)) 80 { 81 continue; 82 } 83 84 if ((indexFind != cfg.end()) && 85 (std::visit(VariantToIntVisitor(), indexFind->second) != 86 *sensorIndex)) 87 { 88 continue; 89 } 90 } 91 92 double hysteresis = std::numeric_limits<double>::quiet_NaN(); 93 auto hysteresisFind = cfg.find("Hysteresis"); 94 if (hysteresisFind != cfg.end()) 95 { 96 hysteresis = 97 std::visit(VariantToDoubleVisitor(), hysteresisFind->second); 98 } 99 100 auto directionFind = cfg.find("Direction"); 101 auto severityFind = cfg.find("Severity"); 102 auto valueFind = cfg.find("Value"); 103 if (valueFind == cfg.end() || severityFind == cfg.end() || 104 directionFind == cfg.end()) 105 { 106 std::cerr << "Malformed threshold on configuration interface " 107 << intf << "\n"; 108 return false; 109 } 110 unsigned int severity = 111 std::visit(VariantToUnsignedIntVisitor(), severityFind->second); 112 113 std::string directions = 114 std::visit(VariantToStringVisitor(), directionFind->second); 115 116 Level level = findThresholdLevel(severity); 117 Direction direction = findThresholdDirection(directions); 118 119 if ((level == Level::ERROR) || (direction == Direction::ERROR)) 120 { 121 continue; 122 } 123 double val = std::visit(VariantToDoubleVisitor(), valueFind->second); 124 125 thresholdVector.emplace_back(level, direction, val, hysteresis); 126 } 127 return true; 128 } 129 130 void persistThreshold(const std::string& path, const std::string& baseInterface, 131 const thresholds::Threshold& threshold, 132 std::shared_ptr<sdbusplus::asio::connection>& conn, 133 size_t thresholdCount, const std::string& labelMatch) 134 { 135 for (size_t ii = 0; ii < thresholdCount; ii++) 136 { 137 std::string thresholdInterface = 138 baseInterface + ".Thresholds" + std::to_string(ii); 139 conn->async_method_call( 140 [&, path, threshold, thresholdInterface, 141 labelMatch](const boost::system::error_code& ec, 142 const SensorBaseConfigMap& result) { 143 if (ec) 144 { 145 return; // threshold not supported 146 } 147 148 if (!labelMatch.empty()) 149 { 150 auto labelFind = result.find("Label"); 151 if (labelFind == result.end()) 152 { 153 std::cerr << "No label in threshold configuration\n"; 154 return; 155 } 156 std::string label = 157 std::visit(VariantToStringVisitor(), labelFind->second); 158 if (label != labelMatch) 159 { 160 return; 161 } 162 } 163 164 auto directionFind = result.find("Direction"); 165 auto severityFind = result.find("Severity"); 166 auto valueFind = result.find("Value"); 167 if (valueFind == result.end() || severityFind == result.end() || 168 directionFind == result.end()) 169 { 170 std::cerr << "Malformed threshold in configuration\n"; 171 return; 172 } 173 unsigned int severity = 174 std::visit(VariantToUnsignedIntVisitor(), severityFind->second); 175 176 std::string dir = 177 std::visit(VariantToStringVisitor(), directionFind->second); 178 if ((findThresholdLevel(severity) != threshold.level) || 179 (findThresholdDirection(dir) != threshold.direction)) 180 { 181 return; // not the droid we're looking for 182 } 183 184 std::variant<double> value(threshold.value); 185 conn->async_method_call( 186 [](const boost::system::error_code& ec) { 187 if (ec) 188 { 189 std::cerr << "Error setting threshold " << ec << "\n"; 190 } 191 }, 192 entityManagerName, path, "org.freedesktop.DBus.Properties", 193 "Set", thresholdInterface, "Value", value); 194 }, 195 entityManagerName, path, "org.freedesktop.DBus.Properties", 196 "GetAll", thresholdInterface); 197 } 198 } 199 200 void updateThresholds(Sensor* sensor) 201 { 202 for (const auto& threshold : sensor->thresholds) 203 { 204 std::shared_ptr<sdbusplus::asio::dbus_interface> interface = 205 sensor->getThresholdInterface(threshold.level); 206 207 if (!interface) 208 { 209 continue; 210 } 211 212 std::string property = 213 Sensor::propertyLevel(threshold.level, threshold.direction); 214 if (property.empty()) 215 { 216 continue; 217 } 218 interface->set_property(property, threshold.value); 219 } 220 } 221 222 // Debugging counters 223 static int cHiTrue = 0; 224 static int cHiFalse = 0; 225 static int cHiMidstate = 0; 226 static int cLoTrue = 0; 227 static int cLoFalse = 0; 228 static int cLoMidstate = 0; 229 static int cDebugThrottle = 0; 230 static constexpr int assertLogCount = 10; 231 232 struct ChangeParam 233 { 234 ChangeParam(Threshold whichThreshold, bool status, double value) : 235 threshold(whichThreshold), asserted(status), assertValue(value) 236 {} 237 238 Threshold threshold; 239 bool asserted; 240 double assertValue; 241 }; 242 243 static std::vector<ChangeParam> checkThresholds(Sensor* sensor, double value) 244 { 245 std::vector<ChangeParam> thresholdChanges; 246 if (sensor->thresholds.empty()) 247 { 248 return thresholdChanges; 249 } 250 251 for (auto& threshold : sensor->thresholds) 252 { 253 // Use "Schmitt trigger" logic to avoid threshold trigger spam, 254 // if value is noisy while hovering very close to a threshold. 255 // When a threshold is crossed, indicate true immediately, 256 // but require more distance to be crossed the other direction, 257 // before resetting the indicator back to false. 258 if (threshold.direction == thresholds::Direction::HIGH) 259 { 260 if (value >= threshold.value) 261 { 262 thresholdChanges.emplace_back(threshold, true, value); 263 if (++cHiTrue < assertLogCount) 264 { 265 std::cerr << "Sensor " << sensor->name << " high threshold " 266 << threshold.value << " assert: value " << value 267 << " raw data " << sensor->rawValue << "\n"; 268 } 269 } 270 else if (value < (threshold.value - threshold.hysteresis)) 271 { 272 thresholdChanges.emplace_back(threshold, false, value); 273 ++cHiFalse; 274 } 275 else 276 { 277 ++cHiMidstate; 278 } 279 } 280 else if (threshold.direction == thresholds::Direction::LOW) 281 { 282 if (value <= threshold.value) 283 { 284 thresholdChanges.emplace_back(threshold, true, value); 285 if (++cLoTrue < assertLogCount) 286 { 287 std::cerr << "Sensor " << sensor->name << " low threshold " 288 << threshold.value << " assert: value " 289 << sensor->value << " raw data " 290 << sensor->rawValue << "\n"; 291 } 292 } 293 else if (value > (threshold.value + threshold.hysteresis)) 294 { 295 thresholdChanges.emplace_back(threshold, false, value); 296 ++cLoFalse; 297 } 298 else 299 { 300 ++cLoMidstate; 301 } 302 } 303 else 304 { 305 std::cerr << "Error determining threshold direction\n"; 306 } 307 } 308 309 // Throttle debug output, so that it does not continuously spam 310 ++cDebugThrottle; 311 if (cDebugThrottle >= 1000) 312 { 313 cDebugThrottle = 0; 314 if constexpr (debug) 315 { 316 std::cerr << "checkThresholds: High T=" << cHiTrue 317 << " F=" << cHiFalse << " M=" << cHiMidstate 318 << ", Low T=" << cLoTrue << " F=" << cLoFalse 319 << " M=" << cLoMidstate << "\n"; 320 } 321 } 322 323 return thresholdChanges; 324 } 325 326 void ThresholdTimer::startTimer(const std::weak_ptr<Sensor>& weakSensor, 327 const Threshold& threshold, bool assert, 328 double assertValue) 329 { 330 struct TimerUsed timerUsed = {}; 331 constexpr const size_t waitTime = 5; 332 TimerPair* pair = nullptr; 333 334 for (TimerPair& timer : timers) 335 { 336 if (!timer.first.used) 337 { 338 pair = &timer; 339 break; 340 } 341 } 342 if (pair == nullptr) 343 { 344 pair = &timers.emplace_back(timerUsed, boost::asio::steady_timer(io)); 345 } 346 347 pair->first.used = true; 348 pair->first.level = threshold.level; 349 pair->first.direction = threshold.direction; 350 pair->first.assert = assert; 351 pair->second.expires_after(std::chrono::seconds(waitTime)); 352 pair->second.async_wait([weakSensor, pair, threshold, assert, 353 assertValue](boost::system::error_code ec) { 354 auto sensorPtr = weakSensor.lock(); 355 if (!sensorPtr) 356 { 357 return; // owner sensor has been destructed 358 } 359 // pair is valid as long as sensor is valid 360 pair->first.used = false; 361 362 if (ec == boost::asio::error::operation_aborted) 363 { 364 return; // we're being canceled 365 } 366 if (ec) 367 { 368 std::cerr << "timer error: " << ec.message() << "\n"; 369 return; 370 } 371 if (sensorPtr->readingStateGood()) 372 { 373 assertThresholds(sensorPtr.get(), assertValue, threshold.level, 374 threshold.direction, assert); 375 } 376 }); 377 } 378 379 bool checkThresholds(Sensor* sensor) 380 { 381 bool status = true; 382 std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value); 383 for (const auto& change : changes) 384 { 385 assertThresholds(sensor, change.assertValue, change.threshold.level, 386 change.threshold.direction, change.asserted); 387 if (change.threshold.level == thresholds::Level::CRITICAL && 388 change.asserted) 389 { 390 status = false; 391 } 392 } 393 394 return status; 395 } 396 397 void checkThresholdsPowerDelay(const std::weak_ptr<Sensor>& weakSensor, 398 ThresholdTimer& thresholdTimer) 399 { 400 auto sensorPtr = weakSensor.lock(); 401 if (!sensorPtr) 402 { 403 return; // sensor is destructed, should never be here 404 } 405 406 Sensor* sensor = sensorPtr.get(); 407 std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value); 408 for (const auto& change : changes) 409 { 410 // When CPU is powered off, some volatges are expected to 411 // go below low thresholds. Filter these events with thresholdTimer. 412 // 1. always delay the assertion of low events to see if they are 413 // caused by power off event. 414 // 2. conditional delay the de-assertion of low events if there is 415 // an existing timer for assertion. 416 // 3. no delays for de-assert of low events if there is an existing 417 // de-assert for low event. This means 2nd de-assert would happen 418 // first and when timer expires for the previous one, no additional 419 // signal will be logged. 420 // 4. no delays for all high events. 421 if (change.threshold.direction == thresholds::Direction::LOW) 422 { 423 if (change.asserted || thresholdTimer.hasActiveTimer( 424 change.threshold, !change.asserted)) 425 { 426 thresholdTimer.startTimer(weakSensor, change.threshold, 427 change.asserted, change.assertValue); 428 continue; 429 } 430 } 431 assertThresholds(sensor, change.assertValue, change.threshold.level, 432 change.threshold.direction, change.asserted); 433 } 434 } 435 436 void assertThresholds(Sensor* sensor, double assertValue, 437 thresholds::Level level, thresholds::Direction direction, 438 bool assert) 439 { 440 std::shared_ptr<sdbusplus::asio::dbus_interface> interface = 441 sensor->getThresholdInterface(level); 442 443 if (!interface) 444 { 445 std::cout << "trying to set uninitialized interface\n"; 446 return; 447 } 448 449 std::string property = Sensor::propertyAlarm(level, direction); 450 if (property.empty()) 451 { 452 std::cout << "Alarm property is empty \n"; 453 return; 454 } 455 if (interface->set_property<bool, true>(property, assert)) 456 { 457 try 458 { 459 // msg.get_path() is interface->get_object_path() 460 sdbusplus::message_t msg = 461 interface->new_signal("ThresholdAsserted"); 462 463 msg.append(sensor->name, interface->get_interface_name(), property, 464 assert, assertValue); 465 msg.signal_send(); 466 } 467 catch (const sdbusplus::exception_t& e) 468 { 469 std::cerr 470 << "Failed to send thresholdAsserted signal with assertValue\n"; 471 } 472 } 473 } 474 475 bool parseThresholdsFromAttr( 476 std::vector<thresholds::Threshold>& thresholdVector, 477 const std::string& inputPath, const double& scaleFactor, 478 const double& offset) 479 { 480 const boost::container::flat_map< 481 std::string, std::vector<std::tuple<const char*, thresholds::Level, 482 thresholds::Direction, double>>> 483 map = { 484 {"average", 485 { 486 std::make_tuple("average_min", Level::WARNING, Direction::LOW, 487 0.0), 488 std::make_tuple("average_max", Level::WARNING, Direction::HIGH, 489 0.0), 490 }}, 491 {"input", 492 { 493 std::make_tuple("min", Level::WARNING, Direction::LOW, 0.0), 494 std::make_tuple("max", Level::WARNING, Direction::HIGH, 0.0), 495 std::make_tuple("lcrit", Level::CRITICAL, Direction::LOW, 0.0), 496 std::make_tuple("crit", Level::CRITICAL, Direction::HIGH, 497 offset), 498 }}, 499 }; 500 501 if (auto fileParts = splitFileName(inputPath)) 502 { 503 auto& [type, nr, item] = *fileParts; 504 if (map.count(item) != 0) 505 { 506 for (const auto& t : map.at(item)) 507 { 508 const auto& [suffix, level, direction, offset] = t; 509 auto attrPath = 510 boost::replace_all_copy(inputPath, item, suffix); 511 if (auto val = readFile(attrPath, scaleFactor)) 512 { 513 *val += offset; 514 if (debug) 515 { 516 std::cout << "Threshold: " << attrPath << ": " << *val 517 << "\n"; 518 } 519 thresholdVector.emplace_back(level, direction, *val, 0); 520 } 521 } 522 } 523 } 524 return true; 525 } 526 527 std::string getInterface(const Level thresholdLevel) 528 { 529 for (const ThresholdDefinition& thresh : thresProp) 530 { 531 if (thresh.level == thresholdLevel) 532 { 533 return std::string("xyz.openbmc_project.Sensor.Threshold.") + 534 thresh.levelName; 535 } 536 } 537 return ""; 538 } 539 } // namespace thresholds 540