1 #include <Thresholds.hpp> 2 #include <VariantVisitors.hpp> 3 #include <boost/algorithm/string/replace.hpp> 4 #include <boost/container/flat_map.hpp> 5 #include <sensor.hpp> 6 7 #include <array> 8 #include <cmath> 9 #include <fstream> 10 #include <iostream> 11 #include <memory> 12 #include <stdexcept> 13 #include <string> 14 #include <tuple> 15 #include <utility> 16 #include <variant> 17 #include <vector> 18 19 static constexpr bool debug = false; 20 namespace thresholds 21 { 22 Level findThresholdLevel(uint8_t sev) 23 { 24 for (const ThresholdDefinition& prop : thresProp) 25 { 26 if (prop.sevOrder == sev) 27 { 28 return prop.level; 29 } 30 } 31 return Level::ERROR; 32 } 33 34 Direction findThresholdDirection(const std::string& direct) 35 { 36 if (direct == "greater than") 37 { 38 return Direction::HIGH; 39 } 40 if (direct == "less than") 41 { 42 return Direction::LOW; 43 } 44 return Direction::ERROR; 45 } 46 47 bool parseThresholdsFromConfig( 48 const SensorData& sensorData, 49 std::vector<thresholds::Threshold>& thresholdVector, 50 const std::string* matchLabel, const int* sensorIndex) 51 { 52 for (const auto& [intf, cfg] : sensorData) 53 { 54 if (intf.find("Thresholds") == std::string::npos) 55 { 56 continue; 57 } 58 if (matchLabel != nullptr) 59 { 60 auto labelFind = cfg.find("Label"); 61 if (labelFind == cfg.end()) 62 { 63 continue; 64 } 65 if (std::visit(VariantToStringVisitor(), labelFind->second) != 66 *matchLabel) 67 { 68 continue; 69 } 70 } 71 72 if (sensorIndex != nullptr) 73 { 74 auto indexFind = cfg.find("Index"); 75 76 // If we're checking for index 1, a missing Index is OK. 77 if ((indexFind == cfg.end()) && (*sensorIndex != 1)) 78 { 79 continue; 80 } 81 82 if ((indexFind != cfg.end()) && 83 (std::visit(VariantToIntVisitor(), indexFind->second) != 84 *sensorIndex)) 85 { 86 continue; 87 } 88 } 89 90 double hysteresis = std::numeric_limits<double>::quiet_NaN(); 91 auto hysteresisFind = cfg.find("Hysteresis"); 92 if (hysteresisFind != cfg.end()) 93 { 94 hysteresis = 95 std::visit(VariantToDoubleVisitor(), hysteresisFind->second); 96 } 97 98 auto directionFind = cfg.find("Direction"); 99 auto severityFind = cfg.find("Severity"); 100 auto valueFind = cfg.find("Value"); 101 if (valueFind == cfg.end() || severityFind == cfg.end() || 102 directionFind == cfg.end()) 103 { 104 std::cerr << "Malformed threshold on configuration interface " 105 << intf << "\n"; 106 return false; 107 } 108 unsigned int severity = 109 std::visit(VariantToUnsignedIntVisitor(), severityFind->second); 110 111 std::string directions = 112 std::visit(VariantToStringVisitor(), directionFind->second); 113 114 Level level = findThresholdLevel(severity); 115 Direction direction = findThresholdDirection(directions); 116 117 if ((level == Level::ERROR) || (direction == Direction::ERROR)) 118 { 119 continue; 120 } 121 double val = std::visit(VariantToDoubleVisitor(), valueFind->second); 122 123 thresholdVector.emplace_back(level, direction, val, hysteresis); 124 } 125 return true; 126 } 127 128 void persistThreshold(const std::string& path, const std::string& baseInterface, 129 const thresholds::Threshold& threshold, 130 std::shared_ptr<sdbusplus::asio::connection>& conn, 131 size_t thresholdCount, const std::string& labelMatch) 132 { 133 for (size_t ii = 0; ii < thresholdCount; ii++) 134 { 135 std::string thresholdInterface = 136 baseInterface + ".Thresholds" + std::to_string(ii); 137 conn->async_method_call( 138 [&, path, threshold, thresholdInterface, 139 labelMatch](const boost::system::error_code& ec, 140 const SensorBaseConfigMap& result) { 141 if (ec) 142 { 143 return; // threshold not supported 144 } 145 146 if (!labelMatch.empty()) 147 { 148 auto labelFind = result.find("Label"); 149 if (labelFind == result.end()) 150 { 151 std::cerr << "No label in threshold configuration\n"; 152 return; 153 } 154 std::string label = 155 std::visit(VariantToStringVisitor(), labelFind->second); 156 if (label != labelMatch) 157 { 158 return; 159 } 160 } 161 162 auto directionFind = result.find("Direction"); 163 auto severityFind = result.find("Severity"); 164 auto valueFind = result.find("Value"); 165 if (valueFind == result.end() || severityFind == result.end() || 166 directionFind == result.end()) 167 { 168 std::cerr << "Malformed threshold in configuration\n"; 169 return; 170 } 171 unsigned int severity = 172 std::visit(VariantToUnsignedIntVisitor(), severityFind->second); 173 174 std::string dir = 175 std::visit(VariantToStringVisitor(), directionFind->second); 176 if ((findThresholdLevel(severity) != threshold.level) || 177 (findThresholdDirection(dir) != threshold.direction)) 178 { 179 return; // not the droid we're looking for 180 } 181 182 std::variant<double> value(threshold.value); 183 conn->async_method_call( 184 [](const boost::system::error_code& ec) { 185 if (ec) 186 { 187 std::cerr << "Error setting threshold " << ec << "\n"; 188 } 189 }, 190 entityManagerName, path, "org.freedesktop.DBus.Properties", 191 "Set", thresholdInterface, "Value", value); 192 }, 193 entityManagerName, path, "org.freedesktop.DBus.Properties", 194 "GetAll", thresholdInterface); 195 } 196 } 197 198 void updateThresholds(Sensor* sensor) 199 { 200 for (const auto& threshold : sensor->thresholds) 201 { 202 std::shared_ptr<sdbusplus::asio::dbus_interface> interface = 203 sensor->getThresholdInterface(threshold.level); 204 205 if (!interface) 206 { 207 continue; 208 } 209 210 std::string property = 211 Sensor::propertyLevel(threshold.level, threshold.direction); 212 if (property.empty()) 213 { 214 continue; 215 } 216 interface->set_property(property, threshold.value); 217 } 218 } 219 220 // Debugging counters 221 static int cHiTrue = 0; 222 static int cHiFalse = 0; 223 static int cHiMidstate = 0; 224 static int cLoTrue = 0; 225 static int cLoFalse = 0; 226 static int cLoMidstate = 0; 227 static int cDebugThrottle = 0; 228 static constexpr int assertLogCount = 10; 229 230 struct ChangeParam 231 { 232 ChangeParam(Threshold whichThreshold, bool status, double value) : 233 threshold(whichThreshold), asserted(status), assertValue(value) 234 {} 235 236 Threshold threshold; 237 bool asserted; 238 double assertValue; 239 }; 240 241 static std::vector<ChangeParam> checkThresholds(Sensor* sensor, double value) 242 { 243 std::vector<ChangeParam> thresholdChanges; 244 if (sensor->thresholds.empty()) 245 { 246 return thresholdChanges; 247 } 248 249 for (auto& threshold : sensor->thresholds) 250 { 251 // Use "Schmitt trigger" logic to avoid threshold trigger spam, 252 // if value is noisy while hovering very close to a threshold. 253 // When a threshold is crossed, indicate true immediately, 254 // but require more distance to be crossed the other direction, 255 // before resetting the indicator back to false. 256 if (threshold.direction == thresholds::Direction::HIGH) 257 { 258 if (value >= threshold.value) 259 { 260 thresholdChanges.emplace_back(threshold, true, value); 261 if (++cHiTrue < assertLogCount) 262 { 263 std::cerr << "Sensor " << sensor->name << " high threshold " 264 << threshold.value << " assert: value " << value 265 << " raw data " << sensor->rawValue << "\n"; 266 } 267 } 268 else if (value < (threshold.value - threshold.hysteresis)) 269 { 270 thresholdChanges.emplace_back(threshold, false, value); 271 ++cHiFalse; 272 } 273 else 274 { 275 ++cHiMidstate; 276 } 277 } 278 else if (threshold.direction == thresholds::Direction::LOW) 279 { 280 if (value <= threshold.value) 281 { 282 thresholdChanges.emplace_back(threshold, true, value); 283 if (++cLoTrue < assertLogCount) 284 { 285 std::cerr << "Sensor " << sensor->name << " low threshold " 286 << threshold.value << " assert: value " 287 << sensor->value << " raw data " 288 << sensor->rawValue << "\n"; 289 } 290 } 291 else if (value > (threshold.value + threshold.hysteresis)) 292 { 293 thresholdChanges.emplace_back(threshold, false, value); 294 ++cLoFalse; 295 } 296 else 297 { 298 ++cLoMidstate; 299 } 300 } 301 else 302 { 303 std::cerr << "Error determining threshold direction\n"; 304 } 305 } 306 307 // Throttle debug output, so that it does not continuously spam 308 ++cDebugThrottle; 309 if (cDebugThrottle >= 1000) 310 { 311 cDebugThrottle = 0; 312 if constexpr (debug) 313 { 314 std::cerr << "checkThresholds: High T=" << cHiTrue 315 << " F=" << cHiFalse << " M=" << cHiMidstate 316 << ", Low T=" << cLoTrue << " F=" << cLoFalse 317 << " M=" << cLoMidstate << "\n"; 318 } 319 } 320 321 return thresholdChanges; 322 } 323 324 void ThresholdTimer::startTimer(const std::weak_ptr<Sensor>& weakSensor, 325 const Threshold& threshold, bool assert, 326 double assertValue) 327 { 328 struct TimerUsed timerUsed = {}; 329 constexpr const size_t waitTime = 5; 330 TimerPair* pair = nullptr; 331 332 for (TimerPair& timer : timers) 333 { 334 if (!timer.first.used) 335 { 336 pair = &timer; 337 break; 338 } 339 } 340 if (pair == nullptr) 341 { 342 pair = &timers.emplace_back(timerUsed, boost::asio::deadline_timer(io)); 343 } 344 345 pair->first.used = true; 346 pair->first.level = threshold.level; 347 pair->first.direction = threshold.direction; 348 pair->first.assert = assert; 349 pair->second.expires_from_now(boost::posix_time::seconds(waitTime)); 350 pair->second.async_wait([weakSensor, pair, threshold, assert, 351 assertValue](boost::system::error_code ec) { 352 auto sensorPtr = weakSensor.lock(); 353 if (!sensorPtr) 354 { 355 return; // owner sensor has been destructed 356 } 357 // pair is valid as long as sensor is valid 358 pair->first.used = false; 359 360 if (ec == boost::asio::error::operation_aborted) 361 { 362 return; // we're being canceled 363 } 364 if (ec) 365 { 366 std::cerr << "timer error: " << ec.message() << "\n"; 367 return; 368 } 369 if (sensorPtr->readingStateGood()) 370 { 371 assertThresholds(sensorPtr.get(), assertValue, threshold.level, 372 threshold.direction, assert); 373 } 374 }); 375 } 376 377 bool checkThresholds(Sensor* sensor) 378 { 379 bool status = true; 380 std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value); 381 for (const auto& change : changes) 382 { 383 assertThresholds(sensor, change.assertValue, change.threshold.level, 384 change.threshold.direction, change.asserted); 385 if (change.threshold.level == thresholds::Level::CRITICAL && 386 change.asserted) 387 { 388 status = false; 389 } 390 } 391 392 return status; 393 } 394 395 void checkThresholdsPowerDelay(const std::weak_ptr<Sensor>& weakSensor, 396 ThresholdTimer& thresholdTimer) 397 { 398 auto sensorPtr = weakSensor.lock(); 399 if (!sensorPtr) 400 { 401 return; // sensor is destructed, should never be here 402 } 403 404 Sensor* sensor = sensorPtr.get(); 405 std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value); 406 for (const auto& change : changes) 407 { 408 // When CPU is powered off, some volatges are expected to 409 // go below low thresholds. Filter these events with thresholdTimer. 410 // 1. always delay the assertion of low events to see if they are 411 // caused by power off event. 412 // 2. conditional delay the de-assertion of low events if there is 413 // an existing timer for assertion. 414 // 3. no delays for de-assert of low events if there is an existing 415 // de-assert for low event. This means 2nd de-assert would happen 416 // first and when timer expires for the previous one, no additional 417 // signal will be logged. 418 // 4. no delays for all high events. 419 if (change.threshold.direction == thresholds::Direction::LOW) 420 { 421 if (change.asserted || thresholdTimer.hasActiveTimer( 422 change.threshold, !change.asserted)) 423 { 424 thresholdTimer.startTimer(weakSensor, change.threshold, 425 change.asserted, change.assertValue); 426 continue; 427 } 428 } 429 assertThresholds(sensor, change.assertValue, change.threshold.level, 430 change.threshold.direction, change.asserted); 431 } 432 } 433 434 void assertThresholds(Sensor* sensor, double assertValue, 435 thresholds::Level level, thresholds::Direction direction, 436 bool assert) 437 { 438 std::shared_ptr<sdbusplus::asio::dbus_interface> interface = 439 sensor->getThresholdInterface(level); 440 441 if (!interface) 442 { 443 std::cout << "trying to set uninitialized interface\n"; 444 return; 445 } 446 447 std::string property = Sensor::propertyAlarm(level, direction); 448 if (property.empty()) 449 { 450 std::cout << "Alarm property is empty \n"; 451 return; 452 } 453 if (interface->set_property<bool, true>(property, assert)) 454 { 455 try 456 { 457 // msg.get_path() is interface->get_object_path() 458 sdbusplus::message_t msg = 459 interface->new_signal("ThresholdAsserted"); 460 461 msg.append(sensor->name, interface->get_interface_name(), property, 462 assert, assertValue); 463 msg.signal_send(); 464 } 465 catch (const sdbusplus::exception_t& e) 466 { 467 std::cerr 468 << "Failed to send thresholdAsserted signal with assertValue\n"; 469 } 470 } 471 } 472 473 bool parseThresholdsFromAttr( 474 std::vector<thresholds::Threshold>& thresholdVector, 475 const std::string& inputPath, const double& scaleFactor, 476 const double& offset) 477 { 478 const boost::container::flat_map< 479 std::string, std::vector<std::tuple<const char*, thresholds::Level, 480 thresholds::Direction, double>>> 481 map = { 482 {"average", 483 { 484 std::make_tuple("average_min", Level::WARNING, Direction::LOW, 485 0.0), 486 std::make_tuple("average_max", Level::WARNING, Direction::HIGH, 487 0.0), 488 }}, 489 {"input", 490 { 491 std::make_tuple("min", Level::WARNING, Direction::LOW, 0.0), 492 std::make_tuple("max", Level::WARNING, Direction::HIGH, 0.0), 493 std::make_tuple("lcrit", Level::CRITICAL, Direction::LOW, 0.0), 494 std::make_tuple("crit", Level::CRITICAL, Direction::HIGH, 495 offset), 496 }}, 497 }; 498 499 if (auto fileParts = splitFileName(inputPath)) 500 { 501 auto& [type, nr, item] = *fileParts; 502 if (map.count(item) != 0) 503 { 504 for (const auto& t : map.at(item)) 505 { 506 const auto& [suffix, level, direction, offset] = t; 507 auto attrPath = 508 boost::replace_all_copy(inputPath, item, suffix); 509 if (auto val = readFile(attrPath, scaleFactor)) 510 { 511 *val += offset; 512 if (debug) 513 { 514 std::cout << "Threshold: " << attrPath << ": " << *val 515 << "\n"; 516 } 517 thresholdVector.emplace_back(level, direction, *val); 518 } 519 } 520 } 521 } 522 return true; 523 } 524 525 std::string getInterface(const Level thresholdLevel) 526 { 527 for (const ThresholdDefinition& thresh : thresProp) 528 { 529 if (thresh.level == thresholdLevel) 530 { 531 return std::string("xyz.openbmc_project.Sensor.Threshold.") + 532 thresh.levelName; 533 } 534 } 535 return ""; 536 } 537 } // namespace thresholds 538