1 #include <Thresholds.hpp> 2 #include <VariantVisitors.hpp> 3 #include <boost/algorithm/string/replace.hpp> 4 #include <boost/container/flat_map.hpp> 5 #include <sensor.hpp> 6 7 #include <array> 8 #include <cmath> 9 #include <fstream> 10 #include <iostream> 11 #include <memory> 12 #include <stdexcept> 13 #include <string> 14 #include <tuple> 15 #include <utility> 16 #include <variant> 17 #include <vector> 18 19 static constexpr bool debug = false; 20 namespace thresholds 21 { 22 Level findThresholdLevel(uint8_t sev) 23 { 24 for (const ThresholdDefinition& prop : thresProp) 25 { 26 if (prop.sevOrder == sev) 27 { 28 return prop.level; 29 } 30 } 31 return Level::ERROR; 32 } 33 34 Direction findThresholdDirection(const std::string& direct) 35 { 36 if (direct == "greater than") 37 { 38 return Direction::HIGH; 39 } 40 if (direct == "less than") 41 { 42 return Direction::LOW; 43 } 44 return Direction::ERROR; 45 } 46 47 bool parseThresholdsFromConfig( 48 const SensorData& sensorData, 49 std::vector<thresholds::Threshold>& thresholdVector, 50 const std::string* matchLabel, const int* sensorIndex) 51 { 52 for (const auto& item : sensorData) 53 { 54 if (item.first.find("Thresholds") == std::string::npos) 55 { 56 continue; 57 } 58 if (matchLabel != nullptr) 59 { 60 auto labelFind = item.second.find("Label"); 61 if (labelFind == item.second.end()) 62 { 63 continue; 64 } 65 if (std::visit(VariantToStringVisitor(), labelFind->second) != 66 *matchLabel) 67 { 68 continue; 69 } 70 } 71 72 if (sensorIndex != nullptr) 73 { 74 auto indexFind = item.second.find("Index"); 75 76 // If we're checking for index 1, a missing Index is OK. 77 if ((indexFind == item.second.end()) && (*sensorIndex != 1)) 78 { 79 continue; 80 } 81 82 if ((indexFind != item.second.end()) && 83 (std::visit(VariantToIntVisitor(), indexFind->second) != 84 *sensorIndex)) 85 { 86 continue; 87 } 88 } 89 90 double hysteresis = std::numeric_limits<double>::quiet_NaN(); 91 auto hysteresisFind = item.second.find("Hysteresis"); 92 if (hysteresisFind != item.second.end()) 93 { 94 hysteresis = 95 std::visit(VariantToDoubleVisitor(), hysteresisFind->second); 96 } 97 98 auto directionFind = item.second.find("Direction"); 99 auto severityFind = item.second.find("Severity"); 100 auto valueFind = item.second.find("Value"); 101 if (valueFind == item.second.end() || 102 severityFind == item.second.end() || 103 directionFind == item.second.end()) 104 { 105 std::cerr << "Malformed threshold on configuration interface " 106 << item.first << "\n"; 107 return false; 108 } 109 unsigned int severity = 110 std::visit(VariantToUnsignedIntVisitor(), severityFind->second); 111 112 std::string directions = 113 std::visit(VariantToStringVisitor(), directionFind->second); 114 115 Level level = findThresholdLevel(severity); 116 Direction direction = findThresholdDirection(directions); 117 118 if ((level == Level::ERROR) || (direction == Direction::ERROR)) 119 { 120 continue; 121 } 122 double val = std::visit(VariantToDoubleVisitor(), valueFind->second); 123 124 thresholdVector.emplace_back(level, direction, val, hysteresis); 125 } 126 return true; 127 } 128 129 void persistThreshold(const std::string& path, const std::string& baseInterface, 130 const thresholds::Threshold& threshold, 131 std::shared_ptr<sdbusplus::asio::connection>& conn, 132 size_t thresholdCount, const std::string& labelMatch) 133 { 134 for (size_t ii = 0; ii < thresholdCount; ii++) 135 { 136 std::string thresholdInterface = 137 baseInterface + ".Thresholds" + std::to_string(ii); 138 conn->async_method_call( 139 [&, path, threshold, thresholdInterface, 140 labelMatch](const boost::system::error_code& ec, 141 const SensorBaseConfigMap& result) { 142 if (ec) 143 { 144 return; // threshold not supported 145 } 146 147 if (!labelMatch.empty()) 148 { 149 auto labelFind = result.find("Label"); 150 if (labelFind == result.end()) 151 { 152 std::cerr << "No label in threshold configuration\n"; 153 return; 154 } 155 std::string label = 156 std::visit(VariantToStringVisitor(), labelFind->second); 157 if (label != labelMatch) 158 { 159 return; 160 } 161 } 162 163 auto directionFind = result.find("Direction"); 164 auto severityFind = result.find("Severity"); 165 auto valueFind = result.find("Value"); 166 if (valueFind == result.end() || severityFind == result.end() || 167 directionFind == result.end()) 168 { 169 std::cerr << "Malformed threshold in configuration\n"; 170 return; 171 } 172 unsigned int severity = 173 std::visit(VariantToUnsignedIntVisitor(), severityFind->second); 174 175 std::string dir = 176 std::visit(VariantToStringVisitor(), directionFind->second); 177 if ((findThresholdLevel(severity) != threshold.level) || 178 (findThresholdDirection(dir) != threshold.direction)) 179 { 180 return; // not the droid we're looking for 181 } 182 183 std::variant<double> value(threshold.value); 184 conn->async_method_call( 185 [](const boost::system::error_code& ec) { 186 if (ec) 187 { 188 std::cerr << "Error setting threshold " << ec << "\n"; 189 } 190 }, 191 entityManagerName, path, "org.freedesktop.DBus.Properties", 192 "Set", thresholdInterface, "Value", value); 193 }, 194 entityManagerName, path, "org.freedesktop.DBus.Properties", 195 "GetAll", thresholdInterface); 196 } 197 } 198 199 void updateThresholds(Sensor* sensor) 200 { 201 for (const auto& threshold : sensor->thresholds) 202 { 203 std::shared_ptr<sdbusplus::asio::dbus_interface> interface = 204 sensor->getThresholdInterface(threshold.level); 205 206 if (!interface) 207 { 208 continue; 209 } 210 211 std::string property = 212 Sensor::propertyLevel(threshold.level, threshold.direction); 213 if (property.empty()) 214 { 215 continue; 216 } 217 interface->set_property(property, threshold.value); 218 } 219 } 220 221 // Debugging counters 222 static int cHiTrue = 0; 223 static int cHiFalse = 0; 224 static int cHiMidstate = 0; 225 static int cLoTrue = 0; 226 static int cLoFalse = 0; 227 static int cLoMidstate = 0; 228 static int cDebugThrottle = 0; 229 static constexpr int assertLogCount = 10; 230 231 struct ChangeParam 232 { 233 ChangeParam(Threshold whichThreshold, bool status, double value) : 234 threshold(whichThreshold), asserted(status), assertValue(value) 235 {} 236 237 Threshold threshold; 238 bool asserted; 239 double assertValue; 240 }; 241 242 static std::vector<ChangeParam> checkThresholds(Sensor* sensor, double value) 243 { 244 std::vector<ChangeParam> thresholdChanges; 245 if (sensor->thresholds.empty()) 246 { 247 return thresholdChanges; 248 } 249 250 for (auto& threshold : sensor->thresholds) 251 { 252 // Use "Schmitt trigger" logic to avoid threshold trigger spam, 253 // if value is noisy while hovering very close to a threshold. 254 // When a threshold is crossed, indicate true immediately, 255 // but require more distance to be crossed the other direction, 256 // before resetting the indicator back to false. 257 if (threshold.direction == thresholds::Direction::HIGH) 258 { 259 if (value >= threshold.value) 260 { 261 thresholdChanges.emplace_back(threshold, true, value); 262 if (++cHiTrue < assertLogCount) 263 { 264 std::cerr << "Sensor " << sensor->name << " high threshold " 265 << threshold.value << " assert: value " << value 266 << " raw data " << sensor->rawValue << "\n"; 267 } 268 } 269 else if (value < (threshold.value - threshold.hysteresis)) 270 { 271 thresholdChanges.emplace_back(threshold, false, value); 272 ++cHiFalse; 273 } 274 else 275 { 276 ++cHiMidstate; 277 } 278 } 279 else if (threshold.direction == thresholds::Direction::LOW) 280 { 281 if (value <= threshold.value) 282 { 283 thresholdChanges.emplace_back(threshold, true, value); 284 if (++cLoTrue < assertLogCount) 285 { 286 std::cerr << "Sensor " << sensor->name << " low threshold " 287 << threshold.value << " assert: value " 288 << sensor->value << " raw data " 289 << sensor->rawValue << "\n"; 290 } 291 } 292 else if (value > (threshold.value + threshold.hysteresis)) 293 { 294 thresholdChanges.emplace_back(threshold, false, value); 295 ++cLoFalse; 296 } 297 else 298 { 299 ++cLoMidstate; 300 } 301 } 302 else 303 { 304 std::cerr << "Error determining threshold direction\n"; 305 } 306 } 307 308 // Throttle debug output, so that it does not continuously spam 309 ++cDebugThrottle; 310 if (cDebugThrottle >= 1000) 311 { 312 cDebugThrottle = 0; 313 if constexpr (debug) 314 { 315 std::cerr << "checkThresholds: High T=" << cHiTrue 316 << " F=" << cHiFalse << " M=" << cHiMidstate 317 << ", Low T=" << cLoTrue << " F=" << cLoFalse 318 << " M=" << cLoMidstate << "\n"; 319 } 320 } 321 322 return thresholdChanges; 323 } 324 325 void ThresholdTimer::startTimer(const std::weak_ptr<Sensor>& weakSensor, 326 const Threshold& threshold, bool assert, 327 double assertValue) 328 { 329 struct TimerUsed timerUsed = {}; 330 constexpr const size_t waitTime = 5; 331 TimerPair* pair = nullptr; 332 333 for (TimerPair& timer : timers) 334 { 335 if (!timer.first.used) 336 { 337 pair = &timer; 338 break; 339 } 340 } 341 if (pair == nullptr) 342 { 343 pair = &timers.emplace_back(timerUsed, boost::asio::deadline_timer(io)); 344 } 345 346 pair->first.used = true; 347 pair->first.level = threshold.level; 348 pair->first.direction = threshold.direction; 349 pair->first.assert = assert; 350 pair->second.expires_from_now(boost::posix_time::seconds(waitTime)); 351 pair->second.async_wait([weakSensor, pair, threshold, assert, 352 assertValue](boost::system::error_code ec) { 353 auto sensorPtr = weakSensor.lock(); 354 if (!sensorPtr) 355 { 356 return; // owner sensor has been destructed 357 } 358 // pair is valid as long as sensor is valid 359 pair->first.used = false; 360 361 if (ec == boost::asio::error::operation_aborted) 362 { 363 return; // we're being canceled 364 } 365 if (ec) 366 { 367 std::cerr << "timer error: " << ec.message() << "\n"; 368 return; 369 } 370 if (sensorPtr->readingStateGood()) 371 { 372 assertThresholds(sensorPtr.get(), assertValue, threshold.level, 373 threshold.direction, assert); 374 } 375 }); 376 } 377 378 bool checkThresholds(Sensor* sensor) 379 { 380 bool status = true; 381 std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value); 382 for (const auto& change : changes) 383 { 384 assertThresholds(sensor, change.assertValue, change.threshold.level, 385 change.threshold.direction, change.asserted); 386 if (change.threshold.level == thresholds::Level::CRITICAL && 387 change.asserted) 388 { 389 status = false; 390 } 391 } 392 393 return status; 394 } 395 396 void checkThresholdsPowerDelay(const std::weak_ptr<Sensor>& weakSensor, 397 ThresholdTimer& thresholdTimer) 398 { 399 auto sensorPtr = weakSensor.lock(); 400 if (!sensorPtr) 401 { 402 return; // sensor is destructed, should never be here 403 } 404 405 Sensor* sensor = sensorPtr.get(); 406 std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value); 407 for (const auto& change : changes) 408 { 409 // When CPU is powered off, some volatges are expected to 410 // go below low thresholds. Filter these events with thresholdTimer. 411 // 1. always delay the assertion of low events to see if they are 412 // caused by power off event. 413 // 2. conditional delay the de-assertion of low events if there is 414 // an existing timer for assertion. 415 // 3. no delays for de-assert of low events if there is an existing 416 // de-assert for low event. This means 2nd de-assert would happen 417 // first and when timer expires for the previous one, no additional 418 // signal will be logged. 419 // 4. no delays for all high events. 420 if (change.threshold.direction == thresholds::Direction::LOW) 421 { 422 if (change.asserted || thresholdTimer.hasActiveTimer( 423 change.threshold, !change.asserted)) 424 { 425 thresholdTimer.startTimer(weakSensor, change.threshold, 426 change.asserted, change.assertValue); 427 continue; 428 } 429 } 430 assertThresholds(sensor, change.assertValue, change.threshold.level, 431 change.threshold.direction, change.asserted); 432 } 433 } 434 435 void assertThresholds(Sensor* sensor, double assertValue, 436 thresholds::Level level, thresholds::Direction direction, 437 bool assert) 438 { 439 std::shared_ptr<sdbusplus::asio::dbus_interface> interface = 440 sensor->getThresholdInterface(level); 441 442 if (!interface) 443 { 444 std::cout << "trying to set uninitialized interface\n"; 445 return; 446 } 447 448 std::string property = Sensor::propertyAlarm(level, direction); 449 if (property.empty()) 450 { 451 std::cout << "Alarm property is empty \n"; 452 return; 453 } 454 if (interface->set_property<bool, true>(property, assert)) 455 { 456 try 457 { 458 // msg.get_path() is interface->get_object_path() 459 sdbusplus::message_t msg = 460 interface->new_signal("ThresholdAsserted"); 461 462 msg.append(sensor->name, interface->get_interface_name(), property, 463 assert, assertValue); 464 msg.signal_send(); 465 } 466 catch (const sdbusplus::exception_t& e) 467 { 468 std::cerr 469 << "Failed to send thresholdAsserted signal with assertValue\n"; 470 } 471 } 472 } 473 474 bool parseThresholdsFromAttr( 475 std::vector<thresholds::Threshold>& thresholdVector, 476 const std::string& inputPath, const double& scaleFactor, 477 const double& offset) 478 { 479 const boost::container::flat_map< 480 std::string, std::vector<std::tuple<const char*, thresholds::Level, 481 thresholds::Direction, double>>> 482 map = { 483 {"average", 484 { 485 std::make_tuple("average_min", Level::WARNING, Direction::LOW, 486 0.0), 487 std::make_tuple("average_max", Level::WARNING, Direction::HIGH, 488 0.0), 489 }}, 490 {"input", 491 { 492 std::make_tuple("min", Level::WARNING, Direction::LOW, 0.0), 493 std::make_tuple("max", Level::WARNING, Direction::HIGH, 0.0), 494 std::make_tuple("lcrit", Level::CRITICAL, Direction::LOW, 0.0), 495 std::make_tuple("crit", Level::CRITICAL, Direction::HIGH, 496 offset), 497 }}, 498 }; 499 500 if (auto fileParts = splitFileName(inputPath)) 501 { 502 auto& [type, nr, item] = *fileParts; 503 if (map.count(item) != 0) 504 { 505 for (const auto& t : map.at(item)) 506 { 507 const auto& [suffix, level, direction, offset] = t; 508 auto attrPath = 509 boost::replace_all_copy(inputPath, item, suffix); 510 if (auto val = readFile(attrPath, scaleFactor)) 511 { 512 *val += offset; 513 if (debug) 514 { 515 std::cout << "Threshold: " << attrPath << ": " << *val 516 << "\n"; 517 } 518 thresholdVector.emplace_back(level, direction, *val); 519 } 520 } 521 } 522 } 523 return true; 524 } 525 526 std::string getInterface(const Level thresholdLevel) 527 { 528 for (const ThresholdDefinition& thresh : thresProp) 529 { 530 if (thresh.level == thresholdLevel) 531 { 532 return std::string("xyz.openbmc_project.Sensor.Threshold.") + 533 thresh.levelName; 534 } 535 } 536 return ""; 537 } 538 } // namespace thresholds 539