1 #include <Thresholds.hpp> 2 #include <VariantVisitors.hpp> 3 #include <boost/algorithm/string/replace.hpp> 4 #include <boost/container/flat_map.hpp> 5 #include <sensor.hpp> 6 7 #include <array> 8 #include <cmath> 9 #include <fstream> 10 #include <iostream> 11 #include <memory> 12 #include <stdexcept> 13 #include <string> 14 #include <tuple> 15 #include <utility> 16 #include <variant> 17 #include <vector> 18 19 static constexpr bool debug = false; 20 namespace thresholds 21 { 22 Level findThresholdLevel(uint8_t sev) 23 { 24 for (const ThresholdDefinition& prop : thresProp) 25 { 26 if (prop.sevOrder == sev) 27 { 28 return prop.level; 29 } 30 } 31 return Level::ERROR; 32 } 33 34 Direction findThresholdDirection(const std::string& direct) 35 { 36 if (direct == "greater than") 37 { 38 return Direction::HIGH; 39 } 40 if (direct == "less than") 41 { 42 return Direction::LOW; 43 } 44 return Direction::ERROR; 45 } 46 47 bool parseThresholdsFromConfig( 48 const SensorData& sensorData, 49 std::vector<thresholds::Threshold>& thresholdVector, 50 const std::string* matchLabel, const int* sensorIndex) 51 { 52 for (const auto& item : sensorData) 53 { 54 if (item.first.find("Thresholds") == std::string::npos) 55 { 56 continue; 57 } 58 if (matchLabel != nullptr) 59 { 60 auto labelFind = item.second.find("Label"); 61 if (labelFind == item.second.end()) 62 { 63 continue; 64 } 65 if (std::visit(VariantToStringVisitor(), labelFind->second) != 66 *matchLabel) 67 { 68 continue; 69 } 70 } 71 72 if (sensorIndex != nullptr) 73 { 74 auto indexFind = item.second.find("Index"); 75 76 // If we're checking for index 1, a missing Index is OK. 77 if ((indexFind == item.second.end()) && (*sensorIndex != 1)) 78 { 79 continue; 80 } 81 82 if ((indexFind != item.second.end()) && 83 (std::visit(VariantToIntVisitor(), indexFind->second) != 84 *sensorIndex)) 85 { 86 continue; 87 } 88 } 89 90 double hysteresis = std::numeric_limits<double>::quiet_NaN(); 91 auto hysteresisFind = item.second.find("Hysteresis"); 92 if (hysteresisFind != item.second.end()) 93 { 94 hysteresis = 95 std::visit(VariantToDoubleVisitor(), hysteresisFind->second); 96 } 97 98 auto directionFind = item.second.find("Direction"); 99 auto severityFind = item.second.find("Severity"); 100 auto valueFind = item.second.find("Value"); 101 if (valueFind == item.second.end() || 102 severityFind == item.second.end() || 103 directionFind == item.second.end()) 104 { 105 std::cerr << "Malformed threshold on configuration interface " 106 << item.first << "\n"; 107 return false; 108 } 109 unsigned int severity = 110 std::visit(VariantToUnsignedIntVisitor(), severityFind->second); 111 112 std::string directions = 113 std::visit(VariantToStringVisitor(), directionFind->second); 114 115 Level level = findThresholdLevel(severity); 116 Direction direction = findThresholdDirection(directions); 117 118 if ((level == Level::ERROR) || (direction == Direction::ERROR)) 119 { 120 continue; 121 } 122 double val = std::visit(VariantToDoubleVisitor(), valueFind->second); 123 124 thresholdVector.emplace_back(level, direction, val, hysteresis); 125 } 126 return true; 127 } 128 129 void persistThreshold(const std::string& path, const std::string& baseInterface, 130 const thresholds::Threshold& threshold, 131 std::shared_ptr<sdbusplus::asio::connection>& conn, 132 size_t thresholdCount, const std::string& labelMatch) 133 { 134 for (size_t ii = 0; ii < thresholdCount; ii++) 135 { 136 std::string thresholdInterface = 137 baseInterface + ".Thresholds" + std::to_string(ii); 138 conn->async_method_call( 139 [&, path, threshold, thresholdInterface, labelMatch]( 140 const boost::system::error_code& ec, 141 const boost::container::flat_map<std::string, BasicVariantType>& 142 result) { 143 if (ec) 144 { 145 return; // threshold not supported 146 } 147 148 if (!labelMatch.empty()) 149 { 150 auto labelFind = result.find("Label"); 151 if (labelFind == result.end()) 152 { 153 std::cerr << "No label in threshold configuration\n"; 154 return; 155 } 156 std::string label = 157 std::visit(VariantToStringVisitor(), labelFind->second); 158 if (label != labelMatch) 159 { 160 return; 161 } 162 } 163 164 auto directionFind = result.find("Direction"); 165 auto severityFind = result.find("Severity"); 166 auto valueFind = result.find("Value"); 167 if (valueFind == result.end() || severityFind == result.end() || 168 directionFind == result.end()) 169 { 170 std::cerr << "Malformed threshold in configuration\n"; 171 return; 172 } 173 unsigned int severity = std::visit( 174 VariantToUnsignedIntVisitor(), severityFind->second); 175 176 std::string dir = 177 std::visit(VariantToStringVisitor(), directionFind->second); 178 if ((findThresholdLevel(severity) != threshold.level) || 179 (findThresholdDirection(dir) != threshold.direction)) 180 { 181 return; // not the droid we're looking for 182 } 183 184 std::variant<double> value(threshold.value); 185 conn->async_method_call( 186 [](const boost::system::error_code& ec) { 187 if (ec) 188 { 189 std::cerr << "Error setting threshold " << ec 190 << "\n"; 191 } 192 }, 193 entityManagerName, path, "org.freedesktop.DBus.Properties", 194 "Set", thresholdInterface, "Value", value); 195 }, 196 entityManagerName, path, "org.freedesktop.DBus.Properties", 197 "GetAll", thresholdInterface); 198 } 199 } 200 201 void updateThresholds(Sensor* sensor) 202 { 203 for (const auto& threshold : sensor->thresholds) 204 { 205 std::shared_ptr<sdbusplus::asio::dbus_interface> interface = 206 sensor->getThresholdInterface(threshold.level); 207 208 if (!interface) 209 { 210 continue; 211 } 212 213 std::string property = 214 sensor->propertyLevel(threshold.level, threshold.direction); 215 if (property.empty()) 216 { 217 continue; 218 } 219 interface->set_property(property, threshold.value); 220 } 221 } 222 223 // Debugging counters 224 static int cHiTrue = 0; 225 static int cHiFalse = 0; 226 static int cHiMidstate = 0; 227 static int cLoTrue = 0; 228 static int cLoFalse = 0; 229 static int cLoMidstate = 0; 230 static int cDebugThrottle = 0; 231 static constexpr int assertLogCount = 10; 232 233 struct ChangeParam 234 { 235 ChangeParam(Threshold whichThreshold, bool status, double value) : 236 threshold(whichThreshold), asserted(status), assertValue(value) 237 {} 238 239 Threshold threshold; 240 bool asserted; 241 double assertValue; 242 }; 243 244 static std::vector<ChangeParam> checkThresholds(Sensor* sensor, double value) 245 { 246 std::vector<ChangeParam> thresholdChanges; 247 if (sensor->thresholds.empty()) 248 { 249 return thresholdChanges; 250 } 251 252 for (auto& threshold : sensor->thresholds) 253 { 254 // Use "Schmitt trigger" logic to avoid threshold trigger spam, 255 // if value is noisy while hovering very close to a threshold. 256 // When a threshold is crossed, indicate true immediately, 257 // but require more distance to be crossed the other direction, 258 // before resetting the indicator back to false. 259 if (threshold.direction == thresholds::Direction::HIGH) 260 { 261 if (value >= threshold.value) 262 { 263 thresholdChanges.emplace_back(threshold, true, value); 264 if (++cHiTrue < assertLogCount) 265 { 266 std::cerr << "Sensor " << sensor->name << " high threshold " 267 << threshold.value << " assert: value " << value 268 << " raw data " << sensor->rawValue << "\n"; 269 } 270 } 271 else if (value < (threshold.value - threshold.hysteresis)) 272 { 273 thresholdChanges.emplace_back(threshold, false, value); 274 ++cHiFalse; 275 } 276 else 277 { 278 ++cHiMidstate; 279 } 280 } 281 else if (threshold.direction == thresholds::Direction::LOW) 282 { 283 if (value <= threshold.value) 284 { 285 thresholdChanges.emplace_back(threshold, true, value); 286 if (++cLoTrue < assertLogCount) 287 { 288 std::cerr << "Sensor " << sensor->name << " low threshold " 289 << threshold.value << " assert: value " 290 << sensor->value << " raw data " 291 << sensor->rawValue << "\n"; 292 } 293 } 294 else if (value > (threshold.value + threshold.hysteresis)) 295 { 296 thresholdChanges.emplace_back(threshold, false, value); 297 ++cLoFalse; 298 } 299 else 300 { 301 ++cLoMidstate; 302 } 303 } 304 else 305 { 306 std::cerr << "Error determining threshold direction\n"; 307 } 308 } 309 310 // Throttle debug output, so that it does not continuously spam 311 ++cDebugThrottle; 312 if (cDebugThrottle >= 1000) 313 { 314 cDebugThrottle = 0; 315 if constexpr (debug) 316 { 317 std::cerr << "checkThresholds: High T=" << cHiTrue 318 << " F=" << cHiFalse << " M=" << cHiMidstate 319 << ", Low T=" << cLoTrue << " F=" << cLoFalse 320 << " M=" << cLoMidstate << "\n"; 321 } 322 } 323 324 return thresholdChanges; 325 } 326 327 void ThresholdTimer::startTimer(const std::weak_ptr<Sensor>& weakSensor, 328 const Threshold& threshold, bool assert, 329 double assertValue) 330 { 331 struct TimerUsed timerUsed = {}; 332 constexpr const size_t waitTime = 5; 333 TimerPair* pair = nullptr; 334 335 for (TimerPair& timer : timers) 336 { 337 if (!timer.first.used) 338 { 339 pair = &timer; 340 break; 341 } 342 } 343 if (pair == nullptr) 344 { 345 pair = &timers.emplace_back(timerUsed, boost::asio::deadline_timer(io)); 346 } 347 348 pair->first.used = true; 349 pair->first.level = threshold.level; 350 pair->first.direction = threshold.direction; 351 pair->first.assert = assert; 352 pair->second.expires_from_now(boost::posix_time::seconds(waitTime)); 353 pair->second.async_wait([weakSensor, pair, threshold, assert, 354 assertValue](boost::system::error_code ec) { 355 auto sensorPtr = weakSensor.lock(); 356 if (!sensorPtr) 357 { 358 return; // owner sensor has been destructed 359 } 360 // pair is valid as long as sensor is valid 361 pair->first.used = false; 362 363 if (ec == boost::asio::error::operation_aborted) 364 { 365 return; // we're being canceled 366 } 367 if (ec) 368 { 369 std::cerr << "timer error: " << ec.message() << "\n"; 370 return; 371 } 372 if (sensorPtr->readingStateGood()) 373 { 374 assertThresholds(sensorPtr.get(), assertValue, threshold.level, 375 threshold.direction, assert); 376 } 377 }); 378 } 379 380 bool checkThresholds(Sensor* sensor) 381 { 382 bool status = true; 383 std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value); 384 for (const auto& change : changes) 385 { 386 assertThresholds(sensor, change.assertValue, change.threshold.level, 387 change.threshold.direction, change.asserted); 388 if (change.threshold.level == thresholds::Level::CRITICAL && 389 change.asserted) 390 { 391 status = false; 392 } 393 } 394 395 return status; 396 } 397 398 void checkThresholdsPowerDelay(const std::weak_ptr<Sensor>& weakSensor, 399 ThresholdTimer& thresholdTimer) 400 { 401 auto sensorPtr = weakSensor.lock(); 402 if (!sensorPtr) 403 { 404 return; // sensor is destructed, should never be here 405 } 406 407 Sensor* sensor = sensorPtr.get(); 408 std::vector<ChangeParam> changes = checkThresholds(sensor, sensor->value); 409 for (const auto& change : changes) 410 { 411 // When CPU is powered off, some volatges are expected to 412 // go below low thresholds. Filter these events with thresholdTimer. 413 // 1. always delay the assertion of low events to see if they are 414 // caused by power off event. 415 // 2. conditional delay the de-assertion of low events if there is 416 // an existing timer for assertion. 417 // 3. no delays for de-assert of low events if there is an existing 418 // de-assert for low event. This means 2nd de-assert would happen 419 // first and when timer expires for the previous one, no additional 420 // signal will be logged. 421 // 4. no delays for all high events. 422 if (change.threshold.direction == thresholds::Direction::LOW) 423 { 424 if (change.asserted || thresholdTimer.hasActiveTimer( 425 change.threshold, !change.asserted)) 426 { 427 thresholdTimer.startTimer(weakSensor, change.threshold, 428 change.asserted, change.assertValue); 429 continue; 430 } 431 } 432 assertThresholds(sensor, change.assertValue, change.threshold.level, 433 change.threshold.direction, change.asserted); 434 } 435 } 436 437 void assertThresholds(Sensor* sensor, double assertValue, 438 thresholds::Level level, thresholds::Direction direction, 439 bool assert) 440 { 441 std::shared_ptr<sdbusplus::asio::dbus_interface> interface = 442 sensor->getThresholdInterface(level); 443 444 if (!interface) 445 { 446 std::cout << "trying to set uninitialized interface\n"; 447 return; 448 } 449 450 std::string property = sensor->propertyAlarm(level, direction); 451 if (property.empty()) 452 { 453 std::cout << "Alarm property is empty \n"; 454 return; 455 } 456 if (interface->set_property<bool, true>(property, assert)) 457 { 458 try 459 { 460 // msg.get_path() is interface->get_object_path() 461 sdbusplus::message::message msg = 462 interface->new_signal("ThresholdAsserted"); 463 464 msg.append(sensor->name, interface->get_interface_name(), property, 465 assert, assertValue); 466 msg.signal_send(); 467 } 468 catch (const sdbusplus::exception::exception& e) 469 { 470 std::cerr 471 << "Failed to send thresholdAsserted signal with assertValue\n"; 472 } 473 } 474 } 475 476 bool parseThresholdsFromAttr( 477 std::vector<thresholds::Threshold>& thresholdVector, 478 const std::string& inputPath, const double& scaleFactor, 479 const double& offset) 480 { 481 const boost::container::flat_map< 482 std::string, std::vector<std::tuple<const char*, thresholds::Level, 483 thresholds::Direction, double>>> 484 map = { 485 {"average", 486 { 487 std::make_tuple("average_min", Level::WARNING, Direction::LOW, 488 0.0), 489 std::make_tuple("average_max", Level::WARNING, Direction::HIGH, 490 0.0), 491 }}, 492 {"input", 493 { 494 std::make_tuple("min", Level::WARNING, Direction::LOW, 0.0), 495 std::make_tuple("max", Level::WARNING, Direction::HIGH, 0.0), 496 std::make_tuple("lcrit", Level::CRITICAL, Direction::LOW, 0.0), 497 std::make_tuple("crit", Level::CRITICAL, Direction::HIGH, 498 offset), 499 }}, 500 }; 501 502 if (auto fileParts = splitFileName(inputPath)) 503 { 504 auto& [type, nr, item] = *fileParts; 505 if (map.count(item) != 0) 506 { 507 for (const auto& t : map.at(item)) 508 { 509 auto& [suffix, level, direction, offset] = t; 510 auto attrPath = 511 boost::replace_all_copy(inputPath, item, suffix); 512 if (auto val = readFile(attrPath, scaleFactor)) 513 { 514 *val += offset; 515 if (debug) 516 { 517 std::cout << "Threshold: " << attrPath << ": " << *val 518 << "\n"; 519 } 520 thresholdVector.emplace_back(level, direction, *val); 521 } 522 } 523 } 524 } 525 return true; 526 } 527 528 std::string getInterface(const Level thresholdLevel) 529 { 530 for (const ThresholdDefinition& thresh : thresProp) 531 { 532 if (thresh.level == thresholdLevel) 533 { 534 return std::string("xyz.openbmc_project.Sensor.Threshold.") + 535 thresh.levelName; 536 } 537 } 538 return ""; 539 } 540 } // namespace thresholds 541