1 /** 2 * Copyright © 2019 IBM Corporation 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #include "repository.hpp" 17 18 #include <sys/stat.h> 19 20 #include <fstream> 21 #include <phosphor-logging/log.hpp> 22 #include <xyz/openbmc_project/Common/File/error.hpp> 23 24 namespace openpower 25 { 26 namespace pels 27 { 28 29 namespace fs = std::filesystem; 30 using namespace phosphor::logging; 31 namespace file_error = sdbusplus::xyz::openbmc_project::Common::File::Error; 32 33 constexpr size_t warningPercentage = 95; 34 35 /** 36 * @brief Returns the amount of space the file uses on disk. 37 * 38 * This is different than just the regular size of the file. 39 * 40 * @param[in] file - The file to get the size of 41 * 42 * @return size_t The disk space the file uses 43 */ 44 size_t getFileDiskSize(const std::filesystem::path& file) 45 { 46 constexpr size_t statBlockSize = 512; 47 struct stat statData; 48 auto rc = stat(file.c_str(), &statData); 49 if (rc != 0) 50 { 51 auto e = errno; 52 std::string msg = "call to stat() failed on " + file.native() + 53 " with errno " + std::to_string(e); 54 log<level::ERR>(msg.c_str()); 55 abort(); 56 } 57 58 return statData.st_blocks * statBlockSize; 59 } 60 61 Repository::Repository(const std::filesystem::path& basePath, size_t repoSize, 62 size_t maxNumPELs) : 63 _logPath(basePath / "logs"), 64 _maxRepoSize(repoSize), _maxNumPELs(maxNumPELs), 65 _archivePath(basePath / "logs" / "archive") 66 { 67 if (!fs::exists(_logPath)) 68 { 69 fs::create_directories(_logPath); 70 } 71 72 if (!fs::exists(_archivePath)) 73 { 74 fs::create_directories(_archivePath); 75 } 76 77 restore(); 78 } 79 80 void Repository::restore() 81 { 82 for (auto& dirEntry : fs::directory_iterator(_logPath)) 83 { 84 try 85 { 86 if (!fs::is_regular_file(dirEntry.path())) 87 { 88 continue; 89 } 90 91 std::ifstream file{dirEntry.path()}; 92 std::vector<uint8_t> data{std::istreambuf_iterator<char>(file), 93 std::istreambuf_iterator<char>()}; 94 file.close(); 95 96 PEL pel{data}; 97 if (pel.valid()) 98 { 99 // If the host hasn't acked it, reset the host state so 100 // it will get sent up again. 101 if (pel.hostTransmissionState() == TransmissionState::sent) 102 { 103 pel.setHostTransmissionState(TransmissionState::newPEL); 104 try 105 { 106 write(pel, dirEntry.path()); 107 } 108 catch (const std::exception& e) 109 { 110 log<level::ERR>( 111 "Failed to save PEL after updating host state", 112 entry("PELID=0x%X", pel.id())); 113 } 114 } 115 116 PELAttributes attributes{dirEntry.path(), 117 getFileDiskSize(dirEntry.path()), 118 pel.privateHeader().creatorID(), 119 pel.userHeader().subsystem(), 120 pel.userHeader().severity(), 121 pel.userHeader().actionFlags(), 122 pel.hostTransmissionState(), 123 pel.hmcTransmissionState()}; 124 125 using pelID = LogID::Pel; 126 using obmcID = LogID::Obmc; 127 _pelAttributes.emplace( 128 LogID(pelID(pel.id()), obmcID(pel.obmcLogID())), 129 attributes); 130 131 updateRepoStats(attributes, true); 132 } 133 else 134 { 135 log<level::ERR>( 136 "Found invalid PEL file while restoring. Removing.", 137 entry("FILENAME=%s", dirEntry.path().c_str())); 138 fs::remove(dirEntry.path()); 139 } 140 } 141 catch (const std::exception& e) 142 { 143 log<level::ERR>("Hit exception while restoring PEL File", 144 entry("FILENAME=%s", dirEntry.path().c_str()), 145 entry("ERROR=%s", e.what())); 146 } 147 } 148 149 // Get size of archive folder 150 for (auto& dirEntry : fs::directory_iterator(_archivePath)) 151 { 152 _archiveSize += getFileDiskSize(dirEntry); 153 } 154 } 155 156 std::string Repository::getPELFilename(uint32_t pelID, const BCDTime& time) 157 { 158 char name[50]; 159 sprintf(name, "%.2X%.2X%.2X%.2X%.2X%.2X%.2X%.2X_%.8X", time.yearMSB, 160 time.yearLSB, time.month, time.day, time.hour, time.minutes, 161 time.seconds, time.hundredths, pelID); 162 return std::string{name}; 163 } 164 165 void Repository::add(std::unique_ptr<PEL>& pel) 166 { 167 pel->setHostTransmissionState(TransmissionState::newPEL); 168 pel->setHMCTransmissionState(TransmissionState::newPEL); 169 170 auto path = _logPath / getPELFilename(pel->id(), pel->commitTime()); 171 172 write(*(pel.get()), path); 173 174 PELAttributes attributes{path, 175 getFileDiskSize(path), 176 pel->privateHeader().creatorID(), 177 pel->userHeader().subsystem(), 178 pel->userHeader().severity(), 179 pel->userHeader().actionFlags(), 180 pel->hostTransmissionState(), 181 pel->hmcTransmissionState()}; 182 183 using pelID = LogID::Pel; 184 using obmcID = LogID::Obmc; 185 _pelAttributes.emplace(LogID(pelID(pel->id()), obmcID(pel->obmcLogID())), 186 attributes); 187 188 _lastPelID = pel->id(); 189 190 updateRepoStats(attributes, true); 191 192 processAddCallbacks(*pel); 193 } 194 195 void Repository::write(const PEL& pel, const fs::path& path) 196 { 197 std::ofstream file{path, std::ios::binary}; 198 199 if (!file.good()) 200 { 201 // If this fails, the filesystem is probably full so it isn't like 202 // we could successfully create yet another error log here. 203 auto e = errno; 204 fs::remove(path); 205 log<level::ERR>("Unable to open PEL file for writing", 206 entry("ERRNO=%d", e), entry("PATH=%s", path.c_str())); 207 throw file_error::Open(); 208 } 209 210 auto data = pel.data(); 211 file.write(reinterpret_cast<const char*>(data.data()), data.size()); 212 213 if (file.fail()) 214 { 215 // Same note as above about not being able to create an error log 216 // for this case even if we wanted. 217 auto e = errno; 218 file.close(); 219 fs::remove(path); 220 log<level::ERR>("Unable to write PEL file", entry("ERRNO=%d", e), 221 entry("PATH=%s", path.c_str())); 222 throw file_error::Write(); 223 } 224 } 225 226 std::optional<Repository::LogID> Repository::remove(const LogID& id) 227 { 228 auto pel = findPEL(id); 229 if (pel == _pelAttributes.end()) 230 { 231 return std::nullopt; 232 } 233 234 LogID actualID = pel->first; 235 updateRepoStats(pel->second, false); 236 237 log<level::DEBUG>("Removing PEL from repository", 238 entry("PEL_ID=0x%X", actualID.pelID.id), 239 entry("OBMC_LOG_ID=%d", actualID.obmcID.id)); 240 241 if (fs::exists(pel->second.path)) 242 { 243 // Check for existense of new archive folder 244 if (!fs::exists(_archivePath)) 245 { 246 fs::create_directories(_archivePath); 247 } 248 249 // Move log file to archive folder 250 auto fileName = _archivePath / pel->second.path.filename(); 251 fs::rename(pel->second.path, fileName); 252 253 // Update size of file 254 _archiveSize += getFileDiskSize(fileName); 255 } 256 257 _pelAttributes.erase(pel); 258 259 processDeleteCallbacks(actualID.pelID.id); 260 261 return actualID; 262 } 263 264 std::optional<std::vector<uint8_t>> Repository::getPELData(const LogID& id) 265 { 266 auto pel = findPEL(id); 267 if (pel != _pelAttributes.end()) 268 { 269 std::ifstream file{pel->second.path.c_str()}; 270 if (!file.good()) 271 { 272 auto e = errno; 273 log<level::ERR>("Unable to open PEL file", entry("ERRNO=%d", e), 274 entry("PATH=%s", pel->second.path.c_str())); 275 throw file_error::Open(); 276 } 277 278 std::vector<uint8_t> data{std::istreambuf_iterator<char>(file), 279 std::istreambuf_iterator<char>()}; 280 return data; 281 } 282 283 return std::nullopt; 284 } 285 286 std::optional<sdbusplus::message::unix_fd> Repository::getPELFD(const LogID& id) 287 { 288 auto pel = findPEL(id); 289 if (pel != _pelAttributes.end()) 290 { 291 FILE* fp = fopen(pel->second.path.c_str(), "rb"); 292 293 if (fp == nullptr) 294 { 295 auto e = errno; 296 log<level::ERR>("Unable to open PEL File", entry("ERRNO=%d", e), 297 entry("PATH=%s", pel->second.path.c_str())); 298 throw file_error::Open(); 299 } 300 301 // Must leave the file open here. It will be closed by sdbusplus 302 // when it sends it back over D-Bus. 303 304 return fileno(fp); 305 } 306 return std::nullopt; 307 } 308 309 void Repository::for_each(ForEachFunc func) const 310 { 311 for (const auto& [id, attributes] : _pelAttributes) 312 { 313 std::ifstream file{attributes.path}; 314 315 if (!file.good()) 316 { 317 auto e = errno; 318 log<level::ERR>("Repository::for_each: Unable to open PEL file", 319 entry("ERRNO=%d", e), 320 entry("PATH=%s", attributes.path.c_str())); 321 continue; 322 } 323 324 std::vector<uint8_t> data{std::istreambuf_iterator<char>(file), 325 std::istreambuf_iterator<char>()}; 326 file.close(); 327 328 PEL pel{data}; 329 330 try 331 { 332 if (func(pel)) 333 { 334 break; 335 } 336 } 337 catch (const std::exception& e) 338 { 339 log<level::ERR>("Repository::for_each function exception", 340 entry("ERROR=%s", e.what())); 341 } 342 } 343 } 344 345 void Repository::processAddCallbacks(const PEL& pel) const 346 { 347 for (auto& [name, func] : _addSubscriptions) 348 { 349 try 350 { 351 func(pel); 352 } 353 catch (const std::exception& e) 354 { 355 log<level::ERR>("PEL Repository add callback exception", 356 entry("NAME=%s", name.c_str()), 357 entry("ERROR=%s", e.what())); 358 } 359 } 360 } 361 362 void Repository::processDeleteCallbacks(uint32_t id) const 363 { 364 for (auto& [name, func] : _deleteSubscriptions) 365 { 366 try 367 { 368 func(id); 369 } 370 catch (const std::exception& e) 371 { 372 log<level::ERR>("PEL Repository delete callback exception", 373 entry("NAME=%s", name.c_str()), 374 entry("ERROR=%s", e.what())); 375 } 376 } 377 } 378 379 std::optional<std::reference_wrapper<const Repository::PELAttributes>> 380 Repository::getPELAttributes(const LogID& id) const 381 { 382 auto pel = findPEL(id); 383 if (pel != _pelAttributes.end()) 384 { 385 return pel->second; 386 } 387 388 return std::nullopt; 389 } 390 391 void Repository::setPELHostTransState(uint32_t pelID, TransmissionState state) 392 { 393 LogID id{LogID::Pel{pelID}}; 394 auto attr = std::find_if(_pelAttributes.begin(), _pelAttributes.end(), 395 [&id](const auto& a) { return a.first == id; }); 396 397 if ((attr != _pelAttributes.end()) && (attr->second.hostState != state)) 398 { 399 PELUpdateFunc func = [state](PEL& pel) { 400 pel.setHostTransmissionState(state); 401 }; 402 403 try 404 { 405 updatePEL(attr->second.path, func); 406 407 attr->second.hostState = state; 408 } 409 catch (const std::exception& e) 410 { 411 log<level::ERR>("Unable to update PEL host transmission state", 412 entry("PATH=%s", attr->second.path.c_str()), 413 entry("ERROR=%s", e.what())); 414 } 415 } 416 } 417 418 void Repository::setPELHMCTransState(uint32_t pelID, TransmissionState state) 419 { 420 LogID id{LogID::Pel{pelID}}; 421 auto attr = std::find_if(_pelAttributes.begin(), _pelAttributes.end(), 422 [&id](const auto& a) { return a.first == id; }); 423 424 if ((attr != _pelAttributes.end()) && (attr->second.hmcState != state)) 425 { 426 PELUpdateFunc func = [state](PEL& pel) { 427 pel.setHMCTransmissionState(state); 428 }; 429 430 try 431 { 432 updatePEL(attr->second.path, func); 433 434 attr->second.hmcState = state; 435 } 436 catch (const std::exception& e) 437 { 438 log<level::ERR>("Unable to update PEL HMC transmission state", 439 entry("PATH=%s", attr->second.path.c_str()), 440 entry("ERROR=%s", e.what())); 441 } 442 } 443 } 444 445 void Repository::updatePEL(const fs::path& path, PELUpdateFunc updateFunc) 446 { 447 std::ifstream file{path}; 448 std::vector<uint8_t> data{std::istreambuf_iterator<char>(file), 449 std::istreambuf_iterator<char>()}; 450 file.close(); 451 452 PEL pel{data}; 453 454 if (pel.valid()) 455 { 456 updateFunc(pel); 457 458 write(pel, path); 459 } 460 else 461 { 462 throw std::runtime_error( 463 "Unable to read a valid PEL when trying to update it"); 464 } 465 } 466 467 bool Repository::isServiceableSev(const PELAttributes& pel) 468 { 469 auto sevType = static_cast<SeverityType>(pel.severity & 0xF0); 470 auto sevPVEntry = 471 pel_values::findByValue(pel.severity, pel_values::severityValues); 472 std::string sevName = std::get<pel_values::registryNamePos>(*sevPVEntry); 473 474 bool check1 = (sevType == SeverityType::predictive) || 475 (sevType == SeverityType::unrecoverable) || 476 (sevType == SeverityType::critical); 477 478 bool check2 = ((sevType == SeverityType::recovered) || 479 (sevName == "symptom_recovered")) && 480 !pel.actionFlags.test(hiddenFlagBit); 481 482 bool check3 = (sevName == "symptom_predictive") || 483 (sevName == "symptom_unrecoverable") || 484 (sevName == "symptom_critical"); 485 486 return check1 || check2 || check3; 487 } 488 489 void Repository::updateRepoStats(const PELAttributes& pel, bool pelAdded) 490 { 491 auto isServiceable = Repository::isServiceableSev(pel); 492 auto bmcPEL = CreatorID::openBMC == static_cast<CreatorID>(pel.creator); 493 494 auto adjustSize = [pelAdded, &pel](auto& runningSize) { 495 if (pelAdded) 496 { 497 runningSize += pel.sizeOnDisk; 498 } 499 else 500 { 501 runningSize = std::max(static_cast<int64_t>(runningSize) - 502 static_cast<int64_t>(pel.sizeOnDisk), 503 static_cast<int64_t>(0)); 504 } 505 }; 506 507 adjustSize(_sizes.total); 508 509 if (bmcPEL) 510 { 511 adjustSize(_sizes.bmc); 512 if (isServiceable) 513 { 514 adjustSize(_sizes.bmcServiceable); 515 } 516 else 517 { 518 adjustSize(_sizes.bmcInfo); 519 } 520 } 521 else 522 { 523 adjustSize(_sizes.nonBMC); 524 if (isServiceable) 525 { 526 adjustSize(_sizes.nonBMCServiceable); 527 } 528 else 529 { 530 adjustSize(_sizes.nonBMCInfo); 531 } 532 } 533 } 534 535 bool Repository::sizeWarning() 536 { 537 std::error_code ec; 538 539 if ((_archiveSize > 0) && ((_sizes.total + _archiveSize) > 540 ((_maxRepoSize * warningPercentage) / 100))) 541 { 542 log<level::INFO>( 543 "Repository::sizeWarning function:Deleting the files in archive"); 544 545 for (const auto& dirEntry : fs::directory_iterator(_archivePath)) 546 { 547 fs::remove(dirEntry.path(), ec); 548 if (ec) 549 { 550 log<level::INFO>( 551 "Repository::sizeWarning function:Could not delete " 552 "a file in PEL archive", 553 entry("FILENAME=%s", dirEntry.path().c_str())); 554 } 555 } 556 557 _archiveSize = 0; 558 } 559 560 return (_sizes.total > (_maxRepoSize * warningPercentage / 100)) || 561 (_pelAttributes.size() > _maxNumPELs); 562 } 563 564 std::vector<Repository::AttributesReference> 565 Repository::getAllPELAttributes(SortOrder order) const 566 { 567 std::vector<Repository::AttributesReference> attributes; 568 569 std::for_each( 570 _pelAttributes.begin(), _pelAttributes.end(), 571 [&attributes](auto& pelEntry) { attributes.push_back(pelEntry); }); 572 573 std::sort(attributes.begin(), attributes.end(), 574 [order](const auto& left, const auto& right) { 575 if (order == SortOrder::ascending) 576 { 577 return left.get().second.path < right.get().second.path; 578 } 579 return left.get().second.path > right.get().second.path; 580 }); 581 582 return attributes; 583 } 584 585 std::vector<uint32_t> 586 Repository::prune(const std::vector<uint32_t>& idsWithHwIsoEntry) 587 { 588 std::vector<uint32_t> obmcLogIDs; 589 std::string msg = "Pruning PEL repository that takes up " + 590 std::to_string(_sizes.total) + " bytes and has " + 591 std::to_string(_pelAttributes.size()) + " PELs"; 592 log<level::INFO>(msg.c_str()); 593 594 // Set up the 5 functions to check if the PEL category 595 // is still over its limits. 596 597 // BMC informational PELs should only take up 15% 598 IsOverLimitFunc overBMCInfoLimit = [this]() { 599 return _sizes.bmcInfo > _maxRepoSize * 15 / 100; 600 }; 601 602 // BMC non informational PELs should only take up 30% 603 IsOverLimitFunc overBMCNonInfoLimit = [this]() { 604 return _sizes.bmcServiceable > _maxRepoSize * 30 / 100; 605 }; 606 607 // Non BMC informational PELs should only take up 15% 608 IsOverLimitFunc overNonBMCInfoLimit = [this]() { 609 return _sizes.nonBMCInfo > _maxRepoSize * 15 / 100; 610 }; 611 612 // Non BMC non informational PELs should only take up 15% 613 IsOverLimitFunc overNonBMCNonInfoLimit = [this]() { 614 return _sizes.nonBMCServiceable > _maxRepoSize * 30 / 100; 615 }; 616 617 // Bring the total number of PELs down to 80% of the max 618 IsOverLimitFunc tooManyPELsLimit = [this]() { 619 return _pelAttributes.size() > _maxNumPELs * 80 / 100; 620 }; 621 622 // Set up the functions to determine which category a PEL is in. 623 // TODO: Return false in these functions if a PEL caused a guard record. 624 625 // A BMC informational PEL 626 IsPELTypeFunc isBMCInfo = [](const PELAttributes& pel) { 627 return (CreatorID::openBMC == static_cast<CreatorID>(pel.creator)) && 628 !Repository::isServiceableSev(pel); 629 }; 630 631 // A BMC non informational PEL 632 IsPELTypeFunc isBMCNonInfo = [](const PELAttributes& pel) { 633 return (CreatorID::openBMC == static_cast<CreatorID>(pel.creator)) && 634 Repository::isServiceableSev(pel); 635 }; 636 637 // A non BMC informational PEL 638 IsPELTypeFunc isNonBMCInfo = [](const PELAttributes& pel) { 639 return (CreatorID::openBMC != static_cast<CreatorID>(pel.creator)) && 640 !Repository::isServiceableSev(pel); 641 }; 642 643 // A non BMC non informational PEL 644 IsPELTypeFunc isNonBMCNonInfo = [](const PELAttributes& pel) { 645 return (CreatorID::openBMC != static_cast<CreatorID>(pel.creator)) && 646 Repository::isServiceableSev(pel); 647 }; 648 649 // When counting PELs, count every PEL 650 IsPELTypeFunc isAnyPEL = [](const PELAttributes& /*pel*/) { return true; }; 651 652 // Check all 4 categories, which will result in at most 90% 653 // usage (15 + 30 + 15 + 30). 654 removePELs(overBMCInfoLimit, isBMCInfo, idsWithHwIsoEntry, obmcLogIDs); 655 removePELs(overBMCNonInfoLimit, isBMCNonInfo, idsWithHwIsoEntry, 656 obmcLogIDs); 657 removePELs(overNonBMCInfoLimit, isNonBMCInfo, idsWithHwIsoEntry, 658 obmcLogIDs); 659 removePELs(overNonBMCNonInfoLimit, isNonBMCNonInfo, idsWithHwIsoEntry, 660 obmcLogIDs); 661 662 // After the above pruning check if there are still too many PELs, 663 // which can happen depending on PEL sizes. 664 if (_pelAttributes.size() > _maxNumPELs) 665 { 666 removePELs(tooManyPELsLimit, isAnyPEL, idsWithHwIsoEntry, obmcLogIDs); 667 } 668 669 if (!obmcLogIDs.empty()) 670 { 671 std::string msg = "Number of PELs removed to save space: " + 672 std::to_string(obmcLogIDs.size()); 673 log<level::INFO>(msg.c_str()); 674 } 675 676 return obmcLogIDs; 677 } 678 679 void Repository::removePELs(IsOverLimitFunc& isOverLimit, 680 IsPELTypeFunc& isPELType, 681 const std::vector<uint32_t>& idsWithHwIsoEntry, 682 std::vector<uint32_t>& removedBMCLogIDs) 683 { 684 if (!isOverLimit()) 685 { 686 return; 687 } 688 689 auto attributes = getAllPELAttributes(SortOrder::ascending); 690 691 // Make 4 passes on the PELs, stopping as soon as isOverLimit 692 // returns false. 693 // Pass 1: only delete HMC acked PELs 694 // Pass 2: only delete OS acked PELs 695 // Pass 3: only delete PHYP sent PELs 696 // Pass 4: delete all PELs 697 static const std::vector<std::function<bool(const PELAttributes& pel)>> 698 stateChecks{[](const auto& pel) { 699 return pel.hmcState == TransmissionState::acked; 700 }, 701 702 [](const auto& pel) { 703 return pel.hostState == TransmissionState::acked; 704 }, 705 706 [](const auto& pel) { 707 return pel.hostState == TransmissionState::sent; 708 }, 709 710 [](const auto& /*pel*/) { return true; }}; 711 712 for (const auto& stateCheck : stateChecks) 713 { 714 for (auto it = attributes.begin(); it != attributes.end();) 715 { 716 const auto& pel = it->get(); 717 if (isPELType(pel.second) && stateCheck(pel.second)) 718 { 719 auto removedID = pel.first.obmcID.id; 720 721 auto idFound = std::find(idsWithHwIsoEntry.begin(), 722 idsWithHwIsoEntry.end(), removedID); 723 if (idFound != idsWithHwIsoEntry.end()) 724 { 725 ++it; 726 continue; 727 } 728 729 remove(pel.first); 730 731 removedBMCLogIDs.push_back(removedID); 732 733 attributes.erase(it); 734 735 if (!isOverLimit()) 736 { 737 break; 738 } 739 } 740 else 741 { 742 ++it; 743 } 744 } 745 746 if (!isOverLimit()) 747 { 748 break; 749 } 750 } 751 } 752 753 void Repository::archivePEL(const PEL& pel) 754 { 755 if (pel.valid()) 756 { 757 auto path = _archivePath / getPELFilename(pel.id(), pel.commitTime()); 758 759 write(pel, path); 760 761 _archiveSize += getFileDiskSize(path); 762 } 763 } 764 765 } // namespace pels 766 } // namespace openpower 767