1 /** 2 * Copyright © 2019 IBM Corporation 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #include "repository.hpp" 17 18 #include <fcntl.h> 19 #include <sys/stat.h> 20 21 #include <phosphor-logging/log.hpp> 22 #include <xyz/openbmc_project/Common/File/error.hpp> 23 24 #include <fstream> 25 26 namespace openpower 27 { 28 namespace pels 29 { 30 31 namespace fs = std::filesystem; 32 using namespace phosphor::logging; 33 namespace file_error = sdbusplus::xyz::openbmc_project::Common::File::Error; 34 35 constexpr size_t warningPercentage = 95; 36 37 /** 38 * @brief Returns the amount of space the file uses on disk. 39 * 40 * This is different than just the regular size of the file. 41 * 42 * @param[in] file - The file to get the size of 43 * 44 * @return size_t The disk space the file uses 45 */ 46 size_t getFileDiskSize(const std::filesystem::path& file) 47 { 48 constexpr size_t statBlockSize = 512; 49 struct stat statData; 50 auto rc = stat(file.c_str(), &statData); 51 if (rc != 0) 52 { 53 auto e = errno; 54 std::string msg = "call to stat() failed on " + file.native() + 55 " with errno " + std::to_string(e); 56 log<level::ERR>(msg.c_str()); 57 abort(); 58 } 59 60 return statData.st_blocks * statBlockSize; 61 } 62 63 Repository::Repository(const std::filesystem::path& basePath, size_t repoSize, 64 size_t maxNumPELs) : 65 _logPath(basePath / "logs"), 66 _maxRepoSize(repoSize), _maxNumPELs(maxNumPELs), 67 _archivePath(basePath / "logs" / "archive") 68 { 69 if (!fs::exists(_logPath)) 70 { 71 fs::create_directories(_logPath); 72 } 73 74 if (!fs::exists(_archivePath)) 75 { 76 fs::create_directories(_archivePath); 77 } 78 79 restore(); 80 } 81 82 void Repository::restore() 83 { 84 for (auto& dirEntry : fs::directory_iterator(_logPath)) 85 { 86 try 87 { 88 if (!fs::is_regular_file(dirEntry.path())) 89 { 90 continue; 91 } 92 93 std::ifstream file{dirEntry.path()}; 94 std::vector<uint8_t> data{std::istreambuf_iterator<char>(file), 95 std::istreambuf_iterator<char>()}; 96 file.close(); 97 98 PEL pel{data}; 99 if (pel.valid()) 100 { 101 // If the host hasn't acked it, reset the host state so 102 // it will get sent up again. 103 if (pel.hostTransmissionState() == TransmissionState::sent) 104 { 105 pel.setHostTransmissionState(TransmissionState::newPEL); 106 try 107 { 108 write(pel, dirEntry.path()); 109 } 110 catch (const std::exception& e) 111 { 112 log<level::ERR>( 113 "Failed to save PEL after updating host state", 114 entry("PELID=0x%X", pel.id())); 115 } 116 } 117 118 PELAttributes attributes{ 119 dirEntry.path(), 120 getFileDiskSize(dirEntry.path()), 121 pel.privateHeader().creatorID(), 122 pel.userHeader().subsystem(), 123 pel.userHeader().severity(), 124 pel.userHeader().actionFlags(), 125 pel.hostTransmissionState(), 126 pel.hmcTransmissionState(), 127 pel.plid(), 128 pel.getDeconfigFlag(), 129 pel.getGuardFlag(), 130 getMillisecondsSinceEpoch( 131 pel.privateHeader().createTimestamp())}; 132 133 using pelID = LogID::Pel; 134 using obmcID = LogID::Obmc; 135 _pelAttributes.emplace( 136 LogID(pelID(pel.id()), obmcID(pel.obmcLogID())), 137 attributes); 138 139 updateRepoStats(attributes, true); 140 } 141 else 142 { 143 log<level::ERR>( 144 "Found invalid PEL file while restoring. Removing.", 145 entry("FILENAME=%s", dirEntry.path().c_str())); 146 fs::remove(dirEntry.path()); 147 } 148 } 149 catch (const std::exception& e) 150 { 151 log<level::ERR>("Hit exception while restoring PEL File", 152 entry("FILENAME=%s", dirEntry.path().c_str()), 153 entry("ERROR=%s", e.what())); 154 } 155 } 156 157 // Get size of archive folder 158 for (auto& dirEntry : fs::directory_iterator(_archivePath)) 159 { 160 _archiveSize += getFileDiskSize(dirEntry); 161 } 162 } 163 164 std::string Repository::getPELFilename(uint32_t pelID, const BCDTime& time) 165 { 166 char name[50]; 167 sprintf(name, "%.2X%.2X%.2X%.2X%.2X%.2X%.2X%.2X_%.8X", time.yearMSB, 168 time.yearLSB, time.month, time.day, time.hour, time.minutes, 169 time.seconds, time.hundredths, pelID); 170 return std::string{name}; 171 } 172 173 void Repository::add(std::unique_ptr<PEL>& pel) 174 { 175 pel->setHostTransmissionState(TransmissionState::newPEL); 176 pel->setHMCTransmissionState(TransmissionState::newPEL); 177 178 auto path = _logPath / getPELFilename(pel->id(), pel->commitTime()); 179 180 write(*(pel.get()), path); 181 182 PELAttributes attributes{ 183 path, 184 getFileDiskSize(path), 185 pel->privateHeader().creatorID(), 186 pel->userHeader().subsystem(), 187 pel->userHeader().severity(), 188 pel->userHeader().actionFlags(), 189 pel->hostTransmissionState(), 190 pel->hmcTransmissionState(), 191 pel->plid(), 192 pel->getDeconfigFlag(), 193 pel->getGuardFlag(), 194 getMillisecondsSinceEpoch(pel->privateHeader().createTimestamp())}; 195 196 using pelID = LogID::Pel; 197 using obmcID = LogID::Obmc; 198 _pelAttributes.emplace(LogID(pelID(pel->id()), obmcID(pel->obmcLogID())), 199 attributes); 200 201 _lastPelID = pel->id(); 202 203 updateRepoStats(attributes, true); 204 205 processAddCallbacks(*pel); 206 } 207 208 void Repository::write(const PEL& pel, const fs::path& path) 209 { 210 std::ofstream file{path, std::ios::binary}; 211 212 if (!file.good()) 213 { 214 // If this fails, the filesystem is probably full so it isn't like 215 // we could successfully create yet another error log here. 216 auto e = errno; 217 fs::remove(path); 218 log<level::ERR>("Unable to open PEL file for writing", 219 entry("ERRNO=%d", e), entry("PATH=%s", path.c_str())); 220 throw file_error::Open(); 221 } 222 223 auto data = pel.data(); 224 file.write(reinterpret_cast<const char*>(data.data()), data.size()); 225 226 if (file.fail()) 227 { 228 // Same note as above about not being able to create an error log 229 // for this case even if we wanted. 230 auto e = errno; 231 file.close(); 232 fs::remove(path); 233 log<level::ERR>("Unable to write PEL file", entry("ERRNO=%d", e), 234 entry("PATH=%s", path.c_str())); 235 throw file_error::Write(); 236 } 237 } 238 239 std::optional<Repository::LogID> Repository::remove(const LogID& id) 240 { 241 auto pel = findPEL(id); 242 if (pel == _pelAttributes.end()) 243 { 244 return std::nullopt; 245 } 246 247 LogID actualID = pel->first; 248 updateRepoStats(pel->second, false); 249 250 log<level::DEBUG>("Removing PEL from repository", 251 entry("PEL_ID=0x%X", actualID.pelID.id), 252 entry("OBMC_LOG_ID=%d", actualID.obmcID.id)); 253 254 if (fs::exists(pel->second.path)) 255 { 256 // Check for existense of new archive folder 257 if (!fs::exists(_archivePath)) 258 { 259 fs::create_directories(_archivePath); 260 } 261 262 // Move log file to archive folder 263 auto fileName = _archivePath / pel->second.path.filename(); 264 fs::rename(pel->second.path, fileName); 265 266 // Update size of file 267 _archiveSize += getFileDiskSize(fileName); 268 } 269 270 _pelAttributes.erase(pel); 271 272 processDeleteCallbacks(actualID.pelID.id); 273 274 return actualID; 275 } 276 277 std::optional<std::vector<uint8_t>> Repository::getPELData(const LogID& id) 278 { 279 auto pel = findPEL(id); 280 if (pel != _pelAttributes.end()) 281 { 282 std::ifstream file{pel->second.path.c_str()}; 283 if (!file.good()) 284 { 285 auto e = errno; 286 log<level::ERR>("Unable to open PEL file", entry("ERRNO=%d", e), 287 entry("PATH=%s", pel->second.path.c_str())); 288 throw file_error::Open(); 289 } 290 291 std::vector<uint8_t> data{std::istreambuf_iterator<char>(file), 292 std::istreambuf_iterator<char>()}; 293 return data; 294 } 295 296 return std::nullopt; 297 } 298 299 std::optional<sdbusplus::message::unix_fd> Repository::getPELFD(const LogID& id) 300 { 301 auto pel = findPEL(id); 302 if (pel != _pelAttributes.end()) 303 { 304 int fd = open(pel->second.path.c_str(), O_RDONLY | O_NONBLOCK); 305 if (fd == -1) 306 { 307 auto e = errno; 308 log<level::ERR>("Unable to open PEL File", entry("ERRNO=%d", e), 309 entry("PATH=%s", pel->second.path.c_str())); 310 throw file_error::Open(); 311 } 312 313 // Must leave the file open here. It will be closed by sdbusplus 314 // when it sends it back over D-Bus. 315 return fd; 316 } 317 return std::nullopt; 318 } 319 320 void Repository::for_each(ForEachFunc func) const 321 { 322 for (const auto& [id, attributes] : _pelAttributes) 323 { 324 std::ifstream file{attributes.path}; 325 326 if (!file.good()) 327 { 328 auto e = errno; 329 log<level::ERR>("Repository::for_each: Unable to open PEL file", 330 entry("ERRNO=%d", e), 331 entry("PATH=%s", attributes.path.c_str())); 332 continue; 333 } 334 335 std::vector<uint8_t> data{std::istreambuf_iterator<char>(file), 336 std::istreambuf_iterator<char>()}; 337 file.close(); 338 339 PEL pel{data}; 340 341 try 342 { 343 if (func(pel)) 344 { 345 break; 346 } 347 } 348 catch (const std::exception& e) 349 { 350 log<level::ERR>("Repository::for_each function exception", 351 entry("ERROR=%s", e.what())); 352 } 353 } 354 } 355 356 void Repository::processAddCallbacks(const PEL& pel) const 357 { 358 for (auto& [name, func] : _addSubscriptions) 359 { 360 try 361 { 362 func(pel); 363 } 364 catch (const std::exception& e) 365 { 366 log<level::ERR>("PEL Repository add callback exception", 367 entry("NAME=%s", name.c_str()), 368 entry("ERROR=%s", e.what())); 369 } 370 } 371 } 372 373 void Repository::processDeleteCallbacks(uint32_t id) const 374 { 375 for (auto& [name, func] : _deleteSubscriptions) 376 { 377 try 378 { 379 func(id); 380 } 381 catch (const std::exception& e) 382 { 383 log<level::ERR>("PEL Repository delete callback exception", 384 entry("NAME=%s", name.c_str()), 385 entry("ERROR=%s", e.what())); 386 } 387 } 388 } 389 390 std::optional<std::reference_wrapper<const Repository::PELAttributes>> 391 Repository::getPELAttributes(const LogID& id) const 392 { 393 auto pel = findPEL(id); 394 if (pel != _pelAttributes.end()) 395 { 396 return pel->second; 397 } 398 399 return std::nullopt; 400 } 401 402 void Repository::setPELHostTransState(uint32_t pelID, TransmissionState state) 403 { 404 LogID id{LogID::Pel{pelID}}; 405 auto attr = std::find_if(_pelAttributes.begin(), _pelAttributes.end(), 406 [&id](const auto& a) { return a.first == id; }); 407 408 if ((attr != _pelAttributes.end()) && (attr->second.hostState != state)) 409 { 410 PELUpdateFunc func = [state](PEL& pel) { 411 pel.setHostTransmissionState(state); 412 }; 413 414 try 415 { 416 updatePEL(attr->second.path, func); 417 418 attr->second.hostState = state; 419 } 420 catch (const std::exception& e) 421 { 422 log<level::ERR>("Unable to update PEL host transmission state", 423 entry("PATH=%s", attr->second.path.c_str()), 424 entry("ERROR=%s", e.what())); 425 } 426 } 427 } 428 429 void Repository::setPELHMCTransState(uint32_t pelID, TransmissionState state) 430 { 431 LogID id{LogID::Pel{pelID}}; 432 auto attr = std::find_if(_pelAttributes.begin(), _pelAttributes.end(), 433 [&id](const auto& a) { return a.first == id; }); 434 435 if ((attr != _pelAttributes.end()) && (attr->second.hmcState != state)) 436 { 437 PELUpdateFunc func = [state](PEL& pel) { 438 pel.setHMCTransmissionState(state); 439 }; 440 441 try 442 { 443 updatePEL(attr->second.path, func); 444 445 attr->second.hmcState = state; 446 } 447 catch (const std::exception& e) 448 { 449 log<level::ERR>("Unable to update PEL HMC transmission state", 450 entry("PATH=%s", attr->second.path.c_str()), 451 entry("ERROR=%s", e.what())); 452 } 453 } 454 } 455 456 void Repository::updatePEL(const fs::path& path, PELUpdateFunc updateFunc) 457 { 458 std::ifstream file{path}; 459 std::vector<uint8_t> data{std::istreambuf_iterator<char>(file), 460 std::istreambuf_iterator<char>()}; 461 file.close(); 462 463 PEL pel{data}; 464 465 if (pel.valid()) 466 { 467 updateFunc(pel); 468 469 write(pel, path); 470 } 471 else 472 { 473 throw std::runtime_error( 474 "Unable to read a valid PEL when trying to update it"); 475 } 476 } 477 478 bool Repository::isServiceableSev(const PELAttributes& pel) 479 { 480 auto sevType = static_cast<SeverityType>(pel.severity & 0xF0); 481 auto sevPVEntry = pel_values::findByValue(pel.severity, 482 pel_values::severityValues); 483 std::string sevName = std::get<pel_values::registryNamePos>(*sevPVEntry); 484 485 bool check1 = (sevType == SeverityType::predictive) || 486 (sevType == SeverityType::unrecoverable) || 487 (sevType == SeverityType::critical); 488 489 bool check2 = ((sevType == SeverityType::recovered) || 490 (sevName == "symptom_recovered")) && 491 !pel.actionFlags.test(hiddenFlagBit); 492 493 bool check3 = (sevName == "symptom_predictive") || 494 (sevName == "symptom_unrecoverable") || 495 (sevName == "symptom_critical"); 496 497 return check1 || check2 || check3; 498 } 499 500 void Repository::updateRepoStats(const PELAttributes& pel, bool pelAdded) 501 { 502 auto isServiceable = Repository::isServiceableSev(pel); 503 auto bmcPEL = CreatorID::openBMC == static_cast<CreatorID>(pel.creator); 504 505 auto adjustSize = [pelAdded, &pel](auto& runningSize) { 506 if (pelAdded) 507 { 508 runningSize += pel.sizeOnDisk; 509 } 510 else 511 { 512 runningSize = std::max(static_cast<int64_t>(runningSize) - 513 static_cast<int64_t>(pel.sizeOnDisk), 514 static_cast<int64_t>(0)); 515 } 516 }; 517 518 adjustSize(_sizes.total); 519 520 if (bmcPEL) 521 { 522 adjustSize(_sizes.bmc); 523 if (isServiceable) 524 { 525 adjustSize(_sizes.bmcServiceable); 526 } 527 else 528 { 529 adjustSize(_sizes.bmcInfo); 530 } 531 } 532 else 533 { 534 adjustSize(_sizes.nonBMC); 535 if (isServiceable) 536 { 537 adjustSize(_sizes.nonBMCServiceable); 538 } 539 else 540 { 541 adjustSize(_sizes.nonBMCInfo); 542 } 543 } 544 } 545 546 bool Repository::sizeWarning() 547 { 548 std::error_code ec; 549 550 if ((_archiveSize > 0) && ((_sizes.total + _archiveSize) > 551 ((_maxRepoSize * warningPercentage) / 100))) 552 { 553 log<level::INFO>( 554 "Repository::sizeWarning function:Deleting the files in archive"); 555 556 for (const auto& dirEntry : fs::directory_iterator(_archivePath)) 557 { 558 fs::remove(dirEntry.path(), ec); 559 if (ec) 560 { 561 log<level::INFO>( 562 "Repository::sizeWarning function:Could not delete " 563 "a file in PEL archive", 564 entry("FILENAME=%s", dirEntry.path().c_str())); 565 } 566 } 567 568 _archiveSize = 0; 569 } 570 571 return (_sizes.total > (_maxRepoSize * warningPercentage / 100)) || 572 (_pelAttributes.size() > _maxNumPELs); 573 } 574 575 std::vector<Repository::AttributesReference> 576 Repository::getAllPELAttributes(SortOrder order) const 577 { 578 std::vector<Repository::AttributesReference> attributes; 579 580 std::for_each(_pelAttributes.begin(), _pelAttributes.end(), 581 [&attributes](auto& pelEntry) { 582 attributes.push_back(pelEntry); 583 }); 584 585 std::sort(attributes.begin(), attributes.end(), 586 [order](const auto& left, const auto& right) { 587 if (order == SortOrder::ascending) 588 { 589 return left.get().second.path < right.get().second.path; 590 } 591 return left.get().second.path > right.get().second.path; 592 }); 593 594 return attributes; 595 } 596 597 std::vector<uint32_t> 598 Repository::prune(const std::vector<uint32_t>& idsWithHwIsoEntry) 599 { 600 std::vector<uint32_t> obmcLogIDs; 601 std::string msg = "Pruning PEL repository that takes up " + 602 std::to_string(_sizes.total) + " bytes and has " + 603 std::to_string(_pelAttributes.size()) + " PELs"; 604 log<level::INFO>(msg.c_str()); 605 606 // Set up the 5 functions to check if the PEL category 607 // is still over its limits. 608 609 // BMC informational PELs should only take up 15% 610 IsOverLimitFunc overBMCInfoLimit = [this]() { 611 return _sizes.bmcInfo > _maxRepoSize * 15 / 100; 612 }; 613 614 // BMC non informational PELs should only take up 30% 615 IsOverLimitFunc overBMCNonInfoLimit = [this]() { 616 return _sizes.bmcServiceable > _maxRepoSize * 30 / 100; 617 }; 618 619 // Non BMC informational PELs should only take up 15% 620 IsOverLimitFunc overNonBMCInfoLimit = [this]() { 621 return _sizes.nonBMCInfo > _maxRepoSize * 15 / 100; 622 }; 623 624 // Non BMC non informational PELs should only take up 15% 625 IsOverLimitFunc overNonBMCNonInfoLimit = [this]() { 626 return _sizes.nonBMCServiceable > _maxRepoSize * 30 / 100; 627 }; 628 629 // Bring the total number of PELs down to 80% of the max 630 IsOverLimitFunc tooManyPELsLimit = [this]() { 631 return _pelAttributes.size() > _maxNumPELs * 80 / 100; 632 }; 633 634 // Set up the functions to determine which category a PEL is in. 635 // TODO: Return false in these functions if a PEL caused a guard record. 636 637 // A BMC informational PEL 638 IsPELTypeFunc isBMCInfo = [](const PELAttributes& pel) { 639 return (CreatorID::openBMC == static_cast<CreatorID>(pel.creator)) && 640 !Repository::isServiceableSev(pel); 641 }; 642 643 // A BMC non informational PEL 644 IsPELTypeFunc isBMCNonInfo = [](const PELAttributes& pel) { 645 return (CreatorID::openBMC == static_cast<CreatorID>(pel.creator)) && 646 Repository::isServiceableSev(pel); 647 }; 648 649 // A non BMC informational PEL 650 IsPELTypeFunc isNonBMCInfo = [](const PELAttributes& pel) { 651 return (CreatorID::openBMC != static_cast<CreatorID>(pel.creator)) && 652 !Repository::isServiceableSev(pel); 653 }; 654 655 // A non BMC non informational PEL 656 IsPELTypeFunc isNonBMCNonInfo = [](const PELAttributes& pel) { 657 return (CreatorID::openBMC != static_cast<CreatorID>(pel.creator)) && 658 Repository::isServiceableSev(pel); 659 }; 660 661 // When counting PELs, count every PEL 662 IsPELTypeFunc isAnyPEL = [](const PELAttributes& /*pel*/) { return true; }; 663 664 // Check all 4 categories, which will result in at most 90% 665 // usage (15 + 30 + 15 + 30). 666 removePELs(overBMCInfoLimit, isBMCInfo, idsWithHwIsoEntry, obmcLogIDs); 667 removePELs(overBMCNonInfoLimit, isBMCNonInfo, idsWithHwIsoEntry, 668 obmcLogIDs); 669 removePELs(overNonBMCInfoLimit, isNonBMCInfo, idsWithHwIsoEntry, 670 obmcLogIDs); 671 removePELs(overNonBMCNonInfoLimit, isNonBMCNonInfo, idsWithHwIsoEntry, 672 obmcLogIDs); 673 674 // After the above pruning check if there are still too many PELs, 675 // which can happen depending on PEL sizes. 676 if (_pelAttributes.size() > _maxNumPELs) 677 { 678 removePELs(tooManyPELsLimit, isAnyPEL, idsWithHwIsoEntry, obmcLogIDs); 679 } 680 681 if (!obmcLogIDs.empty()) 682 { 683 std::string m = "Number of PELs removed to save space: " + 684 std::to_string(obmcLogIDs.size()); 685 log<level::INFO>(m.c_str()); 686 } 687 688 return obmcLogIDs; 689 } 690 691 void Repository::removePELs(const IsOverLimitFunc& isOverLimit, 692 const IsPELTypeFunc& isPELType, 693 const std::vector<uint32_t>& idsWithHwIsoEntry, 694 std::vector<uint32_t>& removedBMCLogIDs) 695 { 696 if (!isOverLimit()) 697 { 698 return; 699 } 700 701 auto attributes = getAllPELAttributes(SortOrder::ascending); 702 703 // Make 4 passes on the PELs, stopping as soon as isOverLimit 704 // returns false. 705 // Pass 1: only delete HMC acked PELs 706 // Pass 2: only delete OS acked PELs 707 // Pass 3: only delete PHYP sent PELs 708 // Pass 4: delete all PELs 709 static const std::vector<std::function<bool(const PELAttributes& pel)>> 710 stateChecks{[](const auto& pel) { 711 return pel.hmcState == TransmissionState::acked; 712 }, 713 714 [](const auto& pel) { 715 return pel.hostState == TransmissionState::acked; 716 }, 717 718 [](const auto& pel) { 719 return pel.hostState == TransmissionState::sent; 720 }, 721 722 [](const auto& /*pel*/) { return true; }}; 723 724 for (const auto& stateCheck : stateChecks) 725 { 726 for (auto it = attributes.begin(); it != attributes.end();) 727 { 728 const auto& pel = it->get(); 729 if (isPELType(pel.second) && stateCheck(pel.second)) 730 { 731 auto removedID = pel.first.obmcID.id; 732 733 auto idFound = std::find(idsWithHwIsoEntry.begin(), 734 idsWithHwIsoEntry.end(), removedID); 735 if (idFound != idsWithHwIsoEntry.end()) 736 { 737 ++it; 738 continue; 739 } 740 741 remove(pel.first); 742 743 removedBMCLogIDs.push_back(removedID); 744 745 attributes.erase(it); 746 747 if (!isOverLimit()) 748 { 749 break; 750 } 751 } 752 else 753 { 754 ++it; 755 } 756 } 757 758 if (!isOverLimit()) 759 { 760 break; 761 } 762 } 763 } 764 765 void Repository::archivePEL(const PEL& pel) 766 { 767 if (pel.valid()) 768 { 769 auto path = _archivePath / getPELFilename(pel.id(), pel.commitTime()); 770 771 write(pel, path); 772 773 _archiveSize += getFileDiskSize(path); 774 } 775 } 776 777 } // namespace pels 778 } // namespace openpower 779