1 /** 2 * Copyright © 2019 IBM Corporation 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #include "repository.hpp" 17 18 #include <sys/stat.h> 19 20 #include <phosphor-logging/log.hpp> 21 #include <xyz/openbmc_project/Common/File/error.hpp> 22 23 #include <fstream> 24 25 namespace openpower 26 { 27 namespace pels 28 { 29 30 namespace fs = std::filesystem; 31 using namespace phosphor::logging; 32 namespace file_error = sdbusplus::xyz::openbmc_project::Common::File::Error; 33 34 constexpr size_t warningPercentage = 95; 35 36 /** 37 * @brief Returns the amount of space the file uses on disk. 38 * 39 * This is different than just the regular size of the file. 40 * 41 * @param[in] file - The file to get the size of 42 * 43 * @return size_t The disk space the file uses 44 */ 45 size_t getFileDiskSize(const std::filesystem::path& file) 46 { 47 constexpr size_t statBlockSize = 512; 48 struct stat statData; 49 auto rc = stat(file.c_str(), &statData); 50 if (rc != 0) 51 { 52 auto e = errno; 53 std::string msg = "call to stat() failed on " + file.native() + 54 " with errno " + std::to_string(e); 55 log<level::ERR>(msg.c_str()); 56 abort(); 57 } 58 59 return statData.st_blocks * statBlockSize; 60 } 61 62 Repository::Repository(const std::filesystem::path& basePath, size_t repoSize, 63 size_t maxNumPELs) : 64 _logPath(basePath / "logs"), 65 _maxRepoSize(repoSize), _maxNumPELs(maxNumPELs), 66 _archivePath(basePath / "logs" / "archive") 67 { 68 if (!fs::exists(_logPath)) 69 { 70 fs::create_directories(_logPath); 71 } 72 73 if (!fs::exists(_archivePath)) 74 { 75 fs::create_directories(_archivePath); 76 } 77 78 restore(); 79 } 80 81 void Repository::restore() 82 { 83 for (auto& dirEntry : fs::directory_iterator(_logPath)) 84 { 85 try 86 { 87 if (!fs::is_regular_file(dirEntry.path())) 88 { 89 continue; 90 } 91 92 std::ifstream file{dirEntry.path()}; 93 std::vector<uint8_t> data{std::istreambuf_iterator<char>(file), 94 std::istreambuf_iterator<char>()}; 95 file.close(); 96 97 PEL pel{data}; 98 if (pel.valid()) 99 { 100 // If the host hasn't acked it, reset the host state so 101 // it will get sent up again. 102 if (pel.hostTransmissionState() == TransmissionState::sent) 103 { 104 pel.setHostTransmissionState(TransmissionState::newPEL); 105 try 106 { 107 write(pel, dirEntry.path()); 108 } 109 catch (const std::exception& e) 110 { 111 log<level::ERR>( 112 "Failed to save PEL after updating host state", 113 entry("PELID=0x%X", pel.id())); 114 } 115 } 116 117 PELAttributes attributes{dirEntry.path(), 118 getFileDiskSize(dirEntry.path()), 119 pel.privateHeader().creatorID(), 120 pel.userHeader().subsystem(), 121 pel.userHeader().severity(), 122 pel.userHeader().actionFlags(), 123 pel.hostTransmissionState(), 124 pel.hmcTransmissionState()}; 125 126 using pelID = LogID::Pel; 127 using obmcID = LogID::Obmc; 128 _pelAttributes.emplace( 129 LogID(pelID(pel.id()), obmcID(pel.obmcLogID())), 130 attributes); 131 132 updateRepoStats(attributes, true); 133 } 134 else 135 { 136 log<level::ERR>( 137 "Found invalid PEL file while restoring. Removing.", 138 entry("FILENAME=%s", dirEntry.path().c_str())); 139 fs::remove(dirEntry.path()); 140 } 141 } 142 catch (const std::exception& e) 143 { 144 log<level::ERR>("Hit exception while restoring PEL File", 145 entry("FILENAME=%s", dirEntry.path().c_str()), 146 entry("ERROR=%s", e.what())); 147 } 148 } 149 150 // Get size of archive folder 151 for (auto& dirEntry : fs::directory_iterator(_archivePath)) 152 { 153 _archiveSize += getFileDiskSize(dirEntry); 154 } 155 } 156 157 std::string Repository::getPELFilename(uint32_t pelID, const BCDTime& time) 158 { 159 char name[50]; 160 sprintf(name, "%.2X%.2X%.2X%.2X%.2X%.2X%.2X%.2X_%.8X", time.yearMSB, 161 time.yearLSB, time.month, time.day, time.hour, time.minutes, 162 time.seconds, time.hundredths, pelID); 163 return std::string{name}; 164 } 165 166 void Repository::add(std::unique_ptr<PEL>& pel) 167 { 168 pel->setHostTransmissionState(TransmissionState::newPEL); 169 pel->setHMCTransmissionState(TransmissionState::newPEL); 170 171 auto path = _logPath / getPELFilename(pel->id(), pel->commitTime()); 172 173 write(*(pel.get()), path); 174 175 PELAttributes attributes{path, 176 getFileDiskSize(path), 177 pel->privateHeader().creatorID(), 178 pel->userHeader().subsystem(), 179 pel->userHeader().severity(), 180 pel->userHeader().actionFlags(), 181 pel->hostTransmissionState(), 182 pel->hmcTransmissionState()}; 183 184 using pelID = LogID::Pel; 185 using obmcID = LogID::Obmc; 186 _pelAttributes.emplace(LogID(pelID(pel->id()), obmcID(pel->obmcLogID())), 187 attributes); 188 189 _lastPelID = pel->id(); 190 191 updateRepoStats(attributes, true); 192 193 processAddCallbacks(*pel); 194 } 195 196 void Repository::write(const PEL& pel, const fs::path& path) 197 { 198 std::ofstream file{path, std::ios::binary}; 199 200 if (!file.good()) 201 { 202 // If this fails, the filesystem is probably full so it isn't like 203 // we could successfully create yet another error log here. 204 auto e = errno; 205 fs::remove(path); 206 log<level::ERR>("Unable to open PEL file for writing", 207 entry("ERRNO=%d", e), entry("PATH=%s", path.c_str())); 208 throw file_error::Open(); 209 } 210 211 auto data = pel.data(); 212 file.write(reinterpret_cast<const char*>(data.data()), data.size()); 213 214 if (file.fail()) 215 { 216 // Same note as above about not being able to create an error log 217 // for this case even if we wanted. 218 auto e = errno; 219 file.close(); 220 fs::remove(path); 221 log<level::ERR>("Unable to write PEL file", entry("ERRNO=%d", e), 222 entry("PATH=%s", path.c_str())); 223 throw file_error::Write(); 224 } 225 } 226 227 std::optional<Repository::LogID> Repository::remove(const LogID& id) 228 { 229 auto pel = findPEL(id); 230 if (pel == _pelAttributes.end()) 231 { 232 return std::nullopt; 233 } 234 235 LogID actualID = pel->first; 236 updateRepoStats(pel->second, false); 237 238 log<level::DEBUG>("Removing PEL from repository", 239 entry("PEL_ID=0x%X", actualID.pelID.id), 240 entry("OBMC_LOG_ID=%d", actualID.obmcID.id)); 241 242 if (fs::exists(pel->second.path)) 243 { 244 // Check for existense of new archive folder 245 if (!fs::exists(_archivePath)) 246 { 247 fs::create_directories(_archivePath); 248 } 249 250 // Move log file to archive folder 251 auto fileName = _archivePath / pel->second.path.filename(); 252 fs::rename(pel->second.path, fileName); 253 254 // Update size of file 255 _archiveSize += getFileDiskSize(fileName); 256 } 257 258 _pelAttributes.erase(pel); 259 260 processDeleteCallbacks(actualID.pelID.id); 261 262 return actualID; 263 } 264 265 std::optional<std::vector<uint8_t>> Repository::getPELData(const LogID& id) 266 { 267 auto pel = findPEL(id); 268 if (pel != _pelAttributes.end()) 269 { 270 std::ifstream file{pel->second.path.c_str()}; 271 if (!file.good()) 272 { 273 auto e = errno; 274 log<level::ERR>("Unable to open PEL file", entry("ERRNO=%d", e), 275 entry("PATH=%s", pel->second.path.c_str())); 276 throw file_error::Open(); 277 } 278 279 std::vector<uint8_t> data{std::istreambuf_iterator<char>(file), 280 std::istreambuf_iterator<char>()}; 281 return data; 282 } 283 284 return std::nullopt; 285 } 286 287 std::optional<sdbusplus::message::unix_fd> Repository::getPELFD(const LogID& id) 288 { 289 auto pel = findPEL(id); 290 if (pel != _pelAttributes.end()) 291 { 292 FILE* fp = fopen(pel->second.path.c_str(), "rb"); 293 294 if (fp == nullptr) 295 { 296 auto e = errno; 297 log<level::ERR>("Unable to open PEL File", entry("ERRNO=%d", e), 298 entry("PATH=%s", pel->second.path.c_str())); 299 throw file_error::Open(); 300 } 301 302 // Must leave the file open here. It will be closed by sdbusplus 303 // when it sends it back over D-Bus. 304 305 return fileno(fp); 306 } 307 return std::nullopt; 308 } 309 310 void Repository::for_each(ForEachFunc func) const 311 { 312 for (const auto& [id, attributes] : _pelAttributes) 313 { 314 std::ifstream file{attributes.path}; 315 316 if (!file.good()) 317 { 318 auto e = errno; 319 log<level::ERR>("Repository::for_each: Unable to open PEL file", 320 entry("ERRNO=%d", e), 321 entry("PATH=%s", attributes.path.c_str())); 322 continue; 323 } 324 325 std::vector<uint8_t> data{std::istreambuf_iterator<char>(file), 326 std::istreambuf_iterator<char>()}; 327 file.close(); 328 329 PEL pel{data}; 330 331 try 332 { 333 if (func(pel)) 334 { 335 break; 336 } 337 } 338 catch (const std::exception& e) 339 { 340 log<level::ERR>("Repository::for_each function exception", 341 entry("ERROR=%s", e.what())); 342 } 343 } 344 } 345 346 void Repository::processAddCallbacks(const PEL& pel) const 347 { 348 for (auto& [name, func] : _addSubscriptions) 349 { 350 try 351 { 352 func(pel); 353 } 354 catch (const std::exception& e) 355 { 356 log<level::ERR>("PEL Repository add callback exception", 357 entry("NAME=%s", name.c_str()), 358 entry("ERROR=%s", e.what())); 359 } 360 } 361 } 362 363 void Repository::processDeleteCallbacks(uint32_t id) const 364 { 365 for (auto& [name, func] : _deleteSubscriptions) 366 { 367 try 368 { 369 func(id); 370 } 371 catch (const std::exception& e) 372 { 373 log<level::ERR>("PEL Repository delete callback exception", 374 entry("NAME=%s", name.c_str()), 375 entry("ERROR=%s", e.what())); 376 } 377 } 378 } 379 380 std::optional<std::reference_wrapper<const Repository::PELAttributes>> 381 Repository::getPELAttributes(const LogID& id) const 382 { 383 auto pel = findPEL(id); 384 if (pel != _pelAttributes.end()) 385 { 386 return pel->second; 387 } 388 389 return std::nullopt; 390 } 391 392 void Repository::setPELHostTransState(uint32_t pelID, TransmissionState state) 393 { 394 LogID id{LogID::Pel{pelID}}; 395 auto attr = std::find_if(_pelAttributes.begin(), _pelAttributes.end(), 396 [&id](const auto& a) { return a.first == id; }); 397 398 if ((attr != _pelAttributes.end()) && (attr->second.hostState != state)) 399 { 400 PELUpdateFunc func = [state](PEL& pel) { 401 pel.setHostTransmissionState(state); 402 }; 403 404 try 405 { 406 updatePEL(attr->second.path, func); 407 408 attr->second.hostState = state; 409 } 410 catch (const std::exception& e) 411 { 412 log<level::ERR>("Unable to update PEL host transmission state", 413 entry("PATH=%s", attr->second.path.c_str()), 414 entry("ERROR=%s", e.what())); 415 } 416 } 417 } 418 419 void Repository::setPELHMCTransState(uint32_t pelID, TransmissionState state) 420 { 421 LogID id{LogID::Pel{pelID}}; 422 auto attr = std::find_if(_pelAttributes.begin(), _pelAttributes.end(), 423 [&id](const auto& a) { return a.first == id; }); 424 425 if ((attr != _pelAttributes.end()) && (attr->second.hmcState != state)) 426 { 427 PELUpdateFunc func = [state](PEL& pel) { 428 pel.setHMCTransmissionState(state); 429 }; 430 431 try 432 { 433 updatePEL(attr->second.path, func); 434 435 attr->second.hmcState = state; 436 } 437 catch (const std::exception& e) 438 { 439 log<level::ERR>("Unable to update PEL HMC transmission state", 440 entry("PATH=%s", attr->second.path.c_str()), 441 entry("ERROR=%s", e.what())); 442 } 443 } 444 } 445 446 void Repository::updatePEL(const fs::path& path, PELUpdateFunc updateFunc) 447 { 448 std::ifstream file{path}; 449 std::vector<uint8_t> data{std::istreambuf_iterator<char>(file), 450 std::istreambuf_iterator<char>()}; 451 file.close(); 452 453 PEL pel{data}; 454 455 if (pel.valid()) 456 { 457 updateFunc(pel); 458 459 write(pel, path); 460 } 461 else 462 { 463 throw std::runtime_error( 464 "Unable to read a valid PEL when trying to update it"); 465 } 466 } 467 468 bool Repository::isServiceableSev(const PELAttributes& pel) 469 { 470 auto sevType = static_cast<SeverityType>(pel.severity & 0xF0); 471 auto sevPVEntry = pel_values::findByValue(pel.severity, 472 pel_values::severityValues); 473 std::string sevName = std::get<pel_values::registryNamePos>(*sevPVEntry); 474 475 bool check1 = (sevType == SeverityType::predictive) || 476 (sevType == SeverityType::unrecoverable) || 477 (sevType == SeverityType::critical); 478 479 bool check2 = ((sevType == SeverityType::recovered) || 480 (sevName == "symptom_recovered")) && 481 !pel.actionFlags.test(hiddenFlagBit); 482 483 bool check3 = (sevName == "symptom_predictive") || 484 (sevName == "symptom_unrecoverable") || 485 (sevName == "symptom_critical"); 486 487 return check1 || check2 || check3; 488 } 489 490 void Repository::updateRepoStats(const PELAttributes& pel, bool pelAdded) 491 { 492 auto isServiceable = Repository::isServiceableSev(pel); 493 auto bmcPEL = CreatorID::openBMC == static_cast<CreatorID>(pel.creator); 494 495 auto adjustSize = [pelAdded, &pel](auto& runningSize) { 496 if (pelAdded) 497 { 498 runningSize += pel.sizeOnDisk; 499 } 500 else 501 { 502 runningSize = std::max(static_cast<int64_t>(runningSize) - 503 static_cast<int64_t>(pel.sizeOnDisk), 504 static_cast<int64_t>(0)); 505 } 506 }; 507 508 adjustSize(_sizes.total); 509 510 if (bmcPEL) 511 { 512 adjustSize(_sizes.bmc); 513 if (isServiceable) 514 { 515 adjustSize(_sizes.bmcServiceable); 516 } 517 else 518 { 519 adjustSize(_sizes.bmcInfo); 520 } 521 } 522 else 523 { 524 adjustSize(_sizes.nonBMC); 525 if (isServiceable) 526 { 527 adjustSize(_sizes.nonBMCServiceable); 528 } 529 else 530 { 531 adjustSize(_sizes.nonBMCInfo); 532 } 533 } 534 } 535 536 bool Repository::sizeWarning() 537 { 538 std::error_code ec; 539 540 if ((_archiveSize > 0) && ((_sizes.total + _archiveSize) > 541 ((_maxRepoSize * warningPercentage) / 100))) 542 { 543 log<level::INFO>( 544 "Repository::sizeWarning function:Deleting the files in archive"); 545 546 for (const auto& dirEntry : fs::directory_iterator(_archivePath)) 547 { 548 fs::remove(dirEntry.path(), ec); 549 if (ec) 550 { 551 log<level::INFO>( 552 "Repository::sizeWarning function:Could not delete " 553 "a file in PEL archive", 554 entry("FILENAME=%s", dirEntry.path().c_str())); 555 } 556 } 557 558 _archiveSize = 0; 559 } 560 561 return (_sizes.total > (_maxRepoSize * warningPercentage / 100)) || 562 (_pelAttributes.size() > _maxNumPELs); 563 } 564 565 std::vector<Repository::AttributesReference> 566 Repository::getAllPELAttributes(SortOrder order) const 567 { 568 std::vector<Repository::AttributesReference> attributes; 569 570 std::for_each( 571 _pelAttributes.begin(), _pelAttributes.end(), 572 [&attributes](auto& pelEntry) { attributes.push_back(pelEntry); }); 573 574 std::sort(attributes.begin(), attributes.end(), 575 [order](const auto& left, const auto& right) { 576 if (order == SortOrder::ascending) 577 { 578 return left.get().second.path < right.get().second.path; 579 } 580 return left.get().second.path > right.get().second.path; 581 }); 582 583 return attributes; 584 } 585 586 std::vector<uint32_t> 587 Repository::prune(const std::vector<uint32_t>& idsWithHwIsoEntry) 588 { 589 std::vector<uint32_t> obmcLogIDs; 590 std::string msg = "Pruning PEL repository that takes up " + 591 std::to_string(_sizes.total) + " bytes and has " + 592 std::to_string(_pelAttributes.size()) + " PELs"; 593 log<level::INFO>(msg.c_str()); 594 595 // Set up the 5 functions to check if the PEL category 596 // is still over its limits. 597 598 // BMC informational PELs should only take up 15% 599 IsOverLimitFunc overBMCInfoLimit = [this]() { 600 return _sizes.bmcInfo > _maxRepoSize * 15 / 100; 601 }; 602 603 // BMC non informational PELs should only take up 30% 604 IsOverLimitFunc overBMCNonInfoLimit = [this]() { 605 return _sizes.bmcServiceable > _maxRepoSize * 30 / 100; 606 }; 607 608 // Non BMC informational PELs should only take up 15% 609 IsOverLimitFunc overNonBMCInfoLimit = [this]() { 610 return _sizes.nonBMCInfo > _maxRepoSize * 15 / 100; 611 }; 612 613 // Non BMC non informational PELs should only take up 15% 614 IsOverLimitFunc overNonBMCNonInfoLimit = [this]() { 615 return _sizes.nonBMCServiceable > _maxRepoSize * 30 / 100; 616 }; 617 618 // Bring the total number of PELs down to 80% of the max 619 IsOverLimitFunc tooManyPELsLimit = [this]() { 620 return _pelAttributes.size() > _maxNumPELs * 80 / 100; 621 }; 622 623 // Set up the functions to determine which category a PEL is in. 624 // TODO: Return false in these functions if a PEL caused a guard record. 625 626 // A BMC informational PEL 627 IsPELTypeFunc isBMCInfo = [](const PELAttributes& pel) { 628 return (CreatorID::openBMC == static_cast<CreatorID>(pel.creator)) && 629 !Repository::isServiceableSev(pel); 630 }; 631 632 // A BMC non informational PEL 633 IsPELTypeFunc isBMCNonInfo = [](const PELAttributes& pel) { 634 return (CreatorID::openBMC == static_cast<CreatorID>(pel.creator)) && 635 Repository::isServiceableSev(pel); 636 }; 637 638 // A non BMC informational PEL 639 IsPELTypeFunc isNonBMCInfo = [](const PELAttributes& pel) { 640 return (CreatorID::openBMC != static_cast<CreatorID>(pel.creator)) && 641 !Repository::isServiceableSev(pel); 642 }; 643 644 // A non BMC non informational PEL 645 IsPELTypeFunc isNonBMCNonInfo = [](const PELAttributes& pel) { 646 return (CreatorID::openBMC != static_cast<CreatorID>(pel.creator)) && 647 Repository::isServiceableSev(pel); 648 }; 649 650 // When counting PELs, count every PEL 651 IsPELTypeFunc isAnyPEL = [](const PELAttributes& /*pel*/) { return true; }; 652 653 // Check all 4 categories, which will result in at most 90% 654 // usage (15 + 30 + 15 + 30). 655 removePELs(overBMCInfoLimit, isBMCInfo, idsWithHwIsoEntry, obmcLogIDs); 656 removePELs(overBMCNonInfoLimit, isBMCNonInfo, idsWithHwIsoEntry, 657 obmcLogIDs); 658 removePELs(overNonBMCInfoLimit, isNonBMCInfo, idsWithHwIsoEntry, 659 obmcLogIDs); 660 removePELs(overNonBMCNonInfoLimit, isNonBMCNonInfo, idsWithHwIsoEntry, 661 obmcLogIDs); 662 663 // After the above pruning check if there are still too many PELs, 664 // which can happen depending on PEL sizes. 665 if (_pelAttributes.size() > _maxNumPELs) 666 { 667 removePELs(tooManyPELsLimit, isAnyPEL, idsWithHwIsoEntry, obmcLogIDs); 668 } 669 670 if (!obmcLogIDs.empty()) 671 { 672 std::string m = "Number of PELs removed to save space: " + 673 std::to_string(obmcLogIDs.size()); 674 log<level::INFO>(m.c_str()); 675 } 676 677 return obmcLogIDs; 678 } 679 680 void Repository::removePELs(const IsOverLimitFunc& isOverLimit, 681 const IsPELTypeFunc& isPELType, 682 const std::vector<uint32_t>& idsWithHwIsoEntry, 683 std::vector<uint32_t>& removedBMCLogIDs) 684 { 685 if (!isOverLimit()) 686 { 687 return; 688 } 689 690 auto attributes = getAllPELAttributes(SortOrder::ascending); 691 692 // Make 4 passes on the PELs, stopping as soon as isOverLimit 693 // returns false. 694 // Pass 1: only delete HMC acked PELs 695 // Pass 2: only delete OS acked PELs 696 // Pass 3: only delete PHYP sent PELs 697 // Pass 4: delete all PELs 698 static const std::vector<std::function<bool(const PELAttributes& pel)>> 699 stateChecks{[](const auto& pel) { 700 return pel.hmcState == TransmissionState::acked; 701 }, 702 703 [](const auto& pel) { 704 return pel.hostState == TransmissionState::acked; 705 }, 706 707 [](const auto& pel) { 708 return pel.hostState == TransmissionState::sent; 709 }, 710 711 [](const auto& /*pel*/) { return true; }}; 712 713 for (const auto& stateCheck : stateChecks) 714 { 715 for (auto it = attributes.begin(); it != attributes.end();) 716 { 717 const auto& pel = it->get(); 718 if (isPELType(pel.second) && stateCheck(pel.second)) 719 { 720 auto removedID = pel.first.obmcID.id; 721 722 auto idFound = std::find(idsWithHwIsoEntry.begin(), 723 idsWithHwIsoEntry.end(), removedID); 724 if (idFound != idsWithHwIsoEntry.end()) 725 { 726 ++it; 727 continue; 728 } 729 730 remove(pel.first); 731 732 removedBMCLogIDs.push_back(removedID); 733 734 attributes.erase(it); 735 736 if (!isOverLimit()) 737 { 738 break; 739 } 740 } 741 else 742 { 743 ++it; 744 } 745 } 746 747 if (!isOverLimit()) 748 { 749 break; 750 } 751 } 752 } 753 754 void Repository::archivePEL(const PEL& pel) 755 { 756 if (pel.valid()) 757 { 758 auto path = _archivePath / getPELFilename(pel.id(), pel.commitTime()); 759 760 write(pel, path); 761 762 _archiveSize += getFileDiskSize(path); 763 } 764 } 765 766 } // namespace pels 767 } // namespace openpower 768