1 /** 2 * Copyright © 2019 IBM Corporation 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #include "repository.hpp" 17 18 #include <fcntl.h> 19 #include <sys/stat.h> 20 21 #include <phosphor-logging/log.hpp> 22 #include <xyz/openbmc_project/Common/File/error.hpp> 23 24 #include <fstream> 25 26 namespace openpower 27 { 28 namespace pels 29 { 30 31 namespace fs = std::filesystem; 32 using namespace phosphor::logging; 33 namespace file_error = sdbusplus::xyz::openbmc_project::Common::File::Error; 34 35 constexpr size_t warningPercentage = 95; 36 37 /** 38 * @brief Returns the amount of space the file uses on disk. 39 * 40 * This is different than just the regular size of the file. 41 * 42 * @param[in] file - The file to get the size of 43 * 44 * @return size_t The disk space the file uses 45 */ 46 size_t getFileDiskSize(const std::filesystem::path& file) 47 { 48 constexpr size_t statBlockSize = 512; 49 struct stat statData; 50 auto rc = stat(file.c_str(), &statData); 51 if (rc != 0) 52 { 53 auto e = errno; 54 std::string msg = "call to stat() failed on " + file.native() + 55 " with errno " + std::to_string(e); 56 log<level::ERR>(msg.c_str()); 57 abort(); 58 } 59 60 return statData.st_blocks * statBlockSize; 61 } 62 63 Repository::Repository(const std::filesystem::path& basePath, size_t repoSize, 64 size_t maxNumPELs) : 65 _logPath(basePath / "logs"), 66 _maxRepoSize(repoSize), _maxNumPELs(maxNumPELs), 67 _archivePath(basePath / "logs" / "archive") 68 { 69 if (!fs::exists(_logPath)) 70 { 71 fs::create_directories(_logPath); 72 } 73 74 if (!fs::exists(_archivePath)) 75 { 76 fs::create_directories(_archivePath); 77 } 78 79 restore(); 80 } 81 82 void Repository::restore() 83 { 84 for (auto& dirEntry : fs::directory_iterator(_logPath)) 85 { 86 try 87 { 88 if (!fs::is_regular_file(dirEntry.path())) 89 { 90 continue; 91 } 92 93 std::ifstream file{dirEntry.path()}; 94 std::vector<uint8_t> data{std::istreambuf_iterator<char>(file), 95 std::istreambuf_iterator<char>()}; 96 file.close(); 97 98 PEL pel{data}; 99 if (pel.valid()) 100 { 101 // If the host hasn't acked it, reset the host state so 102 // it will get sent up again. 103 if (pel.hostTransmissionState() == TransmissionState::sent) 104 { 105 pel.setHostTransmissionState(TransmissionState::newPEL); 106 try 107 { 108 write(pel, dirEntry.path()); 109 } 110 catch (const std::exception& e) 111 { 112 log<level::ERR>( 113 "Failed to save PEL after updating host state", 114 entry("PELID=0x%X", pel.id())); 115 } 116 } 117 118 PELAttributes attributes{dirEntry.path(), 119 getFileDiskSize(dirEntry.path()), 120 pel.privateHeader().creatorID(), 121 pel.userHeader().subsystem(), 122 pel.userHeader().severity(), 123 pel.userHeader().actionFlags(), 124 pel.hostTransmissionState(), 125 pel.hmcTransmissionState()}; 126 127 using pelID = LogID::Pel; 128 using obmcID = LogID::Obmc; 129 _pelAttributes.emplace( 130 LogID(pelID(pel.id()), obmcID(pel.obmcLogID())), 131 attributes); 132 133 updateRepoStats(attributes, true); 134 } 135 else 136 { 137 log<level::ERR>( 138 "Found invalid PEL file while restoring. Removing.", 139 entry("FILENAME=%s", dirEntry.path().c_str())); 140 fs::remove(dirEntry.path()); 141 } 142 } 143 catch (const std::exception& e) 144 { 145 log<level::ERR>("Hit exception while restoring PEL File", 146 entry("FILENAME=%s", dirEntry.path().c_str()), 147 entry("ERROR=%s", e.what())); 148 } 149 } 150 151 // Get size of archive folder 152 for (auto& dirEntry : fs::directory_iterator(_archivePath)) 153 { 154 _archiveSize += getFileDiskSize(dirEntry); 155 } 156 } 157 158 std::string Repository::getPELFilename(uint32_t pelID, const BCDTime& time) 159 { 160 char name[50]; 161 sprintf(name, "%.2X%.2X%.2X%.2X%.2X%.2X%.2X%.2X_%.8X", time.yearMSB, 162 time.yearLSB, time.month, time.day, time.hour, time.minutes, 163 time.seconds, time.hundredths, pelID); 164 return std::string{name}; 165 } 166 167 void Repository::add(std::unique_ptr<PEL>& pel) 168 { 169 pel->setHostTransmissionState(TransmissionState::newPEL); 170 pel->setHMCTransmissionState(TransmissionState::newPEL); 171 172 auto path = _logPath / getPELFilename(pel->id(), pel->commitTime()); 173 174 write(*(pel.get()), path); 175 176 PELAttributes attributes{path, 177 getFileDiskSize(path), 178 pel->privateHeader().creatorID(), 179 pel->userHeader().subsystem(), 180 pel->userHeader().severity(), 181 pel->userHeader().actionFlags(), 182 pel->hostTransmissionState(), 183 pel->hmcTransmissionState()}; 184 185 using pelID = LogID::Pel; 186 using obmcID = LogID::Obmc; 187 _pelAttributes.emplace(LogID(pelID(pel->id()), obmcID(pel->obmcLogID())), 188 attributes); 189 190 _lastPelID = pel->id(); 191 192 updateRepoStats(attributes, true); 193 194 processAddCallbacks(*pel); 195 } 196 197 void Repository::write(const PEL& pel, const fs::path& path) 198 { 199 std::ofstream file{path, std::ios::binary}; 200 201 if (!file.good()) 202 { 203 // If this fails, the filesystem is probably full so it isn't like 204 // we could successfully create yet another error log here. 205 auto e = errno; 206 fs::remove(path); 207 log<level::ERR>("Unable to open PEL file for writing", 208 entry("ERRNO=%d", e), entry("PATH=%s", path.c_str())); 209 throw file_error::Open(); 210 } 211 212 auto data = pel.data(); 213 file.write(reinterpret_cast<const char*>(data.data()), data.size()); 214 215 if (file.fail()) 216 { 217 // Same note as above about not being able to create an error log 218 // for this case even if we wanted. 219 auto e = errno; 220 file.close(); 221 fs::remove(path); 222 log<level::ERR>("Unable to write PEL file", entry("ERRNO=%d", e), 223 entry("PATH=%s", path.c_str())); 224 throw file_error::Write(); 225 } 226 } 227 228 std::optional<Repository::LogID> Repository::remove(const LogID& id) 229 { 230 auto pel = findPEL(id); 231 if (pel == _pelAttributes.end()) 232 { 233 return std::nullopt; 234 } 235 236 LogID actualID = pel->first; 237 updateRepoStats(pel->second, false); 238 239 log<level::DEBUG>("Removing PEL from repository", 240 entry("PEL_ID=0x%X", actualID.pelID.id), 241 entry("OBMC_LOG_ID=%d", actualID.obmcID.id)); 242 243 if (fs::exists(pel->second.path)) 244 { 245 // Check for existense of new archive folder 246 if (!fs::exists(_archivePath)) 247 { 248 fs::create_directories(_archivePath); 249 } 250 251 // Move log file to archive folder 252 auto fileName = _archivePath / pel->second.path.filename(); 253 fs::rename(pel->second.path, fileName); 254 255 // Update size of file 256 _archiveSize += getFileDiskSize(fileName); 257 } 258 259 _pelAttributes.erase(pel); 260 261 processDeleteCallbacks(actualID.pelID.id); 262 263 return actualID; 264 } 265 266 std::optional<std::vector<uint8_t>> Repository::getPELData(const LogID& id) 267 { 268 auto pel = findPEL(id); 269 if (pel != _pelAttributes.end()) 270 { 271 std::ifstream file{pel->second.path.c_str()}; 272 if (!file.good()) 273 { 274 auto e = errno; 275 log<level::ERR>("Unable to open PEL file", entry("ERRNO=%d", e), 276 entry("PATH=%s", pel->second.path.c_str())); 277 throw file_error::Open(); 278 } 279 280 std::vector<uint8_t> data{std::istreambuf_iterator<char>(file), 281 std::istreambuf_iterator<char>()}; 282 return data; 283 } 284 285 return std::nullopt; 286 } 287 288 std::optional<sdbusplus::message::unix_fd> Repository::getPELFD(const LogID& id) 289 { 290 auto pel = findPEL(id); 291 if (pel != _pelAttributes.end()) 292 { 293 int fd = open(pel->second.path.c_str(), O_RDONLY | O_NONBLOCK); 294 if (fd == -1) 295 { 296 auto e = errno; 297 log<level::ERR>("Unable to open PEL File", entry("ERRNO=%d", e), 298 entry("PATH=%s", pel->second.path.c_str())); 299 throw file_error::Open(); 300 } 301 302 // Must leave the file open here. It will be closed by sdbusplus 303 // when it sends it back over D-Bus. 304 return fd; 305 } 306 return std::nullopt; 307 } 308 309 void Repository::for_each(ForEachFunc func) const 310 { 311 for (const auto& [id, attributes] : _pelAttributes) 312 { 313 std::ifstream file{attributes.path}; 314 315 if (!file.good()) 316 { 317 auto e = errno; 318 log<level::ERR>("Repository::for_each: Unable to open PEL file", 319 entry("ERRNO=%d", e), 320 entry("PATH=%s", attributes.path.c_str())); 321 continue; 322 } 323 324 std::vector<uint8_t> data{std::istreambuf_iterator<char>(file), 325 std::istreambuf_iterator<char>()}; 326 file.close(); 327 328 PEL pel{data}; 329 330 try 331 { 332 if (func(pel)) 333 { 334 break; 335 } 336 } 337 catch (const std::exception& e) 338 { 339 log<level::ERR>("Repository::for_each function exception", 340 entry("ERROR=%s", e.what())); 341 } 342 } 343 } 344 345 void Repository::processAddCallbacks(const PEL& pel) const 346 { 347 for (auto& [name, func] : _addSubscriptions) 348 { 349 try 350 { 351 func(pel); 352 } 353 catch (const std::exception& e) 354 { 355 log<level::ERR>("PEL Repository add callback exception", 356 entry("NAME=%s", name.c_str()), 357 entry("ERROR=%s", e.what())); 358 } 359 } 360 } 361 362 void Repository::processDeleteCallbacks(uint32_t id) const 363 { 364 for (auto& [name, func] : _deleteSubscriptions) 365 { 366 try 367 { 368 func(id); 369 } 370 catch (const std::exception& e) 371 { 372 log<level::ERR>("PEL Repository delete callback exception", 373 entry("NAME=%s", name.c_str()), 374 entry("ERROR=%s", e.what())); 375 } 376 } 377 } 378 379 std::optional<std::reference_wrapper<const Repository::PELAttributes>> 380 Repository::getPELAttributes(const LogID& id) const 381 { 382 auto pel = findPEL(id); 383 if (pel != _pelAttributes.end()) 384 { 385 return pel->second; 386 } 387 388 return std::nullopt; 389 } 390 391 void Repository::setPELHostTransState(uint32_t pelID, TransmissionState state) 392 { 393 LogID id{LogID::Pel{pelID}}; 394 auto attr = std::find_if(_pelAttributes.begin(), _pelAttributes.end(), 395 [&id](const auto& a) { return a.first == id; }); 396 397 if ((attr != _pelAttributes.end()) && (attr->second.hostState != state)) 398 { 399 PELUpdateFunc func = [state](PEL& pel) { 400 pel.setHostTransmissionState(state); 401 }; 402 403 try 404 { 405 updatePEL(attr->second.path, func); 406 407 attr->second.hostState = state; 408 } 409 catch (const std::exception& e) 410 { 411 log<level::ERR>("Unable to update PEL host transmission state", 412 entry("PATH=%s", attr->second.path.c_str()), 413 entry("ERROR=%s", e.what())); 414 } 415 } 416 } 417 418 void Repository::setPELHMCTransState(uint32_t pelID, TransmissionState state) 419 { 420 LogID id{LogID::Pel{pelID}}; 421 auto attr = std::find_if(_pelAttributes.begin(), _pelAttributes.end(), 422 [&id](const auto& a) { return a.first == id; }); 423 424 if ((attr != _pelAttributes.end()) && (attr->second.hmcState != state)) 425 { 426 PELUpdateFunc func = [state](PEL& pel) { 427 pel.setHMCTransmissionState(state); 428 }; 429 430 try 431 { 432 updatePEL(attr->second.path, func); 433 434 attr->second.hmcState = state; 435 } 436 catch (const std::exception& e) 437 { 438 log<level::ERR>("Unable to update PEL HMC transmission state", 439 entry("PATH=%s", attr->second.path.c_str()), 440 entry("ERROR=%s", e.what())); 441 } 442 } 443 } 444 445 void Repository::updatePEL(const fs::path& path, PELUpdateFunc updateFunc) 446 { 447 std::ifstream file{path}; 448 std::vector<uint8_t> data{std::istreambuf_iterator<char>(file), 449 std::istreambuf_iterator<char>()}; 450 file.close(); 451 452 PEL pel{data}; 453 454 if (pel.valid()) 455 { 456 updateFunc(pel); 457 458 write(pel, path); 459 } 460 else 461 { 462 throw std::runtime_error( 463 "Unable to read a valid PEL when trying to update it"); 464 } 465 } 466 467 bool Repository::isServiceableSev(const PELAttributes& pel) 468 { 469 auto sevType = static_cast<SeverityType>(pel.severity & 0xF0); 470 auto sevPVEntry = pel_values::findByValue(pel.severity, 471 pel_values::severityValues); 472 std::string sevName = std::get<pel_values::registryNamePos>(*sevPVEntry); 473 474 bool check1 = (sevType == SeverityType::predictive) || 475 (sevType == SeverityType::unrecoverable) || 476 (sevType == SeverityType::critical); 477 478 bool check2 = ((sevType == SeverityType::recovered) || 479 (sevName == "symptom_recovered")) && 480 !pel.actionFlags.test(hiddenFlagBit); 481 482 bool check3 = (sevName == "symptom_predictive") || 483 (sevName == "symptom_unrecoverable") || 484 (sevName == "symptom_critical"); 485 486 return check1 || check2 || check3; 487 } 488 489 void Repository::updateRepoStats(const PELAttributes& pel, bool pelAdded) 490 { 491 auto isServiceable = Repository::isServiceableSev(pel); 492 auto bmcPEL = CreatorID::openBMC == static_cast<CreatorID>(pel.creator); 493 494 auto adjustSize = [pelAdded, &pel](auto& runningSize) { 495 if (pelAdded) 496 { 497 runningSize += pel.sizeOnDisk; 498 } 499 else 500 { 501 runningSize = std::max(static_cast<int64_t>(runningSize) - 502 static_cast<int64_t>(pel.sizeOnDisk), 503 static_cast<int64_t>(0)); 504 } 505 }; 506 507 adjustSize(_sizes.total); 508 509 if (bmcPEL) 510 { 511 adjustSize(_sizes.bmc); 512 if (isServiceable) 513 { 514 adjustSize(_sizes.bmcServiceable); 515 } 516 else 517 { 518 adjustSize(_sizes.bmcInfo); 519 } 520 } 521 else 522 { 523 adjustSize(_sizes.nonBMC); 524 if (isServiceable) 525 { 526 adjustSize(_sizes.nonBMCServiceable); 527 } 528 else 529 { 530 adjustSize(_sizes.nonBMCInfo); 531 } 532 } 533 } 534 535 bool Repository::sizeWarning() 536 { 537 std::error_code ec; 538 539 if ((_archiveSize > 0) && ((_sizes.total + _archiveSize) > 540 ((_maxRepoSize * warningPercentage) / 100))) 541 { 542 log<level::INFO>( 543 "Repository::sizeWarning function:Deleting the files in archive"); 544 545 for (const auto& dirEntry : fs::directory_iterator(_archivePath)) 546 { 547 fs::remove(dirEntry.path(), ec); 548 if (ec) 549 { 550 log<level::INFO>( 551 "Repository::sizeWarning function:Could not delete " 552 "a file in PEL archive", 553 entry("FILENAME=%s", dirEntry.path().c_str())); 554 } 555 } 556 557 _archiveSize = 0; 558 } 559 560 return (_sizes.total > (_maxRepoSize * warningPercentage / 100)) || 561 (_pelAttributes.size() > _maxNumPELs); 562 } 563 564 std::vector<Repository::AttributesReference> 565 Repository::getAllPELAttributes(SortOrder order) const 566 { 567 std::vector<Repository::AttributesReference> attributes; 568 569 std::for_each( 570 _pelAttributes.begin(), _pelAttributes.end(), 571 [&attributes](auto& pelEntry) { attributes.push_back(pelEntry); }); 572 573 std::sort(attributes.begin(), attributes.end(), 574 [order](const auto& left, const auto& right) { 575 if (order == SortOrder::ascending) 576 { 577 return left.get().second.path < right.get().second.path; 578 } 579 return left.get().second.path > right.get().second.path; 580 }); 581 582 return attributes; 583 } 584 585 std::vector<uint32_t> 586 Repository::prune(const std::vector<uint32_t>& idsWithHwIsoEntry) 587 { 588 std::vector<uint32_t> obmcLogIDs; 589 std::string msg = "Pruning PEL repository that takes up " + 590 std::to_string(_sizes.total) + " bytes and has " + 591 std::to_string(_pelAttributes.size()) + " PELs"; 592 log<level::INFO>(msg.c_str()); 593 594 // Set up the 5 functions to check if the PEL category 595 // is still over its limits. 596 597 // BMC informational PELs should only take up 15% 598 IsOverLimitFunc overBMCInfoLimit = [this]() { 599 return _sizes.bmcInfo > _maxRepoSize * 15 / 100; 600 }; 601 602 // BMC non informational PELs should only take up 30% 603 IsOverLimitFunc overBMCNonInfoLimit = [this]() { 604 return _sizes.bmcServiceable > _maxRepoSize * 30 / 100; 605 }; 606 607 // Non BMC informational PELs should only take up 15% 608 IsOverLimitFunc overNonBMCInfoLimit = [this]() { 609 return _sizes.nonBMCInfo > _maxRepoSize * 15 / 100; 610 }; 611 612 // Non BMC non informational PELs should only take up 15% 613 IsOverLimitFunc overNonBMCNonInfoLimit = [this]() { 614 return _sizes.nonBMCServiceable > _maxRepoSize * 30 / 100; 615 }; 616 617 // Bring the total number of PELs down to 80% of the max 618 IsOverLimitFunc tooManyPELsLimit = [this]() { 619 return _pelAttributes.size() > _maxNumPELs * 80 / 100; 620 }; 621 622 // Set up the functions to determine which category a PEL is in. 623 // TODO: Return false in these functions if a PEL caused a guard record. 624 625 // A BMC informational PEL 626 IsPELTypeFunc isBMCInfo = [](const PELAttributes& pel) { 627 return (CreatorID::openBMC == static_cast<CreatorID>(pel.creator)) && 628 !Repository::isServiceableSev(pel); 629 }; 630 631 // A BMC non informational PEL 632 IsPELTypeFunc isBMCNonInfo = [](const PELAttributes& pel) { 633 return (CreatorID::openBMC == static_cast<CreatorID>(pel.creator)) && 634 Repository::isServiceableSev(pel); 635 }; 636 637 // A non BMC informational PEL 638 IsPELTypeFunc isNonBMCInfo = [](const PELAttributes& pel) { 639 return (CreatorID::openBMC != static_cast<CreatorID>(pel.creator)) && 640 !Repository::isServiceableSev(pel); 641 }; 642 643 // A non BMC non informational PEL 644 IsPELTypeFunc isNonBMCNonInfo = [](const PELAttributes& pel) { 645 return (CreatorID::openBMC != static_cast<CreatorID>(pel.creator)) && 646 Repository::isServiceableSev(pel); 647 }; 648 649 // When counting PELs, count every PEL 650 IsPELTypeFunc isAnyPEL = [](const PELAttributes& /*pel*/) { return true; }; 651 652 // Check all 4 categories, which will result in at most 90% 653 // usage (15 + 30 + 15 + 30). 654 removePELs(overBMCInfoLimit, isBMCInfo, idsWithHwIsoEntry, obmcLogIDs); 655 removePELs(overBMCNonInfoLimit, isBMCNonInfo, idsWithHwIsoEntry, 656 obmcLogIDs); 657 removePELs(overNonBMCInfoLimit, isNonBMCInfo, idsWithHwIsoEntry, 658 obmcLogIDs); 659 removePELs(overNonBMCNonInfoLimit, isNonBMCNonInfo, idsWithHwIsoEntry, 660 obmcLogIDs); 661 662 // After the above pruning check if there are still too many PELs, 663 // which can happen depending on PEL sizes. 664 if (_pelAttributes.size() > _maxNumPELs) 665 { 666 removePELs(tooManyPELsLimit, isAnyPEL, idsWithHwIsoEntry, obmcLogIDs); 667 } 668 669 if (!obmcLogIDs.empty()) 670 { 671 std::string m = "Number of PELs removed to save space: " + 672 std::to_string(obmcLogIDs.size()); 673 log<level::INFO>(m.c_str()); 674 } 675 676 return obmcLogIDs; 677 } 678 679 void Repository::removePELs(const IsOverLimitFunc& isOverLimit, 680 const IsPELTypeFunc& isPELType, 681 const std::vector<uint32_t>& idsWithHwIsoEntry, 682 std::vector<uint32_t>& removedBMCLogIDs) 683 { 684 if (!isOverLimit()) 685 { 686 return; 687 } 688 689 auto attributes = getAllPELAttributes(SortOrder::ascending); 690 691 // Make 4 passes on the PELs, stopping as soon as isOverLimit 692 // returns false. 693 // Pass 1: only delete HMC acked PELs 694 // Pass 2: only delete OS acked PELs 695 // Pass 3: only delete PHYP sent PELs 696 // Pass 4: delete all PELs 697 static const std::vector<std::function<bool(const PELAttributes& pel)>> 698 stateChecks{[](const auto& pel) { 699 return pel.hmcState == TransmissionState::acked; 700 }, 701 702 [](const auto& pel) { 703 return pel.hostState == TransmissionState::acked; 704 }, 705 706 [](const auto& pel) { 707 return pel.hostState == TransmissionState::sent; 708 }, 709 710 [](const auto& /*pel*/) { return true; }}; 711 712 for (const auto& stateCheck : stateChecks) 713 { 714 for (auto it = attributes.begin(); it != attributes.end();) 715 { 716 const auto& pel = it->get(); 717 if (isPELType(pel.second) && stateCheck(pel.second)) 718 { 719 auto removedID = pel.first.obmcID.id; 720 721 auto idFound = std::find(idsWithHwIsoEntry.begin(), 722 idsWithHwIsoEntry.end(), removedID); 723 if (idFound != idsWithHwIsoEntry.end()) 724 { 725 ++it; 726 continue; 727 } 728 729 remove(pel.first); 730 731 removedBMCLogIDs.push_back(removedID); 732 733 attributes.erase(it); 734 735 if (!isOverLimit()) 736 { 737 break; 738 } 739 } 740 else 741 { 742 ++it; 743 } 744 } 745 746 if (!isOverLimit()) 747 { 748 break; 749 } 750 } 751 } 752 753 void Repository::archivePEL(const PEL& pel) 754 { 755 if (pel.valid()) 756 { 757 auto path = _archivePath / getPELFilename(pel.id(), pel.commitTime()); 758 759 write(pel, path); 760 761 _archiveSize += getFileDiskSize(path); 762 } 763 } 764 765 } // namespace pels 766 } // namespace openpower 767