1 /* 2 // Copyright (c) 2019 Intel Corporation 3 // 4 // Licensed under the Apache License, Version 2.0 (the "License"); 5 // you may not use this file except in compliance with the License. 6 // You may obtain a copy of the License at 7 // 8 // http://www.apache.org/licenses/LICENSE-2.0 9 // 10 // Unless required by applicable law or agreed to in writing, software 11 // distributed under the License is distributed on an "AS IS" BASIS, 12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 // See the License for the specific language governing permissions and 14 // limitations under the License. 15 */ 16 17 #include "utils.hpp" 18 19 #include <bitset> 20 #include <boost/algorithm/string/replace.hpp> 21 #include <boost/asio/posix/stream_descriptor.hpp> 22 #include <boost/asio/steady_timer.hpp> 23 #include <boost/container/flat_set.hpp> 24 #include <filesystem> 25 #include <fstream> 26 #include <gpiod.hpp> 27 #include <iostream> 28 #include <sdbusplus/asio/connection.hpp> 29 #include <sdbusplus/asio/object_server.hpp> 30 #include <sdbusplus/bus/match.hpp> 31 #include <string> 32 #include <utility> 33 34 extern "C" { 35 #include <i2c/smbus.h> 36 #include <linux/i2c-dev.h> 37 } 38 39 constexpr const char* configType = 40 "xyz.openbmc_project.Configuration.Intel_HSBP_CPLD"; 41 constexpr const char* busName = "xyz.openbmc_project.HsbpManager"; 42 43 constexpr size_t scanRateSeconds = 5; 44 constexpr size_t maxDrives = 8; // only 1 byte alloted 45 46 boost::asio::io_context io; 47 auto conn = std::make_shared<sdbusplus::asio::connection>(io); 48 sdbusplus::asio::object_server objServer(conn); 49 50 // GPIO Lines and Event Descriptors 51 static gpiod::line nvmeLvc3AlertLine; 52 static boost::asio::posix::stream_descriptor nvmeLvc3AlertEvent(io); 53 54 static std::string zeroPad(const uint8_t val) 55 { 56 std::ostringstream version; 57 version << std::setw(2) << std::setfill('0') << static_cast<size_t>(val); 58 return version.str(); 59 } 60 61 struct Mux 62 { 63 Mux(size_t busIn, size_t addressIn, size_t channelsIn, size_t indexIn) : 64 bus(busIn), address(addressIn), channels(channelsIn), index(indexIn) 65 { 66 } 67 size_t bus; 68 size_t address; 69 size_t channels; 70 size_t index; 71 72 // to sort in the flat set 73 bool operator<(const Mux& rhs) const 74 { 75 return index < rhs.index; 76 } 77 }; 78 79 enum class BlinkPattern : uint8_t 80 { 81 off = 0x0, 82 error = 0x2, 83 terminate = 0x3 84 }; 85 86 struct Led : std::enable_shared_from_this<Led> 87 { 88 // led pattern addresses start at 0x10 89 Led(const std::string& path, size_t index, int fd) : 90 address(static_cast<uint8_t>(index + 0x10)), file(fd), 91 ledInterface(objServer.add_interface(path, ledGroup::interface)) 92 { 93 if (index >= maxDrives) 94 { 95 throw std::runtime_error("Invalid drive index"); 96 } 97 98 if (!set(BlinkPattern::off)) 99 { 100 std::cerr << "Cannot initialize LED " << path << "\n"; 101 } 102 } 103 104 // this has to be called outside the constructor for shared_from_this to 105 // work 106 void createInterface(void) 107 { 108 std::shared_ptr<Led> self = shared_from_this(); 109 110 ledInterface->register_property( 111 ledGroup::asserted, false, [self](const bool req, bool& val) { 112 if (req == val) 113 { 114 return 1; 115 } 116 117 if (!isPowerOn()) 118 { 119 std::cerr << "Can't change blink state when power is off\n"; 120 throw std::runtime_error( 121 "Can't change blink state when power is off"); 122 } 123 BlinkPattern pattern = 124 req ? BlinkPattern::error : BlinkPattern::terminate; 125 if (!self->set(pattern)) 126 { 127 std::cerr << "Can't change blink pattern\n"; 128 throw std::runtime_error("Cannot set blink pattern"); 129 } 130 val = req; 131 return 1; 132 }); 133 ledInterface->initialize(); 134 } 135 136 virtual ~Led() 137 { 138 objServer.remove_interface(ledInterface); 139 } 140 141 bool set(BlinkPattern pattern) 142 { 143 int ret = i2c_smbus_write_byte_data(file, address, 144 static_cast<uint8_t>(pattern)); 145 return ret >= 0; 146 } 147 148 uint8_t address; 149 int file; 150 std::shared_ptr<sdbusplus::asio::dbus_interface> ledInterface; 151 }; 152 153 struct Drive 154 { 155 Drive(size_t driveIndex, bool present, bool isOperational, bool nvme, 156 bool rebuilding) : 157 isNvme(nvme), 158 isPresent(present), index(driveIndex) 159 { 160 constexpr const char* basePath = 161 "/xyz/openbmc_project/inventory/item/drive/Drive_"; 162 itemIface = objServer.add_interface( 163 basePath + std::to_string(driveIndex), inventory::interface); 164 itemIface->register_property("Present", isPresent); 165 itemIface->register_property("PrettyName", 166 "Drive " + std::to_string(driveIndex)); 167 itemIface->initialize(); 168 operationalIface = objServer.add_interface( 169 itemIface->get_object_path(), 170 "xyz.openbmc_project.State.Decorator.OperationalStatus"); 171 172 operationalIface->register_property( 173 "Functional", isOperational, 174 [this](const bool req, bool& property) { 175 if (!isPresent) 176 { 177 return 0; 178 } 179 if (property == req) 180 { 181 return 1; 182 } 183 property = req; 184 if (req) 185 { 186 clearFailed(); 187 return 1; 188 } 189 markFailed(); 190 return 1; 191 }); 192 193 operationalIface->initialize(); 194 rebuildingIface = objServer.add_interface( 195 itemIface->get_object_path(), "xyz.openbmc_project.State.Drive"); 196 rebuildingIface->register_property("Rebuilding", rebuilding); 197 rebuildingIface->initialize(); 198 driveIface = 199 objServer.add_interface(itemIface->get_object_path(), 200 "xyz.openbmc_project.Inventory.Item.Drive"); 201 driveIface->initialize(); 202 associations = objServer.add_interface(itemIface->get_object_path(), 203 association::interface); 204 associations->register_property("Associations", 205 std::vector<Association>{}); 206 associations->initialize(); 207 208 if (isPresent && (!isOperational || rebuilding)) 209 { 210 markFailed(); 211 } 212 } 213 virtual ~Drive() 214 { 215 objServer.remove_interface(itemIface); 216 objServer.remove_interface(operationalIface); 217 objServer.remove_interface(rebuildingIface); 218 objServer.remove_interface(assetIface); 219 objServer.remove_interface(driveIface); 220 objServer.remove_interface(associations); 221 } 222 223 void removeAsset() 224 { 225 objServer.remove_interface(assetIface); 226 assetIface = nullptr; 227 } 228 229 void createAsset( 230 const boost::container::flat_map<std::string, std::string>& data) 231 { 232 if (assetIface != nullptr) 233 { 234 return; 235 } 236 assetIface = objServer.add_interface( 237 itemIface->get_object_path(), 238 "xyz.openbmc_project.Inventory.Decorator.Asset"); 239 for (const auto& [key, value] : data) 240 { 241 assetIface->register_property(key, value); 242 if (key == "SerialNumber") 243 { 244 serialNumber = value; 245 serialNumberInitialized = true; 246 } 247 } 248 assetIface->initialize(); 249 } 250 251 void markFailed(void) 252 { 253 // todo: maybe look this up via mapper 254 constexpr const char* globalInventoryPath = 255 "/xyz/openbmc_project/CallbackManager"; 256 257 if (!isPresent) 258 { 259 return; 260 } 261 262 operationalIface->set_property("Functional", false); 263 std::vector<Association> warning = { 264 {"", "warning", globalInventoryPath}}; 265 associations->set_property("Associations", warning); 266 logDriveError("Drive " + std::to_string(index)); 267 } 268 269 void clearFailed(void) 270 { 271 operationalIface->set_property("Functional", true); 272 associations->set_property("Associations", std::vector<Association>{}); 273 } 274 275 void setPresent(bool set) 276 { 277 // nvme drives get detected by their fru 278 if (set == isPresent) 279 { 280 return; 281 } 282 itemIface->set_property("Present", set); 283 isPresent = set; 284 } 285 286 void logPresent() 287 { 288 if (isNvme && !serialNumberInitialized) 289 { 290 // wait until NVMe asset is updated to include the serial number 291 // from the NVMe drive 292 return; 293 } 294 295 if (!isPresent && loggedPresent) 296 { 297 loggedPresent = false; 298 logDeviceRemoved("Drive", std::to_string(index), serialNumber); 299 serialNumber = "N/A"; 300 serialNumberInitialized = false; 301 removeAsset(); 302 } 303 else if (isPresent && !loggedPresent) 304 { 305 loggedPresent = true; 306 logDeviceAdded("Drive", std::to_string(index), serialNumber); 307 } 308 } 309 310 std::shared_ptr<sdbusplus::asio::dbus_interface> itemIface; 311 std::shared_ptr<sdbusplus::asio::dbus_interface> operationalIface; 312 std::shared_ptr<sdbusplus::asio::dbus_interface> rebuildingIface; 313 std::shared_ptr<sdbusplus::asio::dbus_interface> assetIface; 314 std::shared_ptr<sdbusplus::asio::dbus_interface> driveIface; 315 std::shared_ptr<sdbusplus::asio::dbus_interface> associations; 316 317 bool isNvme; 318 bool isPresent; 319 size_t index; 320 std::string serialNumber = "N/A"; 321 bool serialNumberInitialized = false; 322 bool loggedPresent = false; 323 }; 324 325 struct Backplane : std::enable_shared_from_this<Backplane> 326 { 327 328 Backplane(size_t busIn, size_t addressIn, size_t backplaneIndexIn, 329 const std::string& nameIn) : 330 bus(busIn), 331 address(addressIn), backplaneIndex(backplaneIndexIn - 1), name(nameIn), 332 timer(boost::asio::steady_timer(io)), 333 muxes(std::make_shared<boost::container::flat_set<Mux>>()) 334 { 335 } 336 void populateAsset(const std::string& rootPath, const std::string& busname) 337 { 338 conn->async_method_call( 339 [assetIface{assetInterface}, hsbpIface{hsbpItemIface}]( 340 const boost::system::error_code ec, 341 const boost::container::flat_map< 342 std::string, std::variant<std::string>>& values) mutable { 343 if (ec) 344 { 345 std::cerr 346 << "Error getting asset tag from HSBP configuration\n"; 347 348 return; 349 } 350 assetIface = objServer.add_interface( 351 hsbpIface->get_object_path(), assetTag); 352 for (const auto& [key, value] : values) 353 { 354 const std::string* ptr = std::get_if<std::string>(&value); 355 if (ptr == nullptr) 356 { 357 std::cerr << key << " Invalid type!\n"; 358 continue; 359 } 360 assetIface->register_property(key, *ptr); 361 } 362 assetIface->initialize(); 363 }, 364 busname, rootPath, "org.freedesktop.DBus.Properties", "GetAll", 365 assetTag); 366 } 367 368 void run(const std::string& rootPath, const std::string& busname) 369 { 370 file = open(("/dev/i2c-" + std::to_string(bus)).c_str(), 371 O_RDWR | O_CLOEXEC); 372 if (file < 0) 373 { 374 std::cerr << "unable to open bus " << bus << "\n"; 375 return; 376 } 377 378 if (ioctl(file, I2C_SLAVE_FORCE, address) < 0) 379 { 380 std::cerr << "unable to set address to " << address << "\n"; 381 return; 382 } 383 384 if (!getPresent()) 385 { 386 std::cerr << "Cannot detect CPLD\n"; 387 return; 388 } 389 390 getBootVer(bootVer); 391 getFPGAVer(fpgaVer); 392 getSecurityRev(securityRev); 393 std::string dbusName = boost::replace_all_copy(name, " ", "_"); 394 hsbpItemIface = objServer.add_interface( 395 "/xyz/openbmc_project/inventory/item/hsbp/" + dbusName, 396 inventory::interface); 397 hsbpItemIface->register_property("Present", true); 398 hsbpItemIface->register_property("PrettyName", name); 399 hsbpItemIface->initialize(); 400 401 storageInterface = objServer.add_interface( 402 hsbpItemIface->get_object_path(), 403 "xyz.openbmc_project.Inventory.Item.StorageController"); 404 storageInterface->initialize(); 405 406 versionIface = 407 objServer.add_interface("/xyz/openbmc_project/software/" + dbusName, 408 "xyz.openbmc_project.Software.Version"); 409 versionIface->register_property("Version", zeroPad(bootVer) + "." + 410 zeroPad(fpgaVer) + "." + 411 zeroPad(securityRev)); 412 versionIface->register_property( 413 "Purpose", 414 std::string( 415 "xyz.openbmc_project.Software.Version.VersionPurpose.HSBP")); 416 versionIface->initialize(); 417 418 auto activationIface = 419 objServer.add_interface("/xyz/openbmc_project/software/" + dbusName, 420 "xyz.openbmc_project.Software.Activation"); 421 422 activationIface->register_property( 423 "Activation", 424 std::string( 425 "xyz.openbmc_project.Software.Activation.Activations.Active")); 426 activationIface->register_property( 427 "RequestedActivation", 428 std::string("xyz.openbmc_project.Software.Activation." 429 "RequestedActivations.None")); 430 431 activationIface->initialize(); 432 433 getPresence(presence); 434 getIFDET(ifdet); 435 436 populateAsset(rootPath, busname); 437 438 createDrives(); 439 440 runTimer(); 441 } 442 443 void runTimer() 444 { 445 timer.expires_after(std::chrono::seconds(scanRateSeconds)); 446 timer.async_wait([weak{std::weak_ptr<Backplane>(shared_from_this())}]( 447 boost::system::error_code ec) { 448 auto self = weak.lock(); 449 if (!self) 450 { 451 return; 452 } 453 if (ec == boost::asio::error::operation_aborted) 454 { 455 // we're being destroyed 456 return; 457 } 458 else if (ec) 459 { 460 std::cerr << "timer error " << ec.message() << "\n"; 461 return; 462 } 463 464 if (!isPowerOn()) 465 { 466 // can't access hsbp when power is off 467 self->runTimer(); 468 return; 469 } 470 471 self->getPresence(self->presence); 472 self->getIFDET(self->ifdet); 473 self->getFailed(self->failed); 474 self->getRebuild(self->rebuilding); 475 476 self->updateDrives(); 477 self->runTimer(); 478 }); 479 } 480 481 void createDrives() 482 { 483 for (size_t ii = 0; ii < maxDrives; ii++) 484 { 485 uint8_t driveSlot = (1 << ii); 486 bool isNvme = ((ifdet & driveSlot) && !(presence & driveSlot)); 487 bool isPresent = isNvme || (presence & driveSlot); 488 bool isFailed = !isPresent || failed & driveSlot; 489 bool isRebuilding = !isPresent && (rebuilding & driveSlot); 490 491 // +1 to convert from 0 based to 1 based 492 size_t driveIndex = (backplaneIndex * maxDrives) + ii + 1; 493 Drive& drive = drives.emplace_back(driveIndex, isPresent, !isFailed, 494 isNvme, isRebuilding); 495 std::shared_ptr<Led> led = leds.emplace_back(std::make_shared<Led>( 496 drive.itemIface->get_object_path(), ii, file)); 497 led->createInterface(); 498 } 499 } 500 501 void updateDrives() 502 { 503 size_t ii = 0; 504 505 for (auto it = drives.begin(); it != drives.end(); it++, ii++) 506 { 507 uint8_t driveSlot = (1 << ii); 508 bool isNvme = ((ifdet & driveSlot) && !(presence & driveSlot)); 509 bool isPresent = isNvme || (presence & driveSlot); 510 bool isFailed = !isPresent || (failed & driveSlot); 511 bool isRebuilding = isPresent && (rebuilding & driveSlot); 512 513 it->isNvme = isNvme; 514 it->setPresent(isPresent); 515 it->logPresent(); 516 517 it->rebuildingIface->set_property("Rebuilding", isRebuilding); 518 if (isFailed || isRebuilding) 519 { 520 it->markFailed(); 521 } 522 else 523 { 524 it->clearFailed(); 525 } 526 } 527 } 528 529 bool getPresent() 530 { 531 present = i2c_smbus_read_byte(file) >= 0; 532 return present; 533 } 534 535 bool getTypeID(uint8_t& val) 536 { 537 constexpr uint8_t addr = 2; 538 int ret = i2c_smbus_read_byte_data(file, addr); 539 if (ret < 0) 540 { 541 std::cerr << "Error " << __FUNCTION__ << "\n"; 542 return false; 543 } 544 val = static_cast<uint8_t>(ret); 545 return true; 546 } 547 548 bool getBootVer(uint8_t& val) 549 { 550 constexpr uint8_t addr = 3; 551 int ret = i2c_smbus_read_byte_data(file, addr); 552 if (ret < 0) 553 { 554 std::cerr << "Error " << __FUNCTION__ << "\n"; 555 return false; 556 } 557 val = static_cast<uint8_t>(ret); 558 return true; 559 } 560 561 bool getFPGAVer(uint8_t& val) 562 { 563 constexpr uint8_t addr = 4; 564 int ret = i2c_smbus_read_byte_data(file, addr); 565 if (ret < 0) 566 { 567 std::cerr << "Error " << __FUNCTION__ << "\n"; 568 return false; 569 } 570 val = static_cast<uint8_t>(ret); 571 return true; 572 } 573 574 bool getSecurityRev(uint8_t& val) 575 { 576 constexpr uint8_t addr = 5; 577 int ret = i2c_smbus_read_byte_data(file, addr); 578 if (ret < 0) 579 { 580 std::cerr << "Error " << __FUNCTION__ << "\n"; 581 return false; 582 } 583 val = static_cast<uint8_t>(ret); 584 return true; 585 } 586 587 bool getPresence(uint8_t& val) 588 { 589 // NVMe drives do not assert PRSNTn, and as such do not get reported as 590 // PRESENT in this register 591 592 constexpr uint8_t addr = 8; 593 594 int ret = i2c_smbus_read_byte_data(file, addr); 595 if (ret < 0) 596 { 597 std::cerr << "Error " << __FUNCTION__ << "\n"; 598 return false; 599 } 600 // presence is inverted 601 val = static_cast<uint8_t>(~ret); 602 return true; 603 } 604 605 bool getIFDET(uint8_t& val) 606 { 607 // This register is a bitmap of parallel GPIO pins connected to the 608 // IFDETn pin of a drive slot. SATA, SAS, and NVMe drives all assert 609 // IFDETn low when they are inserted into the HSBP.This register, in 610 // combination with the PRESENCE register, are used by the BMC to detect 611 // the presence of NVMe drives. 612 613 constexpr uint8_t addr = 9; 614 615 int ret = i2c_smbus_read_byte_data(file, addr); 616 if (ret < 0) 617 { 618 std::cerr << "Error " << __FUNCTION__ << "\n"; 619 return false; 620 } 621 // ifdet is inverted 622 val = static_cast<uint8_t>(~ret); 623 return true; 624 } 625 626 bool getFailed(uint8_t& val) 627 { 628 constexpr uint8_t addr = 0xC; 629 int ret = i2c_smbus_read_byte_data(file, addr); 630 if (ret < 0) 631 { 632 std::cerr << "Error " << __FUNCTION__ << "\n"; 633 return false; 634 } 635 val = static_cast<uint8_t>(ret); 636 return true; 637 } 638 639 bool getRebuild(uint8_t& val) 640 { 641 constexpr uint8_t addr = 0xD; 642 int ret = i2c_smbus_read_byte_data(file, addr); 643 if (ret < 0) 644 { 645 std::cerr << "Error " << __FUNCTION__ << " " << strerror(ret) 646 << "\n"; 647 return false; 648 } 649 val = static_cast<uint8_t>(ret); 650 return true; 651 } 652 653 virtual ~Backplane() 654 { 655 objServer.remove_interface(hsbpItemIface); 656 objServer.remove_interface(versionIface); 657 timer.cancel(); 658 if (file >= 0) 659 { 660 close(file); 661 } 662 } 663 664 size_t bus; 665 size_t address; 666 size_t backplaneIndex; 667 std::string name; 668 boost::asio::steady_timer timer; 669 bool present = false; 670 uint8_t typeId = 0; 671 uint8_t bootVer = 0; 672 uint8_t fpgaVer = 0; 673 uint8_t securityRev = 0; 674 uint8_t funSupported = 0; 675 uint8_t presence = 0; 676 uint8_t ifdet = 0; 677 uint8_t failed = 0; 678 uint8_t rebuilding = 0; 679 680 int file = -1; 681 682 std::string type; 683 684 std::shared_ptr<sdbusplus::asio::dbus_interface> hsbpItemIface; 685 std::shared_ptr<sdbusplus::asio::dbus_interface> versionIface; 686 std::shared_ptr<sdbusplus::asio::dbus_interface> storageInterface; 687 std::shared_ptr<sdbusplus::asio::dbus_interface> assetInterface; 688 689 std::list<Drive> drives; 690 std::vector<std::shared_ptr<Led>> leds; 691 std::shared_ptr<boost::container::flat_set<Mux>> muxes; 692 }; 693 694 std::unordered_map<std::string, std::shared_ptr<Backplane>> backplanes; 695 std::list<Drive> ownerlessDrives; // drives without a backplane 696 697 static size_t getDriveCount() 698 { 699 size_t count = 0; 700 for (const auto& [key, backplane] : backplanes) 701 { 702 count += backplane->drives.size(); 703 } 704 return count + ownerlessDrives.size(); 705 } 706 707 void updateAssets() 708 { 709 static constexpr const char* nvmeType = 710 "xyz.openbmc_project.Inventory.Item.NVMe"; 711 712 conn->async_method_call( 713 [](const boost::system::error_code ec, const GetSubTreeType& subtree) { 714 if (ec) 715 { 716 std::cerr << "Error contacting mapper " << ec.message() << "\n"; 717 return; 718 } 719 720 // drives may get an owner during this, or we might disover more 721 // drives 722 ownerlessDrives.clear(); 723 for (const auto& [path, objDict] : subtree) 724 { 725 if (objDict.empty()) 726 { 727 continue; 728 } 729 730 const std::string& owner = objDict.begin()->first; 731 // we export this interface too 732 if (owner == busName) 733 { 734 continue; 735 } 736 if (std::find(objDict.begin()->second.begin(), 737 objDict.begin()->second.end(), 738 assetTag) == objDict.begin()->second.end()) 739 { 740 // no asset tag to associate to 741 continue; 742 } 743 744 conn->async_method_call( 745 [path](const boost::system::error_code ec2, 746 const boost::container::flat_map< 747 std::string, 748 std::variant<uint64_t, std::string>>& values) { 749 if (ec2) 750 { 751 std::cerr << "Error Getting Config " 752 << ec2.message() << " " << __FUNCTION__ 753 << "\n"; 754 return; 755 } 756 auto findBus = values.find("Bus"); 757 758 if (findBus == values.end()) 759 { 760 std::cerr << "Illegal interface at " << path 761 << "\n"; 762 return; 763 } 764 765 // find the mux bus and addr 766 size_t muxBus = static_cast<size_t>( 767 std::get<uint64_t>(findBus->second)); 768 std::filesystem::path muxPath = 769 "/sys/bus/i2c/devices/i2c-" + 770 std::to_string(muxBus) + "/mux_device"; 771 if (!std::filesystem::is_symlink(muxPath)) 772 { 773 std::cerr << path << " mux does not exist\n"; 774 return; 775 } 776 777 // we should be getting something of the form 7-0052 778 // for bus 7 addr 52 779 std::string fname = 780 std::filesystem::read_symlink(muxPath).filename(); 781 auto findDash = fname.find('-'); 782 783 if (findDash == std::string::npos || 784 findDash + 1 >= fname.size()) 785 { 786 std::cerr << path << " mux path invalid\n"; 787 return; 788 } 789 790 std::string busStr = fname.substr(0, findDash); 791 std::string muxStr = fname.substr(findDash + 1); 792 793 size_t bus = static_cast<size_t>(std::stoi(busStr)); 794 size_t addr = 795 static_cast<size_t>(std::stoi(muxStr, nullptr, 16)); 796 size_t muxIndex = 0; 797 798 // find the channel of the mux the drive is on 799 std::ifstream nameFile("/sys/bus/i2c/devices/i2c-" + 800 std::to_string(muxBus) + 801 "/name"); 802 if (!nameFile) 803 { 804 std::cerr << "Unable to open name file of bus " 805 << muxBus << "\n"; 806 return; 807 } 808 809 std::string nameStr; 810 std::getline(nameFile, nameStr); 811 812 // file is of the form "i2c-4-mux (chan_id 1)", get chan 813 // assume single digit chan 814 const std::string prefix = "chan_id "; 815 size_t findId = nameStr.find(prefix); 816 if (findId == std::string::npos || 817 findId + 1 >= nameStr.size()) 818 { 819 std::cerr << "Illegal name file on bus " << muxBus 820 << "\n"; 821 } 822 823 std::string indexStr = 824 nameStr.substr(findId + prefix.size(), 1); 825 826 size_t driveIndex = std::stoi(indexStr); 827 828 Backplane* parent = nullptr; 829 for (auto& [name, backplane] : backplanes) 830 { 831 muxIndex = 0; 832 for (const Mux& mux : *(backplane->muxes)) 833 { 834 if (bus == mux.bus && addr == mux.address) 835 { 836 parent = backplane.get(); 837 break; 838 } 839 muxIndex += mux.channels; 840 } 841 } 842 boost::container::flat_map<std::string, std::string> 843 assetInventory; 844 const std::array<const char*, 4> assetKeys = { 845 "PartNumber", "SerialNumber", "Manufacturer", 846 "Model"}; 847 for (const auto& [key, value] : values) 848 { 849 if (std::find(assetKeys.begin(), assetKeys.end(), 850 key) == assetKeys.end()) 851 { 852 continue; 853 } 854 assetInventory[key] = std::get<std::string>(value); 855 } 856 857 // assume its a M.2 or something without a hsbp 858 if (parent == nullptr) 859 { 860 auto& drive = ownerlessDrives.emplace_back( 861 getDriveCount() + 1, true, true, true, false); 862 drive.createAsset(assetInventory); 863 return; 864 } 865 866 driveIndex += muxIndex; 867 868 if (parent->drives.size() <= driveIndex) 869 { 870 std::cerr << "Illegal drive index at " << path 871 << " " << driveIndex << "\n"; 872 return; 873 } 874 auto it = parent->drives.begin(); 875 std::advance(it, driveIndex); 876 877 it->createAsset(assetInventory); 878 }, 879 owner, path, "org.freedesktop.DBus.Properties", "GetAll", 880 "" /*all interface items*/); 881 } 882 }, 883 mapper::busName, mapper::path, mapper::interface, mapper::subtree, "/", 884 0, std::array<const char*, 1>{nvmeType}); 885 } 886 887 void populateMuxes(std::shared_ptr<boost::container::flat_set<Mux>> muxes, 888 std::string& rootPath) 889 { 890 const static std::array<const std::string, 4> muxTypes = { 891 "xyz.openbmc_project.Configuration.PCA9543Mux", 892 "xyz.openbmc_project.Configuration.PCA9544Mux", 893 "xyz.openbmc_project.Configuration.PCA9545Mux", 894 "xyz.openbmc_project.Configuration.PCA9546Mux"}; 895 conn->async_method_call( 896 [muxes](const boost::system::error_code ec, 897 const GetSubTreeType& subtree) { 898 if (ec) 899 { 900 std::cerr << "Error contacting mapper " << ec.message() << "\n"; 901 return; 902 } 903 std::shared_ptr<std::function<void()>> callback = 904 std::make_shared<std::function<void()>>( 905 []() { updateAssets(); }); 906 size_t index = 0; // as we use a flat map, these are sorted 907 for (const auto& [path, objDict] : subtree) 908 { 909 if (objDict.empty() || objDict.begin()->second.empty()) 910 { 911 continue; 912 } 913 914 const std::string& owner = objDict.begin()->first; 915 const std::vector<std::string>& interfaces = 916 objDict.begin()->second; 917 918 const std::string* interface = nullptr; 919 for (const std::string& iface : interfaces) 920 { 921 if (std::find(muxTypes.begin(), muxTypes.end(), iface) != 922 muxTypes.end()) 923 { 924 interface = &iface; 925 break; 926 } 927 } 928 if (interface == nullptr) 929 { 930 std::cerr << "Cannot get mux type\n"; 931 continue; 932 } 933 934 conn->async_method_call( 935 [path, muxes, callback, index]( 936 const boost::system::error_code ec2, 937 const boost::container::flat_map< 938 std::string, 939 std::variant<uint64_t, std::vector<std::string>>>& 940 values) { 941 if (ec2) 942 { 943 std::cerr << "Error Getting Config " 944 << ec2.message() << " " << __FUNCTION__ 945 << "\n"; 946 return; 947 } 948 auto findBus = values.find("Bus"); 949 auto findAddress = values.find("Address"); 950 auto findChannelNames = values.find("ChannelNames"); 951 if (findBus == values.end() || 952 findAddress == values.end()) 953 { 954 std::cerr << "Illegal configuration at " << path 955 << "\n"; 956 return; 957 } 958 size_t bus = static_cast<size_t>( 959 std::get<uint64_t>(findBus->second)); 960 size_t address = static_cast<size_t>( 961 std::get<uint64_t>(findAddress->second)); 962 std::vector<std::string> channels = 963 std::get<std::vector<std::string>>( 964 findChannelNames->second); 965 muxes->emplace(bus, address, channels.size(), index); 966 if (callback.use_count() == 1) 967 { 968 (*callback)(); 969 } 970 }, 971 owner, path, "org.freedesktop.DBus.Properties", "GetAll", 972 *interface); 973 index++; 974 } 975 }, 976 mapper::busName, mapper::path, mapper::interface, mapper::subtree, 977 rootPath, 1, muxTypes); 978 } 979 980 void populate() 981 { 982 backplanes.clear(); 983 conn->async_method_call( 984 [](const boost::system::error_code ec, const GetSubTreeType& subtree) { 985 if (ec) 986 { 987 std::cerr << "Error contacting mapper " << ec.message() << "\n"; 988 return; 989 } 990 for (const auto& [path, objDict] : subtree) 991 { 992 if (objDict.empty()) 993 { 994 continue; 995 } 996 997 const std::string& owner = objDict.begin()->first; 998 conn->async_method_call( 999 [path, owner](const boost::system::error_code ec2, 1000 const boost::container::flat_map< 1001 std::string, BasicVariantType>& resp) { 1002 if (ec2) 1003 { 1004 std::cerr << "Error Getting Config " 1005 << ec2.message() << "\n"; 1006 return; 1007 } 1008 std::optional<size_t> bus; 1009 std::optional<size_t> address; 1010 std::optional<size_t> backplaneIndex; 1011 std::optional<std::string> name; 1012 for (const auto& [key, value] : resp) 1013 { 1014 if (key == "Bus") 1015 { 1016 bus = std::get<uint64_t>(value); 1017 } 1018 else if (key == "Address") 1019 { 1020 address = std::get<uint64_t>(value); 1021 } 1022 else if (key == "Index") 1023 { 1024 backplaneIndex = std::get<uint64_t>(value); 1025 } 1026 else if (key == "Name") 1027 { 1028 name = std::get<std::string>(value); 1029 } 1030 } 1031 if (!bus || !address || !name || !backplaneIndex) 1032 { 1033 std::cerr << "Illegal configuration at " << path 1034 << "\n"; 1035 return; 1036 } 1037 std::string parentPath = 1038 std::filesystem::path(path).parent_path(); 1039 const auto& [backplane, status] = backplanes.emplace( 1040 *name, std::make_shared<Backplane>( 1041 *bus, *address, *backplaneIndex, *name)); 1042 backplane->second->run(parentPath, owner); 1043 populateMuxes(backplane->second->muxes, parentPath); 1044 }, 1045 owner, path, "org.freedesktop.DBus.Properties", "GetAll", 1046 configType); 1047 } 1048 }, 1049 mapper::busName, mapper::path, mapper::interface, mapper::subtree, "/", 1050 0, std::array<const char*, 1>{configType}); 1051 } 1052 1053 static bool hsbpRequestAlertGpioEvents( 1054 const std::string& name, const std::function<void()>& handler, 1055 gpiod::line& gpioLine, 1056 boost::asio::posix::stream_descriptor& gpioEventDescriptor) 1057 { 1058 // Find the GPIO line 1059 gpioLine = gpiod::find_line(name); 1060 if (!gpioLine) 1061 { 1062 std::cerr << "Failed to find the " << name << " line\n"; 1063 return false; 1064 } 1065 1066 try 1067 { 1068 gpioLine.request( 1069 {"hsbp-manager", gpiod::line_request::EVENT_BOTH_EDGES, 0}); 1070 } 1071 catch (std::exception&) 1072 { 1073 std::cerr << "Failed to request events for " << name << "\n"; 1074 return false; 1075 } 1076 1077 int gpioLineFd = gpioLine.event_get_fd(); 1078 if (gpioLineFd < 0) 1079 { 1080 std::cerr << "Failed to get " << name << " fd\n"; 1081 return false; 1082 } 1083 1084 gpioEventDescriptor.assign(gpioLineFd); 1085 1086 gpioEventDescriptor.async_wait( 1087 boost::asio::posix::stream_descriptor::wait_read, 1088 [&name, handler](const boost::system::error_code ec) { 1089 if (ec) 1090 { 1091 std::cerr << name << " fd handler error: " << ec.message() 1092 << "\n"; 1093 return; 1094 } 1095 handler(); 1096 }); 1097 return true; 1098 } 1099 1100 /****************************************************************************************** 1101 * HSBP Position CPLD SMB Address 1102 * 1 0xD0(0x68 7 bit) 1103 * 2 0xD2(0x69 7 bit) 1104 * we have max 2 HSBP per system. Closed chassis systems will either have 0 or 1105 * 2 HSBP's. 1106 *******************************************************************************************/ 1107 static constexpr uint8_t hsbpI2cBus = 4; 1108 static constexpr uint8_t allDrivesWithStatusBit = 17; 1109 static constexpr uint8_t statusAllDrives = (allDrivesWithStatusBit - 1); 1110 static constexpr uint8_t allClockBitsDb2000 = 25; 1111 static constexpr uint8_t statusAllClocksDb2000 = (allClockBitsDb2000 - 1); 1112 static constexpr uint8_t singleDriveWithStatusBit = 9; 1113 static constexpr uint8_t statusSingleDrive = (singleDriveWithStatusBit - 1); 1114 static constexpr uint8_t maxDrivesPerHsbp = 8; 1115 1116 static std::bitset<allDrivesWithStatusBit> drivePresenceStatus; 1117 static std::bitset<allClockBitsDb2000> driveClockStatus; 1118 static constexpr uint8_t hsbpCpldSmbaddr1 = 0x68; 1119 static constexpr uint8_t hsbpCpldSmbaddr2 = 0x69; 1120 static constexpr uint8_t hsbpCpldReg8 = 0x8; 1121 static constexpr uint8_t hsbpCpldReg9 = 0x9; 1122 static constexpr uint8_t db2000SlaveAddr = 0x6d; 1123 static constexpr uint8_t db2000RegByte0 = 0x80; 1124 static constexpr uint8_t db2000RegByte1 = 0x81; 1125 static constexpr uint8_t db2000RegByte2 = 0x82; 1126 static int hsbpFd; 1127 1128 /******************************************************************** 1129 * DB2000 Programming guide for PCIe Clocks enable/disable 1130 * CPU 0 1131 * ================================================================= 1132 * slot Byte bit number 1133 * Position position 1134 * ================================================================= 1135 * 7 0 5 1136 * 6 0 4 1137 * 5 0 3 1138 * 4 2 7 1139 * 3 1 3 1140 * 2 1 2 1141 * 1 1 1 1142 * 0 1 0 1143 * 1144 * CPU 1 1145 * ================================================================= 1146 * slot Byte bit number 1147 * Position position 1148 * ================================================================= 1149 * 7 1 6 1150 * 6 1 7 1151 * 5 2 0 1152 * 4 2 1 1153 * 3 1 5 1154 * 2 2 4 1155 * 1 2 2 1156 * 0 2 3 1157 *********************************************************************/ 1158 std::optional<int> 1159 updateClocksStatus(std::bitset<allDrivesWithStatusBit> nvmeDriveStatus) 1160 { 1161 std::bitset<allClockBitsDb2000> nvmeClockStatus; 1162 /* mapping table for nvme drive index(0-15) to DB2000 register bit fields */ 1163 constexpr std::array<int, statusAllDrives> slotToClockTable = { 1164 8, 9, 10, 11, 23, 3, 4, 5, 19, 18, 20, 13, 17, 16, 15, 14}; 1165 1166 /* scan through all drives(except the status bit) and update corresponding 1167 * clock bit */ 1168 for (std::size_t i = 0; i < (nvmeDriveStatus.size() - 1); i++) 1169 { 1170 if (nvmeDriveStatus.test(i)) 1171 { 1172 nvmeClockStatus.set(slotToClockTable[i]); 1173 } 1174 else 1175 { 1176 nvmeClockStatus.reset(slotToClockTable[i]); 1177 } 1178 } 1179 1180 if (ioctl(hsbpFd, I2C_SLAVE_FORCE, db2000SlaveAddr) < 0) 1181 { 1182 std::cerr << "unable to set DB2000 address to " << db2000SlaveAddr 1183 << "\n"; 1184 return std::nullopt; 1185 } 1186 int ret = i2c_smbus_write_byte_data( 1187 hsbpFd, db2000RegByte0, 1188 static_cast<unsigned char>(nvmeClockStatus.to_ulong())); 1189 1190 if (ret < 0) 1191 { 1192 std::cerr << "Error: unable to write data to clock register " 1193 << __FUNCTION__ << __LINE__ << "\n"; 1194 return ret; 1195 } 1196 1197 ret = i2c_smbus_write_byte_data( 1198 hsbpFd, db2000RegByte1, 1199 static_cast<unsigned char>((nvmeClockStatus >> 8).to_ulong())); 1200 1201 if (ret < 0) 1202 { 1203 std::cerr << "Error: unable to write data to clock register " 1204 << __FUNCTION__ << __LINE__ << "\n"; 1205 return ret; 1206 } 1207 1208 ret = i2c_smbus_write_byte_data( 1209 hsbpFd, db2000RegByte2, 1210 static_cast<unsigned char>((nvmeClockStatus >> 16).to_ulong())); 1211 1212 if (ret < 0) 1213 { 1214 std::cerr << "Error: unable to write data to clock register " 1215 << __FUNCTION__ << __LINE__ << "\n"; 1216 return ret; 1217 } 1218 // Update global clock status 1219 driveClockStatus = nvmeClockStatus; 1220 driveClockStatus.set(statusAllClocksDb2000, 1); 1221 return 0; 1222 } 1223 1224 std::bitset<singleDriveWithStatusBit> 1225 getSingleHsbpDriveStatus(const uint8_t cpldSmbaddr) 1226 { 1227 std::bitset<singleDriveWithStatusBit> singleDriveStatus; 1228 1229 // probe 1230 if (ioctl(hsbpFd, I2C_SLAVE_FORCE, cpldSmbaddr) < 0) 1231 { 1232 std::cerr << "Failed to talk to cpldSmbaddr : " << cpldSmbaddr << "\n"; 1233 return singleDriveStatus; 1234 } 1235 1236 // read status of lower four drive connectivity 1237 int valueReg8 = i2c_smbus_read_byte_data(hsbpFd, hsbpCpldReg8); 1238 if (valueReg8 < 0) 1239 { 1240 std::cerr << "Error: Unable to read cpld reg 0x8 " << __FUNCTION__ 1241 << __LINE__ << "\n"; 1242 return singleDriveStatus; 1243 } 1244 1245 // read status of upper four drive connectivity 1246 int valueReg9 = i2c_smbus_read_byte_data(hsbpFd, hsbpCpldReg9); 1247 if (valueReg9 < 0) 1248 { 1249 std::cerr << "Error: Unable to read cpld reg 0x9 " << __FUNCTION__ 1250 << __LINE__ << "\n"; 1251 return singleDriveStatus; 1252 } 1253 1254 // Find drives which have NVMe drive connected 1255 for (int loop = 0; loop < (singleDriveWithStatusBit - 1); loop++) 1256 { 1257 // Check if NVME drive detected(corresponding bit numbers of reg8 and 1258 // reg9 are 1 and 0 resp) 1259 if (valueReg8 & (1U << loop)) 1260 { 1261 if ((valueReg9 & (1U << loop)) == 0) 1262 { 1263 singleDriveStatus.set(loop, 1); 1264 } 1265 } 1266 } 1267 1268 // Reading successful, set the statusok bit 1269 singleDriveStatus.set(statusSingleDrive, 1); 1270 return singleDriveStatus; 1271 } 1272 1273 /* Try reading both HSBP and report back if atleast one of them is found to be 1274 connected. Status bit is set by the function even if one HSBP is responding 1275 */ 1276 std::bitset<allDrivesWithStatusBit> getCompleteDriveStatus(void) 1277 { 1278 std::bitset<singleDriveWithStatusBit> singleDrvStatus; 1279 std::bitset<allDrivesWithStatusBit> currDriveStatus; 1280 1281 singleDrvStatus = getSingleHsbpDriveStatus(hsbpCpldSmbaddr1); 1282 1283 if (singleDrvStatus[statusSingleDrive] == 1) 1284 { 1285 for (int i = 0; i < maxDrivesPerHsbp; i++) 1286 { 1287 currDriveStatus[i] = singleDrvStatus[i]; 1288 } 1289 // set valid bit if a single hsbp drive status is valid 1290 currDriveStatus.set(statusAllDrives); 1291 } 1292 else 1293 { 1294 currDriveStatus &= (~0xFF); 1295 } 1296 1297 singleDrvStatus = getSingleHsbpDriveStatus(hsbpCpldSmbaddr2); 1298 if (singleDrvStatus[statusSingleDrive] == 1) 1299 { 1300 for (int i = maxDrivesPerHsbp, j = 0; i < (allDrivesWithStatusBit - 1); 1301 i++, j++) 1302 { 1303 currDriveStatus[i] = singleDrvStatus[j]; 1304 } 1305 // set valid bit if a single hsbp drive status is valid 1306 currDriveStatus.set(statusAllDrives); 1307 } 1308 else 1309 { 1310 currDriveStatus &= (~(0xFF << maxDrivesPerHsbp)); 1311 } 1312 return currDriveStatus; 1313 } 1314 1315 void cpldReadingInit(void) 1316 { 1317 hsbpFd = open(("/dev/i2c-" + std::to_string(hsbpI2cBus)).c_str(), 1318 O_RDWR | O_CLOEXEC); 1319 if (hsbpFd < 0) 1320 { 1321 std::cerr << "unable to open hsbpI2cBus " << hsbpI2cBus << "\n"; 1322 return; 1323 } 1324 1325 std::bitset<allDrivesWithStatusBit> currDrvStatus = 1326 getCompleteDriveStatus(); 1327 if (currDrvStatus[statusAllDrives] == 1) 1328 { 1329 // update global drive presence for next time comparison 1330 drivePresenceStatus = currDrvStatus; 1331 std::optional<int> updateStatus = 1332 updateClocksStatus(drivePresenceStatus); 1333 if (updateStatus.has_value()) 1334 { 1335 if (updateStatus == -1) 1336 { 1337 std::cerr << "error: DB2000 register read issue " 1338 << "\n"; 1339 close(hsbpFd); 1340 hsbpFd = -1; 1341 } 1342 } 1343 else 1344 { 1345 std::cerr << "error: DB2000 i2c access issue " 1346 << "\n"; 1347 close(hsbpFd); 1348 hsbpFd = -1; 1349 } 1350 } 1351 else 1352 { 1353 close(hsbpFd); 1354 hsbpFd = -1; 1355 } 1356 } 1357 1358 // Callback handler passed to hsbpRequestAlertGpioEvents: 1359 static void nvmeLvc3AlertHandler() 1360 { 1361 if (hsbpFd >= 0) 1362 { 1363 gpiod::line_event gpioLineEvent = nvmeLvc3AlertLine.event_read(); 1364 1365 if (gpioLineEvent.event_type == gpiod::line_event::FALLING_EDGE) 1366 { 1367 /* Step 1: Either drive is removed or inserted; read the CPLD reg 8 1368 and 9 to determine if drive is added or removed. Need to compare 1369 current number of drives with previous state to determine 1370 it. 1371 */ 1372 std::bitset<allDrivesWithStatusBit> currDrvStat = 1373 getCompleteDriveStatus(); 1374 if (currDrvStat[statusAllDrives] == 1) 1375 { 1376 if (drivePresenceStatus != currDrvStat) 1377 { 1378 uint32_t tmpVar = static_cast<uint32_t>( 1379 (drivePresenceStatus ^ currDrvStat).to_ulong()); 1380 uint32_t indexDrive = 0; 1381 while (tmpVar > 0) 1382 { 1383 if (tmpVar & 1) 1384 { 1385 if (drivePresenceStatus[indexDrive] == 0) 1386 { 1387 logDeviceAdded( 1388 "Drive", std::to_string(indexDrive), "N/A"); 1389 } 1390 else 1391 { 1392 logDeviceRemoved( 1393 "Drive", std::to_string(indexDrive), "N/A"); 1394 } 1395 } 1396 indexDrive++; 1397 tmpVar >>= 1; 1398 } 1399 // update global drive presence for next time comparison 1400 drivePresenceStatus = currDrvStat; 1401 1402 // Step 2: disable or enable the pcie clock for 1403 // corresponding drive 1404 std::optional<int> tmpUpdStatus = 1405 updateClocksStatus(currDrvStat); 1406 if (tmpUpdStatus.has_value()) 1407 { 1408 if (tmpUpdStatus == -1) 1409 { 1410 std::cerr << "error: DB2000 register read issue " 1411 << "\n"; 1412 close(hsbpFd); 1413 hsbpFd = -1; 1414 } 1415 } 1416 else 1417 { 1418 std::cerr << "error: DB2000 i2c access issue " 1419 << "\n"; 1420 close(hsbpFd); 1421 hsbpFd = -1; 1422 } 1423 } 1424 // false alarm 1425 else 1426 { 1427 std::cerr 1428 << "False alarm detected by HSBP; no action taken \n"; 1429 } 1430 } 1431 else 1432 { 1433 close(hsbpFd); 1434 hsbpFd = -1; 1435 } 1436 } 1437 1438 nvmeLvc3AlertEvent.async_wait( 1439 boost::asio::posix::stream_descriptor::wait_read, 1440 [](const boost::system::error_code ec) { 1441 if (ec) 1442 { 1443 std::cerr << "nvmealert handler error: " << ec.message() 1444 << "\n"; 1445 return; 1446 } 1447 nvmeLvc3AlertHandler(); 1448 }); 1449 } 1450 } 1451 1452 int main() 1453 { 1454 boost::asio::steady_timer callbackTimer(io); 1455 1456 conn->request_name(busName); 1457 1458 sdbusplus::bus::match_t match( 1459 *conn, 1460 "type='signal',member='PropertiesChanged',arg0='" + 1461 std::string(configType) + "'", 1462 [&callbackTimer](sdbusplus::message_t&) { 1463 callbackTimer.expires_after(std::chrono::seconds(2)); 1464 callbackTimer.async_wait([](const boost::system::error_code ec) { 1465 if (ec == boost::asio::error::operation_aborted) 1466 { 1467 // timer was restarted 1468 return; 1469 } 1470 else if (ec) 1471 { 1472 std::cerr << "Timer error" << ec.message() << "\n"; 1473 return; 1474 } 1475 populate(); 1476 }); 1477 }); 1478 1479 sdbusplus::bus::match_t drive( 1480 *conn, 1481 "type='signal',member='PropertiesChanged',arg0='xyz.openbmc_project." 1482 "Inventory.Item.NVMe'", 1483 [&callbackTimer](sdbusplus::message_t& message) { 1484 callbackTimer.expires_after(std::chrono::seconds(2)); 1485 if (message.get_sender() == conn->get_unique_name()) 1486 { 1487 return; 1488 } 1489 callbackTimer.async_wait([](const boost::system::error_code ec) { 1490 if (ec == boost::asio::error::operation_aborted) 1491 { 1492 // timer was restarted 1493 return; 1494 } 1495 else if (ec) 1496 { 1497 std::cerr << "Timer error" << ec.message() << "\n"; 1498 return; 1499 } 1500 updateAssets(); 1501 }); 1502 }); 1503 1504 cpldReadingInit(); 1505 1506 if (hsbpFd >= 0) 1507 { 1508 if (!hsbpRequestAlertGpioEvents("FM_SMB_BMC_NVME_LVC3_ALERT_N", 1509 nvmeLvc3AlertHandler, nvmeLvc3AlertLine, 1510 nvmeLvc3AlertEvent)) 1511 { 1512 std::cerr << "error: Unable to monitor events on HSBP Alert line " 1513 << "\n"; 1514 } 1515 } 1516 1517 auto iface = 1518 objServer.add_interface("/xyz/openbmc_project/inventory/item/storage", 1519 "xyz.openbmc_project.inventory.item.storage"); 1520 1521 io.post([]() { populate(); }); 1522 setupPowerMatch(conn); 1523 io.run(); 1524 close(hsbpFd); 1525 hsbpFd = -1; 1526 } 1527