1 // Copyright (c) 2022 Intel Corporation 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 #include "cpuinfo_utils.hpp" 16 #include "speed_select.hpp" 17 18 #include <iostream> 19 20 namespace cpu_info 21 { 22 namespace sst 23 { 24 25 /** 26 * Convenience RAII object for Wake-On-PECI (WOP) management, since PECI Config 27 * Local accesses to the OS Mailbox require the package to pop up to PC2. Also 28 * provides PCode OS Mailbox routine. 29 * 30 * Since multiple applications may be modifying WOP, we'll use this algorithm: 31 * Whenever a PECI command fails with associated error code, set WOP bit and 32 * retry command. Upon manager destruction, clear WOP bit only if we previously 33 * set it. 34 */ 35 struct PECIManager 36 { 37 uint8_t peciAddress; 38 bool peciWoken; 39 CPUModel cpuModel; 40 uint8_t mbBus; 41 WakePolicy wakePolicy; 42 PECIManagercpu_info::sst::PECIManager43 PECIManager(uint8_t address, CPUModel model, WakePolicy wakePolicy_) : 44 peciAddress(address), peciWoken(false), cpuModel(model), 45 wakePolicy(wakePolicy_) 46 { 47 mbBus = (model == iceLake) ? mbBusIceLake : mbBusOther; 48 } 49 ~PECIManagercpu_info::sst::PECIManager50 ~PECIManager() 51 { 52 // If we're being destroyed due to a PECIError, try to clear the mode 53 // bit, but catch and ignore any duplicate error it might raise to 54 // prevent termination. 55 try 56 { 57 if (peciWoken) 58 { 59 setWakeOnPECI(false); 60 } 61 } 62 catch (const PECIError& err) 63 {} 64 } 65 isSleepingcpu_info::sst::PECIManager66 static bool isSleeping(EPECIStatus libStatus, uint8_t completionCode) 67 { 68 // PECI completion code defined in peci-ioctl.h which is not available 69 // for us to include. 70 constexpr int PECI_DEV_CC_UNAVAIL_RESOURCE = 0x82; 71 // Observed library returning DRIVER_ERR for reads and TIMEOUT for 72 // writes while PECI is sleeping. Either way, the completion code from 73 // PECI client should be reliable indicator of need to set WOP. 74 return libStatus != PECI_CC_SUCCESS && 75 completionCode == PECI_DEV_CC_UNAVAIL_RESOURCE; 76 } 77 78 /** 79 * Send a single PECI PCS write to modify the Wake-On-PECI mode bit 80 */ setWakeOnPECIcpu_info::sst::PECIManager81 void setWakeOnPECI(bool enable) 82 { 83 uint8_t completionCode; 84 EPECIStatus libStatus = 85 peci_WrPkgConfig(peciAddress, 5, enable ? 1 : 0, 0, 86 sizeof(uint32_t), &completionCode); 87 if (!checkPECIStatus(libStatus, completionCode)) 88 { 89 throw PECIError("Failed to set Wake-On-PECI mode bit"); 90 } 91 92 if (enable) 93 { 94 peciWoken = true; 95 } 96 } 97 98 // PCode OS Mailbox interface register locations 99 static constexpr int mbBusIceLake = 14; 100 static constexpr int mbBusOther = 31; 101 static constexpr int mbSegment = 0; 102 static constexpr int mbDevice = 30; 103 static constexpr int mbFunction = 1; 104 static constexpr int mbDataReg = 0xA0; 105 static constexpr int mbInterfaceReg = 0xA4; 106 static constexpr int mbRegSize = sizeof(uint32_t); 107 108 enum class MailboxStatus 109 { 110 NoError = 0x0, 111 InvalidCommand = 0x1, 112 IllegalData = 0x16 113 }; 114 115 /** 116 * Send a single Write PCI Config Local command, targeting the PCU CR1 117 * register block. 118 * 119 * @param[in] regAddress PCI Offset of register. 120 * @param[in] data Data to write. 121 */ wrMailboxRegcpu_info::sst::PECIManager122 void wrMailboxReg(uint16_t regAddress, uint32_t data) 123 { 124 uint8_t completionCode; 125 bool tryWaking = (wakePolicy == wakeAllowed); 126 while (true) 127 { 128 EPECIStatus libStatus = peci_WrEndPointPCIConfigLocal( 129 peciAddress, mbSegment, mbBus, mbDevice, mbFunction, regAddress, 130 mbRegSize, data, &completionCode); 131 if (tryWaking && isSleeping(libStatus, completionCode)) 132 { 133 setWakeOnPECI(true); 134 tryWaking = false; 135 continue; 136 } 137 else if (!checkPECIStatus(libStatus, completionCode)) 138 { 139 throw PECIError("Failed to write mailbox reg"); 140 } 141 break; 142 } 143 } 144 145 /** 146 * Send a single Read PCI Config Local command, targeting the PCU CR1 147 * register block. 148 * 149 * @param[in] regAddress PCI offset of register. 150 * 151 * @return Register value 152 */ rdMailboxRegcpu_info::sst::PECIManager153 uint32_t rdMailboxReg(uint16_t regAddress) 154 { 155 uint8_t completionCode; 156 uint32_t outputData; 157 bool tryWaking = (wakePolicy == wakeAllowed); 158 while (true) 159 { 160 EPECIStatus libStatus = peci_RdEndPointConfigPciLocal( 161 peciAddress, mbSegment, mbBus, mbDevice, mbFunction, regAddress, 162 mbRegSize, reinterpret_cast<uint8_t*>(&outputData), 163 &completionCode); 164 if (tryWaking && isSleeping(libStatus, completionCode)) 165 { 166 setWakeOnPECI(true); 167 tryWaking = false; 168 continue; 169 } 170 if (!checkPECIStatus(libStatus, completionCode)) 171 { 172 throw PECIError("Failed to read mailbox reg"); 173 } 174 break; 175 } 176 return outputData; 177 } 178 179 /** 180 * Send command on PCode OS Mailbox interface. 181 * 182 * @param[in] command Main command ID. 183 * @param[in] subCommand Sub command ID. 184 * @param[in] inputData Data to put in mailbox. Is always written, but 185 * will be ignored by PCode if command is a 186 * "getter". 187 * @param[out] responseCode Optional parameter to receive the 188 * mailbox-level response status. If null, a 189 * PECIError will be thrown for error status. 190 * 191 * @return Data returned in mailbox. Value is undefined if command is a 192 * "setter". 193 */ sendPECIOSMailboxCmdcpu_info::sst::PECIManager194 uint32_t sendPECIOSMailboxCmd(uint8_t command, uint8_t subCommand, 195 uint32_t inputData = 0, 196 MailboxStatus* responseCode = nullptr) 197 { 198 // The simple mailbox algorithm just says to wait until the busy bit 199 // is clear, but we'll give up after 10 tries. It's arbitrary but that's 200 // quite long wall clock time. 201 constexpr int mbRetries = 10; 202 constexpr uint32_t mbBusyBit = bit(31); 203 204 // Wait until RUN_BUSY == 0 205 int attempts = mbRetries; 206 while ((rdMailboxReg(mbInterfaceReg) & mbBusyBit) != 0 && 207 --attempts > 0) 208 ; 209 if (attempts == 0) 210 { 211 throw PECIError("OS Mailbox failed to become free"); 212 } 213 214 // Write required command specific input data to data register 215 wrMailboxReg(mbDataReg, inputData); 216 217 // Write required command specific command/sub-command values and set 218 // RUN_BUSY bit in interface register. 219 uint32_t interfaceReg = 220 mbBusyBit | (static_cast<uint32_t>(subCommand) << 8) | command; 221 wrMailboxReg(mbInterfaceReg, interfaceReg); 222 223 // Wait until RUN_BUSY == 0 224 attempts = mbRetries; 225 do 226 { 227 interfaceReg = rdMailboxReg(mbInterfaceReg); 228 } while ((interfaceReg & mbBusyBit) != 0 && --attempts > 0); 229 if (attempts == 0) 230 { 231 throw PECIError("OS Mailbox failed to return"); 232 } 233 234 // Read command return status or error code from interface register 235 auto status = static_cast<MailboxStatus>(interfaceReg & 0xFF); 236 if (responseCode != nullptr) 237 { 238 *responseCode = status; 239 } 240 else if (status != MailboxStatus::NoError) 241 { 242 throw PECIError(std::string("OS Mailbox returned with error: ") + 243 std::to_string(static_cast<int>(status))); 244 } 245 246 // Read command return data from the data register 247 return rdMailboxReg(mbDataReg); 248 } 249 }; 250 251 /** 252 * Base class for set of PECI OS Mailbox commands. 253 * Constructing it runs the command and stores the value for use by derived 254 * class accessor methods. 255 */ 256 template <uint8_t subcommand> 257 struct OsMailboxCommand 258 { 259 enum ErrorPolicy 260 { 261 Throw, 262 NoThrow 263 }; 264 265 uint32_t value; 266 PECIManager::MailboxStatus status = PECIManager::MailboxStatus::NoError; 267 /** 268 * Construct the command object with required PECI address and up to 4 269 * optional 1-byte input data parameters. 270 */ OsMailboxCommandcpu_info::sst::OsMailboxCommand271 OsMailboxCommand(PECIManager& pm, uint8_t param1 = 0, uint8_t param2 = 0, 272 uint8_t param3 = 0, uint8_t param4 = 0) : 273 OsMailboxCommand(pm, ErrorPolicy::Throw, param1, param2, param3, param4) 274 {} 275 OsMailboxCommandcpu_info::sst::OsMailboxCommand276 OsMailboxCommand(PECIManager& pm, ErrorPolicy errorPolicy, 277 uint8_t param1 = 0, uint8_t param2 = 0, uint8_t param3 = 0, 278 uint8_t param4 = 0) 279 { 280 DEBUG_PRINT << "Running OS Mailbox command " 281 << static_cast<int>(subcommand) << '\n'; 282 PECIManager::MailboxStatus* callStatus = 283 errorPolicy == Throw ? nullptr : &status; 284 uint32_t param = (static_cast<uint32_t>(param4) << 24) | 285 (static_cast<uint32_t>(param3) << 16) | 286 (static_cast<uint32_t>(param2) << 8) | param1; 287 value = pm.sendPECIOSMailboxCmd(0x7F, subcommand, param, callStatus); 288 } 289 290 /** Return whether the mailbox status indicated success or not. */ successcpu_info::sst::OsMailboxCommand291 bool success() const 292 { 293 return status == PECIManager::MailboxStatus::NoError; 294 } 295 }; 296 297 /** 298 * Macro to define a derived class accessor method. 299 * 300 * @param[in] type Return type of accessor method. 301 * @param[in] name Name of accessor method. 302 * @param[in] hibit Most significant bit of field to access. 303 * @param[in] lobit Least significant bit of field to access. 304 */ 305 #define FIELD(type, name, hibit, lobit) \ 306 type name() const \ 307 { \ 308 return (value >> lobit) & (bit(hibit - lobit + 1) - 1); \ 309 } 310 311 struct GetLevelsInfo : OsMailboxCommand<0x0> 312 { 313 using OsMailboxCommand::OsMailboxCommand; 314 FIELD(bool, enabled, 31, 31) 315 FIELD(bool, lock, 24, 24) 316 FIELD(unsigned, currentConfigTdpLevel, 23, 16) 317 FIELD(unsigned, configTdpLevels, 15, 8) 318 FIELD(unsigned, version, 7, 0) 319 }; 320 321 struct GetConfigTdpControl : OsMailboxCommand<0x1> 322 { 323 using OsMailboxCommand::OsMailboxCommand; 324 FIELD(bool, pbfEnabled, 17, 17); 325 FIELD(bool, factEnabled, 16, 16); 326 FIELD(bool, pbfSupport, 1, 1); 327 FIELD(bool, factSupport, 0, 0); 328 }; 329 330 struct SetConfigTdpControl : OsMailboxCommand<0x2> 331 { 332 using OsMailboxCommand::OsMailboxCommand; 333 }; 334 335 struct GetTdpInfo : OsMailboxCommand<0x3> 336 { 337 using OsMailboxCommand::OsMailboxCommand; 338 FIELD(unsigned, tdpRatio, 23, 16); 339 FIELD(unsigned, pkgTdp, 14, 0); 340 }; 341 342 struct GetCoreMask : OsMailboxCommand<0x6> 343 { 344 using OsMailboxCommand::OsMailboxCommand; 345 FIELD(uint32_t, coresMask, 31, 0); 346 }; 347 348 struct GetTurboLimitRatios : OsMailboxCommand<0x7> 349 { 350 using OsMailboxCommand::OsMailboxCommand; 351 }; 352 353 struct SetLevel : OsMailboxCommand<0x8> 354 { 355 using OsMailboxCommand::OsMailboxCommand; 356 }; 357 358 struct GetRatioInfo : OsMailboxCommand<0xC> 359 { 360 using OsMailboxCommand::OsMailboxCommand; 361 FIELD(unsigned, pm, 31, 24); 362 FIELD(unsigned, pn, 23, 16); 363 FIELD(unsigned, p1, 15, 8); 364 FIELD(unsigned, p0, 7, 0); 365 }; 366 367 struct GetTjmaxInfo : OsMailboxCommand<0x5> 368 { 369 using OsMailboxCommand::OsMailboxCommand; 370 FIELD(unsigned, tProchot, 7, 0); 371 }; 372 373 struct PbfGetCoreMaskInfo : OsMailboxCommand<0x20> 374 { 375 using OsMailboxCommand::OsMailboxCommand; 376 FIELD(uint32_t, p1HiCoreMask, 31, 0); 377 }; 378 379 struct PbfGetP1HiP1LoInfo : OsMailboxCommand<0x21> 380 { 381 using OsMailboxCommand::OsMailboxCommand; 382 FIELD(unsigned, p1Hi, 15, 8); 383 FIELD(unsigned, p1Lo, 7, 0); 384 }; 385 386 /** 387 * Implementation of SSTInterface based on OS Mailbox interface supported on ICX 388 * and SPR processors. 389 * It's expected that an instance of this class will be created for each 390 * "atomic" set of operations. 391 */ 392 class SSTMailbox : public SSTInterface 393 { 394 private: 395 uint8_t address; 396 CPUModel model; 397 PECIManager pm; 398 399 static constexpr int mhzPerRatio = 100; 400 401 public: SSTMailbox(uint8_t _address,CPUModel _model,WakePolicy wakePolicy)402 SSTMailbox(uint8_t _address, CPUModel _model, WakePolicy wakePolicy) : 403 address(_address), model(_model), 404 pm(static_cast<uint8_t>(address), model, wakePolicy) 405 {} ~SSTMailbox()406 ~SSTMailbox() {} 407 ready()408 bool ready() override 409 { 410 return true; 411 } 412 supportsControl()413 bool supportsControl() override 414 { 415 switch (model) 416 { 417 case sapphireRapids: 418 case emeraldRapids: 419 return true; 420 default: 421 return false; 422 } 423 } 424 currentLevel()425 unsigned int currentLevel() override 426 { 427 return GetLevelsInfo(pm).currentConfigTdpLevel(); 428 } maxLevel()429 unsigned int maxLevel() override 430 { 431 return GetLevelsInfo(pm).configTdpLevels(); 432 } ppEnabled()433 bool ppEnabled() override 434 { 435 return GetLevelsInfo(pm).enabled(); 436 } 437 levelSupported(unsigned int level)438 bool levelSupported(unsigned int level) override 439 { 440 GetConfigTdpControl tdpControl( 441 pm, GetConfigTdpControl::ErrorPolicy::NoThrow, 442 static_cast<uint8_t>(level)); 443 return tdpControl.success(); 444 } bfSupported(unsigned int level)445 bool bfSupported(unsigned int level) override 446 { 447 return GetConfigTdpControl(pm, static_cast<uint8_t>(level)) 448 .pbfSupport(); 449 } tfSupported(unsigned int level)450 bool tfSupported(unsigned int level) override 451 { 452 return GetConfigTdpControl(pm, static_cast<uint8_t>(level)) 453 .factSupport(); 454 } bfEnabled(unsigned int level)455 bool bfEnabled(unsigned int level) override 456 { 457 return GetConfigTdpControl(pm, static_cast<uint8_t>(level)) 458 .pbfEnabled(); 459 } tfEnabled(unsigned int level)460 bool tfEnabled(unsigned int level) override 461 { 462 return GetConfigTdpControl(pm, static_cast<uint8_t>(level)) 463 .factEnabled(); 464 } tdp(unsigned int level)465 unsigned int tdp(unsigned int level) override 466 { 467 return GetTdpInfo(pm, static_cast<uint8_t>(level)).pkgTdp(); 468 } coreCount(unsigned int level)469 unsigned int coreCount(unsigned int level) override 470 { 471 return enabledCoreList(level).size(); 472 } enabledCoreList(unsigned int level)473 std::vector<unsigned int> enabledCoreList(unsigned int level) override 474 { 475 uint64_t coreMaskLo = 476 GetCoreMask(pm, static_cast<uint8_t>(level), 0).coresMask(); 477 uint64_t coreMaskHi = 478 GetCoreMask(pm, static_cast<uint8_t>(level), 1).coresMask(); 479 std::bitset<64> coreMask = (coreMaskHi << 32 | coreMaskLo); 480 return convertMaskToList(coreMask); 481 } sseTurboProfile(unsigned int level)482 std::vector<TurboEntry> sseTurboProfile(unsigned int level) override 483 { 484 // Read the Turbo Ratio Limit Cores MSR which is used to generate the 485 // Turbo Profile for each profile. This is a package scope MSR, so just 486 // read thread 0. 487 uint64_t trlCores; 488 uint8_t cc; 489 EPECIStatus status = peci_RdIAMSR(static_cast<uint8_t>(address), 0, 490 0x1AE, &trlCores, &cc); 491 if (!checkPECIStatus(status, cc)) 492 { 493 throw PECIError("Failed to read TRL MSR"); 494 } 495 496 std::vector<TurboEntry> turboSpeeds; 497 uint64_t limitRatioLo = 498 GetTurboLimitRatios(pm, static_cast<uint8_t>(level), 0, 0).value; 499 uint64_t limitRatioHi = 500 GetTurboLimitRatios(pm, static_cast<uint8_t>(level), 1, 0).value; 501 uint64_t limitRatios = (limitRatioHi << 32) | limitRatioLo; 502 503 constexpr int maxTFBuckets = 8; 504 for (int i = 0; i < maxTFBuckets; ++i) 505 { 506 size_t bucketCount = trlCores & 0xFF; 507 int bucketSpeed = limitRatios & 0xFF; 508 if (bucketCount != 0 && bucketSpeed != 0) 509 { 510 turboSpeeds.push_back({bucketSpeed * mhzPerRatio, bucketCount}); 511 } 512 513 trlCores >>= 8; 514 limitRatios >>= 8; 515 } 516 return turboSpeeds; 517 } p1Freq(unsigned int level)518 unsigned int p1Freq(unsigned int level) override 519 { 520 return GetRatioInfo(pm, static_cast<uint8_t>(level)).p1() * mhzPerRatio; 521 } p0Freq(unsigned int level)522 unsigned int p0Freq(unsigned int level) override 523 { 524 return GetRatioInfo(pm, static_cast<uint8_t>(level)).p0() * mhzPerRatio; 525 } prochotTemp(unsigned int level)526 unsigned int prochotTemp(unsigned int level) override 527 { 528 return GetTjmaxInfo(pm, static_cast<uint8_t>(level)).tProchot(); 529 } 530 std::vector<unsigned int> bfHighPriorityCoreList(unsigned int level)531 bfHighPriorityCoreList(unsigned int level) override 532 { 533 uint64_t coreMaskLo = 534 PbfGetCoreMaskInfo(pm, static_cast<uint8_t>(level), 0) 535 .p1HiCoreMask(); 536 uint64_t coreMaskHi = 537 PbfGetCoreMaskInfo(pm, static_cast<uint8_t>(level), 1) 538 .p1HiCoreMask(); 539 std::bitset<64> hiFreqCoreList = (coreMaskHi << 32) | coreMaskLo; 540 return convertMaskToList(hiFreqCoreList); 541 } bfHighPriorityFreq(unsigned int level)542 unsigned int bfHighPriorityFreq(unsigned int level) override 543 { 544 return PbfGetP1HiP1LoInfo(pm, static_cast<uint8_t>(level)).p1Hi() * 545 mhzPerRatio; 546 } bfLowPriorityFreq(unsigned int level)547 unsigned int bfLowPriorityFreq(unsigned int level) override 548 { 549 return PbfGetP1HiP1LoInfo(pm, static_cast<uint8_t>(level)).p1Lo() * 550 mhzPerRatio; 551 } 552 setBfEnabled(bool enable)553 void setBfEnabled(bool enable) override 554 { 555 GetConfigTdpControl getTDPControl(pm); 556 bool tfEnabled = false; 557 uint8_t param = (enable ? bit(1) : 0) | (tfEnabled ? bit(0) : 0); 558 SetConfigTdpControl(pm, 0, 0, param); 559 } setTfEnabled(bool enable)560 void setTfEnabled(bool enable) override 561 { 562 // TODO: use cached BF value 563 bool bfEnabled = false; 564 uint8_t param = (bfEnabled ? bit(1) : 0) | (enable ? bit(0) : 0); 565 SetConfigTdpControl(pm, 0, 0, param); 566 } setCurrentLevel(unsigned int level)567 void setCurrentLevel(unsigned int level) override 568 { 569 SetLevel(pm, static_cast<uint8_t>(level)); 570 } 571 }; 572 573 static std::unique_ptr<SSTInterface> createMailbox(uint8_t address,CPUModel model,WakePolicy wakePolicy)574 createMailbox(uint8_t address, CPUModel model, WakePolicy wakePolicy) 575 { 576 DEBUG_PRINT << "createMailbox\n"; 577 switch (model) 578 { 579 case iceLake: 580 case iceLakeD: 581 case sapphireRapids: 582 case emeraldRapids: 583 return std::make_unique<SSTMailbox>(address, model, wakePolicy); 584 default: 585 return nullptr; 586 } 587 } 588 589 SSTProviderRegistration(createMailbox); 590 591 } // namespace sst 592 } // namespace cpu_info 593