1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2020 Intel Corporation. */ 3 #include <linux/io-64-nonatomic-lo-hi.h> 4 #include <linux/device.h> 5 #include <linux/slab.h> 6 #include <linux/pci.h> 7 #include <cxlmem.h> 8 #include <cxlpci.h> 9 10 #include "core.h" 11 12 /** 13 * DOC: cxl registers 14 * 15 * CXL device capabilities are enumerated by PCI DVSEC (Designated 16 * Vendor-specific) and / or descriptors provided by platform firmware. 17 * They can be defined as a set like the device and component registers 18 * mandated by CXL Section 8.1.12.2 Memory Device PCIe Capabilities and 19 * Extended Capabilities, or they can be individual capabilities 20 * appended to bridged and endpoint devices. 21 * 22 * Provide common infrastructure for enumerating and mapping these 23 * discrete capabilities. 24 */ 25 26 /** 27 * cxl_probe_component_regs() - Detect CXL Component register blocks 28 * @dev: Host device of the @base mapping 29 * @base: Mapping containing the HDM Decoder Capability Header 30 * @map: Map object describing the register block information found 31 * 32 * See CXL 2.0 8.2.4 Component Register Layout and Definition 33 * See CXL 2.0 8.2.5.5 CXL Device Register Interface 34 * 35 * Probe for component register information and return it in map object. 36 */ 37 void cxl_probe_component_regs(struct device *dev, void __iomem *base, 38 struct cxl_component_reg_map *map) 39 { 40 int cap, cap_count; 41 u32 cap_array; 42 43 *map = (struct cxl_component_reg_map) { 0 }; 44 45 /* 46 * CXL.cache and CXL.mem registers are at offset 0x1000 as defined in 47 * CXL 2.0 8.2.4 Table 141. 48 */ 49 base += CXL_CM_OFFSET; 50 51 cap_array = readl(base + CXL_CM_CAP_HDR_OFFSET); 52 53 if (FIELD_GET(CXL_CM_CAP_HDR_ID_MASK, cap_array) != CM_CAP_HDR_CAP_ID) { 54 dev_err(dev, 55 "Couldn't locate the CXL.cache and CXL.mem capability array header.\n"); 56 return; 57 } 58 59 /* It's assumed that future versions will be backward compatible */ 60 cap_count = FIELD_GET(CXL_CM_CAP_HDR_ARRAY_SIZE_MASK, cap_array); 61 62 for (cap = 1; cap <= cap_count; cap++) { 63 void __iomem *register_block; 64 struct cxl_reg_map *rmap; 65 u16 cap_id, offset; 66 u32 length, hdr; 67 68 hdr = readl(base + cap * 0x4); 69 70 cap_id = FIELD_GET(CXL_CM_CAP_HDR_ID_MASK, hdr); 71 offset = FIELD_GET(CXL_CM_CAP_PTR_MASK, hdr); 72 register_block = base + offset; 73 hdr = readl(register_block); 74 75 rmap = NULL; 76 switch (cap_id) { 77 case CXL_CM_CAP_CAP_ID_HDM: { 78 int decoder_cnt; 79 80 dev_dbg(dev, "found HDM decoder capability (0x%x)\n", 81 offset); 82 83 decoder_cnt = cxl_hdm_decoder_count(hdr); 84 length = 0x20 * decoder_cnt + 0x10; 85 rmap = &map->hdm_decoder; 86 break; 87 } 88 case CXL_CM_CAP_CAP_ID_RAS: 89 dev_dbg(dev, "found RAS capability (0x%x)\n", 90 offset); 91 length = CXL_RAS_CAPABILITY_LENGTH; 92 rmap = &map->ras; 93 break; 94 default: 95 dev_dbg(dev, "Unknown CM cap ID: %d (0x%x)\n", cap_id, 96 offset); 97 break; 98 } 99 100 if (!rmap) 101 continue; 102 rmap->valid = true; 103 rmap->id = cap_id; 104 rmap->offset = CXL_CM_OFFSET + offset; 105 rmap->size = length; 106 } 107 } 108 EXPORT_SYMBOL_NS_GPL(cxl_probe_component_regs, CXL); 109 110 /** 111 * cxl_probe_device_regs() - Detect CXL Device register blocks 112 * @dev: Host device of the @base mapping 113 * @base: Mapping of CXL 2.0 8.2.8 CXL Device Register Interface 114 * @map: Map object describing the register block information found 115 * 116 * Probe for device register information and return it in map object. 117 */ 118 void cxl_probe_device_regs(struct device *dev, void __iomem *base, 119 struct cxl_device_reg_map *map) 120 { 121 int cap, cap_count; 122 u64 cap_array; 123 124 *map = (struct cxl_device_reg_map){ 0 }; 125 126 cap_array = readq(base + CXLDEV_CAP_ARRAY_OFFSET); 127 if (FIELD_GET(CXLDEV_CAP_ARRAY_ID_MASK, cap_array) != 128 CXLDEV_CAP_ARRAY_CAP_ID) 129 return; 130 131 cap_count = FIELD_GET(CXLDEV_CAP_ARRAY_COUNT_MASK, cap_array); 132 133 for (cap = 1; cap <= cap_count; cap++) { 134 struct cxl_reg_map *rmap; 135 u32 offset, length; 136 u16 cap_id; 137 138 cap_id = FIELD_GET(CXLDEV_CAP_HDR_CAP_ID_MASK, 139 readl(base + cap * 0x10)); 140 offset = readl(base + cap * 0x10 + 0x4); 141 length = readl(base + cap * 0x10 + 0x8); 142 143 rmap = NULL; 144 switch (cap_id) { 145 case CXLDEV_CAP_CAP_ID_DEVICE_STATUS: 146 dev_dbg(dev, "found Status capability (0x%x)\n", offset); 147 rmap = &map->status; 148 break; 149 case CXLDEV_CAP_CAP_ID_PRIMARY_MAILBOX: 150 dev_dbg(dev, "found Mailbox capability (0x%x)\n", offset); 151 rmap = &map->mbox; 152 break; 153 case CXLDEV_CAP_CAP_ID_SECONDARY_MAILBOX: 154 dev_dbg(dev, "found Secondary Mailbox capability (0x%x)\n", offset); 155 break; 156 case CXLDEV_CAP_CAP_ID_MEMDEV: 157 dev_dbg(dev, "found Memory Device capability (0x%x)\n", offset); 158 rmap = &map->memdev; 159 break; 160 default: 161 if (cap_id >= 0x8000) 162 dev_dbg(dev, "Vendor cap ID: %#x offset: %#x\n", cap_id, offset); 163 else 164 dev_dbg(dev, "Unknown cap ID: %#x offset: %#x\n", cap_id, offset); 165 break; 166 } 167 168 if (!rmap) 169 continue; 170 rmap->valid = true; 171 rmap->id = cap_id; 172 rmap->offset = offset; 173 rmap->size = length; 174 } 175 } 176 EXPORT_SYMBOL_NS_GPL(cxl_probe_device_regs, CXL); 177 178 void __iomem *devm_cxl_iomap_block(struct device *dev, resource_size_t addr, 179 resource_size_t length) 180 { 181 void __iomem *ret_val; 182 struct resource *res; 183 184 if (WARN_ON_ONCE(addr == CXL_RESOURCE_NONE)) 185 return NULL; 186 187 res = devm_request_mem_region(dev, addr, length, dev_name(dev)); 188 if (!res) { 189 resource_size_t end = addr + length - 1; 190 191 dev_err(dev, "Failed to request region %pa-%pa\n", &addr, &end); 192 return NULL; 193 } 194 195 ret_val = devm_ioremap(dev, addr, length); 196 if (!ret_val) 197 dev_err(dev, "Failed to map region %pr\n", res); 198 199 return ret_val; 200 } 201 202 int cxl_map_component_regs(struct device *dev, struct cxl_component_regs *regs, 203 struct cxl_register_map *map, unsigned long map_mask) 204 { 205 struct mapinfo { 206 struct cxl_reg_map *rmap; 207 void __iomem **addr; 208 } mapinfo[] = { 209 { &map->component_map.hdm_decoder, ®s->hdm_decoder }, 210 { &map->component_map.ras, ®s->ras }, 211 }; 212 int i; 213 214 for (i = 0; i < ARRAY_SIZE(mapinfo); i++) { 215 struct mapinfo *mi = &mapinfo[i]; 216 resource_size_t phys_addr; 217 resource_size_t length; 218 219 if (!mi->rmap->valid) 220 continue; 221 if (!test_bit(mi->rmap->id, &map_mask)) 222 continue; 223 phys_addr = map->resource + mi->rmap->offset; 224 length = mi->rmap->size; 225 *(mi->addr) = devm_cxl_iomap_block(dev, phys_addr, length); 226 if (!*(mi->addr)) 227 return -ENOMEM; 228 } 229 230 return 0; 231 } 232 EXPORT_SYMBOL_NS_GPL(cxl_map_component_regs, CXL); 233 234 int cxl_map_device_regs(struct device *dev, 235 struct cxl_device_regs *regs, 236 struct cxl_register_map *map) 237 { 238 resource_size_t phys_addr = map->resource; 239 struct mapinfo { 240 struct cxl_reg_map *rmap; 241 void __iomem **addr; 242 } mapinfo[] = { 243 { &map->device_map.status, ®s->status, }, 244 { &map->device_map.mbox, ®s->mbox, }, 245 { &map->device_map.memdev, ®s->memdev, }, 246 }; 247 int i; 248 249 for (i = 0; i < ARRAY_SIZE(mapinfo); i++) { 250 struct mapinfo *mi = &mapinfo[i]; 251 resource_size_t length; 252 resource_size_t addr; 253 254 if (!mi->rmap->valid) 255 continue; 256 257 addr = phys_addr + mi->rmap->offset; 258 length = mi->rmap->size; 259 *(mi->addr) = devm_cxl_iomap_block(dev, addr, length); 260 if (!*(mi->addr)) 261 return -ENOMEM; 262 } 263 264 return 0; 265 } 266 EXPORT_SYMBOL_NS_GPL(cxl_map_device_regs, CXL); 267 268 static bool cxl_decode_regblock(struct pci_dev *pdev, u32 reg_lo, u32 reg_hi, 269 struct cxl_register_map *map) 270 { 271 int bar = FIELD_GET(CXL_DVSEC_REG_LOCATOR_BIR_MASK, reg_lo); 272 u64 offset = ((u64)reg_hi << 32) | 273 (reg_lo & CXL_DVSEC_REG_LOCATOR_BLOCK_OFF_LOW_MASK); 274 275 if (offset > pci_resource_len(pdev, bar)) { 276 dev_warn(&pdev->dev, 277 "BAR%d: %pr: too small (offset: %pa, type: %d)\n", bar, 278 &pdev->resource[bar], &offset, map->reg_type); 279 return false; 280 } 281 282 map->reg_type = FIELD_GET(CXL_DVSEC_REG_LOCATOR_BLOCK_ID_MASK, reg_lo); 283 map->resource = pci_resource_start(pdev, bar) + offset; 284 map->max_size = pci_resource_len(pdev, bar) - offset; 285 return true; 286 } 287 288 /** 289 * cxl_find_regblock() - Locate register blocks by type 290 * @pdev: The CXL PCI device to enumerate. 291 * @type: Register Block Indicator id 292 * @map: Enumeration output, clobbered on error 293 * 294 * Return: 0 if register block enumerated, negative error code otherwise 295 * 296 * A CXL DVSEC may point to one or more register blocks, search for them 297 * by @type. 298 */ 299 int cxl_find_regblock(struct pci_dev *pdev, enum cxl_regloc_type type, 300 struct cxl_register_map *map) 301 { 302 u32 regloc_size, regblocks; 303 int regloc, i; 304 305 map->resource = CXL_RESOURCE_NONE; 306 regloc = pci_find_dvsec_capability(pdev, PCI_DVSEC_VENDOR_ID_CXL, 307 CXL_DVSEC_REG_LOCATOR); 308 if (!regloc) 309 return -ENXIO; 310 311 pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, ®loc_size); 312 regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size); 313 314 regloc += CXL_DVSEC_REG_LOCATOR_BLOCK1_OFFSET; 315 regblocks = (regloc_size - CXL_DVSEC_REG_LOCATOR_BLOCK1_OFFSET) / 8; 316 317 for (i = 0; i < regblocks; i++, regloc += 8) { 318 u32 reg_lo, reg_hi; 319 320 pci_read_config_dword(pdev, regloc, ®_lo); 321 pci_read_config_dword(pdev, regloc + 4, ®_hi); 322 323 if (!cxl_decode_regblock(pdev, reg_lo, reg_hi, map)) 324 continue; 325 326 if (map->reg_type == type) 327 return 0; 328 } 329 330 map->resource = CXL_RESOURCE_NONE; 331 return -ENODEV; 332 } 333 EXPORT_SYMBOL_NS_GPL(cxl_find_regblock, CXL); 334 335 resource_size_t cxl_rcrb_to_component(struct device *dev, 336 resource_size_t rcrb, 337 enum cxl_rcrb which) 338 { 339 resource_size_t component_reg_phys; 340 void __iomem *addr; 341 u32 bar0, bar1; 342 u16 cmd; 343 u32 id; 344 345 if (which == CXL_RCRB_UPSTREAM) 346 rcrb += SZ_4K; 347 348 /* 349 * RCRB's BAR[0..1] point to component block containing CXL 350 * subsystem component registers. MEMBAR extraction follows 351 * the PCI Base spec here, esp. 64 bit extraction and memory 352 * ranges alignment (6.0, 7.5.1.2.1). 353 */ 354 if (!request_mem_region(rcrb, SZ_4K, "CXL RCRB")) 355 return CXL_RESOURCE_NONE; 356 addr = ioremap(rcrb, SZ_4K); 357 if (!addr) { 358 dev_err(dev, "Failed to map region %pr\n", addr); 359 release_mem_region(rcrb, SZ_4K); 360 return CXL_RESOURCE_NONE; 361 } 362 363 id = readl(addr + PCI_VENDOR_ID); 364 cmd = readw(addr + PCI_COMMAND); 365 bar0 = readl(addr + PCI_BASE_ADDRESS_0); 366 bar1 = readl(addr + PCI_BASE_ADDRESS_1); 367 iounmap(addr); 368 release_mem_region(rcrb, SZ_4K); 369 370 /* 371 * Sanity check, see CXL 3.0 Figure 9-8 CXL Device that Does Not 372 * Remap Upstream Port and Component Registers 373 */ 374 if (id == U32_MAX) { 375 if (which == CXL_RCRB_DOWNSTREAM) 376 dev_err(dev, "Failed to access Downstream Port RCRB\n"); 377 return CXL_RESOURCE_NONE; 378 } 379 if (!(cmd & PCI_COMMAND_MEMORY)) 380 return CXL_RESOURCE_NONE; 381 /* The RCRB is a Memory Window, and the MEM_TYPE_1M bit is obsolete */ 382 if (bar0 & (PCI_BASE_ADDRESS_MEM_TYPE_1M | PCI_BASE_ADDRESS_SPACE_IO)) 383 return CXL_RESOURCE_NONE; 384 385 component_reg_phys = bar0 & PCI_BASE_ADDRESS_MEM_MASK; 386 if (bar0 & PCI_BASE_ADDRESS_MEM_TYPE_64) 387 component_reg_phys |= ((u64)bar1) << 32; 388 389 if (!component_reg_phys) 390 return CXL_RESOURCE_NONE; 391 392 /* MEMBAR is block size (64k) aligned. */ 393 if (!IS_ALIGNED(component_reg_phys, CXL_COMPONENT_REG_BLOCK_SIZE)) 394 return CXL_RESOURCE_NONE; 395 396 return component_reg_phys; 397 } 398 EXPORT_SYMBOL_NS_GPL(cxl_rcrb_to_component, CXL); 399