1 /* pci_sun4v.c: SUN4V specific PCI controller support. 2 * 3 * Copyright (C) 2006, 2007, 2008 David S. Miller (davem@davemloft.net) 4 */ 5 6 #include <linux/kernel.h> 7 #include <linux/types.h> 8 #include <linux/pci.h> 9 #include <linux/init.h> 10 #include <linux/slab.h> 11 #include <linux/interrupt.h> 12 #include <linux/percpu.h> 13 #include <linux/irq.h> 14 #include <linux/msi.h> 15 #include <linux/export.h> 16 #include <linux/log2.h> 17 #include <linux/of_device.h> 18 #include <linux/iommu-common.h> 19 20 #include <asm/iommu.h> 21 #include <asm/irq.h> 22 #include <asm/hypervisor.h> 23 #include <asm/prom.h> 24 25 #include "pci_impl.h" 26 #include "iommu_common.h" 27 28 #include "pci_sun4v.h" 29 30 #define DRIVER_NAME "pci_sun4v" 31 #define PFX DRIVER_NAME ": " 32 33 static unsigned long vpci_major; 34 static unsigned long vpci_minor; 35 36 struct vpci_version { 37 unsigned long major; 38 unsigned long minor; 39 }; 40 41 /* Ordered from largest major to lowest */ 42 static struct vpci_version vpci_versions[] = { 43 { .major = 2, .minor = 0 }, 44 { .major = 1, .minor = 1 }, 45 }; 46 47 #define PGLIST_NENTS (PAGE_SIZE / sizeof(u64)) 48 49 struct iommu_batch { 50 struct device *dev; /* Device mapping is for. */ 51 unsigned long prot; /* IOMMU page protections */ 52 unsigned long entry; /* Index into IOTSB. */ 53 u64 *pglist; /* List of physical pages */ 54 unsigned long npages; /* Number of pages in list. */ 55 }; 56 57 static DEFINE_PER_CPU(struct iommu_batch, iommu_batch); 58 static int iommu_batch_initialized; 59 60 /* Interrupts must be disabled. */ 61 static inline void iommu_batch_start(struct device *dev, unsigned long prot, unsigned long entry) 62 { 63 struct iommu_batch *p = this_cpu_ptr(&iommu_batch); 64 65 p->dev = dev; 66 p->prot = prot; 67 p->entry = entry; 68 p->npages = 0; 69 } 70 71 /* Interrupts must be disabled. */ 72 static long iommu_batch_flush(struct iommu_batch *p) 73 { 74 struct pci_pbm_info *pbm = p->dev->archdata.host_controller; 75 unsigned long devhandle = pbm->devhandle; 76 unsigned long prot = p->prot; 77 unsigned long entry = p->entry; 78 u64 *pglist = p->pglist; 79 unsigned long npages = p->npages; 80 81 /* VPCI maj=1, min=[0,1] only supports read and write */ 82 if (vpci_major < 2) 83 prot &= (HV_PCI_MAP_ATTR_READ | HV_PCI_MAP_ATTR_WRITE); 84 85 while (npages != 0) { 86 long num; 87 88 num = pci_sun4v_iommu_map(devhandle, HV_PCI_TSBID(0, entry), 89 npages, prot, __pa(pglist)); 90 if (unlikely(num < 0)) { 91 if (printk_ratelimit()) 92 printk("iommu_batch_flush: IOMMU map of " 93 "[%08lx:%08llx:%lx:%lx:%lx] failed with " 94 "status %ld\n", 95 devhandle, HV_PCI_TSBID(0, entry), 96 npages, prot, __pa(pglist), num); 97 return -1; 98 } 99 100 entry += num; 101 npages -= num; 102 pglist += num; 103 } 104 105 p->entry = entry; 106 p->npages = 0; 107 108 return 0; 109 } 110 111 static inline void iommu_batch_new_entry(unsigned long entry) 112 { 113 struct iommu_batch *p = this_cpu_ptr(&iommu_batch); 114 115 if (p->entry + p->npages == entry) 116 return; 117 if (p->entry != ~0UL) 118 iommu_batch_flush(p); 119 p->entry = entry; 120 } 121 122 /* Interrupts must be disabled. */ 123 static inline long iommu_batch_add(u64 phys_page) 124 { 125 struct iommu_batch *p = this_cpu_ptr(&iommu_batch); 126 127 BUG_ON(p->npages >= PGLIST_NENTS); 128 129 p->pglist[p->npages++] = phys_page; 130 if (p->npages == PGLIST_NENTS) 131 return iommu_batch_flush(p); 132 133 return 0; 134 } 135 136 /* Interrupts must be disabled. */ 137 static inline long iommu_batch_end(void) 138 { 139 struct iommu_batch *p = this_cpu_ptr(&iommu_batch); 140 141 BUG_ON(p->npages >= PGLIST_NENTS); 142 143 return iommu_batch_flush(p); 144 } 145 146 static void *dma_4v_alloc_coherent(struct device *dev, size_t size, 147 dma_addr_t *dma_addrp, gfp_t gfp, 148 unsigned long attrs) 149 { 150 unsigned long flags, order, first_page, npages, n; 151 unsigned long prot = 0; 152 struct iommu *iommu; 153 struct page *page; 154 void *ret; 155 long entry; 156 int nid; 157 158 size = IO_PAGE_ALIGN(size); 159 order = get_order(size); 160 if (unlikely(order >= MAX_ORDER)) 161 return NULL; 162 163 npages = size >> IO_PAGE_SHIFT; 164 165 if (attrs & DMA_ATTR_WEAK_ORDERING) 166 prot = HV_PCI_MAP_ATTR_RELAXED_ORDER; 167 168 nid = dev->archdata.numa_node; 169 page = alloc_pages_node(nid, gfp, order); 170 if (unlikely(!page)) 171 return NULL; 172 173 first_page = (unsigned long) page_address(page); 174 memset((char *)first_page, 0, PAGE_SIZE << order); 175 176 iommu = dev->archdata.iommu; 177 178 entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, NULL, 179 (unsigned long)(-1), 0); 180 181 if (unlikely(entry == IOMMU_ERROR_CODE)) 182 goto range_alloc_fail; 183 184 *dma_addrp = (iommu->tbl.table_map_base + (entry << IO_PAGE_SHIFT)); 185 ret = (void *) first_page; 186 first_page = __pa(first_page); 187 188 local_irq_save(flags); 189 190 iommu_batch_start(dev, 191 (HV_PCI_MAP_ATTR_READ | prot | 192 HV_PCI_MAP_ATTR_WRITE), 193 entry); 194 195 for (n = 0; n < npages; n++) { 196 long err = iommu_batch_add(first_page + (n * PAGE_SIZE)); 197 if (unlikely(err < 0L)) 198 goto iommu_map_fail; 199 } 200 201 if (unlikely(iommu_batch_end() < 0L)) 202 goto iommu_map_fail; 203 204 local_irq_restore(flags); 205 206 return ret; 207 208 iommu_map_fail: 209 iommu_tbl_range_free(&iommu->tbl, *dma_addrp, npages, IOMMU_ERROR_CODE); 210 211 range_alloc_fail: 212 free_pages(first_page, order); 213 return NULL; 214 } 215 216 static void dma_4v_iommu_demap(void *demap_arg, unsigned long entry, 217 unsigned long npages) 218 { 219 u32 devhandle = *(u32 *)demap_arg; 220 unsigned long num, flags; 221 222 local_irq_save(flags); 223 do { 224 num = pci_sun4v_iommu_demap(devhandle, 225 HV_PCI_TSBID(0, entry), 226 npages); 227 228 entry += num; 229 npages -= num; 230 } while (npages != 0); 231 local_irq_restore(flags); 232 } 233 234 static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu, 235 dma_addr_t dvma, unsigned long attrs) 236 { 237 struct pci_pbm_info *pbm; 238 struct iommu *iommu; 239 unsigned long order, npages, entry; 240 u32 devhandle; 241 242 npages = IO_PAGE_ALIGN(size) >> IO_PAGE_SHIFT; 243 iommu = dev->archdata.iommu; 244 pbm = dev->archdata.host_controller; 245 devhandle = pbm->devhandle; 246 entry = ((dvma - iommu->tbl.table_map_base) >> IO_PAGE_SHIFT); 247 dma_4v_iommu_demap(&devhandle, entry, npages); 248 iommu_tbl_range_free(&iommu->tbl, dvma, npages, IOMMU_ERROR_CODE); 249 order = get_order(size); 250 if (order < 10) 251 free_pages((unsigned long)cpu, order); 252 } 253 254 static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page, 255 unsigned long offset, size_t sz, 256 enum dma_data_direction direction, 257 unsigned long attrs) 258 { 259 struct iommu *iommu; 260 unsigned long flags, npages, oaddr; 261 unsigned long i, base_paddr; 262 u32 bus_addr, ret; 263 unsigned long prot; 264 long entry; 265 266 iommu = dev->archdata.iommu; 267 268 if (unlikely(direction == DMA_NONE)) 269 goto bad; 270 271 oaddr = (unsigned long)(page_address(page) + offset); 272 npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); 273 npages >>= IO_PAGE_SHIFT; 274 275 entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, NULL, 276 (unsigned long)(-1), 0); 277 278 if (unlikely(entry == IOMMU_ERROR_CODE)) 279 goto bad; 280 281 bus_addr = (iommu->tbl.table_map_base + (entry << IO_PAGE_SHIFT)); 282 ret = bus_addr | (oaddr & ~IO_PAGE_MASK); 283 base_paddr = __pa(oaddr & IO_PAGE_MASK); 284 prot = HV_PCI_MAP_ATTR_READ; 285 if (direction != DMA_TO_DEVICE) 286 prot |= HV_PCI_MAP_ATTR_WRITE; 287 288 if (attrs & DMA_ATTR_WEAK_ORDERING) 289 prot |= HV_PCI_MAP_ATTR_RELAXED_ORDER; 290 291 local_irq_save(flags); 292 293 iommu_batch_start(dev, prot, entry); 294 295 for (i = 0; i < npages; i++, base_paddr += IO_PAGE_SIZE) { 296 long err = iommu_batch_add(base_paddr); 297 if (unlikely(err < 0L)) 298 goto iommu_map_fail; 299 } 300 if (unlikely(iommu_batch_end() < 0L)) 301 goto iommu_map_fail; 302 303 local_irq_restore(flags); 304 305 return ret; 306 307 bad: 308 if (printk_ratelimit()) 309 WARN_ON(1); 310 return DMA_ERROR_CODE; 311 312 iommu_map_fail: 313 iommu_tbl_range_free(&iommu->tbl, bus_addr, npages, IOMMU_ERROR_CODE); 314 return DMA_ERROR_CODE; 315 } 316 317 static void dma_4v_unmap_page(struct device *dev, dma_addr_t bus_addr, 318 size_t sz, enum dma_data_direction direction, 319 unsigned long attrs) 320 { 321 struct pci_pbm_info *pbm; 322 struct iommu *iommu; 323 unsigned long npages; 324 long entry; 325 u32 devhandle; 326 327 if (unlikely(direction == DMA_NONE)) { 328 if (printk_ratelimit()) 329 WARN_ON(1); 330 return; 331 } 332 333 iommu = dev->archdata.iommu; 334 pbm = dev->archdata.host_controller; 335 devhandle = pbm->devhandle; 336 337 npages = IO_PAGE_ALIGN(bus_addr + sz) - (bus_addr & IO_PAGE_MASK); 338 npages >>= IO_PAGE_SHIFT; 339 bus_addr &= IO_PAGE_MASK; 340 entry = (bus_addr - iommu->tbl.table_map_base) >> IO_PAGE_SHIFT; 341 dma_4v_iommu_demap(&devhandle, entry, npages); 342 iommu_tbl_range_free(&iommu->tbl, bus_addr, npages, IOMMU_ERROR_CODE); 343 } 344 345 static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, 346 int nelems, enum dma_data_direction direction, 347 unsigned long attrs) 348 { 349 struct scatterlist *s, *outs, *segstart; 350 unsigned long flags, handle, prot; 351 dma_addr_t dma_next = 0, dma_addr; 352 unsigned int max_seg_size; 353 unsigned long seg_boundary_size; 354 int outcount, incount, i; 355 struct iommu *iommu; 356 unsigned long base_shift; 357 long err; 358 359 BUG_ON(direction == DMA_NONE); 360 361 iommu = dev->archdata.iommu; 362 if (nelems == 0 || !iommu) 363 return 0; 364 365 prot = HV_PCI_MAP_ATTR_READ; 366 if (direction != DMA_TO_DEVICE) 367 prot |= HV_PCI_MAP_ATTR_WRITE; 368 369 if (attrs & DMA_ATTR_WEAK_ORDERING) 370 prot |= HV_PCI_MAP_ATTR_RELAXED_ORDER; 371 372 outs = s = segstart = &sglist[0]; 373 outcount = 1; 374 incount = nelems; 375 handle = 0; 376 377 /* Init first segment length for backout at failure */ 378 outs->dma_length = 0; 379 380 local_irq_save(flags); 381 382 iommu_batch_start(dev, prot, ~0UL); 383 384 max_seg_size = dma_get_max_seg_size(dev); 385 seg_boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, 386 IO_PAGE_SIZE) >> IO_PAGE_SHIFT; 387 base_shift = iommu->tbl.table_map_base >> IO_PAGE_SHIFT; 388 for_each_sg(sglist, s, nelems, i) { 389 unsigned long paddr, npages, entry, out_entry = 0, slen; 390 391 slen = s->length; 392 /* Sanity check */ 393 if (slen == 0) { 394 dma_next = 0; 395 continue; 396 } 397 /* Allocate iommu entries for that segment */ 398 paddr = (unsigned long) SG_ENT_PHYS_ADDRESS(s); 399 npages = iommu_num_pages(paddr, slen, IO_PAGE_SIZE); 400 entry = iommu_tbl_range_alloc(dev, &iommu->tbl, npages, 401 &handle, (unsigned long)(-1), 0); 402 403 /* Handle failure */ 404 if (unlikely(entry == IOMMU_ERROR_CODE)) { 405 if (printk_ratelimit()) 406 printk(KERN_INFO "iommu_alloc failed, iommu %p paddr %lx" 407 " npages %lx\n", iommu, paddr, npages); 408 goto iommu_map_failed; 409 } 410 411 iommu_batch_new_entry(entry); 412 413 /* Convert entry to a dma_addr_t */ 414 dma_addr = iommu->tbl.table_map_base + (entry << IO_PAGE_SHIFT); 415 dma_addr |= (s->offset & ~IO_PAGE_MASK); 416 417 /* Insert into HW table */ 418 paddr &= IO_PAGE_MASK; 419 while (npages--) { 420 err = iommu_batch_add(paddr); 421 if (unlikely(err < 0L)) 422 goto iommu_map_failed; 423 paddr += IO_PAGE_SIZE; 424 } 425 426 /* If we are in an open segment, try merging */ 427 if (segstart != s) { 428 /* We cannot merge if: 429 * - allocated dma_addr isn't contiguous to previous allocation 430 */ 431 if ((dma_addr != dma_next) || 432 (outs->dma_length + s->length > max_seg_size) || 433 (is_span_boundary(out_entry, base_shift, 434 seg_boundary_size, outs, s))) { 435 /* Can't merge: create a new segment */ 436 segstart = s; 437 outcount++; 438 outs = sg_next(outs); 439 } else { 440 outs->dma_length += s->length; 441 } 442 } 443 444 if (segstart == s) { 445 /* This is a new segment, fill entries */ 446 outs->dma_address = dma_addr; 447 outs->dma_length = slen; 448 out_entry = entry; 449 } 450 451 /* Calculate next page pointer for contiguous check */ 452 dma_next = dma_addr + slen; 453 } 454 455 err = iommu_batch_end(); 456 457 if (unlikely(err < 0L)) 458 goto iommu_map_failed; 459 460 local_irq_restore(flags); 461 462 if (outcount < incount) { 463 outs = sg_next(outs); 464 outs->dma_address = DMA_ERROR_CODE; 465 outs->dma_length = 0; 466 } 467 468 return outcount; 469 470 iommu_map_failed: 471 for_each_sg(sglist, s, nelems, i) { 472 if (s->dma_length != 0) { 473 unsigned long vaddr, npages; 474 475 vaddr = s->dma_address & IO_PAGE_MASK; 476 npages = iommu_num_pages(s->dma_address, s->dma_length, 477 IO_PAGE_SIZE); 478 iommu_tbl_range_free(&iommu->tbl, vaddr, npages, 479 IOMMU_ERROR_CODE); 480 /* XXX demap? XXX */ 481 s->dma_address = DMA_ERROR_CODE; 482 s->dma_length = 0; 483 } 484 if (s == outs) 485 break; 486 } 487 local_irq_restore(flags); 488 489 return 0; 490 } 491 492 static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist, 493 int nelems, enum dma_data_direction direction, 494 unsigned long attrs) 495 { 496 struct pci_pbm_info *pbm; 497 struct scatterlist *sg; 498 struct iommu *iommu; 499 unsigned long flags, entry; 500 u32 devhandle; 501 502 BUG_ON(direction == DMA_NONE); 503 504 iommu = dev->archdata.iommu; 505 pbm = dev->archdata.host_controller; 506 devhandle = pbm->devhandle; 507 508 local_irq_save(flags); 509 510 sg = sglist; 511 while (nelems--) { 512 dma_addr_t dma_handle = sg->dma_address; 513 unsigned int len = sg->dma_length; 514 unsigned long npages; 515 struct iommu_map_table *tbl = &iommu->tbl; 516 unsigned long shift = IO_PAGE_SHIFT; 517 518 if (!len) 519 break; 520 npages = iommu_num_pages(dma_handle, len, IO_PAGE_SIZE); 521 entry = ((dma_handle - tbl->table_map_base) >> shift); 522 dma_4v_iommu_demap(&devhandle, entry, npages); 523 iommu_tbl_range_free(&iommu->tbl, dma_handle, npages, 524 IOMMU_ERROR_CODE); 525 sg = sg_next(sg); 526 } 527 528 local_irq_restore(flags); 529 } 530 531 static struct dma_map_ops sun4v_dma_ops = { 532 .alloc = dma_4v_alloc_coherent, 533 .free = dma_4v_free_coherent, 534 .map_page = dma_4v_map_page, 535 .unmap_page = dma_4v_unmap_page, 536 .map_sg = dma_4v_map_sg, 537 .unmap_sg = dma_4v_unmap_sg, 538 }; 539 540 static void pci_sun4v_scan_bus(struct pci_pbm_info *pbm, struct device *parent) 541 { 542 struct property *prop; 543 struct device_node *dp; 544 545 dp = pbm->op->dev.of_node; 546 prop = of_find_property(dp, "66mhz-capable", NULL); 547 pbm->is_66mhz_capable = (prop != NULL); 548 pbm->pci_bus = pci_scan_one_pbm(pbm, parent); 549 550 /* XXX register error interrupt handlers XXX */ 551 } 552 553 static unsigned long probe_existing_entries(struct pci_pbm_info *pbm, 554 struct iommu_map_table *iommu) 555 { 556 struct iommu_pool *pool; 557 unsigned long i, pool_nr, cnt = 0; 558 u32 devhandle; 559 560 devhandle = pbm->devhandle; 561 for (pool_nr = 0; pool_nr < iommu->nr_pools; pool_nr++) { 562 pool = &(iommu->pools[pool_nr]); 563 for (i = pool->start; i <= pool->end; i++) { 564 unsigned long ret, io_attrs, ra; 565 566 ret = pci_sun4v_iommu_getmap(devhandle, 567 HV_PCI_TSBID(0, i), 568 &io_attrs, &ra); 569 if (ret == HV_EOK) { 570 if (page_in_phys_avail(ra)) { 571 pci_sun4v_iommu_demap(devhandle, 572 HV_PCI_TSBID(0, 573 i), 1); 574 } else { 575 cnt++; 576 __set_bit(i, iommu->map); 577 } 578 } 579 } 580 } 581 return cnt; 582 } 583 584 static int pci_sun4v_iommu_init(struct pci_pbm_info *pbm) 585 { 586 static const u32 vdma_default[] = { 0x80000000, 0x80000000 }; 587 struct iommu *iommu = pbm->iommu; 588 unsigned long num_tsb_entries, sz; 589 u32 dma_mask, dma_offset; 590 const u32 *vdma; 591 592 vdma = of_get_property(pbm->op->dev.of_node, "virtual-dma", NULL); 593 if (!vdma) 594 vdma = vdma_default; 595 596 if ((vdma[0] | vdma[1]) & ~IO_PAGE_MASK) { 597 printk(KERN_ERR PFX "Strange virtual-dma[%08x:%08x].\n", 598 vdma[0], vdma[1]); 599 return -EINVAL; 600 } 601 602 dma_mask = (roundup_pow_of_two(vdma[1]) - 1UL); 603 num_tsb_entries = vdma[1] / IO_PAGE_SIZE; 604 605 dma_offset = vdma[0]; 606 607 /* Setup initial software IOMMU state. */ 608 spin_lock_init(&iommu->lock); 609 iommu->ctx_lowest_free = 1; 610 iommu->tbl.table_map_base = dma_offset; 611 iommu->dma_addr_mask = dma_mask; 612 613 /* Allocate and initialize the free area map. */ 614 sz = (num_tsb_entries + 7) / 8; 615 sz = (sz + 7UL) & ~7UL; 616 iommu->tbl.map = kzalloc(sz, GFP_KERNEL); 617 if (!iommu->tbl.map) { 618 printk(KERN_ERR PFX "Error, kmalloc(arena.map) failed.\n"); 619 return -ENOMEM; 620 } 621 iommu_tbl_pool_init(&iommu->tbl, num_tsb_entries, IO_PAGE_SHIFT, 622 NULL, false /* no large_pool */, 623 0 /* default npools */, 624 false /* want span boundary checking */); 625 sz = probe_existing_entries(pbm, &iommu->tbl); 626 if (sz) 627 printk("%s: Imported %lu TSB entries from OBP\n", 628 pbm->name, sz); 629 630 return 0; 631 } 632 633 #ifdef CONFIG_PCI_MSI 634 struct pci_sun4v_msiq_entry { 635 u64 version_type; 636 #define MSIQ_VERSION_MASK 0xffffffff00000000UL 637 #define MSIQ_VERSION_SHIFT 32 638 #define MSIQ_TYPE_MASK 0x00000000000000ffUL 639 #define MSIQ_TYPE_SHIFT 0 640 #define MSIQ_TYPE_NONE 0x00 641 #define MSIQ_TYPE_MSG 0x01 642 #define MSIQ_TYPE_MSI32 0x02 643 #define MSIQ_TYPE_MSI64 0x03 644 #define MSIQ_TYPE_INTX 0x08 645 #define MSIQ_TYPE_NONE2 0xff 646 647 u64 intx_sysino; 648 u64 reserved1; 649 u64 stick; 650 u64 req_id; /* bus/device/func */ 651 #define MSIQ_REQID_BUS_MASK 0xff00UL 652 #define MSIQ_REQID_BUS_SHIFT 8 653 #define MSIQ_REQID_DEVICE_MASK 0x00f8UL 654 #define MSIQ_REQID_DEVICE_SHIFT 3 655 #define MSIQ_REQID_FUNC_MASK 0x0007UL 656 #define MSIQ_REQID_FUNC_SHIFT 0 657 658 u64 msi_address; 659 660 /* The format of this value is message type dependent. 661 * For MSI bits 15:0 are the data from the MSI packet. 662 * For MSI-X bits 31:0 are the data from the MSI packet. 663 * For MSG, the message code and message routing code where: 664 * bits 39:32 is the bus/device/fn of the msg target-id 665 * bits 18:16 is the message routing code 666 * bits 7:0 is the message code 667 * For INTx the low order 2-bits are: 668 * 00 - INTA 669 * 01 - INTB 670 * 10 - INTC 671 * 11 - INTD 672 */ 673 u64 msi_data; 674 675 u64 reserved2; 676 }; 677 678 static int pci_sun4v_get_head(struct pci_pbm_info *pbm, unsigned long msiqid, 679 unsigned long *head) 680 { 681 unsigned long err, limit; 682 683 err = pci_sun4v_msiq_gethead(pbm->devhandle, msiqid, head); 684 if (unlikely(err)) 685 return -ENXIO; 686 687 limit = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry); 688 if (unlikely(*head >= limit)) 689 return -EFBIG; 690 691 return 0; 692 } 693 694 static int pci_sun4v_dequeue_msi(struct pci_pbm_info *pbm, 695 unsigned long msiqid, unsigned long *head, 696 unsigned long *msi) 697 { 698 struct pci_sun4v_msiq_entry *ep; 699 unsigned long err, type; 700 701 /* Note: void pointer arithmetic, 'head' is a byte offset */ 702 ep = (pbm->msi_queues + ((msiqid - pbm->msiq_first) * 703 (pbm->msiq_ent_count * 704 sizeof(struct pci_sun4v_msiq_entry))) + 705 *head); 706 707 if ((ep->version_type & MSIQ_TYPE_MASK) == 0) 708 return 0; 709 710 type = (ep->version_type & MSIQ_TYPE_MASK) >> MSIQ_TYPE_SHIFT; 711 if (unlikely(type != MSIQ_TYPE_MSI32 && 712 type != MSIQ_TYPE_MSI64)) 713 return -EINVAL; 714 715 *msi = ep->msi_data; 716 717 err = pci_sun4v_msi_setstate(pbm->devhandle, 718 ep->msi_data /* msi_num */, 719 HV_MSISTATE_IDLE); 720 if (unlikely(err)) 721 return -ENXIO; 722 723 /* Clear the entry. */ 724 ep->version_type &= ~MSIQ_TYPE_MASK; 725 726 (*head) += sizeof(struct pci_sun4v_msiq_entry); 727 if (*head >= 728 (pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry))) 729 *head = 0; 730 731 return 1; 732 } 733 734 static int pci_sun4v_set_head(struct pci_pbm_info *pbm, unsigned long msiqid, 735 unsigned long head) 736 { 737 unsigned long err; 738 739 err = pci_sun4v_msiq_sethead(pbm->devhandle, msiqid, head); 740 if (unlikely(err)) 741 return -EINVAL; 742 743 return 0; 744 } 745 746 static int pci_sun4v_msi_setup(struct pci_pbm_info *pbm, unsigned long msiqid, 747 unsigned long msi, int is_msi64) 748 { 749 if (pci_sun4v_msi_setmsiq(pbm->devhandle, msi, msiqid, 750 (is_msi64 ? 751 HV_MSITYPE_MSI64 : HV_MSITYPE_MSI32))) 752 return -ENXIO; 753 if (pci_sun4v_msi_setstate(pbm->devhandle, msi, HV_MSISTATE_IDLE)) 754 return -ENXIO; 755 if (pci_sun4v_msi_setvalid(pbm->devhandle, msi, HV_MSIVALID_VALID)) 756 return -ENXIO; 757 return 0; 758 } 759 760 static int pci_sun4v_msi_teardown(struct pci_pbm_info *pbm, unsigned long msi) 761 { 762 unsigned long err, msiqid; 763 764 err = pci_sun4v_msi_getmsiq(pbm->devhandle, msi, &msiqid); 765 if (err) 766 return -ENXIO; 767 768 pci_sun4v_msi_setvalid(pbm->devhandle, msi, HV_MSIVALID_INVALID); 769 770 return 0; 771 } 772 773 static int pci_sun4v_msiq_alloc(struct pci_pbm_info *pbm) 774 { 775 unsigned long q_size, alloc_size, pages, order; 776 int i; 777 778 q_size = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry); 779 alloc_size = (pbm->msiq_num * q_size); 780 order = get_order(alloc_size); 781 pages = __get_free_pages(GFP_KERNEL | __GFP_COMP, order); 782 if (pages == 0UL) { 783 printk(KERN_ERR "MSI: Cannot allocate MSI queues (o=%lu).\n", 784 order); 785 return -ENOMEM; 786 } 787 memset((char *)pages, 0, PAGE_SIZE << order); 788 pbm->msi_queues = (void *) pages; 789 790 for (i = 0; i < pbm->msiq_num; i++) { 791 unsigned long err, base = __pa(pages + (i * q_size)); 792 unsigned long ret1, ret2; 793 794 err = pci_sun4v_msiq_conf(pbm->devhandle, 795 pbm->msiq_first + i, 796 base, pbm->msiq_ent_count); 797 if (err) { 798 printk(KERN_ERR "MSI: msiq register fails (err=%lu)\n", 799 err); 800 goto h_error; 801 } 802 803 err = pci_sun4v_msiq_info(pbm->devhandle, 804 pbm->msiq_first + i, 805 &ret1, &ret2); 806 if (err) { 807 printk(KERN_ERR "MSI: Cannot read msiq (err=%lu)\n", 808 err); 809 goto h_error; 810 } 811 if (ret1 != base || ret2 != pbm->msiq_ent_count) { 812 printk(KERN_ERR "MSI: Bogus qconf " 813 "expected[%lx:%x] got[%lx:%lx]\n", 814 base, pbm->msiq_ent_count, 815 ret1, ret2); 816 goto h_error; 817 } 818 } 819 820 return 0; 821 822 h_error: 823 free_pages(pages, order); 824 return -EINVAL; 825 } 826 827 static void pci_sun4v_msiq_free(struct pci_pbm_info *pbm) 828 { 829 unsigned long q_size, alloc_size, pages, order; 830 int i; 831 832 for (i = 0; i < pbm->msiq_num; i++) { 833 unsigned long msiqid = pbm->msiq_first + i; 834 835 (void) pci_sun4v_msiq_conf(pbm->devhandle, msiqid, 0UL, 0); 836 } 837 838 q_size = pbm->msiq_ent_count * sizeof(struct pci_sun4v_msiq_entry); 839 alloc_size = (pbm->msiq_num * q_size); 840 order = get_order(alloc_size); 841 842 pages = (unsigned long) pbm->msi_queues; 843 844 free_pages(pages, order); 845 846 pbm->msi_queues = NULL; 847 } 848 849 static int pci_sun4v_msiq_build_irq(struct pci_pbm_info *pbm, 850 unsigned long msiqid, 851 unsigned long devino) 852 { 853 unsigned int irq = sun4v_build_irq(pbm->devhandle, devino); 854 855 if (!irq) 856 return -ENOMEM; 857 858 if (pci_sun4v_msiq_setvalid(pbm->devhandle, msiqid, HV_MSIQ_VALID)) 859 return -EINVAL; 860 if (pci_sun4v_msiq_setstate(pbm->devhandle, msiqid, HV_MSIQSTATE_IDLE)) 861 return -EINVAL; 862 863 return irq; 864 } 865 866 static const struct sparc64_msiq_ops pci_sun4v_msiq_ops = { 867 .get_head = pci_sun4v_get_head, 868 .dequeue_msi = pci_sun4v_dequeue_msi, 869 .set_head = pci_sun4v_set_head, 870 .msi_setup = pci_sun4v_msi_setup, 871 .msi_teardown = pci_sun4v_msi_teardown, 872 .msiq_alloc = pci_sun4v_msiq_alloc, 873 .msiq_free = pci_sun4v_msiq_free, 874 .msiq_build_irq = pci_sun4v_msiq_build_irq, 875 }; 876 877 static void pci_sun4v_msi_init(struct pci_pbm_info *pbm) 878 { 879 sparc64_pbm_msi_init(pbm, &pci_sun4v_msiq_ops); 880 } 881 #else /* CONFIG_PCI_MSI */ 882 static void pci_sun4v_msi_init(struct pci_pbm_info *pbm) 883 { 884 } 885 #endif /* !(CONFIG_PCI_MSI) */ 886 887 static int pci_sun4v_pbm_init(struct pci_pbm_info *pbm, 888 struct platform_device *op, u32 devhandle) 889 { 890 struct device_node *dp = op->dev.of_node; 891 int err; 892 893 pbm->numa_node = of_node_to_nid(dp); 894 895 pbm->pci_ops = &sun4v_pci_ops; 896 pbm->config_space_reg_bits = 12; 897 898 pbm->index = pci_num_pbms++; 899 900 pbm->op = op; 901 902 pbm->devhandle = devhandle; 903 904 pbm->name = dp->full_name; 905 906 printk("%s: SUN4V PCI Bus Module\n", pbm->name); 907 printk("%s: On NUMA node %d\n", pbm->name, pbm->numa_node); 908 909 pci_determine_mem_io_space(pbm); 910 911 pci_get_pbm_props(pbm); 912 913 err = pci_sun4v_iommu_init(pbm); 914 if (err) 915 return err; 916 917 pci_sun4v_msi_init(pbm); 918 919 pci_sun4v_scan_bus(pbm, &op->dev); 920 921 pbm->next = pci_pbm_root; 922 pci_pbm_root = pbm; 923 924 return 0; 925 } 926 927 static int pci_sun4v_probe(struct platform_device *op) 928 { 929 const struct linux_prom64_registers *regs; 930 static int hvapi_negotiated = 0; 931 struct pci_pbm_info *pbm; 932 struct device_node *dp; 933 struct iommu *iommu; 934 u32 devhandle; 935 int i, err = -ENODEV; 936 937 dp = op->dev.of_node; 938 939 if (!hvapi_negotiated++) { 940 for (i = 0; i < ARRAY_SIZE(vpci_versions); i++) { 941 vpci_major = vpci_versions[i].major; 942 vpci_minor = vpci_versions[i].minor; 943 944 err = sun4v_hvapi_register(HV_GRP_PCI, vpci_major, 945 &vpci_minor); 946 if (!err) 947 break; 948 } 949 950 if (err) { 951 pr_err(PFX "Could not register hvapi, err=%d\n", err); 952 return err; 953 } 954 pr_info(PFX "Registered hvapi major[%lu] minor[%lu]\n", 955 vpci_major, vpci_minor); 956 957 dma_ops = &sun4v_dma_ops; 958 } 959 960 regs = of_get_property(dp, "reg", NULL); 961 err = -ENODEV; 962 if (!regs) { 963 printk(KERN_ERR PFX "Could not find config registers\n"); 964 goto out_err; 965 } 966 devhandle = (regs->phys_addr >> 32UL) & 0x0fffffff; 967 968 err = -ENOMEM; 969 if (!iommu_batch_initialized) { 970 for_each_possible_cpu(i) { 971 unsigned long page = get_zeroed_page(GFP_KERNEL); 972 973 if (!page) 974 goto out_err; 975 976 per_cpu(iommu_batch, i).pglist = (u64 *) page; 977 } 978 iommu_batch_initialized = 1; 979 } 980 981 pbm = kzalloc(sizeof(*pbm), GFP_KERNEL); 982 if (!pbm) { 983 printk(KERN_ERR PFX "Could not allocate pci_pbm_info\n"); 984 goto out_err; 985 } 986 987 iommu = kzalloc(sizeof(struct iommu), GFP_KERNEL); 988 if (!iommu) { 989 printk(KERN_ERR PFX "Could not allocate pbm iommu\n"); 990 goto out_free_controller; 991 } 992 993 pbm->iommu = iommu; 994 995 err = pci_sun4v_pbm_init(pbm, op, devhandle); 996 if (err) 997 goto out_free_iommu; 998 999 dev_set_drvdata(&op->dev, pbm); 1000 1001 return 0; 1002 1003 out_free_iommu: 1004 kfree(pbm->iommu); 1005 1006 out_free_controller: 1007 kfree(pbm); 1008 1009 out_err: 1010 return err; 1011 } 1012 1013 static const struct of_device_id pci_sun4v_match[] = { 1014 { 1015 .name = "pci", 1016 .compatible = "SUNW,sun4v-pci", 1017 }, 1018 {}, 1019 }; 1020 1021 static struct platform_driver pci_sun4v_driver = { 1022 .driver = { 1023 .name = DRIVER_NAME, 1024 .of_match_table = pci_sun4v_match, 1025 }, 1026 .probe = pci_sun4v_probe, 1027 }; 1028 1029 static int __init pci_sun4v_init(void) 1030 { 1031 return platform_driver_register(&pci_sun4v_driver); 1032 } 1033 1034 subsys_initcall(pci_sun4v_init); 1035