1 /** 2 * IBM Accelerator Family 'GenWQE' 3 * 4 * (C) Copyright IBM Corp. 2013 5 * 6 * Author: Frank Haverkamp <haver@linux.vnet.ibm.com> 7 * Author: Joerg-Stephan Vogt <jsvogt@de.ibm.com> 8 * Author: Michael Jung <mijung@gmx.net> 9 * Author: Michael Ruettger <michael@ibmra.de> 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of the GNU General Public License (version 2 only) 13 * as published by the Free Software Foundation. 14 * 15 * This program is distributed in the hope that it will be useful, 16 * but WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 * GNU General Public License for more details. 19 */ 20 21 /* 22 * Miscelanous functionality used in the other GenWQE driver parts. 23 */ 24 25 #include <linux/kernel.h> 26 #include <linux/sched.h> 27 #include <linux/vmalloc.h> 28 #include <linux/page-flags.h> 29 #include <linux/scatterlist.h> 30 #include <linux/hugetlb.h> 31 #include <linux/iommu.h> 32 #include <linux/pci.h> 33 #include <linux/dma-mapping.h> 34 #include <linux/ctype.h> 35 #include <linux/module.h> 36 #include <linux/platform_device.h> 37 #include <linux/delay.h> 38 #include <asm/pgtable.h> 39 40 #include "genwqe_driver.h" 41 #include "card_base.h" 42 #include "card_ddcb.h" 43 44 /** 45 * __genwqe_writeq() - Write 64-bit register 46 * @cd: genwqe device descriptor 47 * @byte_offs: byte offset within BAR 48 * @val: 64-bit value 49 * 50 * Return: 0 if success; < 0 if error 51 */ 52 int __genwqe_writeq(struct genwqe_dev *cd, u64 byte_offs, u64 val) 53 { 54 struct pci_dev *pci_dev = cd->pci_dev; 55 56 if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE) 57 return -EIO; 58 59 if (cd->mmio == NULL) 60 return -EIO; 61 62 if (pci_channel_offline(pci_dev)) 63 return -EIO; 64 65 __raw_writeq((__force u64)cpu_to_be64(val), cd->mmio + byte_offs); 66 return 0; 67 } 68 69 /** 70 * __genwqe_readq() - Read 64-bit register 71 * @cd: genwqe device descriptor 72 * @byte_offs: offset within BAR 73 * 74 * Return: value from register 75 */ 76 u64 __genwqe_readq(struct genwqe_dev *cd, u64 byte_offs) 77 { 78 if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE) 79 return 0xffffffffffffffffull; 80 81 if ((cd->err_inject & GENWQE_INJECT_GFIR_FATAL) && 82 (byte_offs == IO_SLC_CFGREG_GFIR)) 83 return 0x000000000000ffffull; 84 85 if ((cd->err_inject & GENWQE_INJECT_GFIR_INFO) && 86 (byte_offs == IO_SLC_CFGREG_GFIR)) 87 return 0x00000000ffff0000ull; 88 89 if (cd->mmio == NULL) 90 return 0xffffffffffffffffull; 91 92 return be64_to_cpu((__force __be64)__raw_readq(cd->mmio + byte_offs)); 93 } 94 95 /** 96 * __genwqe_writel() - Write 32-bit register 97 * @cd: genwqe device descriptor 98 * @byte_offs: byte offset within BAR 99 * @val: 32-bit value 100 * 101 * Return: 0 if success; < 0 if error 102 */ 103 int __genwqe_writel(struct genwqe_dev *cd, u64 byte_offs, u32 val) 104 { 105 struct pci_dev *pci_dev = cd->pci_dev; 106 107 if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE) 108 return -EIO; 109 110 if (cd->mmio == NULL) 111 return -EIO; 112 113 if (pci_channel_offline(pci_dev)) 114 return -EIO; 115 116 __raw_writel((__force u32)cpu_to_be32(val), cd->mmio + byte_offs); 117 return 0; 118 } 119 120 /** 121 * __genwqe_readl() - Read 32-bit register 122 * @cd: genwqe device descriptor 123 * @byte_offs: offset within BAR 124 * 125 * Return: Value from register 126 */ 127 u32 __genwqe_readl(struct genwqe_dev *cd, u64 byte_offs) 128 { 129 if (cd->err_inject & GENWQE_INJECT_HARDWARE_FAILURE) 130 return 0xffffffff; 131 132 if (cd->mmio == NULL) 133 return 0xffffffff; 134 135 return be32_to_cpu((__force __be32)__raw_readl(cd->mmio + byte_offs)); 136 } 137 138 /** 139 * genwqe_read_app_id() - Extract app_id 140 * 141 * app_unitcfg need to be filled with valid data first 142 */ 143 int genwqe_read_app_id(struct genwqe_dev *cd, char *app_name, int len) 144 { 145 int i, j; 146 u32 app_id = (u32)cd->app_unitcfg; 147 148 memset(app_name, 0, len); 149 for (i = 0, j = 0; j < min(len, 4); j++) { 150 char ch = (char)((app_id >> (24 - j*8)) & 0xff); 151 152 if (ch == ' ') 153 continue; 154 app_name[i++] = isprint(ch) ? ch : 'X'; 155 } 156 return i; 157 } 158 159 /** 160 * genwqe_init_crc32() - Prepare a lookup table for fast crc32 calculations 161 * 162 * Existing kernel functions seem to use a different polynom, 163 * therefore we could not use them here. 164 * 165 * Genwqe's Polynomial = 0x20044009 166 */ 167 #define CRC32_POLYNOMIAL 0x20044009 168 static u32 crc32_tab[256]; /* crc32 lookup table */ 169 170 void genwqe_init_crc32(void) 171 { 172 int i, j; 173 u32 crc; 174 175 for (i = 0; i < 256; i++) { 176 crc = i << 24; 177 for (j = 0; j < 8; j++) { 178 if (crc & 0x80000000) 179 crc = (crc << 1) ^ CRC32_POLYNOMIAL; 180 else 181 crc = (crc << 1); 182 } 183 crc32_tab[i] = crc; 184 } 185 } 186 187 /** 188 * genwqe_crc32() - Generate 32-bit crc as required for DDCBs 189 * @buff: pointer to data buffer 190 * @len: length of data for calculation 191 * @init: initial crc (0xffffffff at start) 192 * 193 * polynomial = x^32 * + x^29 + x^18 + x^14 + x^3 + 1 (0x20044009) 194 195 * Example: 4 bytes 0x01 0x02 0x03 0x04 with init=0xffffffff should 196 * result in a crc32 of 0xf33cb7d3. 197 * 198 * The existing kernel crc functions did not cover this polynom yet. 199 * 200 * Return: crc32 checksum. 201 */ 202 u32 genwqe_crc32(u8 *buff, size_t len, u32 init) 203 { 204 int i; 205 u32 crc; 206 207 crc = init; 208 while (len--) { 209 i = ((crc >> 24) ^ *buff++) & 0xFF; 210 crc = (crc << 8) ^ crc32_tab[i]; 211 } 212 return crc; 213 } 214 215 void *__genwqe_alloc_consistent(struct genwqe_dev *cd, size_t size, 216 dma_addr_t *dma_handle) 217 { 218 if (get_order(size) >= MAX_ORDER) 219 return NULL; 220 221 return dma_alloc_coherent(&cd->pci_dev->dev, size, dma_handle, 222 GFP_KERNEL); 223 } 224 225 void __genwqe_free_consistent(struct genwqe_dev *cd, size_t size, 226 void *vaddr, dma_addr_t dma_handle) 227 { 228 if (vaddr == NULL) 229 return; 230 231 dma_free_coherent(&cd->pci_dev->dev, size, vaddr, dma_handle); 232 } 233 234 static void genwqe_unmap_pages(struct genwqe_dev *cd, dma_addr_t *dma_list, 235 int num_pages) 236 { 237 int i; 238 struct pci_dev *pci_dev = cd->pci_dev; 239 240 for (i = 0; (i < num_pages) && (dma_list[i] != 0x0); i++) { 241 pci_unmap_page(pci_dev, dma_list[i], 242 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); 243 dma_list[i] = 0x0; 244 } 245 } 246 247 static int genwqe_map_pages(struct genwqe_dev *cd, 248 struct page **page_list, int num_pages, 249 dma_addr_t *dma_list) 250 { 251 int i; 252 struct pci_dev *pci_dev = cd->pci_dev; 253 254 /* establish DMA mapping for requested pages */ 255 for (i = 0; i < num_pages; i++) { 256 dma_addr_t daddr; 257 258 dma_list[i] = 0x0; 259 daddr = pci_map_page(pci_dev, page_list[i], 260 0, /* map_offs */ 261 PAGE_SIZE, 262 PCI_DMA_BIDIRECTIONAL); /* FIXME rd/rw */ 263 264 if (pci_dma_mapping_error(pci_dev, daddr)) { 265 dev_err(&pci_dev->dev, 266 "[%s] err: no dma addr daddr=%016llx!\n", 267 __func__, (long long)daddr); 268 goto err; 269 } 270 271 dma_list[i] = daddr; 272 } 273 return 0; 274 275 err: 276 genwqe_unmap_pages(cd, dma_list, num_pages); 277 return -EIO; 278 } 279 280 static int genwqe_sgl_size(int num_pages) 281 { 282 int len, num_tlb = num_pages / 7; 283 284 len = sizeof(struct sg_entry) * (num_pages+num_tlb + 1); 285 return roundup(len, PAGE_SIZE); 286 } 287 288 /** 289 * genwqe_alloc_sync_sgl() - Allocate memory for sgl and overlapping pages 290 * 291 * Allocates memory for sgl and overlapping pages. Pages which might 292 * overlap other user-space memory blocks are being cached for DMAs, 293 * such that we do not run into syncronization issues. Data is copied 294 * from user-space into the cached pages. 295 */ 296 int genwqe_alloc_sync_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl, 297 void __user *user_addr, size_t user_size, int write) 298 { 299 int ret = -ENOMEM; 300 struct pci_dev *pci_dev = cd->pci_dev; 301 302 sgl->fpage_offs = offset_in_page((unsigned long)user_addr); 303 sgl->fpage_size = min_t(size_t, PAGE_SIZE-sgl->fpage_offs, user_size); 304 sgl->nr_pages = DIV_ROUND_UP(sgl->fpage_offs + user_size, PAGE_SIZE); 305 sgl->lpage_size = (user_size - sgl->fpage_size) % PAGE_SIZE; 306 307 dev_dbg(&pci_dev->dev, "[%s] uaddr=%p usize=%8ld nr_pages=%ld fpage_offs=%lx fpage_size=%ld lpage_size=%ld\n", 308 __func__, user_addr, user_size, sgl->nr_pages, 309 sgl->fpage_offs, sgl->fpage_size, sgl->lpage_size); 310 311 sgl->user_addr = user_addr; 312 sgl->user_size = user_size; 313 sgl->write = write; 314 sgl->sgl_size = genwqe_sgl_size(sgl->nr_pages); 315 316 if (get_order(sgl->sgl_size) > MAX_ORDER) { 317 dev_err(&pci_dev->dev, 318 "[%s] err: too much memory requested!\n", __func__); 319 return ret; 320 } 321 322 sgl->sgl = __genwqe_alloc_consistent(cd, sgl->sgl_size, 323 &sgl->sgl_dma_addr); 324 if (sgl->sgl == NULL) { 325 dev_err(&pci_dev->dev, 326 "[%s] err: no memory available!\n", __func__); 327 return ret; 328 } 329 330 /* Only use buffering on incomplete pages */ 331 if ((sgl->fpage_size != 0) && (sgl->fpage_size != PAGE_SIZE)) { 332 sgl->fpage = __genwqe_alloc_consistent(cd, PAGE_SIZE, 333 &sgl->fpage_dma_addr); 334 if (sgl->fpage == NULL) 335 goto err_out; 336 337 /* Sync with user memory */ 338 if (copy_from_user(sgl->fpage + sgl->fpage_offs, 339 user_addr, sgl->fpage_size)) { 340 ret = -EFAULT; 341 goto err_out; 342 } 343 } 344 if (sgl->lpage_size != 0) { 345 sgl->lpage = __genwqe_alloc_consistent(cd, PAGE_SIZE, 346 &sgl->lpage_dma_addr); 347 if (sgl->lpage == NULL) 348 goto err_out1; 349 350 /* Sync with user memory */ 351 if (copy_from_user(sgl->lpage, user_addr + user_size - 352 sgl->lpage_size, sgl->lpage_size)) { 353 ret = -EFAULT; 354 goto err_out2; 355 } 356 } 357 return 0; 358 359 err_out2: 360 __genwqe_free_consistent(cd, PAGE_SIZE, sgl->lpage, 361 sgl->lpage_dma_addr); 362 sgl->lpage = NULL; 363 sgl->lpage_dma_addr = 0; 364 err_out1: 365 __genwqe_free_consistent(cd, PAGE_SIZE, sgl->fpage, 366 sgl->fpage_dma_addr); 367 sgl->fpage = NULL; 368 sgl->fpage_dma_addr = 0; 369 err_out: 370 __genwqe_free_consistent(cd, sgl->sgl_size, sgl->sgl, 371 sgl->sgl_dma_addr); 372 sgl->sgl = NULL; 373 sgl->sgl_dma_addr = 0; 374 sgl->sgl_size = 0; 375 376 return ret; 377 } 378 379 int genwqe_setup_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl, 380 dma_addr_t *dma_list) 381 { 382 int i = 0, j = 0, p; 383 unsigned long dma_offs, map_offs; 384 dma_addr_t prev_daddr = 0; 385 struct sg_entry *s, *last_s = NULL; 386 size_t size = sgl->user_size; 387 388 dma_offs = 128; /* next block if needed/dma_offset */ 389 map_offs = sgl->fpage_offs; /* offset in first page */ 390 391 s = &sgl->sgl[0]; /* first set of 8 entries */ 392 p = 0; /* page */ 393 while (p < sgl->nr_pages) { 394 dma_addr_t daddr; 395 unsigned int size_to_map; 396 397 /* always write the chaining entry, cleanup is done later */ 398 j = 0; 399 s[j].target_addr = cpu_to_be64(sgl->sgl_dma_addr + dma_offs); 400 s[j].len = cpu_to_be32(128); 401 s[j].flags = cpu_to_be32(SG_CHAINED); 402 j++; 403 404 while (j < 8) { 405 /* DMA mapping for requested page, offs, size */ 406 size_to_map = min(size, PAGE_SIZE - map_offs); 407 408 if ((p == 0) && (sgl->fpage != NULL)) { 409 daddr = sgl->fpage_dma_addr + map_offs; 410 411 } else if ((p == sgl->nr_pages - 1) && 412 (sgl->lpage != NULL)) { 413 daddr = sgl->lpage_dma_addr; 414 } else { 415 daddr = dma_list[p] + map_offs; 416 } 417 418 size -= size_to_map; 419 map_offs = 0; 420 421 if (prev_daddr == daddr) { 422 u32 prev_len = be32_to_cpu(last_s->len); 423 424 /* pr_info("daddr combining: " 425 "%016llx/%08x -> %016llx\n", 426 prev_daddr, prev_len, daddr); */ 427 428 last_s->len = cpu_to_be32(prev_len + 429 size_to_map); 430 431 p++; /* process next page */ 432 if (p == sgl->nr_pages) 433 goto fixup; /* nothing to do */ 434 435 prev_daddr = daddr + size_to_map; 436 continue; 437 } 438 439 /* start new entry */ 440 s[j].target_addr = cpu_to_be64(daddr); 441 s[j].len = cpu_to_be32(size_to_map); 442 s[j].flags = cpu_to_be32(SG_DATA); 443 prev_daddr = daddr + size_to_map; 444 last_s = &s[j]; 445 j++; 446 447 p++; /* process next page */ 448 if (p == sgl->nr_pages) 449 goto fixup; /* nothing to do */ 450 } 451 dma_offs += 128; 452 s += 8; /* continue 8 elements further */ 453 } 454 fixup: 455 if (j == 1) { /* combining happened on last entry! */ 456 s -= 8; /* full shift needed on previous sgl block */ 457 j = 7; /* shift all elements */ 458 } 459 460 for (i = 0; i < j; i++) /* move elements 1 up */ 461 s[i] = s[i + 1]; 462 463 s[i].target_addr = cpu_to_be64(0); 464 s[i].len = cpu_to_be32(0); 465 s[i].flags = cpu_to_be32(SG_END_LIST); 466 return 0; 467 } 468 469 /** 470 * genwqe_free_sync_sgl() - Free memory for sgl and overlapping pages 471 * 472 * After the DMA transfer has been completed we free the memory for 473 * the sgl and the cached pages. Data is being transferred from cached 474 * pages into user-space buffers. 475 */ 476 int genwqe_free_sync_sgl(struct genwqe_dev *cd, struct genwqe_sgl *sgl) 477 { 478 int rc = 0; 479 size_t offset; 480 unsigned long res; 481 struct pci_dev *pci_dev = cd->pci_dev; 482 483 if (sgl->fpage) { 484 if (sgl->write) { 485 res = copy_to_user(sgl->user_addr, 486 sgl->fpage + sgl->fpage_offs, sgl->fpage_size); 487 if (res) { 488 dev_err(&pci_dev->dev, 489 "[%s] err: copying fpage! (res=%lu)\n", 490 __func__, res); 491 rc = -EFAULT; 492 } 493 } 494 __genwqe_free_consistent(cd, PAGE_SIZE, sgl->fpage, 495 sgl->fpage_dma_addr); 496 sgl->fpage = NULL; 497 sgl->fpage_dma_addr = 0; 498 } 499 if (sgl->lpage) { 500 if (sgl->write) { 501 offset = sgl->user_size - sgl->lpage_size; 502 res = copy_to_user(sgl->user_addr + offset, sgl->lpage, 503 sgl->lpage_size); 504 if (res) { 505 dev_err(&pci_dev->dev, 506 "[%s] err: copying lpage! (res=%lu)\n", 507 __func__, res); 508 rc = -EFAULT; 509 } 510 } 511 __genwqe_free_consistent(cd, PAGE_SIZE, sgl->lpage, 512 sgl->lpage_dma_addr); 513 sgl->lpage = NULL; 514 sgl->lpage_dma_addr = 0; 515 } 516 __genwqe_free_consistent(cd, sgl->sgl_size, sgl->sgl, 517 sgl->sgl_dma_addr); 518 519 sgl->sgl = NULL; 520 sgl->sgl_dma_addr = 0x0; 521 sgl->sgl_size = 0; 522 return rc; 523 } 524 525 /** 526 * genwqe_free_user_pages() - Give pinned pages back 527 * 528 * Documentation of get_user_pages is in mm/gup.c: 529 * 530 * If the page is written to, set_page_dirty (or set_page_dirty_lock, 531 * as appropriate) must be called after the page is finished with, and 532 * before put_page is called. 533 */ 534 static int genwqe_free_user_pages(struct page **page_list, 535 unsigned int nr_pages, int dirty) 536 { 537 unsigned int i; 538 539 for (i = 0; i < nr_pages; i++) { 540 if (page_list[i] != NULL) { 541 if (dirty) 542 set_page_dirty_lock(page_list[i]); 543 put_page(page_list[i]); 544 } 545 } 546 return 0; 547 } 548 549 /** 550 * genwqe_user_vmap() - Map user-space memory to virtual kernel memory 551 * @cd: pointer to genwqe device 552 * @m: mapping params 553 * @uaddr: user virtual address 554 * @size: size of memory to be mapped 555 * 556 * We need to think about how we could speed this up. Of course it is 557 * not a good idea to do this over and over again, like we are 558 * currently doing it. Nevertheless, I am curious where on the path 559 * the performance is spend. Most probably within the memory 560 * allocation functions, but maybe also in the DMA mapping code. 561 * 562 * Restrictions: The maximum size of the possible mapping currently depends 563 * on the amount of memory we can get using kzalloc() for the 564 * page_list and pci_alloc_consistent for the sg_list. 565 * The sg_list is currently itself not scattered, which could 566 * be fixed with some effort. The page_list must be split into 567 * PAGE_SIZE chunks too. All that will make the complicated 568 * code more complicated. 569 * 570 * Return: 0 if success 571 */ 572 int genwqe_user_vmap(struct genwqe_dev *cd, struct dma_mapping *m, void *uaddr, 573 unsigned long size) 574 { 575 int rc = -EINVAL; 576 unsigned long data, offs; 577 struct pci_dev *pci_dev = cd->pci_dev; 578 579 if ((uaddr == NULL) || (size == 0)) { 580 m->size = 0; /* mark unused and not added */ 581 return -EINVAL; 582 } 583 m->u_vaddr = uaddr; 584 m->size = size; 585 586 /* determine space needed for page_list. */ 587 data = (unsigned long)uaddr; 588 offs = offset_in_page(data); 589 m->nr_pages = DIV_ROUND_UP(offs + size, PAGE_SIZE); 590 591 m->page_list = kcalloc(m->nr_pages, 592 sizeof(struct page *) + sizeof(dma_addr_t), 593 GFP_KERNEL); 594 if (!m->page_list) { 595 dev_err(&pci_dev->dev, "err: alloc page_list failed\n"); 596 m->nr_pages = 0; 597 m->u_vaddr = NULL; 598 m->size = 0; /* mark unused and not added */ 599 return -ENOMEM; 600 } 601 m->dma_list = (dma_addr_t *)(m->page_list + m->nr_pages); 602 603 /* pin user pages in memory */ 604 rc = get_user_pages_fast(data & PAGE_MASK, /* page aligned addr */ 605 m->nr_pages, 606 m->write, /* readable/writable */ 607 m->page_list); /* ptrs to pages */ 608 if (rc < 0) 609 goto fail_get_user_pages; 610 611 /* assumption: get_user_pages can be killed by signals. */ 612 if (rc < m->nr_pages) { 613 genwqe_free_user_pages(m->page_list, rc, m->write); 614 rc = -EFAULT; 615 goto fail_get_user_pages; 616 } 617 618 rc = genwqe_map_pages(cd, m->page_list, m->nr_pages, m->dma_list); 619 if (rc != 0) 620 goto fail_free_user_pages; 621 622 return 0; 623 624 fail_free_user_pages: 625 genwqe_free_user_pages(m->page_list, m->nr_pages, m->write); 626 627 fail_get_user_pages: 628 kfree(m->page_list); 629 m->page_list = NULL; 630 m->dma_list = NULL; 631 m->nr_pages = 0; 632 m->u_vaddr = NULL; 633 m->size = 0; /* mark unused and not added */ 634 return rc; 635 } 636 637 /** 638 * genwqe_user_vunmap() - Undo mapping of user-space mem to virtual kernel 639 * memory 640 * @cd: pointer to genwqe device 641 * @m: mapping params 642 */ 643 int genwqe_user_vunmap(struct genwqe_dev *cd, struct dma_mapping *m) 644 { 645 struct pci_dev *pci_dev = cd->pci_dev; 646 647 if (!dma_mapping_used(m)) { 648 dev_err(&pci_dev->dev, "[%s] err: mapping %p not used!\n", 649 __func__, m); 650 return -EINVAL; 651 } 652 653 if (m->dma_list) 654 genwqe_unmap_pages(cd, m->dma_list, m->nr_pages); 655 656 if (m->page_list) { 657 genwqe_free_user_pages(m->page_list, m->nr_pages, m->write); 658 659 kfree(m->page_list); 660 m->page_list = NULL; 661 m->dma_list = NULL; 662 m->nr_pages = 0; 663 } 664 665 m->u_vaddr = NULL; 666 m->size = 0; /* mark as unused and not added */ 667 return 0; 668 } 669 670 /** 671 * genwqe_card_type() - Get chip type SLU Configuration Register 672 * @cd: pointer to the genwqe device descriptor 673 * Return: 0: Altera Stratix-IV 230 674 * 1: Altera Stratix-IV 530 675 * 2: Altera Stratix-V A4 676 * 3: Altera Stratix-V A7 677 */ 678 u8 genwqe_card_type(struct genwqe_dev *cd) 679 { 680 u64 card_type = cd->slu_unitcfg; 681 682 return (u8)((card_type & IO_SLU_UNITCFG_TYPE_MASK) >> 20); 683 } 684 685 /** 686 * genwqe_card_reset() - Reset the card 687 * @cd: pointer to the genwqe device descriptor 688 */ 689 int genwqe_card_reset(struct genwqe_dev *cd) 690 { 691 u64 softrst; 692 struct pci_dev *pci_dev = cd->pci_dev; 693 694 if (!genwqe_is_privileged(cd)) 695 return -ENODEV; 696 697 /* new SL */ 698 __genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, 0x1ull); 699 msleep(1000); 700 __genwqe_readq(cd, IO_HSU_FIR_CLR); 701 __genwqe_readq(cd, IO_APP_FIR_CLR); 702 __genwqe_readq(cd, IO_SLU_FIR_CLR); 703 704 /* 705 * Read-modify-write to preserve the stealth bits 706 * 707 * For SL >= 039, Stealth WE bit allows removing 708 * the read-modify-wrote. 709 * r-m-w may require a mask 0x3C to avoid hitting hard 710 * reset again for error reset (should be 0, chicken). 711 */ 712 softrst = __genwqe_readq(cd, IO_SLC_CFGREG_SOFTRESET) & 0x3cull; 713 __genwqe_writeq(cd, IO_SLC_CFGREG_SOFTRESET, softrst | 0x2ull); 714 715 /* give ERRORRESET some time to finish */ 716 msleep(50); 717 718 if (genwqe_need_err_masking(cd)) { 719 dev_info(&pci_dev->dev, 720 "[%s] masking errors for old bitstreams\n", __func__); 721 __genwqe_writeq(cd, IO_SLC_MISC_DEBUG, 0x0aull); 722 } 723 return 0; 724 } 725 726 int genwqe_read_softreset(struct genwqe_dev *cd) 727 { 728 u64 bitstream; 729 730 if (!genwqe_is_privileged(cd)) 731 return -ENODEV; 732 733 bitstream = __genwqe_readq(cd, IO_SLU_BITSTREAM) & 0x1; 734 cd->softreset = (bitstream == 0) ? 0x8ull : 0xcull; 735 return 0; 736 } 737 738 /** 739 * genwqe_set_interrupt_capability() - Configure MSI capability structure 740 * @cd: pointer to the device 741 * Return: 0 if no error 742 */ 743 int genwqe_set_interrupt_capability(struct genwqe_dev *cd, int count) 744 { 745 int rc; 746 747 rc = pci_alloc_irq_vectors(cd->pci_dev, 1, count, PCI_IRQ_MSI); 748 if (rc < 0) 749 return rc; 750 return 0; 751 } 752 753 /** 754 * genwqe_reset_interrupt_capability() - Undo genwqe_set_interrupt_capability() 755 * @cd: pointer to the device 756 */ 757 void genwqe_reset_interrupt_capability(struct genwqe_dev *cd) 758 { 759 pci_free_irq_vectors(cd->pci_dev); 760 } 761 762 /** 763 * set_reg_idx() - Fill array with data. Ignore illegal offsets. 764 * @cd: card device 765 * @r: debug register array 766 * @i: index to desired entry 767 * @m: maximum possible entries 768 * @addr: addr which is read 769 * @index: index in debug array 770 * @val: read value 771 */ 772 static int set_reg_idx(struct genwqe_dev *cd, struct genwqe_reg *r, 773 unsigned int *i, unsigned int m, u32 addr, u32 idx, 774 u64 val) 775 { 776 if (WARN_ON_ONCE(*i >= m)) 777 return -EFAULT; 778 779 r[*i].addr = addr; 780 r[*i].idx = idx; 781 r[*i].val = val; 782 ++*i; 783 return 0; 784 } 785 786 static int set_reg(struct genwqe_dev *cd, struct genwqe_reg *r, 787 unsigned int *i, unsigned int m, u32 addr, u64 val) 788 { 789 return set_reg_idx(cd, r, i, m, addr, 0, val); 790 } 791 792 int genwqe_read_ffdc_regs(struct genwqe_dev *cd, struct genwqe_reg *regs, 793 unsigned int max_regs, int all) 794 { 795 unsigned int i, j, idx = 0; 796 u32 ufir_addr, ufec_addr, sfir_addr, sfec_addr; 797 u64 gfir, sluid, appid, ufir, ufec, sfir, sfec; 798 799 /* Global FIR */ 800 gfir = __genwqe_readq(cd, IO_SLC_CFGREG_GFIR); 801 set_reg(cd, regs, &idx, max_regs, IO_SLC_CFGREG_GFIR, gfir); 802 803 /* UnitCfg for SLU */ 804 sluid = __genwqe_readq(cd, IO_SLU_UNITCFG); /* 0x00000000 */ 805 set_reg(cd, regs, &idx, max_regs, IO_SLU_UNITCFG, sluid); 806 807 /* UnitCfg for APP */ 808 appid = __genwqe_readq(cd, IO_APP_UNITCFG); /* 0x02000000 */ 809 set_reg(cd, regs, &idx, max_regs, IO_APP_UNITCFG, appid); 810 811 /* Check all chip Units */ 812 for (i = 0; i < GENWQE_MAX_UNITS; i++) { 813 814 /* Unit FIR */ 815 ufir_addr = (i << 24) | 0x008; 816 ufir = __genwqe_readq(cd, ufir_addr); 817 set_reg(cd, regs, &idx, max_regs, ufir_addr, ufir); 818 819 /* Unit FEC */ 820 ufec_addr = (i << 24) | 0x018; 821 ufec = __genwqe_readq(cd, ufec_addr); 822 set_reg(cd, regs, &idx, max_regs, ufec_addr, ufec); 823 824 for (j = 0; j < 64; j++) { 825 /* wherever there is a primary 1, read the 2ndary */ 826 if (!all && (!(ufir & (1ull << j)))) 827 continue; 828 829 sfir_addr = (i << 24) | (0x100 + 8 * j); 830 sfir = __genwqe_readq(cd, sfir_addr); 831 set_reg(cd, regs, &idx, max_regs, sfir_addr, sfir); 832 833 sfec_addr = (i << 24) | (0x300 + 8 * j); 834 sfec = __genwqe_readq(cd, sfec_addr); 835 set_reg(cd, regs, &idx, max_regs, sfec_addr, sfec); 836 } 837 } 838 839 /* fill with invalid data until end */ 840 for (i = idx; i < max_regs; i++) { 841 regs[i].addr = 0xffffffff; 842 regs[i].val = 0xffffffffffffffffull; 843 } 844 return idx; 845 } 846 847 /** 848 * genwqe_ffdc_buff_size() - Calculates the number of dump registers 849 */ 850 int genwqe_ffdc_buff_size(struct genwqe_dev *cd, int uid) 851 { 852 int entries = 0, ring, traps, traces, trace_entries; 853 u32 eevptr_addr, l_addr, d_len, d_type; 854 u64 eevptr, val, addr; 855 856 eevptr_addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_ERROR_POINTER; 857 eevptr = __genwqe_readq(cd, eevptr_addr); 858 859 if ((eevptr != 0x0) && (eevptr != -1ull)) { 860 l_addr = GENWQE_UID_OFFS(uid) | eevptr; 861 862 while (1) { 863 val = __genwqe_readq(cd, l_addr); 864 865 if ((val == 0x0) || (val == -1ull)) 866 break; 867 868 /* 38:24 */ 869 d_len = (val & 0x0000007fff000000ull) >> 24; 870 871 /* 39 */ 872 d_type = (val & 0x0000008000000000ull) >> 36; 873 874 if (d_type) { /* repeat */ 875 entries += d_len; 876 } else { /* size in bytes! */ 877 entries += d_len >> 3; 878 } 879 880 l_addr += 8; 881 } 882 } 883 884 for (ring = 0; ring < 8; ring++) { 885 addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_DIAG_MAP(ring); 886 val = __genwqe_readq(cd, addr); 887 888 if ((val == 0x0ull) || (val == -1ull)) 889 continue; 890 891 traps = (val >> 24) & 0xff; 892 traces = (val >> 16) & 0xff; 893 trace_entries = val & 0xffff; 894 895 entries += traps + (traces * trace_entries); 896 } 897 return entries; 898 } 899 900 /** 901 * genwqe_ffdc_buff_read() - Implements LogoutExtendedErrorRegisters procedure 902 */ 903 int genwqe_ffdc_buff_read(struct genwqe_dev *cd, int uid, 904 struct genwqe_reg *regs, unsigned int max_regs) 905 { 906 int i, traps, traces, trace, trace_entries, trace_entry, ring; 907 unsigned int idx = 0; 908 u32 eevptr_addr, l_addr, d_addr, d_len, d_type; 909 u64 eevptr, e, val, addr; 910 911 eevptr_addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_ERROR_POINTER; 912 eevptr = __genwqe_readq(cd, eevptr_addr); 913 914 if ((eevptr != 0x0) && (eevptr != 0xffffffffffffffffull)) { 915 l_addr = GENWQE_UID_OFFS(uid) | eevptr; 916 while (1) { 917 e = __genwqe_readq(cd, l_addr); 918 if ((e == 0x0) || (e == 0xffffffffffffffffull)) 919 break; 920 921 d_addr = (e & 0x0000000000ffffffull); /* 23:0 */ 922 d_len = (e & 0x0000007fff000000ull) >> 24; /* 38:24 */ 923 d_type = (e & 0x0000008000000000ull) >> 36; /* 39 */ 924 d_addr |= GENWQE_UID_OFFS(uid); 925 926 if (d_type) { 927 for (i = 0; i < (int)d_len; i++) { 928 val = __genwqe_readq(cd, d_addr); 929 set_reg_idx(cd, regs, &idx, max_regs, 930 d_addr, i, val); 931 } 932 } else { 933 d_len >>= 3; /* Size in bytes! */ 934 for (i = 0; i < (int)d_len; i++, d_addr += 8) { 935 val = __genwqe_readq(cd, d_addr); 936 set_reg_idx(cd, regs, &idx, max_regs, 937 d_addr, 0, val); 938 } 939 } 940 l_addr += 8; 941 } 942 } 943 944 /* 945 * To save time, there are only 6 traces poplulated on Uid=2, 946 * Ring=1. each with iters=512. 947 */ 948 for (ring = 0; ring < 8; ring++) { /* 0 is fls, 1 is fds, 949 2...7 are ASI rings */ 950 addr = GENWQE_UID_OFFS(uid) | IO_EXTENDED_DIAG_MAP(ring); 951 val = __genwqe_readq(cd, addr); 952 953 if ((val == 0x0ull) || (val == -1ull)) 954 continue; 955 956 traps = (val >> 24) & 0xff; /* Number of Traps */ 957 traces = (val >> 16) & 0xff; /* Number of Traces */ 958 trace_entries = val & 0xffff; /* Entries per trace */ 959 960 /* Note: This is a combined loop that dumps both the traps */ 961 /* (for the trace == 0 case) as well as the traces 1 to */ 962 /* 'traces'. */ 963 for (trace = 0; trace <= traces; trace++) { 964 u32 diag_sel = 965 GENWQE_EXTENDED_DIAG_SELECTOR(ring, trace); 966 967 addr = (GENWQE_UID_OFFS(uid) | 968 IO_EXTENDED_DIAG_SELECTOR); 969 __genwqe_writeq(cd, addr, diag_sel); 970 971 for (trace_entry = 0; 972 trace_entry < (trace ? trace_entries : traps); 973 trace_entry++) { 974 addr = (GENWQE_UID_OFFS(uid) | 975 IO_EXTENDED_DIAG_READ_MBX); 976 val = __genwqe_readq(cd, addr); 977 set_reg_idx(cd, regs, &idx, max_regs, addr, 978 (diag_sel<<16) | trace_entry, val); 979 } 980 } 981 } 982 return 0; 983 } 984 985 /** 986 * genwqe_write_vreg() - Write register in virtual window 987 * 988 * Note, these registers are only accessible to the PF through the 989 * VF-window. It is not intended for the VF to access. 990 */ 991 int genwqe_write_vreg(struct genwqe_dev *cd, u32 reg, u64 val, int func) 992 { 993 __genwqe_writeq(cd, IO_PF_SLC_VIRTUAL_WINDOW, func & 0xf); 994 __genwqe_writeq(cd, reg, val); 995 return 0; 996 } 997 998 /** 999 * genwqe_read_vreg() - Read register in virtual window 1000 * 1001 * Note, these registers are only accessible to the PF through the 1002 * VF-window. It is not intended for the VF to access. 1003 */ 1004 u64 genwqe_read_vreg(struct genwqe_dev *cd, u32 reg, int func) 1005 { 1006 __genwqe_writeq(cd, IO_PF_SLC_VIRTUAL_WINDOW, func & 0xf); 1007 return __genwqe_readq(cd, reg); 1008 } 1009 1010 /** 1011 * genwqe_base_clock_frequency() - Deteremine base clock frequency of the card 1012 * 1013 * Note: From a design perspective it turned out to be a bad idea to 1014 * use codes here to specifiy the frequency/speed values. An old 1015 * driver cannot understand new codes and is therefore always a 1016 * problem. Better is to measure out the value or put the 1017 * speed/frequency directly into a register which is always a valid 1018 * value for old as well as for new software. 1019 * 1020 * Return: Card clock in MHz 1021 */ 1022 int genwqe_base_clock_frequency(struct genwqe_dev *cd) 1023 { 1024 u16 speed; /* MHz MHz MHz MHz */ 1025 static const int speed_grade[] = { 250, 200, 166, 175 }; 1026 1027 speed = (u16)((cd->slu_unitcfg >> 28) & 0x0full); 1028 if (speed >= ARRAY_SIZE(speed_grade)) 1029 return 0; /* illegal value */ 1030 1031 return speed_grade[speed]; 1032 } 1033 1034 /** 1035 * genwqe_stop_traps() - Stop traps 1036 * 1037 * Before reading out the analysis data, we need to stop the traps. 1038 */ 1039 void genwqe_stop_traps(struct genwqe_dev *cd) 1040 { 1041 __genwqe_writeq(cd, IO_SLC_MISC_DEBUG_SET, 0xcull); 1042 } 1043 1044 /** 1045 * genwqe_start_traps() - Start traps 1046 * 1047 * After having read the data, we can/must enable the traps again. 1048 */ 1049 void genwqe_start_traps(struct genwqe_dev *cd) 1050 { 1051 __genwqe_writeq(cd, IO_SLC_MISC_DEBUG_CLR, 0xcull); 1052 1053 if (genwqe_need_err_masking(cd)) 1054 __genwqe_writeq(cd, IO_SLC_MISC_DEBUG, 0x0aull); 1055 } 1056