1 /* 2 * This file is provided under a dual BSD/GPLv2 license. When using or 3 * redistributing this file, you may do so under either license. 4 * 5 * GPL LICENSE SUMMARY 6 * 7 * Copyright(c) 2012 Intel Corporation. All rights reserved. 8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 9 * Copyright (C) 2016 T-Platforms. All Rights Reserved. 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of version 2 of the GNU General Public License as 13 * published by the Free Software Foundation. 14 * 15 * BSD LICENSE 16 * 17 * Copyright(c) 2012 Intel Corporation. All rights reserved. 18 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 19 * Copyright (C) 2016 T-Platforms. All Rights Reserved. 20 * 21 * Redistribution and use in source and binary forms, with or without 22 * modification, are permitted provided that the following conditions 23 * are met: 24 * 25 * * Redistributions of source code must retain the above copyright 26 * notice, this list of conditions and the following disclaimer. 27 * * Redistributions in binary form must reproduce the above copy 28 * notice, this list of conditions and the following disclaimer in 29 * the documentation and/or other materials provided with the 30 * distribution. 31 * * Neither the name of Intel Corporation nor the names of its 32 * contributors may be used to endorse or promote products derived 33 * from this software without specific prior written permission. 34 * 35 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 36 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 37 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 38 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 39 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 40 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 41 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 42 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 43 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 44 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 45 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 46 * 47 * Intel PCIe NTB Linux driver 48 */ 49 50 #include <linux/debugfs.h> 51 #include <linux/delay.h> 52 #include <linux/init.h> 53 #include <linux/interrupt.h> 54 #include <linux/module.h> 55 #include <linux/pci.h> 56 #include <linux/random.h> 57 #include <linux/slab.h> 58 #include <linux/ntb.h> 59 60 #include "ntb_hw_intel.h" 61 #include "ntb_hw_gen1.h" 62 #include "ntb_hw_gen3.h" 63 #include "ntb_hw_gen4.h" 64 65 #define NTB_NAME "ntb_hw_intel" 66 #define NTB_DESC "Intel(R) PCI-E Non-Transparent Bridge Driver" 67 #define NTB_VER "2.0" 68 69 MODULE_DESCRIPTION(NTB_DESC); 70 MODULE_VERSION(NTB_VER); 71 MODULE_LICENSE("Dual BSD/GPL"); 72 MODULE_AUTHOR("Intel Corporation"); 73 74 #define bar0_off(base, bar) ((base) + ((bar) << 2)) 75 #define bar2_off(base, bar) bar0_off(base, (bar) - 2) 76 77 static const struct intel_ntb_reg xeon_reg; 78 static const struct intel_ntb_alt_reg xeon_pri_reg; 79 static const struct intel_ntb_alt_reg xeon_sec_reg; 80 static const struct intel_ntb_alt_reg xeon_b2b_reg; 81 static const struct intel_ntb_xlat_reg xeon_pri_xlat; 82 static const struct intel_ntb_xlat_reg xeon_sec_xlat; 83 static const struct ntb_dev_ops intel_ntb_ops; 84 85 static const struct file_operations intel_ntb_debugfs_info; 86 static struct dentry *debugfs_dir; 87 88 static int b2b_mw_idx = -1; 89 module_param(b2b_mw_idx, int, 0644); 90 MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb. A " 91 "value of zero or positive starts from first mw idx, and a " 92 "negative value starts from last mw idx. Both sides MUST " 93 "set the same value here!"); 94 95 static unsigned int b2b_mw_share; 96 module_param(b2b_mw_share, uint, 0644); 97 MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the " 98 "ntb so that the peer ntb only occupies the first half of " 99 "the mw, so the second half can still be used as a mw. Both " 100 "sides MUST set the same value here!"); 101 102 module_param_named(xeon_b2b_usd_bar2_addr64, 103 xeon_b2b_usd_addr.bar2_addr64, ullong, 0644); 104 MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64, 105 "XEON B2B USD BAR 2 64-bit address"); 106 107 module_param_named(xeon_b2b_usd_bar4_addr64, 108 xeon_b2b_usd_addr.bar4_addr64, ullong, 0644); 109 MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr64, 110 "XEON B2B USD BAR 4 64-bit address"); 111 112 module_param_named(xeon_b2b_usd_bar4_addr32, 113 xeon_b2b_usd_addr.bar4_addr32, ullong, 0644); 114 MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr32, 115 "XEON B2B USD split-BAR 4 32-bit address"); 116 117 module_param_named(xeon_b2b_usd_bar5_addr32, 118 xeon_b2b_usd_addr.bar5_addr32, ullong, 0644); 119 MODULE_PARM_DESC(xeon_b2b_usd_bar5_addr32, 120 "XEON B2B USD split-BAR 5 32-bit address"); 121 122 module_param_named(xeon_b2b_dsd_bar2_addr64, 123 xeon_b2b_dsd_addr.bar2_addr64, ullong, 0644); 124 MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64, 125 "XEON B2B DSD BAR 2 64-bit address"); 126 127 module_param_named(xeon_b2b_dsd_bar4_addr64, 128 xeon_b2b_dsd_addr.bar4_addr64, ullong, 0644); 129 MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr64, 130 "XEON B2B DSD BAR 4 64-bit address"); 131 132 module_param_named(xeon_b2b_dsd_bar4_addr32, 133 xeon_b2b_dsd_addr.bar4_addr32, ullong, 0644); 134 MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr32, 135 "XEON B2B DSD split-BAR 4 32-bit address"); 136 137 module_param_named(xeon_b2b_dsd_bar5_addr32, 138 xeon_b2b_dsd_addr.bar5_addr32, ullong, 0644); 139 MODULE_PARM_DESC(xeon_b2b_dsd_bar5_addr32, 140 "XEON B2B DSD split-BAR 5 32-bit address"); 141 142 143 static int xeon_init_isr(struct intel_ntb_dev *ndev); 144 145 static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev) 146 { 147 ndev->unsafe_flags = 0; 148 ndev->unsafe_flags_ignore = 0; 149 150 /* Only B2B has a workaround to avoid SDOORBELL */ 151 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) 152 if (!ntb_topo_is_b2b(ndev->ntb.topo)) 153 ndev->unsafe_flags |= NTB_UNSAFE_DB; 154 155 /* No low level workaround to avoid SB01BASE */ 156 if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) { 157 ndev->unsafe_flags |= NTB_UNSAFE_DB; 158 ndev->unsafe_flags |= NTB_UNSAFE_SPAD; 159 } 160 } 161 162 static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev, 163 unsigned long flag) 164 { 165 return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore); 166 } 167 168 static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev, 169 unsigned long flag) 170 { 171 flag &= ndev->unsafe_flags; 172 ndev->unsafe_flags_ignore |= flag; 173 174 return !!flag; 175 } 176 177 int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx) 178 { 179 if (idx < 0 || idx >= ndev->mw_count) 180 return -EINVAL; 181 return ndev->reg->mw_bar[idx]; 182 } 183 184 void ndev_db_addr(struct intel_ntb_dev *ndev, 185 phys_addr_t *db_addr, resource_size_t *db_size, 186 phys_addr_t reg_addr, unsigned long reg) 187 { 188 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) 189 pr_warn_once("%s: NTB unsafe doorbell access", __func__); 190 191 if (db_addr) { 192 *db_addr = reg_addr + reg; 193 dev_dbg(&ndev->ntb.pdev->dev, "Peer db addr %llx\n", *db_addr); 194 } 195 196 if (db_size) { 197 *db_size = ndev->reg->db_size; 198 dev_dbg(&ndev->ntb.pdev->dev, "Peer db size %llx\n", *db_size); 199 } 200 } 201 202 u64 ndev_db_read(struct intel_ntb_dev *ndev, 203 void __iomem *mmio) 204 { 205 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) 206 pr_warn_once("%s: NTB unsafe doorbell access", __func__); 207 208 return ndev->reg->db_ioread(mmio); 209 } 210 211 int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits, 212 void __iomem *mmio) 213 { 214 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) 215 pr_warn_once("%s: NTB unsafe doorbell access", __func__); 216 217 if (db_bits & ~ndev->db_valid_mask) 218 return -EINVAL; 219 220 ndev->reg->db_iowrite(db_bits, mmio); 221 222 return 0; 223 } 224 225 static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits, 226 void __iomem *mmio) 227 { 228 unsigned long irqflags; 229 230 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) 231 pr_warn_once("%s: NTB unsafe doorbell access", __func__); 232 233 if (db_bits & ~ndev->db_valid_mask) 234 return -EINVAL; 235 236 spin_lock_irqsave(&ndev->db_mask_lock, irqflags); 237 { 238 ndev->db_mask |= db_bits; 239 ndev->reg->db_iowrite(ndev->db_mask, mmio); 240 } 241 spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags); 242 243 return 0; 244 } 245 246 static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits, 247 void __iomem *mmio) 248 { 249 unsigned long irqflags; 250 251 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) 252 pr_warn_once("%s: NTB unsafe doorbell access", __func__); 253 254 if (db_bits & ~ndev->db_valid_mask) 255 return -EINVAL; 256 257 spin_lock_irqsave(&ndev->db_mask_lock, irqflags); 258 { 259 ndev->db_mask &= ~db_bits; 260 ndev->reg->db_iowrite(ndev->db_mask, mmio); 261 } 262 spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags); 263 264 return 0; 265 } 266 267 static inline u64 ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector) 268 { 269 u64 shift, mask; 270 271 shift = ndev->db_vec_shift; 272 mask = BIT_ULL(shift) - 1; 273 274 return mask << (shift * db_vector); 275 } 276 277 static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx, 278 phys_addr_t *spad_addr, phys_addr_t reg_addr, 279 unsigned long reg) 280 { 281 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD)) 282 pr_warn_once("%s: NTB unsafe scratchpad access", __func__); 283 284 if (idx < 0 || idx >= ndev->spad_count) 285 return -EINVAL; 286 287 if (spad_addr) { 288 *spad_addr = reg_addr + reg + (idx << 2); 289 dev_dbg(&ndev->ntb.pdev->dev, "Peer spad addr %llx\n", 290 *spad_addr); 291 } 292 293 return 0; 294 } 295 296 static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx, 297 void __iomem *mmio) 298 { 299 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD)) 300 pr_warn_once("%s: NTB unsafe scratchpad access", __func__); 301 302 if (idx < 0 || idx >= ndev->spad_count) 303 return 0; 304 305 return ioread32(mmio + (idx << 2)); 306 } 307 308 static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val, 309 void __iomem *mmio) 310 { 311 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD)) 312 pr_warn_once("%s: NTB unsafe scratchpad access", __func__); 313 314 if (idx < 0 || idx >= ndev->spad_count) 315 return -EINVAL; 316 317 iowrite32(val, mmio + (idx << 2)); 318 319 return 0; 320 } 321 322 static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec) 323 { 324 u64 vec_mask; 325 326 vec_mask = ndev_vec_mask(ndev, vec); 327 328 if ((ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) && (vec == 31)) 329 vec_mask |= ndev->db_link_mask; 330 331 dev_dbg(&ndev->ntb.pdev->dev, "vec %d vec_mask %llx\n", vec, vec_mask); 332 333 ndev->last_ts = jiffies; 334 335 if (vec_mask & ndev->db_link_mask) { 336 if (ndev->reg->poll_link(ndev)) 337 ntb_link_event(&ndev->ntb); 338 } 339 340 if (vec_mask & ndev->db_valid_mask) 341 ntb_db_event(&ndev->ntb, vec); 342 343 return IRQ_HANDLED; 344 } 345 346 static irqreturn_t ndev_vec_isr(int irq, void *dev) 347 { 348 struct intel_ntb_vec *nvec = dev; 349 350 dev_dbg(&nvec->ndev->ntb.pdev->dev, "irq: %d nvec->num: %d\n", 351 irq, nvec->num); 352 353 return ndev_interrupt(nvec->ndev, nvec->num); 354 } 355 356 static irqreturn_t ndev_irq_isr(int irq, void *dev) 357 { 358 struct intel_ntb_dev *ndev = dev; 359 360 return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq); 361 } 362 363 int ndev_init_isr(struct intel_ntb_dev *ndev, 364 int msix_min, int msix_max, 365 int msix_shift, int total_shift) 366 { 367 struct pci_dev *pdev; 368 int rc, i, msix_count, node; 369 370 pdev = ndev->ntb.pdev; 371 372 node = dev_to_node(&pdev->dev); 373 374 /* Mask all doorbell interrupts */ 375 ndev->db_mask = ndev->db_valid_mask; 376 ndev->reg->db_iowrite(ndev->db_mask, 377 ndev->self_mmio + 378 ndev->self_reg->db_mask); 379 380 /* Try to set up msix irq */ 381 382 ndev->vec = kcalloc_node(msix_max, sizeof(*ndev->vec), 383 GFP_KERNEL, node); 384 if (!ndev->vec) 385 goto err_msix_vec_alloc; 386 387 ndev->msix = kcalloc_node(msix_max, sizeof(*ndev->msix), 388 GFP_KERNEL, node); 389 if (!ndev->msix) 390 goto err_msix_alloc; 391 392 for (i = 0; i < msix_max; ++i) 393 ndev->msix[i].entry = i; 394 395 msix_count = pci_enable_msix_range(pdev, ndev->msix, 396 msix_min, msix_max); 397 if (msix_count < 0) 398 goto err_msix_enable; 399 400 for (i = 0; i < msix_count; ++i) { 401 ndev->vec[i].ndev = ndev; 402 ndev->vec[i].num = i; 403 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0, 404 "ndev_vec_isr", &ndev->vec[i]); 405 if (rc) 406 goto err_msix_request; 407 } 408 409 dev_dbg(&pdev->dev, "Using %d msix interrupts\n", msix_count); 410 ndev->db_vec_count = msix_count; 411 ndev->db_vec_shift = msix_shift; 412 return 0; 413 414 err_msix_request: 415 while (i-- > 0) 416 free_irq(ndev->msix[i].vector, &ndev->vec[i]); 417 pci_disable_msix(pdev); 418 err_msix_enable: 419 kfree(ndev->msix); 420 err_msix_alloc: 421 kfree(ndev->vec); 422 err_msix_vec_alloc: 423 ndev->msix = NULL; 424 ndev->vec = NULL; 425 426 /* Try to set up msi irq */ 427 428 rc = pci_enable_msi(pdev); 429 if (rc) 430 goto err_msi_enable; 431 432 rc = request_irq(pdev->irq, ndev_irq_isr, 0, 433 "ndev_irq_isr", ndev); 434 if (rc) 435 goto err_msi_request; 436 437 dev_dbg(&pdev->dev, "Using msi interrupts\n"); 438 ndev->db_vec_count = 1; 439 ndev->db_vec_shift = total_shift; 440 return 0; 441 442 err_msi_request: 443 pci_disable_msi(pdev); 444 err_msi_enable: 445 446 /* Try to set up intx irq */ 447 448 pci_intx(pdev, 1); 449 450 rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED, 451 "ndev_irq_isr", ndev); 452 if (rc) 453 goto err_intx_request; 454 455 dev_dbg(&pdev->dev, "Using intx interrupts\n"); 456 ndev->db_vec_count = 1; 457 ndev->db_vec_shift = total_shift; 458 return 0; 459 460 err_intx_request: 461 return rc; 462 } 463 464 static void ndev_deinit_isr(struct intel_ntb_dev *ndev) 465 { 466 struct pci_dev *pdev; 467 int i; 468 469 pdev = ndev->ntb.pdev; 470 471 /* Mask all doorbell interrupts */ 472 ndev->db_mask = ndev->db_valid_mask; 473 ndev->reg->db_iowrite(ndev->db_mask, 474 ndev->self_mmio + 475 ndev->self_reg->db_mask); 476 477 if (ndev->msix) { 478 i = ndev->db_vec_count; 479 while (i--) 480 free_irq(ndev->msix[i].vector, &ndev->vec[i]); 481 pci_disable_msix(pdev); 482 kfree(ndev->msix); 483 kfree(ndev->vec); 484 } else { 485 free_irq(pdev->irq, ndev); 486 if (pci_dev_msi_enabled(pdev)) 487 pci_disable_msi(pdev); 488 } 489 } 490 491 static ssize_t ndev_ntb_debugfs_read(struct file *filp, char __user *ubuf, 492 size_t count, loff_t *offp) 493 { 494 struct intel_ntb_dev *ndev; 495 struct pci_dev *pdev; 496 void __iomem *mmio; 497 char *buf; 498 size_t buf_size; 499 ssize_t ret, off; 500 union { u64 v64; u32 v32; u16 v16; u8 v8; } u; 501 502 ndev = filp->private_data; 503 pdev = ndev->ntb.pdev; 504 mmio = ndev->self_mmio; 505 506 buf_size = min(count, 0x800ul); 507 508 buf = kmalloc(buf_size, GFP_KERNEL); 509 if (!buf) 510 return -ENOMEM; 511 512 off = 0; 513 514 off += scnprintf(buf + off, buf_size - off, 515 "NTB Device Information:\n"); 516 517 off += scnprintf(buf + off, buf_size - off, 518 "Connection Topology -\t%s\n", 519 ntb_topo_string(ndev->ntb.topo)); 520 521 if (ndev->b2b_idx != UINT_MAX) { 522 off += scnprintf(buf + off, buf_size - off, 523 "B2B MW Idx -\t\t%u\n", ndev->b2b_idx); 524 off += scnprintf(buf + off, buf_size - off, 525 "B2B Offset -\t\t%#lx\n", ndev->b2b_off); 526 } 527 528 off += scnprintf(buf + off, buf_size - off, 529 "BAR4 Split -\t\t%s\n", 530 ndev->bar4_split ? "yes" : "no"); 531 532 off += scnprintf(buf + off, buf_size - off, 533 "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl); 534 off += scnprintf(buf + off, buf_size - off, 535 "LNK STA -\t\t%#06x\n", ndev->lnk_sta); 536 537 if (!ndev->reg->link_is_up(ndev)) { 538 off += scnprintf(buf + off, buf_size - off, 539 "Link Status -\t\tDown\n"); 540 } else { 541 off += scnprintf(buf + off, buf_size - off, 542 "Link Status -\t\tUp\n"); 543 off += scnprintf(buf + off, buf_size - off, 544 "Link Speed -\t\tPCI-E Gen %u\n", 545 NTB_LNK_STA_SPEED(ndev->lnk_sta)); 546 off += scnprintf(buf + off, buf_size - off, 547 "Link Width -\t\tx%u\n", 548 NTB_LNK_STA_WIDTH(ndev->lnk_sta)); 549 } 550 551 off += scnprintf(buf + off, buf_size - off, 552 "Memory Window Count -\t%u\n", ndev->mw_count); 553 off += scnprintf(buf + off, buf_size - off, 554 "Scratchpad Count -\t%u\n", ndev->spad_count); 555 off += scnprintf(buf + off, buf_size - off, 556 "Doorbell Count -\t%u\n", ndev->db_count); 557 off += scnprintf(buf + off, buf_size - off, 558 "Doorbell Vector Count -\t%u\n", ndev->db_vec_count); 559 off += scnprintf(buf + off, buf_size - off, 560 "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift); 561 562 off += scnprintf(buf + off, buf_size - off, 563 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask); 564 off += scnprintf(buf + off, buf_size - off, 565 "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask); 566 off += scnprintf(buf + off, buf_size - off, 567 "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask); 568 569 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask); 570 off += scnprintf(buf + off, buf_size - off, 571 "Doorbell Mask -\t\t%#llx\n", u.v64); 572 573 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell); 574 off += scnprintf(buf + off, buf_size - off, 575 "Doorbell Bell -\t\t%#llx\n", u.v64); 576 577 off += scnprintf(buf + off, buf_size - off, 578 "\nNTB Window Size:\n"); 579 580 pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &u.v8); 581 off += scnprintf(buf + off, buf_size - off, 582 "PBAR23SZ %hhu\n", u.v8); 583 if (!ndev->bar4_split) { 584 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &u.v8); 585 off += scnprintf(buf + off, buf_size - off, 586 "PBAR45SZ %hhu\n", u.v8); 587 } else { 588 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &u.v8); 589 off += scnprintf(buf + off, buf_size - off, 590 "PBAR4SZ %hhu\n", u.v8); 591 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &u.v8); 592 off += scnprintf(buf + off, buf_size - off, 593 "PBAR5SZ %hhu\n", u.v8); 594 } 595 596 pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &u.v8); 597 off += scnprintf(buf + off, buf_size - off, 598 "SBAR23SZ %hhu\n", u.v8); 599 if (!ndev->bar4_split) { 600 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &u.v8); 601 off += scnprintf(buf + off, buf_size - off, 602 "SBAR45SZ %hhu\n", u.v8); 603 } else { 604 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &u.v8); 605 off += scnprintf(buf + off, buf_size - off, 606 "SBAR4SZ %hhu\n", u.v8); 607 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &u.v8); 608 off += scnprintf(buf + off, buf_size - off, 609 "SBAR5SZ %hhu\n", u.v8); 610 } 611 612 off += scnprintf(buf + off, buf_size - off, 613 "\nNTB Incoming XLAT:\n"); 614 615 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2)); 616 off += scnprintf(buf + off, buf_size - off, 617 "XLAT23 -\t\t%#018llx\n", u.v64); 618 619 if (ndev->bar4_split) { 620 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4)); 621 off += scnprintf(buf + off, buf_size - off, 622 "XLAT4 -\t\t\t%#06x\n", u.v32); 623 624 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 5)); 625 off += scnprintf(buf + off, buf_size - off, 626 "XLAT5 -\t\t\t%#06x\n", u.v32); 627 } else { 628 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4)); 629 off += scnprintf(buf + off, buf_size - off, 630 "XLAT45 -\t\t%#018llx\n", u.v64); 631 } 632 633 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2)); 634 off += scnprintf(buf + off, buf_size - off, 635 "LMT23 -\t\t\t%#018llx\n", u.v64); 636 637 if (ndev->bar4_split) { 638 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4)); 639 off += scnprintf(buf + off, buf_size - off, 640 "LMT4 -\t\t\t%#06x\n", u.v32); 641 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 5)); 642 off += scnprintf(buf + off, buf_size - off, 643 "LMT5 -\t\t\t%#06x\n", u.v32); 644 } else { 645 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4)); 646 off += scnprintf(buf + off, buf_size - off, 647 "LMT45 -\t\t\t%#018llx\n", u.v64); 648 } 649 650 if (pdev_is_gen1(pdev)) { 651 if (ntb_topo_is_b2b(ndev->ntb.topo)) { 652 off += scnprintf(buf + off, buf_size - off, 653 "\nNTB Outgoing B2B XLAT:\n"); 654 655 u.v64 = ioread64(mmio + XEON_PBAR23XLAT_OFFSET); 656 off += scnprintf(buf + off, buf_size - off, 657 "B2B XLAT23 -\t\t%#018llx\n", u.v64); 658 659 if (ndev->bar4_split) { 660 u.v32 = ioread32(mmio + XEON_PBAR4XLAT_OFFSET); 661 off += scnprintf(buf + off, buf_size - off, 662 "B2B XLAT4 -\t\t%#06x\n", 663 u.v32); 664 u.v32 = ioread32(mmio + XEON_PBAR5XLAT_OFFSET); 665 off += scnprintf(buf + off, buf_size - off, 666 "B2B XLAT5 -\t\t%#06x\n", 667 u.v32); 668 } else { 669 u.v64 = ioread64(mmio + XEON_PBAR45XLAT_OFFSET); 670 off += scnprintf(buf + off, buf_size - off, 671 "B2B XLAT45 -\t\t%#018llx\n", 672 u.v64); 673 } 674 675 u.v64 = ioread64(mmio + XEON_PBAR23LMT_OFFSET); 676 off += scnprintf(buf + off, buf_size - off, 677 "B2B LMT23 -\t\t%#018llx\n", u.v64); 678 679 if (ndev->bar4_split) { 680 u.v32 = ioread32(mmio + XEON_PBAR4LMT_OFFSET); 681 off += scnprintf(buf + off, buf_size - off, 682 "B2B LMT4 -\t\t%#06x\n", 683 u.v32); 684 u.v32 = ioread32(mmio + XEON_PBAR5LMT_OFFSET); 685 off += scnprintf(buf + off, buf_size - off, 686 "B2B LMT5 -\t\t%#06x\n", 687 u.v32); 688 } else { 689 u.v64 = ioread64(mmio + XEON_PBAR45LMT_OFFSET); 690 off += scnprintf(buf + off, buf_size - off, 691 "B2B LMT45 -\t\t%#018llx\n", 692 u.v64); 693 } 694 695 off += scnprintf(buf + off, buf_size - off, 696 "\nNTB Secondary BAR:\n"); 697 698 u.v64 = ioread64(mmio + XEON_SBAR0BASE_OFFSET); 699 off += scnprintf(buf + off, buf_size - off, 700 "SBAR01 -\t\t%#018llx\n", u.v64); 701 702 u.v64 = ioread64(mmio + XEON_SBAR23BASE_OFFSET); 703 off += scnprintf(buf + off, buf_size - off, 704 "SBAR23 -\t\t%#018llx\n", u.v64); 705 706 if (ndev->bar4_split) { 707 u.v32 = ioread32(mmio + XEON_SBAR4BASE_OFFSET); 708 off += scnprintf(buf + off, buf_size - off, 709 "SBAR4 -\t\t\t%#06x\n", u.v32); 710 u.v32 = ioread32(mmio + XEON_SBAR5BASE_OFFSET); 711 off += scnprintf(buf + off, buf_size - off, 712 "SBAR5 -\t\t\t%#06x\n", u.v32); 713 } else { 714 u.v64 = ioread64(mmio + XEON_SBAR45BASE_OFFSET); 715 off += scnprintf(buf + off, buf_size - off, 716 "SBAR45 -\t\t%#018llx\n", 717 u.v64); 718 } 719 } 720 721 off += scnprintf(buf + off, buf_size - off, 722 "\nXEON NTB Statistics:\n"); 723 724 u.v16 = ioread16(mmio + XEON_USMEMMISS_OFFSET); 725 off += scnprintf(buf + off, buf_size - off, 726 "Upstream Memory Miss -\t%u\n", u.v16); 727 728 off += scnprintf(buf + off, buf_size - off, 729 "\nXEON NTB Hardware Errors:\n"); 730 731 if (!pci_read_config_word(pdev, 732 XEON_DEVSTS_OFFSET, &u.v16)) 733 off += scnprintf(buf + off, buf_size - off, 734 "DEVSTS -\t\t%#06x\n", u.v16); 735 736 if (!pci_read_config_word(pdev, 737 XEON_LINK_STATUS_OFFSET, &u.v16)) 738 off += scnprintf(buf + off, buf_size - off, 739 "LNKSTS -\t\t%#06x\n", u.v16); 740 741 if (!pci_read_config_dword(pdev, 742 XEON_UNCERRSTS_OFFSET, &u.v32)) 743 off += scnprintf(buf + off, buf_size - off, 744 "UNCERRSTS -\t\t%#06x\n", u.v32); 745 746 if (!pci_read_config_dword(pdev, 747 XEON_CORERRSTS_OFFSET, &u.v32)) 748 off += scnprintf(buf + off, buf_size - off, 749 "CORERRSTS -\t\t%#06x\n", u.v32); 750 } 751 752 ret = simple_read_from_buffer(ubuf, count, offp, buf, off); 753 kfree(buf); 754 return ret; 755 } 756 757 static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf, 758 size_t count, loff_t *offp) 759 { 760 struct intel_ntb_dev *ndev = filp->private_data; 761 762 if (pdev_is_gen1(ndev->ntb.pdev)) 763 return ndev_ntb_debugfs_read(filp, ubuf, count, offp); 764 else if (pdev_is_gen3(ndev->ntb.pdev)) 765 return ndev_ntb3_debugfs_read(filp, ubuf, count, offp); 766 else if (pdev_is_gen4(ndev->ntb.pdev) || pdev_is_gen5(ndev->ntb.pdev)) 767 return ndev_ntb4_debugfs_read(filp, ubuf, count, offp); 768 769 return -ENXIO; 770 } 771 772 static void ndev_init_debugfs(struct intel_ntb_dev *ndev) 773 { 774 if (!debugfs_dir) { 775 ndev->debugfs_dir = NULL; 776 ndev->debugfs_info = NULL; 777 } else { 778 ndev->debugfs_dir = 779 debugfs_create_dir(pci_name(ndev->ntb.pdev), 780 debugfs_dir); 781 if (!ndev->debugfs_dir) 782 ndev->debugfs_info = NULL; 783 else 784 ndev->debugfs_info = 785 debugfs_create_file("info", S_IRUSR, 786 ndev->debugfs_dir, ndev, 787 &intel_ntb_debugfs_info); 788 } 789 } 790 791 static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev) 792 { 793 debugfs_remove_recursive(ndev->debugfs_dir); 794 } 795 796 int intel_ntb_mw_count(struct ntb_dev *ntb, int pidx) 797 { 798 if (pidx != NTB_DEF_PEER_IDX) 799 return -EINVAL; 800 801 return ntb_ndev(ntb)->mw_count; 802 } 803 804 int intel_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx, 805 resource_size_t *addr_align, 806 resource_size_t *size_align, 807 resource_size_t *size_max) 808 { 809 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 810 resource_size_t bar_size, mw_size; 811 int bar; 812 813 if (pidx != NTB_DEF_PEER_IDX) 814 return -EINVAL; 815 816 if (idx >= ndev->b2b_idx && !ndev->b2b_off) 817 idx += 1; 818 819 bar = ndev_mw_to_bar(ndev, idx); 820 if (bar < 0) 821 return bar; 822 823 bar_size = pci_resource_len(ndev->ntb.pdev, bar); 824 825 if (idx == ndev->b2b_idx) 826 mw_size = bar_size - ndev->b2b_off; 827 else 828 mw_size = bar_size; 829 830 if (addr_align) 831 *addr_align = pci_resource_len(ndev->ntb.pdev, bar); 832 833 if (size_align) 834 *size_align = 1; 835 836 if (size_max) 837 *size_max = mw_size; 838 839 return 0; 840 } 841 842 static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, 843 dma_addr_t addr, resource_size_t size) 844 { 845 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 846 unsigned long base_reg, xlat_reg, limit_reg; 847 resource_size_t bar_size, mw_size; 848 void __iomem *mmio; 849 u64 base, limit, reg_val; 850 int bar; 851 852 if (pidx != NTB_DEF_PEER_IDX) 853 return -EINVAL; 854 855 if (idx >= ndev->b2b_idx && !ndev->b2b_off) 856 idx += 1; 857 858 bar = ndev_mw_to_bar(ndev, idx); 859 if (bar < 0) 860 return bar; 861 862 bar_size = pci_resource_len(ndev->ntb.pdev, bar); 863 864 if (idx == ndev->b2b_idx) 865 mw_size = bar_size - ndev->b2b_off; 866 else 867 mw_size = bar_size; 868 869 /* hardware requires that addr is aligned to bar size */ 870 if (addr & (bar_size - 1)) 871 return -EINVAL; 872 873 /* make sure the range fits in the usable mw size */ 874 if (size > mw_size) 875 return -EINVAL; 876 877 mmio = ndev->self_mmio; 878 base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar); 879 xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar); 880 limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar); 881 882 if (bar < 4 || !ndev->bar4_split) { 883 base = ioread64(mmio + base_reg) & NTB_BAR_MASK_64; 884 885 /* Set the limit if supported, if size is not mw_size */ 886 if (limit_reg && size != mw_size) 887 limit = base + size; 888 else 889 limit = 0; 890 891 /* set and verify setting the translation address */ 892 iowrite64(addr, mmio + xlat_reg); 893 reg_val = ioread64(mmio + xlat_reg); 894 if (reg_val != addr) { 895 iowrite64(0, mmio + xlat_reg); 896 return -EIO; 897 } 898 899 /* set and verify setting the limit */ 900 iowrite64(limit, mmio + limit_reg); 901 reg_val = ioread64(mmio + limit_reg); 902 if (reg_val != limit) { 903 iowrite64(base, mmio + limit_reg); 904 iowrite64(0, mmio + xlat_reg); 905 return -EIO; 906 } 907 } else { 908 /* split bar addr range must all be 32 bit */ 909 if (addr & (~0ull << 32)) 910 return -EINVAL; 911 if ((addr + size) & (~0ull << 32)) 912 return -EINVAL; 913 914 base = ioread32(mmio + base_reg) & NTB_BAR_MASK_32; 915 916 /* Set the limit if supported, if size is not mw_size */ 917 if (limit_reg && size != mw_size) 918 limit = base + size; 919 else 920 limit = 0; 921 922 /* set and verify setting the translation address */ 923 iowrite32(addr, mmio + xlat_reg); 924 reg_val = ioread32(mmio + xlat_reg); 925 if (reg_val != addr) { 926 iowrite32(0, mmio + xlat_reg); 927 return -EIO; 928 } 929 930 /* set and verify setting the limit */ 931 iowrite32(limit, mmio + limit_reg); 932 reg_val = ioread32(mmio + limit_reg); 933 if (reg_val != limit) { 934 iowrite32(base, mmio + limit_reg); 935 iowrite32(0, mmio + xlat_reg); 936 return -EIO; 937 } 938 } 939 940 return 0; 941 } 942 943 u64 intel_ntb_link_is_up(struct ntb_dev *ntb, enum ntb_speed *speed, 944 enum ntb_width *width) 945 { 946 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 947 948 if (ndev->reg->link_is_up(ndev)) { 949 if (speed) 950 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta); 951 if (width) 952 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta); 953 return 1; 954 } else { 955 /* TODO MAYBE: is it possible to observe the link speed and 956 * width while link is training? */ 957 if (speed) 958 *speed = NTB_SPEED_NONE; 959 if (width) 960 *width = NTB_WIDTH_NONE; 961 return 0; 962 } 963 } 964 965 static int intel_ntb_link_enable(struct ntb_dev *ntb, 966 enum ntb_speed max_speed, 967 enum ntb_width max_width) 968 { 969 struct intel_ntb_dev *ndev; 970 u32 ntb_ctl; 971 972 ndev = container_of(ntb, struct intel_ntb_dev, ntb); 973 974 if (ndev->ntb.topo == NTB_TOPO_SEC) 975 return -EINVAL; 976 977 dev_dbg(&ntb->pdev->dev, 978 "Enabling link with max_speed %d max_width %d\n", 979 max_speed, max_width); 980 if (max_speed != NTB_SPEED_AUTO) 981 dev_dbg(&ntb->pdev->dev, "ignoring max_speed %d\n", max_speed); 982 if (max_width != NTB_WIDTH_AUTO) 983 dev_dbg(&ntb->pdev->dev, "ignoring max_width %d\n", max_width); 984 985 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl); 986 ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK); 987 ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP; 988 ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP; 989 if (ndev->bar4_split) 990 ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP; 991 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl); 992 993 return 0; 994 } 995 996 int intel_ntb_link_disable(struct ntb_dev *ntb) 997 { 998 struct intel_ntb_dev *ndev; 999 u32 ntb_cntl; 1000 1001 ndev = container_of(ntb, struct intel_ntb_dev, ntb); 1002 1003 if (ndev->ntb.topo == NTB_TOPO_SEC) 1004 return -EINVAL; 1005 1006 dev_dbg(&ntb->pdev->dev, "Disabling link\n"); 1007 1008 /* Bring NTB link down */ 1009 ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl); 1010 ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP); 1011 ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP); 1012 if (ndev->bar4_split) 1013 ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP); 1014 ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK; 1015 iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl); 1016 1017 return 0; 1018 } 1019 1020 int intel_ntb_peer_mw_count(struct ntb_dev *ntb) 1021 { 1022 /* Numbers of inbound and outbound memory windows match */ 1023 return ntb_ndev(ntb)->mw_count; 1024 } 1025 1026 int intel_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx, 1027 phys_addr_t *base, resource_size_t *size) 1028 { 1029 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1030 int bar; 1031 1032 if (idx >= ndev->b2b_idx && !ndev->b2b_off) 1033 idx += 1; 1034 1035 bar = ndev_mw_to_bar(ndev, idx); 1036 if (bar < 0) 1037 return bar; 1038 1039 if (base) 1040 *base = pci_resource_start(ndev->ntb.pdev, bar) + 1041 (idx == ndev->b2b_idx ? ndev->b2b_off : 0); 1042 1043 if (size) 1044 *size = pci_resource_len(ndev->ntb.pdev, bar) - 1045 (idx == ndev->b2b_idx ? ndev->b2b_off : 0); 1046 1047 return 0; 1048 } 1049 1050 static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb) 1051 { 1052 return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB); 1053 } 1054 1055 u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb) 1056 { 1057 return ntb_ndev(ntb)->db_valid_mask; 1058 } 1059 1060 int intel_ntb_db_vector_count(struct ntb_dev *ntb) 1061 { 1062 struct intel_ntb_dev *ndev; 1063 1064 ndev = container_of(ntb, struct intel_ntb_dev, ntb); 1065 1066 return ndev->db_vec_count; 1067 } 1068 1069 u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) 1070 { 1071 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1072 1073 if (db_vector < 0 || db_vector > ndev->db_vec_count) 1074 return 0; 1075 1076 return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector); 1077 } 1078 1079 static u64 intel_ntb_db_read(struct ntb_dev *ntb) 1080 { 1081 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1082 1083 return ndev_db_read(ndev, 1084 ndev->self_mmio + 1085 ndev->self_reg->db_bell); 1086 } 1087 1088 static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) 1089 { 1090 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1091 1092 return ndev_db_write(ndev, db_bits, 1093 ndev->self_mmio + 1094 ndev->self_reg->db_bell); 1095 } 1096 1097 int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) 1098 { 1099 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1100 1101 return ndev_db_set_mask(ndev, db_bits, 1102 ndev->self_mmio + 1103 ndev->self_reg->db_mask); 1104 } 1105 1106 int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) 1107 { 1108 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1109 1110 return ndev_db_clear_mask(ndev, db_bits, 1111 ndev->self_mmio + 1112 ndev->self_reg->db_mask); 1113 } 1114 1115 static int intel_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr, 1116 resource_size_t *db_size, u64 *db_data, int db_bit) 1117 { 1118 u64 db_bits; 1119 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1120 1121 if (unlikely(db_bit >= BITS_PER_LONG_LONG)) 1122 return -EINVAL; 1123 1124 db_bits = BIT_ULL(db_bit); 1125 1126 if (unlikely(db_bits & ~ntb_ndev(ntb)->db_valid_mask)) 1127 return -EINVAL; 1128 1129 ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr, 1130 ndev->peer_reg->db_bell); 1131 1132 if (db_data) 1133 *db_data = db_bits; 1134 1135 1136 return 0; 1137 } 1138 1139 static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) 1140 { 1141 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1142 1143 return ndev_db_write(ndev, db_bits, 1144 ndev->peer_mmio + 1145 ndev->peer_reg->db_bell); 1146 } 1147 1148 int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb) 1149 { 1150 return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD); 1151 } 1152 1153 int intel_ntb_spad_count(struct ntb_dev *ntb) 1154 { 1155 struct intel_ntb_dev *ndev; 1156 1157 ndev = container_of(ntb, struct intel_ntb_dev, ntb); 1158 1159 return ndev->spad_count; 1160 } 1161 1162 u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx) 1163 { 1164 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1165 1166 return ndev_spad_read(ndev, idx, 1167 ndev->self_mmio + 1168 ndev->self_reg->spad); 1169 } 1170 1171 int intel_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val) 1172 { 1173 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1174 1175 return ndev_spad_write(ndev, idx, val, 1176 ndev->self_mmio + 1177 ndev->self_reg->spad); 1178 } 1179 1180 int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx, 1181 phys_addr_t *spad_addr) 1182 { 1183 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1184 1185 return ndev_spad_addr(ndev, sidx, spad_addr, ndev->peer_addr, 1186 ndev->peer_reg->spad); 1187 } 1188 1189 u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx) 1190 { 1191 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1192 1193 return ndev_spad_read(ndev, sidx, 1194 ndev->peer_mmio + 1195 ndev->peer_reg->spad); 1196 } 1197 1198 int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, int sidx, 1199 u32 val) 1200 { 1201 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1202 1203 return ndev_spad_write(ndev, sidx, val, 1204 ndev->peer_mmio + 1205 ndev->peer_reg->spad); 1206 } 1207 1208 static u64 xeon_db_ioread(const void __iomem *mmio) 1209 { 1210 return (u64)ioread16(mmio); 1211 } 1212 1213 static void xeon_db_iowrite(u64 bits, void __iomem *mmio) 1214 { 1215 iowrite16((u16)bits, mmio); 1216 } 1217 1218 static int xeon_poll_link(struct intel_ntb_dev *ndev) 1219 { 1220 u16 reg_val; 1221 int rc; 1222 1223 ndev->reg->db_iowrite(ndev->db_link_mask, 1224 ndev->self_mmio + 1225 ndev->self_reg->db_bell); 1226 1227 rc = pci_read_config_word(ndev->ntb.pdev, 1228 XEON_LINK_STATUS_OFFSET, ®_val); 1229 if (rc) 1230 return 0; 1231 1232 if (reg_val == ndev->lnk_sta) 1233 return 0; 1234 1235 ndev->lnk_sta = reg_val; 1236 1237 return 1; 1238 } 1239 1240 int xeon_link_is_up(struct intel_ntb_dev *ndev) 1241 { 1242 if (ndev->ntb.topo == NTB_TOPO_SEC) 1243 return 1; 1244 1245 return NTB_LNK_STA_ACTIVE(ndev->lnk_sta); 1246 } 1247 1248 enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd) 1249 { 1250 switch (ppd & XEON_PPD_TOPO_MASK) { 1251 case XEON_PPD_TOPO_B2B_USD: 1252 return NTB_TOPO_B2B_USD; 1253 1254 case XEON_PPD_TOPO_B2B_DSD: 1255 return NTB_TOPO_B2B_DSD; 1256 1257 case XEON_PPD_TOPO_PRI_USD: 1258 case XEON_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */ 1259 return NTB_TOPO_PRI; 1260 1261 case XEON_PPD_TOPO_SEC_USD: 1262 case XEON_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */ 1263 return NTB_TOPO_SEC; 1264 } 1265 1266 return NTB_TOPO_NONE; 1267 } 1268 1269 static inline int xeon_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd) 1270 { 1271 if (ppd & XEON_PPD_SPLIT_BAR_MASK) { 1272 dev_dbg(&ndev->ntb.pdev->dev, "PPD %d split bar\n", ppd); 1273 return 1; 1274 } 1275 return 0; 1276 } 1277 1278 static int xeon_init_isr(struct intel_ntb_dev *ndev) 1279 { 1280 return ndev_init_isr(ndev, XEON_DB_MSIX_VECTOR_COUNT, 1281 XEON_DB_MSIX_VECTOR_COUNT, 1282 XEON_DB_MSIX_VECTOR_SHIFT, 1283 XEON_DB_TOTAL_SHIFT); 1284 } 1285 1286 static void xeon_deinit_isr(struct intel_ntb_dev *ndev) 1287 { 1288 ndev_deinit_isr(ndev); 1289 } 1290 1291 static int xeon_setup_b2b_mw(struct intel_ntb_dev *ndev, 1292 const struct intel_b2b_addr *addr, 1293 const struct intel_b2b_addr *peer_addr) 1294 { 1295 struct pci_dev *pdev; 1296 void __iomem *mmio; 1297 resource_size_t bar_size; 1298 phys_addr_t bar_addr; 1299 int b2b_bar; 1300 u8 bar_sz; 1301 1302 pdev = ndev->ntb.pdev; 1303 mmio = ndev->self_mmio; 1304 1305 if (ndev->b2b_idx == UINT_MAX) { 1306 dev_dbg(&pdev->dev, "not using b2b mw\n"); 1307 b2b_bar = 0; 1308 ndev->b2b_off = 0; 1309 } else { 1310 b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx); 1311 if (b2b_bar < 0) 1312 return -EIO; 1313 1314 dev_dbg(&pdev->dev, "using b2b mw bar %d\n", b2b_bar); 1315 1316 bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar); 1317 1318 dev_dbg(&pdev->dev, "b2b bar size %#llx\n", bar_size); 1319 1320 if (b2b_mw_share && XEON_B2B_MIN_SIZE <= bar_size >> 1) { 1321 dev_dbg(&pdev->dev, "b2b using first half of bar\n"); 1322 ndev->b2b_off = bar_size >> 1; 1323 } else if (XEON_B2B_MIN_SIZE <= bar_size) { 1324 dev_dbg(&pdev->dev, "b2b using whole bar\n"); 1325 ndev->b2b_off = 0; 1326 --ndev->mw_count; 1327 } else { 1328 dev_dbg(&pdev->dev, "b2b bar size is too small\n"); 1329 return -EIO; 1330 } 1331 } 1332 1333 /* Reset the secondary bar sizes to match the primary bar sizes, 1334 * except disable or halve the size of the b2b secondary bar. 1335 * 1336 * Note: code for each specific bar size register, because the register 1337 * offsets are not in a consistent order (bar5sz comes after ppd, odd). 1338 */ 1339 pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &bar_sz); 1340 dev_dbg(&pdev->dev, "PBAR23SZ %#x\n", bar_sz); 1341 if (b2b_bar == 2) { 1342 if (ndev->b2b_off) 1343 bar_sz -= 1; 1344 else 1345 bar_sz = 0; 1346 } 1347 pci_write_config_byte(pdev, XEON_SBAR23SZ_OFFSET, bar_sz); 1348 pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &bar_sz); 1349 dev_dbg(&pdev->dev, "SBAR23SZ %#x\n", bar_sz); 1350 1351 if (!ndev->bar4_split) { 1352 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &bar_sz); 1353 dev_dbg(&pdev->dev, "PBAR45SZ %#x\n", bar_sz); 1354 if (b2b_bar == 4) { 1355 if (ndev->b2b_off) 1356 bar_sz -= 1; 1357 else 1358 bar_sz = 0; 1359 } 1360 pci_write_config_byte(pdev, XEON_SBAR45SZ_OFFSET, bar_sz); 1361 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &bar_sz); 1362 dev_dbg(&pdev->dev, "SBAR45SZ %#x\n", bar_sz); 1363 } else { 1364 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &bar_sz); 1365 dev_dbg(&pdev->dev, "PBAR4SZ %#x\n", bar_sz); 1366 if (b2b_bar == 4) { 1367 if (ndev->b2b_off) 1368 bar_sz -= 1; 1369 else 1370 bar_sz = 0; 1371 } 1372 pci_write_config_byte(pdev, XEON_SBAR4SZ_OFFSET, bar_sz); 1373 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &bar_sz); 1374 dev_dbg(&pdev->dev, "SBAR4SZ %#x\n", bar_sz); 1375 1376 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &bar_sz); 1377 dev_dbg(&pdev->dev, "PBAR5SZ %#x\n", bar_sz); 1378 if (b2b_bar == 5) { 1379 if (ndev->b2b_off) 1380 bar_sz -= 1; 1381 else 1382 bar_sz = 0; 1383 } 1384 pci_write_config_byte(pdev, XEON_SBAR5SZ_OFFSET, bar_sz); 1385 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &bar_sz); 1386 dev_dbg(&pdev->dev, "SBAR5SZ %#x\n", bar_sz); 1387 } 1388 1389 /* SBAR01 hit by first part of the b2b bar */ 1390 if (b2b_bar == 0) 1391 bar_addr = addr->bar0_addr; 1392 else if (b2b_bar == 2) 1393 bar_addr = addr->bar2_addr64; 1394 else if (b2b_bar == 4 && !ndev->bar4_split) 1395 bar_addr = addr->bar4_addr64; 1396 else if (b2b_bar == 4) 1397 bar_addr = addr->bar4_addr32; 1398 else if (b2b_bar == 5) 1399 bar_addr = addr->bar5_addr32; 1400 else 1401 return -EIO; 1402 1403 dev_dbg(&pdev->dev, "SBAR01 %#018llx\n", bar_addr); 1404 iowrite64(bar_addr, mmio + XEON_SBAR0BASE_OFFSET); 1405 1406 /* Other SBAR are normally hit by the PBAR xlat, except for b2b bar. 1407 * The b2b bar is either disabled above, or configured half-size, and 1408 * it starts at the PBAR xlat + offset. 1409 */ 1410 1411 bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0); 1412 iowrite64(bar_addr, mmio + XEON_SBAR23BASE_OFFSET); 1413 bar_addr = ioread64(mmio + XEON_SBAR23BASE_OFFSET); 1414 dev_dbg(&pdev->dev, "SBAR23 %#018llx\n", bar_addr); 1415 1416 if (!ndev->bar4_split) { 1417 bar_addr = addr->bar4_addr64 + 1418 (b2b_bar == 4 ? ndev->b2b_off : 0); 1419 iowrite64(bar_addr, mmio + XEON_SBAR45BASE_OFFSET); 1420 bar_addr = ioread64(mmio + XEON_SBAR45BASE_OFFSET); 1421 dev_dbg(&pdev->dev, "SBAR45 %#018llx\n", bar_addr); 1422 } else { 1423 bar_addr = addr->bar4_addr32 + 1424 (b2b_bar == 4 ? ndev->b2b_off : 0); 1425 iowrite32(bar_addr, mmio + XEON_SBAR4BASE_OFFSET); 1426 bar_addr = ioread32(mmio + XEON_SBAR4BASE_OFFSET); 1427 dev_dbg(&pdev->dev, "SBAR4 %#010llx\n", bar_addr); 1428 1429 bar_addr = addr->bar5_addr32 + 1430 (b2b_bar == 5 ? ndev->b2b_off : 0); 1431 iowrite32(bar_addr, mmio + XEON_SBAR5BASE_OFFSET); 1432 bar_addr = ioread32(mmio + XEON_SBAR5BASE_OFFSET); 1433 dev_dbg(&pdev->dev, "SBAR5 %#010llx\n", bar_addr); 1434 } 1435 1436 /* setup incoming bar limits == base addrs (zero length windows) */ 1437 1438 bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0); 1439 iowrite64(bar_addr, mmio + XEON_SBAR23LMT_OFFSET); 1440 bar_addr = ioread64(mmio + XEON_SBAR23LMT_OFFSET); 1441 dev_dbg(&pdev->dev, "SBAR23LMT %#018llx\n", bar_addr); 1442 1443 if (!ndev->bar4_split) { 1444 bar_addr = addr->bar4_addr64 + 1445 (b2b_bar == 4 ? ndev->b2b_off : 0); 1446 iowrite64(bar_addr, mmio + XEON_SBAR45LMT_OFFSET); 1447 bar_addr = ioread64(mmio + XEON_SBAR45LMT_OFFSET); 1448 dev_dbg(&pdev->dev, "SBAR45LMT %#018llx\n", bar_addr); 1449 } else { 1450 bar_addr = addr->bar4_addr32 + 1451 (b2b_bar == 4 ? ndev->b2b_off : 0); 1452 iowrite32(bar_addr, mmio + XEON_SBAR4LMT_OFFSET); 1453 bar_addr = ioread32(mmio + XEON_SBAR4LMT_OFFSET); 1454 dev_dbg(&pdev->dev, "SBAR4LMT %#010llx\n", bar_addr); 1455 1456 bar_addr = addr->bar5_addr32 + 1457 (b2b_bar == 5 ? ndev->b2b_off : 0); 1458 iowrite32(bar_addr, mmio + XEON_SBAR5LMT_OFFSET); 1459 bar_addr = ioread32(mmio + XEON_SBAR5LMT_OFFSET); 1460 dev_dbg(&pdev->dev, "SBAR5LMT %#05llx\n", bar_addr); 1461 } 1462 1463 /* zero incoming translation addrs */ 1464 iowrite64(0, mmio + XEON_SBAR23XLAT_OFFSET); 1465 1466 if (!ndev->bar4_split) { 1467 iowrite64(0, mmio + XEON_SBAR45XLAT_OFFSET); 1468 } else { 1469 iowrite32(0, mmio + XEON_SBAR4XLAT_OFFSET); 1470 iowrite32(0, mmio + XEON_SBAR5XLAT_OFFSET); 1471 } 1472 1473 /* zero outgoing translation limits (whole bar size windows) */ 1474 iowrite64(0, mmio + XEON_PBAR23LMT_OFFSET); 1475 if (!ndev->bar4_split) { 1476 iowrite64(0, mmio + XEON_PBAR45LMT_OFFSET); 1477 } else { 1478 iowrite32(0, mmio + XEON_PBAR4LMT_OFFSET); 1479 iowrite32(0, mmio + XEON_PBAR5LMT_OFFSET); 1480 } 1481 1482 /* set outgoing translation offsets */ 1483 bar_addr = peer_addr->bar2_addr64; 1484 iowrite64(bar_addr, mmio + XEON_PBAR23XLAT_OFFSET); 1485 bar_addr = ioread64(mmio + XEON_PBAR23XLAT_OFFSET); 1486 dev_dbg(&pdev->dev, "PBAR23XLAT %#018llx\n", bar_addr); 1487 1488 if (!ndev->bar4_split) { 1489 bar_addr = peer_addr->bar4_addr64; 1490 iowrite64(bar_addr, mmio + XEON_PBAR45XLAT_OFFSET); 1491 bar_addr = ioread64(mmio + XEON_PBAR45XLAT_OFFSET); 1492 dev_dbg(&pdev->dev, "PBAR45XLAT %#018llx\n", bar_addr); 1493 } else { 1494 bar_addr = peer_addr->bar4_addr32; 1495 iowrite32(bar_addr, mmio + XEON_PBAR4XLAT_OFFSET); 1496 bar_addr = ioread32(mmio + XEON_PBAR4XLAT_OFFSET); 1497 dev_dbg(&pdev->dev, "PBAR4XLAT %#010llx\n", bar_addr); 1498 1499 bar_addr = peer_addr->bar5_addr32; 1500 iowrite32(bar_addr, mmio + XEON_PBAR5XLAT_OFFSET); 1501 bar_addr = ioread32(mmio + XEON_PBAR5XLAT_OFFSET); 1502 dev_dbg(&pdev->dev, "PBAR5XLAT %#010llx\n", bar_addr); 1503 } 1504 1505 /* set the translation offset for b2b registers */ 1506 if (b2b_bar == 0) 1507 bar_addr = peer_addr->bar0_addr; 1508 else if (b2b_bar == 2) 1509 bar_addr = peer_addr->bar2_addr64; 1510 else if (b2b_bar == 4 && !ndev->bar4_split) 1511 bar_addr = peer_addr->bar4_addr64; 1512 else if (b2b_bar == 4) 1513 bar_addr = peer_addr->bar4_addr32; 1514 else if (b2b_bar == 5) 1515 bar_addr = peer_addr->bar5_addr32; 1516 else 1517 return -EIO; 1518 1519 /* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */ 1520 dev_dbg(&pdev->dev, "B2BXLAT %#018llx\n", bar_addr); 1521 iowrite32(bar_addr, mmio + XEON_B2B_XLAT_OFFSETL); 1522 iowrite32(bar_addr >> 32, mmio + XEON_B2B_XLAT_OFFSETU); 1523 1524 if (b2b_bar) { 1525 /* map peer ntb mmio config space registers */ 1526 ndev->peer_mmio = pci_iomap(pdev, b2b_bar, 1527 XEON_B2B_MIN_SIZE); 1528 if (!ndev->peer_mmio) 1529 return -EIO; 1530 1531 ndev->peer_addr = pci_resource_start(pdev, b2b_bar); 1532 } 1533 1534 return 0; 1535 } 1536 1537 static int xeon_init_ntb(struct intel_ntb_dev *ndev) 1538 { 1539 struct device *dev = &ndev->ntb.pdev->dev; 1540 int rc; 1541 u32 ntb_ctl; 1542 1543 if (ndev->bar4_split) 1544 ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT; 1545 else 1546 ndev->mw_count = XEON_MW_COUNT; 1547 1548 ndev->spad_count = XEON_SPAD_COUNT; 1549 ndev->db_count = XEON_DB_COUNT; 1550 ndev->db_link_mask = XEON_DB_LINK_BIT; 1551 1552 switch (ndev->ntb.topo) { 1553 case NTB_TOPO_PRI: 1554 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) { 1555 dev_err(dev, "NTB Primary config disabled\n"); 1556 return -EINVAL; 1557 } 1558 1559 /* enable link to allow secondary side device to appear */ 1560 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl); 1561 ntb_ctl &= ~NTB_CTL_DISABLE; 1562 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl); 1563 1564 /* use half the spads for the peer */ 1565 ndev->spad_count >>= 1; 1566 ndev->self_reg = &xeon_pri_reg; 1567 ndev->peer_reg = &xeon_sec_reg; 1568 ndev->xlat_reg = &xeon_sec_xlat; 1569 break; 1570 1571 case NTB_TOPO_SEC: 1572 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) { 1573 dev_err(dev, "NTB Secondary config disabled\n"); 1574 return -EINVAL; 1575 } 1576 /* use half the spads for the peer */ 1577 ndev->spad_count >>= 1; 1578 ndev->self_reg = &xeon_sec_reg; 1579 ndev->peer_reg = &xeon_pri_reg; 1580 ndev->xlat_reg = &xeon_pri_xlat; 1581 break; 1582 1583 case NTB_TOPO_B2B_USD: 1584 case NTB_TOPO_B2B_DSD: 1585 ndev->self_reg = &xeon_pri_reg; 1586 ndev->peer_reg = &xeon_b2b_reg; 1587 ndev->xlat_reg = &xeon_sec_xlat; 1588 1589 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) { 1590 ndev->peer_reg = &xeon_pri_reg; 1591 1592 if (b2b_mw_idx < 0) 1593 ndev->b2b_idx = b2b_mw_idx + ndev->mw_count; 1594 else 1595 ndev->b2b_idx = b2b_mw_idx; 1596 1597 if (ndev->b2b_idx >= ndev->mw_count) { 1598 dev_dbg(dev, 1599 "b2b_mw_idx %d invalid for mw_count %u\n", 1600 b2b_mw_idx, ndev->mw_count); 1601 return -EINVAL; 1602 } 1603 1604 dev_dbg(dev, "setting up b2b mw idx %d means %d\n", 1605 b2b_mw_idx, ndev->b2b_idx); 1606 1607 } else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) { 1608 dev_warn(dev, "Reduce doorbell count by 1\n"); 1609 ndev->db_count -= 1; 1610 } 1611 1612 if (ndev->ntb.topo == NTB_TOPO_B2B_USD) { 1613 rc = xeon_setup_b2b_mw(ndev, 1614 &xeon_b2b_dsd_addr, 1615 &xeon_b2b_usd_addr); 1616 } else { 1617 rc = xeon_setup_b2b_mw(ndev, 1618 &xeon_b2b_usd_addr, 1619 &xeon_b2b_dsd_addr); 1620 } 1621 if (rc) 1622 return rc; 1623 1624 /* Enable Bus Master and Memory Space on the secondary side */ 1625 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER, 1626 ndev->self_mmio + XEON_SPCICMD_OFFSET); 1627 1628 break; 1629 1630 default: 1631 return -EINVAL; 1632 } 1633 1634 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; 1635 1636 ndev->reg->db_iowrite(ndev->db_valid_mask, 1637 ndev->self_mmio + 1638 ndev->self_reg->db_mask); 1639 1640 return 0; 1641 } 1642 1643 static int xeon_init_dev(struct intel_ntb_dev *ndev) 1644 { 1645 struct pci_dev *pdev; 1646 u8 ppd; 1647 int rc, mem; 1648 1649 pdev = ndev->ntb.pdev; 1650 1651 switch (pdev->device) { 1652 /* There is a Xeon hardware errata related to writes to SDOORBELL or 1653 * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space, 1654 * which may hang the system. To workaround this use the second memory 1655 * window to access the interrupt and scratch pad registers on the 1656 * remote system. 1657 */ 1658 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF: 1659 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF: 1660 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF: 1661 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB: 1662 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB: 1663 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB: 1664 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: 1665 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: 1666 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: 1667 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: 1668 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: 1669 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: 1670 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX: 1671 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX: 1672 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX: 1673 ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP; 1674 break; 1675 } 1676 1677 switch (pdev->device) { 1678 /* There is a hardware errata related to accessing any register in 1679 * SB01BASE in the presence of bidirectional traffic crossing the NTB. 1680 */ 1681 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: 1682 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: 1683 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: 1684 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: 1685 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: 1686 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: 1687 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX: 1688 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX: 1689 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX: 1690 ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP; 1691 break; 1692 } 1693 1694 switch (pdev->device) { 1695 /* HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1696 * mirrored to the remote system. Shrink the number of bits by one, 1697 * since bit 14 is the last bit. 1698 */ 1699 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF: 1700 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF: 1701 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF: 1702 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB: 1703 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB: 1704 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB: 1705 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: 1706 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: 1707 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: 1708 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: 1709 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: 1710 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: 1711 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX: 1712 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX: 1713 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX: 1714 ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14; 1715 break; 1716 } 1717 1718 ndev->reg = &xeon_reg; 1719 1720 rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd); 1721 if (rc) 1722 return -EIO; 1723 1724 ndev->ntb.topo = xeon_ppd_topo(ndev, ppd); 1725 dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd, 1726 ntb_topo_string(ndev->ntb.topo)); 1727 if (ndev->ntb.topo == NTB_TOPO_NONE) 1728 return -EINVAL; 1729 1730 if (ndev->ntb.topo != NTB_TOPO_SEC) { 1731 ndev->bar4_split = xeon_ppd_bar4_split(ndev, ppd); 1732 dev_dbg(&pdev->dev, "ppd %#x bar4_split %d\n", 1733 ppd, ndev->bar4_split); 1734 } else { 1735 /* This is a way for transparent BAR to figure out if we are 1736 * doing split BAR or not. There is no way for the hw on the 1737 * transparent side to know and set the PPD. 1738 */ 1739 mem = pci_select_bars(pdev, IORESOURCE_MEM); 1740 ndev->bar4_split = hweight32(mem) == 1741 HSX_SPLIT_BAR_MW_COUNT + 1; 1742 dev_dbg(&pdev->dev, "mem %#x bar4_split %d\n", 1743 mem, ndev->bar4_split); 1744 } 1745 1746 rc = xeon_init_ntb(ndev); 1747 if (rc) 1748 return rc; 1749 1750 return xeon_init_isr(ndev); 1751 } 1752 1753 static void xeon_deinit_dev(struct intel_ntb_dev *ndev) 1754 { 1755 xeon_deinit_isr(ndev); 1756 } 1757 1758 static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev) 1759 { 1760 int rc; 1761 1762 pci_set_drvdata(pdev, ndev); 1763 1764 rc = pci_enable_device(pdev); 1765 if (rc) 1766 goto err_pci_enable; 1767 1768 rc = pci_request_regions(pdev, NTB_NAME); 1769 if (rc) 1770 goto err_pci_regions; 1771 1772 pci_set_master(pdev); 1773 1774 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 1775 if (rc) { 1776 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 1777 if (rc) 1778 goto err_dma_mask; 1779 dev_warn(&pdev->dev, "Cannot DMA highmem\n"); 1780 } 1781 1782 ndev->self_mmio = pci_iomap(pdev, 0, 0); 1783 if (!ndev->self_mmio) { 1784 rc = -EIO; 1785 goto err_mmio; 1786 } 1787 ndev->peer_mmio = ndev->self_mmio; 1788 ndev->peer_addr = pci_resource_start(pdev, 0); 1789 1790 return 0; 1791 1792 err_mmio: 1793 err_dma_mask: 1794 pci_release_regions(pdev); 1795 err_pci_regions: 1796 pci_disable_device(pdev); 1797 err_pci_enable: 1798 pci_set_drvdata(pdev, NULL); 1799 return rc; 1800 } 1801 1802 static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev) 1803 { 1804 struct pci_dev *pdev = ndev->ntb.pdev; 1805 1806 if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio) 1807 pci_iounmap(pdev, ndev->peer_mmio); 1808 pci_iounmap(pdev, ndev->self_mmio); 1809 1810 pci_release_regions(pdev); 1811 pci_disable_device(pdev); 1812 pci_set_drvdata(pdev, NULL); 1813 } 1814 1815 static inline void ndev_init_struct(struct intel_ntb_dev *ndev, 1816 struct pci_dev *pdev) 1817 { 1818 ndev->ntb.pdev = pdev; 1819 ndev->ntb.topo = NTB_TOPO_NONE; 1820 ndev->ntb.ops = &intel_ntb_ops; 1821 1822 ndev->b2b_off = 0; 1823 ndev->b2b_idx = UINT_MAX; 1824 1825 ndev->bar4_split = 0; 1826 1827 ndev->mw_count = 0; 1828 ndev->spad_count = 0; 1829 ndev->db_count = 0; 1830 ndev->db_vec_count = 0; 1831 ndev->db_vec_shift = 0; 1832 1833 ndev->ntb_ctl = 0; 1834 ndev->lnk_sta = 0; 1835 1836 ndev->db_valid_mask = 0; 1837 ndev->db_link_mask = 0; 1838 ndev->db_mask = 0; 1839 1840 spin_lock_init(&ndev->db_mask_lock); 1841 } 1842 1843 static int intel_ntb_pci_probe(struct pci_dev *pdev, 1844 const struct pci_device_id *id) 1845 { 1846 struct intel_ntb_dev *ndev; 1847 int rc, node; 1848 1849 node = dev_to_node(&pdev->dev); 1850 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node); 1851 if (!ndev) { 1852 rc = -ENOMEM; 1853 goto err_ndev; 1854 } 1855 1856 ndev_init_struct(ndev, pdev); 1857 1858 if (pdev_is_gen1(pdev)) { 1859 rc = intel_ntb_init_pci(ndev, pdev); 1860 if (rc) 1861 goto err_init_pci; 1862 1863 rc = xeon_init_dev(ndev); 1864 if (rc) 1865 goto err_init_dev; 1866 } else if (pdev_is_gen3(pdev)) { 1867 ndev->ntb.ops = &intel_ntb3_ops; 1868 rc = intel_ntb_init_pci(ndev, pdev); 1869 if (rc) 1870 goto err_init_pci; 1871 1872 rc = gen3_init_dev(ndev); 1873 if (rc) 1874 goto err_init_dev; 1875 } else if (pdev_is_gen4(pdev) || pdev_is_gen5(pdev)) { 1876 ndev->ntb.ops = &intel_ntb4_ops; 1877 rc = intel_ntb_init_pci(ndev, pdev); 1878 if (rc) 1879 goto err_init_pci; 1880 1881 rc = gen4_init_dev(ndev); 1882 if (rc) 1883 goto err_init_dev; 1884 } else { 1885 rc = -EINVAL; 1886 goto err_init_pci; 1887 } 1888 1889 ndev_reset_unsafe_flags(ndev); 1890 1891 ndev->reg->poll_link(ndev); 1892 1893 ndev_init_debugfs(ndev); 1894 1895 rc = ntb_register_device(&ndev->ntb); 1896 if (rc) 1897 goto err_register; 1898 1899 dev_info(&pdev->dev, "NTB device registered.\n"); 1900 1901 return 0; 1902 1903 err_register: 1904 ndev_deinit_debugfs(ndev); 1905 if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev) || 1906 pdev_is_gen4(pdev) || pdev_is_gen5(pdev)) 1907 xeon_deinit_dev(ndev); 1908 err_init_dev: 1909 intel_ntb_deinit_pci(ndev); 1910 err_init_pci: 1911 kfree(ndev); 1912 err_ndev: 1913 return rc; 1914 } 1915 1916 static void intel_ntb_pci_remove(struct pci_dev *pdev) 1917 { 1918 struct intel_ntb_dev *ndev = pci_get_drvdata(pdev); 1919 1920 ntb_unregister_device(&ndev->ntb); 1921 ndev_deinit_debugfs(ndev); 1922 if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev) || 1923 pdev_is_gen4(pdev) || pdev_is_gen5(pdev)) 1924 xeon_deinit_dev(ndev); 1925 intel_ntb_deinit_pci(ndev); 1926 kfree(ndev); 1927 } 1928 1929 static const struct intel_ntb_reg xeon_reg = { 1930 .poll_link = xeon_poll_link, 1931 .link_is_up = xeon_link_is_up, 1932 .db_ioread = xeon_db_ioread, 1933 .db_iowrite = xeon_db_iowrite, 1934 .db_size = sizeof(u32), 1935 .ntb_ctl = XEON_NTBCNTL_OFFSET, 1936 .mw_bar = {2, 4, 5}, 1937 }; 1938 1939 static const struct intel_ntb_alt_reg xeon_pri_reg = { 1940 .db_bell = XEON_PDOORBELL_OFFSET, 1941 .db_mask = XEON_PDBMSK_OFFSET, 1942 .spad = XEON_SPAD_OFFSET, 1943 }; 1944 1945 static const struct intel_ntb_alt_reg xeon_sec_reg = { 1946 .db_bell = XEON_SDOORBELL_OFFSET, 1947 .db_mask = XEON_SDBMSK_OFFSET, 1948 /* second half of the scratchpads */ 1949 .spad = XEON_SPAD_OFFSET + (XEON_SPAD_COUNT << 1), 1950 }; 1951 1952 static const struct intel_ntb_alt_reg xeon_b2b_reg = { 1953 .db_bell = XEON_B2B_DOORBELL_OFFSET, 1954 .spad = XEON_B2B_SPAD_OFFSET, 1955 }; 1956 1957 static const struct intel_ntb_xlat_reg xeon_pri_xlat = { 1958 /* Note: no primary .bar0_base visible to the secondary side. 1959 * 1960 * The secondary side cannot get the base address stored in primary 1961 * bars. The base address is necessary to set the limit register to 1962 * any value other than zero, or unlimited. 1963 * 1964 * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the 1965 * window by setting the limit equal to base, nor can it limit the size 1966 * of the memory window by setting the limit to base + size. 1967 */ 1968 .bar2_limit = XEON_PBAR23LMT_OFFSET, 1969 .bar2_xlat = XEON_PBAR23XLAT_OFFSET, 1970 }; 1971 1972 static const struct intel_ntb_xlat_reg xeon_sec_xlat = { 1973 .bar0_base = XEON_SBAR0BASE_OFFSET, 1974 .bar2_limit = XEON_SBAR23LMT_OFFSET, 1975 .bar2_xlat = XEON_SBAR23XLAT_OFFSET, 1976 }; 1977 1978 struct intel_b2b_addr xeon_b2b_usd_addr = { 1979 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 1980 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 1981 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 1982 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 1983 }; 1984 1985 struct intel_b2b_addr xeon_b2b_dsd_addr = { 1986 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 1987 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 1988 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 1989 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 1990 }; 1991 1992 /* operations for primary side of local ntb */ 1993 static const struct ntb_dev_ops intel_ntb_ops = { 1994 .mw_count = intel_ntb_mw_count, 1995 .mw_get_align = intel_ntb_mw_get_align, 1996 .mw_set_trans = intel_ntb_mw_set_trans, 1997 .peer_mw_count = intel_ntb_peer_mw_count, 1998 .peer_mw_get_addr = intel_ntb_peer_mw_get_addr, 1999 .link_is_up = intel_ntb_link_is_up, 2000 .link_enable = intel_ntb_link_enable, 2001 .link_disable = intel_ntb_link_disable, 2002 .db_is_unsafe = intel_ntb_db_is_unsafe, 2003 .db_valid_mask = intel_ntb_db_valid_mask, 2004 .db_vector_count = intel_ntb_db_vector_count, 2005 .db_vector_mask = intel_ntb_db_vector_mask, 2006 .db_read = intel_ntb_db_read, 2007 .db_clear = intel_ntb_db_clear, 2008 .db_set_mask = intel_ntb_db_set_mask, 2009 .db_clear_mask = intel_ntb_db_clear_mask, 2010 .peer_db_addr = intel_ntb_peer_db_addr, 2011 .peer_db_set = intel_ntb_peer_db_set, 2012 .spad_is_unsafe = intel_ntb_spad_is_unsafe, 2013 .spad_count = intel_ntb_spad_count, 2014 .spad_read = intel_ntb_spad_read, 2015 .spad_write = intel_ntb_spad_write, 2016 .peer_spad_addr = intel_ntb_peer_spad_addr, 2017 .peer_spad_read = intel_ntb_peer_spad_read, 2018 .peer_spad_write = intel_ntb_peer_spad_write, 2019 }; 2020 2021 static const struct file_operations intel_ntb_debugfs_info = { 2022 .owner = THIS_MODULE, 2023 .open = simple_open, 2024 .read = ndev_debugfs_read, 2025 }; 2026 2027 static const struct pci_device_id intel_ntb_pci_tbl[] = { 2028 /* GEN1 */ 2029 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)}, 2030 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)}, 2031 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)}, 2032 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)}, 2033 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BDX)}, 2034 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)}, 2035 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)}, 2036 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)}, 2037 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)}, 2038 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_BDX)}, 2039 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)}, 2040 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)}, 2041 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)}, 2042 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)}, 2043 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_BDX)}, 2044 2045 /* GEN3 */ 2046 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)}, 2047 2048 /* GEN4 */ 2049 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_ICX)}, 2050 /* GEN5 PCIe */ 2051 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_GNR)}, 2052 {0} 2053 }; 2054 MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl); 2055 2056 static struct pci_driver intel_ntb_pci_driver = { 2057 .name = KBUILD_MODNAME, 2058 .id_table = intel_ntb_pci_tbl, 2059 .probe = intel_ntb_pci_probe, 2060 .remove = intel_ntb_pci_remove, 2061 }; 2062 2063 static int __init intel_ntb_pci_driver_init(void) 2064 { 2065 int ret; 2066 pr_info("%s %s\n", NTB_DESC, NTB_VER); 2067 2068 if (debugfs_initialized()) 2069 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 2070 2071 ret = pci_register_driver(&intel_ntb_pci_driver); 2072 if (ret) 2073 debugfs_remove_recursive(debugfs_dir); 2074 2075 return ret; 2076 } 2077 module_init(intel_ntb_pci_driver_init); 2078 2079 static void __exit intel_ntb_pci_driver_exit(void) 2080 { 2081 pci_unregister_driver(&intel_ntb_pci_driver); 2082 2083 debugfs_remove_recursive(debugfs_dir); 2084 } 2085 module_exit(intel_ntb_pci_driver_exit); 2086