1 /* 2 * This file is provided under a dual BSD/GPLv2 license. When using or 3 * redistributing this file, you may do so under either license. 4 * 5 * GPL LICENSE SUMMARY 6 * 7 * Copyright(c) 2012 Intel Corporation. All rights reserved. 8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 9 * Copyright (C) 2016 T-Platforms. All Rights Reserved. 10 * 11 * This program is free software; you can redistribute it and/or modify 12 * it under the terms of version 2 of the GNU General Public License as 13 * published by the Free Software Foundation. 14 * 15 * BSD LICENSE 16 * 17 * Copyright(c) 2012 Intel Corporation. All rights reserved. 18 * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 19 * Copyright (C) 2016 T-Platforms. All Rights Reserved. 20 * 21 * Redistribution and use in source and binary forms, with or without 22 * modification, are permitted provided that the following conditions 23 * are met: 24 * 25 * * Redistributions of source code must retain the above copyright 26 * notice, this list of conditions and the following disclaimer. 27 * * Redistributions in binary form must reproduce the above copy 28 * notice, this list of conditions and the following disclaimer in 29 * the documentation and/or other materials provided with the 30 * distribution. 31 * * Neither the name of Intel Corporation nor the names of its 32 * contributors may be used to endorse or promote products derived 33 * from this software without specific prior written permission. 34 * 35 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 36 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 37 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 38 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 39 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 40 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 41 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 42 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 43 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 44 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 45 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 46 * 47 * Intel PCIe NTB Linux driver 48 */ 49 50 #include <linux/debugfs.h> 51 #include <linux/delay.h> 52 #include <linux/init.h> 53 #include <linux/interrupt.h> 54 #include <linux/module.h> 55 #include <linux/pci.h> 56 #include <linux/random.h> 57 #include <linux/slab.h> 58 #include <linux/ntb.h> 59 60 #include "ntb_hw_intel.h" 61 #include "ntb_hw_gen1.h" 62 #include "ntb_hw_gen3.h" 63 64 #define NTB_NAME "ntb_hw_intel" 65 #define NTB_DESC "Intel(R) PCI-E Non-Transparent Bridge Driver" 66 #define NTB_VER "2.0" 67 68 MODULE_DESCRIPTION(NTB_DESC); 69 MODULE_VERSION(NTB_VER); 70 MODULE_LICENSE("Dual BSD/GPL"); 71 MODULE_AUTHOR("Intel Corporation"); 72 73 #define bar0_off(base, bar) ((base) + ((bar) << 2)) 74 #define bar2_off(base, bar) bar0_off(base, (bar) - 2) 75 76 static const struct intel_ntb_reg xeon_reg; 77 static const struct intel_ntb_alt_reg xeon_pri_reg; 78 static const struct intel_ntb_alt_reg xeon_sec_reg; 79 static const struct intel_ntb_alt_reg xeon_b2b_reg; 80 static const struct intel_ntb_xlat_reg xeon_pri_xlat; 81 static const struct intel_ntb_xlat_reg xeon_sec_xlat; 82 static const struct ntb_dev_ops intel_ntb_ops; 83 84 static const struct file_operations intel_ntb_debugfs_info; 85 static struct dentry *debugfs_dir; 86 87 static int b2b_mw_idx = -1; 88 module_param(b2b_mw_idx, int, 0644); 89 MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb. A " 90 "value of zero or positive starts from first mw idx, and a " 91 "negative value starts from last mw idx. Both sides MUST " 92 "set the same value here!"); 93 94 static unsigned int b2b_mw_share; 95 module_param(b2b_mw_share, uint, 0644); 96 MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the " 97 "ntb so that the peer ntb only occupies the first half of " 98 "the mw, so the second half can still be used as a mw. Both " 99 "sides MUST set the same value here!"); 100 101 module_param_named(xeon_b2b_usd_bar2_addr64, 102 xeon_b2b_usd_addr.bar2_addr64, ullong, 0644); 103 MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64, 104 "XEON B2B USD BAR 2 64-bit address"); 105 106 module_param_named(xeon_b2b_usd_bar4_addr64, 107 xeon_b2b_usd_addr.bar4_addr64, ullong, 0644); 108 MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr64, 109 "XEON B2B USD BAR 4 64-bit address"); 110 111 module_param_named(xeon_b2b_usd_bar4_addr32, 112 xeon_b2b_usd_addr.bar4_addr32, ullong, 0644); 113 MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr32, 114 "XEON B2B USD split-BAR 4 32-bit address"); 115 116 module_param_named(xeon_b2b_usd_bar5_addr32, 117 xeon_b2b_usd_addr.bar5_addr32, ullong, 0644); 118 MODULE_PARM_DESC(xeon_b2b_usd_bar5_addr32, 119 "XEON B2B USD split-BAR 5 32-bit address"); 120 121 module_param_named(xeon_b2b_dsd_bar2_addr64, 122 xeon_b2b_dsd_addr.bar2_addr64, ullong, 0644); 123 MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64, 124 "XEON B2B DSD BAR 2 64-bit address"); 125 126 module_param_named(xeon_b2b_dsd_bar4_addr64, 127 xeon_b2b_dsd_addr.bar4_addr64, ullong, 0644); 128 MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr64, 129 "XEON B2B DSD BAR 4 64-bit address"); 130 131 module_param_named(xeon_b2b_dsd_bar4_addr32, 132 xeon_b2b_dsd_addr.bar4_addr32, ullong, 0644); 133 MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr32, 134 "XEON B2B DSD split-BAR 4 32-bit address"); 135 136 module_param_named(xeon_b2b_dsd_bar5_addr32, 137 xeon_b2b_dsd_addr.bar5_addr32, ullong, 0644); 138 MODULE_PARM_DESC(xeon_b2b_dsd_bar5_addr32, 139 "XEON B2B DSD split-BAR 5 32-bit address"); 140 141 142 static int xeon_init_isr(struct intel_ntb_dev *ndev); 143 144 static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev) 145 { 146 ndev->unsafe_flags = 0; 147 ndev->unsafe_flags_ignore = 0; 148 149 /* Only B2B has a workaround to avoid SDOORBELL */ 150 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) 151 if (!ntb_topo_is_b2b(ndev->ntb.topo)) 152 ndev->unsafe_flags |= NTB_UNSAFE_DB; 153 154 /* No low level workaround to avoid SB01BASE */ 155 if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) { 156 ndev->unsafe_flags |= NTB_UNSAFE_DB; 157 ndev->unsafe_flags |= NTB_UNSAFE_SPAD; 158 } 159 } 160 161 static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev, 162 unsigned long flag) 163 { 164 return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore); 165 } 166 167 static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev, 168 unsigned long flag) 169 { 170 flag &= ndev->unsafe_flags; 171 ndev->unsafe_flags_ignore |= flag; 172 173 return !!flag; 174 } 175 176 int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx) 177 { 178 if (idx < 0 || idx >= ndev->mw_count) 179 return -EINVAL; 180 return ndev->reg->mw_bar[idx]; 181 } 182 183 static inline int ndev_db_addr(struct intel_ntb_dev *ndev, 184 phys_addr_t *db_addr, resource_size_t *db_size, 185 phys_addr_t reg_addr, unsigned long reg) 186 { 187 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) 188 pr_warn_once("%s: NTB unsafe doorbell access", __func__); 189 190 if (db_addr) { 191 *db_addr = reg_addr + reg; 192 dev_dbg(&ndev->ntb.pdev->dev, "Peer db addr %llx\n", *db_addr); 193 } 194 195 if (db_size) { 196 *db_size = ndev->reg->db_size; 197 dev_dbg(&ndev->ntb.pdev->dev, "Peer db size %llx\n", *db_size); 198 } 199 200 return 0; 201 } 202 203 u64 ndev_db_read(struct intel_ntb_dev *ndev, 204 void __iomem *mmio) 205 { 206 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) 207 pr_warn_once("%s: NTB unsafe doorbell access", __func__); 208 209 return ndev->reg->db_ioread(mmio); 210 } 211 212 int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits, 213 void __iomem *mmio) 214 { 215 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) 216 pr_warn_once("%s: NTB unsafe doorbell access", __func__); 217 218 if (db_bits & ~ndev->db_valid_mask) 219 return -EINVAL; 220 221 ndev->reg->db_iowrite(db_bits, mmio); 222 223 return 0; 224 } 225 226 static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits, 227 void __iomem *mmio) 228 { 229 unsigned long irqflags; 230 231 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) 232 pr_warn_once("%s: NTB unsafe doorbell access", __func__); 233 234 if (db_bits & ~ndev->db_valid_mask) 235 return -EINVAL; 236 237 spin_lock_irqsave(&ndev->db_mask_lock, irqflags); 238 { 239 ndev->db_mask |= db_bits; 240 ndev->reg->db_iowrite(ndev->db_mask, mmio); 241 } 242 spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags); 243 244 return 0; 245 } 246 247 static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits, 248 void __iomem *mmio) 249 { 250 unsigned long irqflags; 251 252 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) 253 pr_warn_once("%s: NTB unsafe doorbell access", __func__); 254 255 if (db_bits & ~ndev->db_valid_mask) 256 return -EINVAL; 257 258 spin_lock_irqsave(&ndev->db_mask_lock, irqflags); 259 { 260 ndev->db_mask &= ~db_bits; 261 ndev->reg->db_iowrite(ndev->db_mask, mmio); 262 } 263 spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags); 264 265 return 0; 266 } 267 268 static inline int ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector) 269 { 270 u64 shift, mask; 271 272 shift = ndev->db_vec_shift; 273 mask = BIT_ULL(shift) - 1; 274 275 return mask << (shift * db_vector); 276 } 277 278 static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx, 279 phys_addr_t *spad_addr, phys_addr_t reg_addr, 280 unsigned long reg) 281 { 282 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD)) 283 pr_warn_once("%s: NTB unsafe scratchpad access", __func__); 284 285 if (idx < 0 || idx >= ndev->spad_count) 286 return -EINVAL; 287 288 if (spad_addr) { 289 *spad_addr = reg_addr + reg + (idx << 2); 290 dev_dbg(&ndev->ntb.pdev->dev, "Peer spad addr %llx\n", 291 *spad_addr); 292 } 293 294 return 0; 295 } 296 297 static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx, 298 void __iomem *mmio) 299 { 300 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD)) 301 pr_warn_once("%s: NTB unsafe scratchpad access", __func__); 302 303 if (idx < 0 || idx >= ndev->spad_count) 304 return 0; 305 306 return ioread32(mmio + (idx << 2)); 307 } 308 309 static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val, 310 void __iomem *mmio) 311 { 312 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD)) 313 pr_warn_once("%s: NTB unsafe scratchpad access", __func__); 314 315 if (idx < 0 || idx >= ndev->spad_count) 316 return -EINVAL; 317 318 iowrite32(val, mmio + (idx << 2)); 319 320 return 0; 321 } 322 323 static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec) 324 { 325 u64 vec_mask; 326 327 vec_mask = ndev_vec_mask(ndev, vec); 328 329 if ((ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) && (vec == 31)) 330 vec_mask |= ndev->db_link_mask; 331 332 dev_dbg(&ndev->ntb.pdev->dev, "vec %d vec_mask %llx\n", vec, vec_mask); 333 334 ndev->last_ts = jiffies; 335 336 if (vec_mask & ndev->db_link_mask) { 337 if (ndev->reg->poll_link(ndev)) 338 ntb_link_event(&ndev->ntb); 339 } 340 341 if (vec_mask & ndev->db_valid_mask) 342 ntb_db_event(&ndev->ntb, vec); 343 344 return IRQ_HANDLED; 345 } 346 347 static irqreturn_t ndev_vec_isr(int irq, void *dev) 348 { 349 struct intel_ntb_vec *nvec = dev; 350 351 dev_dbg(&nvec->ndev->ntb.pdev->dev, "irq: %d nvec->num: %d\n", 352 irq, nvec->num); 353 354 return ndev_interrupt(nvec->ndev, nvec->num); 355 } 356 357 static irqreturn_t ndev_irq_isr(int irq, void *dev) 358 { 359 struct intel_ntb_dev *ndev = dev; 360 361 return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq); 362 } 363 364 int ndev_init_isr(struct intel_ntb_dev *ndev, 365 int msix_min, int msix_max, 366 int msix_shift, int total_shift) 367 { 368 struct pci_dev *pdev; 369 int rc, i, msix_count, node; 370 371 pdev = ndev->ntb.pdev; 372 373 node = dev_to_node(&pdev->dev); 374 375 /* Mask all doorbell interrupts */ 376 ndev->db_mask = ndev->db_valid_mask; 377 ndev->reg->db_iowrite(ndev->db_mask, 378 ndev->self_mmio + 379 ndev->self_reg->db_mask); 380 381 /* Try to set up msix irq */ 382 383 ndev->vec = kcalloc_node(msix_max, sizeof(*ndev->vec), 384 GFP_KERNEL, node); 385 if (!ndev->vec) 386 goto err_msix_vec_alloc; 387 388 ndev->msix = kcalloc_node(msix_max, sizeof(*ndev->msix), 389 GFP_KERNEL, node); 390 if (!ndev->msix) 391 goto err_msix_alloc; 392 393 for (i = 0; i < msix_max; ++i) 394 ndev->msix[i].entry = i; 395 396 msix_count = pci_enable_msix_range(pdev, ndev->msix, 397 msix_min, msix_max); 398 if (msix_count < 0) 399 goto err_msix_enable; 400 401 for (i = 0; i < msix_count; ++i) { 402 ndev->vec[i].ndev = ndev; 403 ndev->vec[i].num = i; 404 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0, 405 "ndev_vec_isr", &ndev->vec[i]); 406 if (rc) 407 goto err_msix_request; 408 } 409 410 dev_dbg(&pdev->dev, "Using %d msix interrupts\n", msix_count); 411 ndev->db_vec_count = msix_count; 412 ndev->db_vec_shift = msix_shift; 413 return 0; 414 415 err_msix_request: 416 while (i-- > 0) 417 free_irq(ndev->msix[i].vector, &ndev->vec[i]); 418 pci_disable_msix(pdev); 419 err_msix_enable: 420 kfree(ndev->msix); 421 err_msix_alloc: 422 kfree(ndev->vec); 423 err_msix_vec_alloc: 424 ndev->msix = NULL; 425 ndev->vec = NULL; 426 427 /* Try to set up msi irq */ 428 429 rc = pci_enable_msi(pdev); 430 if (rc) 431 goto err_msi_enable; 432 433 rc = request_irq(pdev->irq, ndev_irq_isr, 0, 434 "ndev_irq_isr", ndev); 435 if (rc) 436 goto err_msi_request; 437 438 dev_dbg(&pdev->dev, "Using msi interrupts\n"); 439 ndev->db_vec_count = 1; 440 ndev->db_vec_shift = total_shift; 441 return 0; 442 443 err_msi_request: 444 pci_disable_msi(pdev); 445 err_msi_enable: 446 447 /* Try to set up intx irq */ 448 449 pci_intx(pdev, 1); 450 451 rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED, 452 "ndev_irq_isr", ndev); 453 if (rc) 454 goto err_intx_request; 455 456 dev_dbg(&pdev->dev, "Using intx interrupts\n"); 457 ndev->db_vec_count = 1; 458 ndev->db_vec_shift = total_shift; 459 return 0; 460 461 err_intx_request: 462 return rc; 463 } 464 465 static void ndev_deinit_isr(struct intel_ntb_dev *ndev) 466 { 467 struct pci_dev *pdev; 468 int i; 469 470 pdev = ndev->ntb.pdev; 471 472 /* Mask all doorbell interrupts */ 473 ndev->db_mask = ndev->db_valid_mask; 474 ndev->reg->db_iowrite(ndev->db_mask, 475 ndev->self_mmio + 476 ndev->self_reg->db_mask); 477 478 if (ndev->msix) { 479 i = ndev->db_vec_count; 480 while (i--) 481 free_irq(ndev->msix[i].vector, &ndev->vec[i]); 482 pci_disable_msix(pdev); 483 kfree(ndev->msix); 484 kfree(ndev->vec); 485 } else { 486 free_irq(pdev->irq, ndev); 487 if (pci_dev_msi_enabled(pdev)) 488 pci_disable_msi(pdev); 489 } 490 } 491 492 static ssize_t ndev_ntb_debugfs_read(struct file *filp, char __user *ubuf, 493 size_t count, loff_t *offp) 494 { 495 struct intel_ntb_dev *ndev; 496 struct pci_dev *pdev; 497 void __iomem *mmio; 498 char *buf; 499 size_t buf_size; 500 ssize_t ret, off; 501 union { u64 v64; u32 v32; u16 v16; u8 v8; } u; 502 503 ndev = filp->private_data; 504 pdev = ndev->ntb.pdev; 505 mmio = ndev->self_mmio; 506 507 buf_size = min(count, 0x800ul); 508 509 buf = kmalloc(buf_size, GFP_KERNEL); 510 if (!buf) 511 return -ENOMEM; 512 513 off = 0; 514 515 off += scnprintf(buf + off, buf_size - off, 516 "NTB Device Information:\n"); 517 518 off += scnprintf(buf + off, buf_size - off, 519 "Connection Topology -\t%s\n", 520 ntb_topo_string(ndev->ntb.topo)); 521 522 if (ndev->b2b_idx != UINT_MAX) { 523 off += scnprintf(buf + off, buf_size - off, 524 "B2B MW Idx -\t\t%u\n", ndev->b2b_idx); 525 off += scnprintf(buf + off, buf_size - off, 526 "B2B Offset -\t\t%#lx\n", ndev->b2b_off); 527 } 528 529 off += scnprintf(buf + off, buf_size - off, 530 "BAR4 Split -\t\t%s\n", 531 ndev->bar4_split ? "yes" : "no"); 532 533 off += scnprintf(buf + off, buf_size - off, 534 "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl); 535 off += scnprintf(buf + off, buf_size - off, 536 "LNK STA -\t\t%#06x\n", ndev->lnk_sta); 537 538 if (!ndev->reg->link_is_up(ndev)) { 539 off += scnprintf(buf + off, buf_size - off, 540 "Link Status -\t\tDown\n"); 541 } else { 542 off += scnprintf(buf + off, buf_size - off, 543 "Link Status -\t\tUp\n"); 544 off += scnprintf(buf + off, buf_size - off, 545 "Link Speed -\t\tPCI-E Gen %u\n", 546 NTB_LNK_STA_SPEED(ndev->lnk_sta)); 547 off += scnprintf(buf + off, buf_size - off, 548 "Link Width -\t\tx%u\n", 549 NTB_LNK_STA_WIDTH(ndev->lnk_sta)); 550 } 551 552 off += scnprintf(buf + off, buf_size - off, 553 "Memory Window Count -\t%u\n", ndev->mw_count); 554 off += scnprintf(buf + off, buf_size - off, 555 "Scratchpad Count -\t%u\n", ndev->spad_count); 556 off += scnprintf(buf + off, buf_size - off, 557 "Doorbell Count -\t%u\n", ndev->db_count); 558 off += scnprintf(buf + off, buf_size - off, 559 "Doorbell Vector Count -\t%u\n", ndev->db_vec_count); 560 off += scnprintf(buf + off, buf_size - off, 561 "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift); 562 563 off += scnprintf(buf + off, buf_size - off, 564 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask); 565 off += scnprintf(buf + off, buf_size - off, 566 "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask); 567 off += scnprintf(buf + off, buf_size - off, 568 "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask); 569 570 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask); 571 off += scnprintf(buf + off, buf_size - off, 572 "Doorbell Mask -\t\t%#llx\n", u.v64); 573 574 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell); 575 off += scnprintf(buf + off, buf_size - off, 576 "Doorbell Bell -\t\t%#llx\n", u.v64); 577 578 off += scnprintf(buf + off, buf_size - off, 579 "\nNTB Window Size:\n"); 580 581 pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &u.v8); 582 off += scnprintf(buf + off, buf_size - off, 583 "PBAR23SZ %hhu\n", u.v8); 584 if (!ndev->bar4_split) { 585 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &u.v8); 586 off += scnprintf(buf + off, buf_size - off, 587 "PBAR45SZ %hhu\n", u.v8); 588 } else { 589 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &u.v8); 590 off += scnprintf(buf + off, buf_size - off, 591 "PBAR4SZ %hhu\n", u.v8); 592 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &u.v8); 593 off += scnprintf(buf + off, buf_size - off, 594 "PBAR5SZ %hhu\n", u.v8); 595 } 596 597 pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &u.v8); 598 off += scnprintf(buf + off, buf_size - off, 599 "SBAR23SZ %hhu\n", u.v8); 600 if (!ndev->bar4_split) { 601 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &u.v8); 602 off += scnprintf(buf + off, buf_size - off, 603 "SBAR45SZ %hhu\n", u.v8); 604 } else { 605 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &u.v8); 606 off += scnprintf(buf + off, buf_size - off, 607 "SBAR4SZ %hhu\n", u.v8); 608 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &u.v8); 609 off += scnprintf(buf + off, buf_size - off, 610 "SBAR5SZ %hhu\n", u.v8); 611 } 612 613 off += scnprintf(buf + off, buf_size - off, 614 "\nNTB Incoming XLAT:\n"); 615 616 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2)); 617 off += scnprintf(buf + off, buf_size - off, 618 "XLAT23 -\t\t%#018llx\n", u.v64); 619 620 if (ndev->bar4_split) { 621 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4)); 622 off += scnprintf(buf + off, buf_size - off, 623 "XLAT4 -\t\t\t%#06x\n", u.v32); 624 625 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 5)); 626 off += scnprintf(buf + off, buf_size - off, 627 "XLAT5 -\t\t\t%#06x\n", u.v32); 628 } else { 629 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4)); 630 off += scnprintf(buf + off, buf_size - off, 631 "XLAT45 -\t\t%#018llx\n", u.v64); 632 } 633 634 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2)); 635 off += scnprintf(buf + off, buf_size - off, 636 "LMT23 -\t\t\t%#018llx\n", u.v64); 637 638 if (ndev->bar4_split) { 639 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4)); 640 off += scnprintf(buf + off, buf_size - off, 641 "LMT4 -\t\t\t%#06x\n", u.v32); 642 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 5)); 643 off += scnprintf(buf + off, buf_size - off, 644 "LMT5 -\t\t\t%#06x\n", u.v32); 645 } else { 646 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4)); 647 off += scnprintf(buf + off, buf_size - off, 648 "LMT45 -\t\t\t%#018llx\n", u.v64); 649 } 650 651 if (pdev_is_gen1(pdev)) { 652 if (ntb_topo_is_b2b(ndev->ntb.topo)) { 653 off += scnprintf(buf + off, buf_size - off, 654 "\nNTB Outgoing B2B XLAT:\n"); 655 656 u.v64 = ioread64(mmio + XEON_PBAR23XLAT_OFFSET); 657 off += scnprintf(buf + off, buf_size - off, 658 "B2B XLAT23 -\t\t%#018llx\n", u.v64); 659 660 if (ndev->bar4_split) { 661 u.v32 = ioread32(mmio + XEON_PBAR4XLAT_OFFSET); 662 off += scnprintf(buf + off, buf_size - off, 663 "B2B XLAT4 -\t\t%#06x\n", 664 u.v32); 665 u.v32 = ioread32(mmio + XEON_PBAR5XLAT_OFFSET); 666 off += scnprintf(buf + off, buf_size - off, 667 "B2B XLAT5 -\t\t%#06x\n", 668 u.v32); 669 } else { 670 u.v64 = ioread64(mmio + XEON_PBAR45XLAT_OFFSET); 671 off += scnprintf(buf + off, buf_size - off, 672 "B2B XLAT45 -\t\t%#018llx\n", 673 u.v64); 674 } 675 676 u.v64 = ioread64(mmio + XEON_PBAR23LMT_OFFSET); 677 off += scnprintf(buf + off, buf_size - off, 678 "B2B LMT23 -\t\t%#018llx\n", u.v64); 679 680 if (ndev->bar4_split) { 681 u.v32 = ioread32(mmio + XEON_PBAR4LMT_OFFSET); 682 off += scnprintf(buf + off, buf_size - off, 683 "B2B LMT4 -\t\t%#06x\n", 684 u.v32); 685 u.v32 = ioread32(mmio + XEON_PBAR5LMT_OFFSET); 686 off += scnprintf(buf + off, buf_size - off, 687 "B2B LMT5 -\t\t%#06x\n", 688 u.v32); 689 } else { 690 u.v64 = ioread64(mmio + XEON_PBAR45LMT_OFFSET); 691 off += scnprintf(buf + off, buf_size - off, 692 "B2B LMT45 -\t\t%#018llx\n", 693 u.v64); 694 } 695 696 off += scnprintf(buf + off, buf_size - off, 697 "\nNTB Secondary BAR:\n"); 698 699 u.v64 = ioread64(mmio + XEON_SBAR0BASE_OFFSET); 700 off += scnprintf(buf + off, buf_size - off, 701 "SBAR01 -\t\t%#018llx\n", u.v64); 702 703 u.v64 = ioread64(mmio + XEON_SBAR23BASE_OFFSET); 704 off += scnprintf(buf + off, buf_size - off, 705 "SBAR23 -\t\t%#018llx\n", u.v64); 706 707 if (ndev->bar4_split) { 708 u.v32 = ioread32(mmio + XEON_SBAR4BASE_OFFSET); 709 off += scnprintf(buf + off, buf_size - off, 710 "SBAR4 -\t\t\t%#06x\n", u.v32); 711 u.v32 = ioread32(mmio + XEON_SBAR5BASE_OFFSET); 712 off += scnprintf(buf + off, buf_size - off, 713 "SBAR5 -\t\t\t%#06x\n", u.v32); 714 } else { 715 u.v64 = ioread64(mmio + XEON_SBAR45BASE_OFFSET); 716 off += scnprintf(buf + off, buf_size - off, 717 "SBAR45 -\t\t%#018llx\n", 718 u.v64); 719 } 720 } 721 722 off += scnprintf(buf + off, buf_size - off, 723 "\nXEON NTB Statistics:\n"); 724 725 u.v16 = ioread16(mmio + XEON_USMEMMISS_OFFSET); 726 off += scnprintf(buf + off, buf_size - off, 727 "Upstream Memory Miss -\t%u\n", u.v16); 728 729 off += scnprintf(buf + off, buf_size - off, 730 "\nXEON NTB Hardware Errors:\n"); 731 732 if (!pci_read_config_word(pdev, 733 XEON_DEVSTS_OFFSET, &u.v16)) 734 off += scnprintf(buf + off, buf_size - off, 735 "DEVSTS -\t\t%#06x\n", u.v16); 736 737 if (!pci_read_config_word(pdev, 738 XEON_LINK_STATUS_OFFSET, &u.v16)) 739 off += scnprintf(buf + off, buf_size - off, 740 "LNKSTS -\t\t%#06x\n", u.v16); 741 742 if (!pci_read_config_dword(pdev, 743 XEON_UNCERRSTS_OFFSET, &u.v32)) 744 off += scnprintf(buf + off, buf_size - off, 745 "UNCERRSTS -\t\t%#06x\n", u.v32); 746 747 if (!pci_read_config_dword(pdev, 748 XEON_CORERRSTS_OFFSET, &u.v32)) 749 off += scnprintf(buf + off, buf_size - off, 750 "CORERRSTS -\t\t%#06x\n", u.v32); 751 } 752 753 ret = simple_read_from_buffer(ubuf, count, offp, buf, off); 754 kfree(buf); 755 return ret; 756 } 757 758 static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf, 759 size_t count, loff_t *offp) 760 { 761 struct intel_ntb_dev *ndev = filp->private_data; 762 763 if (pdev_is_gen1(ndev->ntb.pdev)) 764 return ndev_ntb_debugfs_read(filp, ubuf, count, offp); 765 else if (pdev_is_gen3(ndev->ntb.pdev)) 766 return ndev_ntb3_debugfs_read(filp, ubuf, count, offp); 767 768 return -ENXIO; 769 } 770 771 static void ndev_init_debugfs(struct intel_ntb_dev *ndev) 772 { 773 if (!debugfs_dir) { 774 ndev->debugfs_dir = NULL; 775 ndev->debugfs_info = NULL; 776 } else { 777 ndev->debugfs_dir = 778 debugfs_create_dir(pci_name(ndev->ntb.pdev), 779 debugfs_dir); 780 if (!ndev->debugfs_dir) 781 ndev->debugfs_info = NULL; 782 else 783 ndev->debugfs_info = 784 debugfs_create_file("info", S_IRUSR, 785 ndev->debugfs_dir, ndev, 786 &intel_ntb_debugfs_info); 787 } 788 } 789 790 static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev) 791 { 792 debugfs_remove_recursive(ndev->debugfs_dir); 793 } 794 795 int intel_ntb_mw_count(struct ntb_dev *ntb, int pidx) 796 { 797 if (pidx != NTB_DEF_PEER_IDX) 798 return -EINVAL; 799 800 return ntb_ndev(ntb)->mw_count; 801 } 802 803 int intel_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx, 804 resource_size_t *addr_align, 805 resource_size_t *size_align, 806 resource_size_t *size_max) 807 { 808 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 809 resource_size_t bar_size, mw_size; 810 int bar; 811 812 if (pidx != NTB_DEF_PEER_IDX) 813 return -EINVAL; 814 815 if (idx >= ndev->b2b_idx && !ndev->b2b_off) 816 idx += 1; 817 818 bar = ndev_mw_to_bar(ndev, idx); 819 if (bar < 0) 820 return bar; 821 822 bar_size = pci_resource_len(ndev->ntb.pdev, bar); 823 824 if (idx == ndev->b2b_idx) 825 mw_size = bar_size - ndev->b2b_off; 826 else 827 mw_size = bar_size; 828 829 if (addr_align) 830 *addr_align = pci_resource_len(ndev->ntb.pdev, bar); 831 832 if (size_align) 833 *size_align = 1; 834 835 if (size_max) 836 *size_max = mw_size; 837 838 return 0; 839 } 840 841 static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, 842 dma_addr_t addr, resource_size_t size) 843 { 844 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 845 unsigned long base_reg, xlat_reg, limit_reg; 846 resource_size_t bar_size, mw_size; 847 void __iomem *mmio; 848 u64 base, limit, reg_val; 849 int bar; 850 851 if (pidx != NTB_DEF_PEER_IDX) 852 return -EINVAL; 853 854 if (idx >= ndev->b2b_idx && !ndev->b2b_off) 855 idx += 1; 856 857 bar = ndev_mw_to_bar(ndev, idx); 858 if (bar < 0) 859 return bar; 860 861 bar_size = pci_resource_len(ndev->ntb.pdev, bar); 862 863 if (idx == ndev->b2b_idx) 864 mw_size = bar_size - ndev->b2b_off; 865 else 866 mw_size = bar_size; 867 868 /* hardware requires that addr is aligned to bar size */ 869 if (addr & (bar_size - 1)) 870 return -EINVAL; 871 872 /* make sure the range fits in the usable mw size */ 873 if (size > mw_size) 874 return -EINVAL; 875 876 mmio = ndev->self_mmio; 877 base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar); 878 xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar); 879 limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar); 880 881 if (bar < 4 || !ndev->bar4_split) { 882 base = ioread64(mmio + base_reg) & NTB_BAR_MASK_64; 883 884 /* Set the limit if supported, if size is not mw_size */ 885 if (limit_reg && size != mw_size) 886 limit = base + size; 887 else 888 limit = 0; 889 890 /* set and verify setting the translation address */ 891 iowrite64(addr, mmio + xlat_reg); 892 reg_val = ioread64(mmio + xlat_reg); 893 if (reg_val != addr) { 894 iowrite64(0, mmio + xlat_reg); 895 return -EIO; 896 } 897 898 /* set and verify setting the limit */ 899 iowrite64(limit, mmio + limit_reg); 900 reg_val = ioread64(mmio + limit_reg); 901 if (reg_val != limit) { 902 iowrite64(base, mmio + limit_reg); 903 iowrite64(0, mmio + xlat_reg); 904 return -EIO; 905 } 906 } else { 907 /* split bar addr range must all be 32 bit */ 908 if (addr & (~0ull << 32)) 909 return -EINVAL; 910 if ((addr + size) & (~0ull << 32)) 911 return -EINVAL; 912 913 base = ioread32(mmio + base_reg) & NTB_BAR_MASK_32; 914 915 /* Set the limit if supported, if size is not mw_size */ 916 if (limit_reg && size != mw_size) 917 limit = base + size; 918 else 919 limit = 0; 920 921 /* set and verify setting the translation address */ 922 iowrite32(addr, mmio + xlat_reg); 923 reg_val = ioread32(mmio + xlat_reg); 924 if (reg_val != addr) { 925 iowrite32(0, mmio + xlat_reg); 926 return -EIO; 927 } 928 929 /* set and verify setting the limit */ 930 iowrite32(limit, mmio + limit_reg); 931 reg_val = ioread32(mmio + limit_reg); 932 if (reg_val != limit) { 933 iowrite32(base, mmio + limit_reg); 934 iowrite32(0, mmio + xlat_reg); 935 return -EIO; 936 } 937 } 938 939 return 0; 940 } 941 942 u64 intel_ntb_link_is_up(struct ntb_dev *ntb, enum ntb_speed *speed, 943 enum ntb_width *width) 944 { 945 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 946 947 if (ndev->reg->link_is_up(ndev)) { 948 if (speed) 949 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta); 950 if (width) 951 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta); 952 return 1; 953 } else { 954 /* TODO MAYBE: is it possible to observe the link speed and 955 * width while link is training? */ 956 if (speed) 957 *speed = NTB_SPEED_NONE; 958 if (width) 959 *width = NTB_WIDTH_NONE; 960 return 0; 961 } 962 } 963 964 static int intel_ntb_link_enable(struct ntb_dev *ntb, 965 enum ntb_speed max_speed, 966 enum ntb_width max_width) 967 { 968 struct intel_ntb_dev *ndev; 969 u32 ntb_ctl; 970 971 ndev = container_of(ntb, struct intel_ntb_dev, ntb); 972 973 if (ndev->ntb.topo == NTB_TOPO_SEC) 974 return -EINVAL; 975 976 dev_dbg(&ntb->pdev->dev, 977 "Enabling link with max_speed %d max_width %d\n", 978 max_speed, max_width); 979 if (max_speed != NTB_SPEED_AUTO) 980 dev_dbg(&ntb->pdev->dev, "ignoring max_speed %d\n", max_speed); 981 if (max_width != NTB_WIDTH_AUTO) 982 dev_dbg(&ntb->pdev->dev, "ignoring max_width %d\n", max_width); 983 984 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl); 985 ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK); 986 ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP; 987 ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP; 988 if (ndev->bar4_split) 989 ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP; 990 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl); 991 992 return 0; 993 } 994 995 int intel_ntb_link_disable(struct ntb_dev *ntb) 996 { 997 struct intel_ntb_dev *ndev; 998 u32 ntb_cntl; 999 1000 ndev = container_of(ntb, struct intel_ntb_dev, ntb); 1001 1002 if (ndev->ntb.topo == NTB_TOPO_SEC) 1003 return -EINVAL; 1004 1005 dev_dbg(&ntb->pdev->dev, "Disabling link\n"); 1006 1007 /* Bring NTB link down */ 1008 ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl); 1009 ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP); 1010 ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP); 1011 if (ndev->bar4_split) 1012 ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP); 1013 ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK; 1014 iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl); 1015 1016 return 0; 1017 } 1018 1019 int intel_ntb_peer_mw_count(struct ntb_dev *ntb) 1020 { 1021 /* Numbers of inbound and outbound memory windows match */ 1022 return ntb_ndev(ntb)->mw_count; 1023 } 1024 1025 int intel_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx, 1026 phys_addr_t *base, resource_size_t *size) 1027 { 1028 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1029 int bar; 1030 1031 if (idx >= ndev->b2b_idx && !ndev->b2b_off) 1032 idx += 1; 1033 1034 bar = ndev_mw_to_bar(ndev, idx); 1035 if (bar < 0) 1036 return bar; 1037 1038 if (base) 1039 *base = pci_resource_start(ndev->ntb.pdev, bar) + 1040 (idx == ndev->b2b_idx ? ndev->b2b_off : 0); 1041 1042 if (size) 1043 *size = pci_resource_len(ndev->ntb.pdev, bar) - 1044 (idx == ndev->b2b_idx ? ndev->b2b_off : 0); 1045 1046 return 0; 1047 } 1048 1049 static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb) 1050 { 1051 return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB); 1052 } 1053 1054 u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb) 1055 { 1056 return ntb_ndev(ntb)->db_valid_mask; 1057 } 1058 1059 int intel_ntb_db_vector_count(struct ntb_dev *ntb) 1060 { 1061 struct intel_ntb_dev *ndev; 1062 1063 ndev = container_of(ntb, struct intel_ntb_dev, ntb); 1064 1065 return ndev->db_vec_count; 1066 } 1067 1068 u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) 1069 { 1070 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1071 1072 if (db_vector < 0 || db_vector > ndev->db_vec_count) 1073 return 0; 1074 1075 return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector); 1076 } 1077 1078 static u64 intel_ntb_db_read(struct ntb_dev *ntb) 1079 { 1080 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1081 1082 return ndev_db_read(ndev, 1083 ndev->self_mmio + 1084 ndev->self_reg->db_bell); 1085 } 1086 1087 static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) 1088 { 1089 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1090 1091 return ndev_db_write(ndev, db_bits, 1092 ndev->self_mmio + 1093 ndev->self_reg->db_bell); 1094 } 1095 1096 int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) 1097 { 1098 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1099 1100 return ndev_db_set_mask(ndev, db_bits, 1101 ndev->self_mmio + 1102 ndev->self_reg->db_mask); 1103 } 1104 1105 int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) 1106 { 1107 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1108 1109 return ndev_db_clear_mask(ndev, db_bits, 1110 ndev->self_mmio + 1111 ndev->self_reg->db_mask); 1112 } 1113 1114 int intel_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr, 1115 resource_size_t *db_size) 1116 { 1117 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1118 1119 return ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr, 1120 ndev->peer_reg->db_bell); 1121 } 1122 1123 static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) 1124 { 1125 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1126 1127 return ndev_db_write(ndev, db_bits, 1128 ndev->peer_mmio + 1129 ndev->peer_reg->db_bell); 1130 } 1131 1132 int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb) 1133 { 1134 return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD); 1135 } 1136 1137 int intel_ntb_spad_count(struct ntb_dev *ntb) 1138 { 1139 struct intel_ntb_dev *ndev; 1140 1141 ndev = container_of(ntb, struct intel_ntb_dev, ntb); 1142 1143 return ndev->spad_count; 1144 } 1145 1146 u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx) 1147 { 1148 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1149 1150 return ndev_spad_read(ndev, idx, 1151 ndev->self_mmio + 1152 ndev->self_reg->spad); 1153 } 1154 1155 int intel_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val) 1156 { 1157 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1158 1159 return ndev_spad_write(ndev, idx, val, 1160 ndev->self_mmio + 1161 ndev->self_reg->spad); 1162 } 1163 1164 int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx, 1165 phys_addr_t *spad_addr) 1166 { 1167 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1168 1169 return ndev_spad_addr(ndev, sidx, spad_addr, ndev->peer_addr, 1170 ndev->peer_reg->spad); 1171 } 1172 1173 u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx) 1174 { 1175 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1176 1177 return ndev_spad_read(ndev, sidx, 1178 ndev->peer_mmio + 1179 ndev->peer_reg->spad); 1180 } 1181 1182 int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, int sidx, 1183 u32 val) 1184 { 1185 struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1186 1187 return ndev_spad_write(ndev, sidx, val, 1188 ndev->peer_mmio + 1189 ndev->peer_reg->spad); 1190 } 1191 1192 static u64 xeon_db_ioread(void __iomem *mmio) 1193 { 1194 return (u64)ioread16(mmio); 1195 } 1196 1197 static void xeon_db_iowrite(u64 bits, void __iomem *mmio) 1198 { 1199 iowrite16((u16)bits, mmio); 1200 } 1201 1202 static int xeon_poll_link(struct intel_ntb_dev *ndev) 1203 { 1204 u16 reg_val; 1205 int rc; 1206 1207 ndev->reg->db_iowrite(ndev->db_link_mask, 1208 ndev->self_mmio + 1209 ndev->self_reg->db_bell); 1210 1211 rc = pci_read_config_word(ndev->ntb.pdev, 1212 XEON_LINK_STATUS_OFFSET, ®_val); 1213 if (rc) 1214 return 0; 1215 1216 if (reg_val == ndev->lnk_sta) 1217 return 0; 1218 1219 ndev->lnk_sta = reg_val; 1220 1221 return 1; 1222 } 1223 1224 int xeon_link_is_up(struct intel_ntb_dev *ndev) 1225 { 1226 if (ndev->ntb.topo == NTB_TOPO_SEC) 1227 return 1; 1228 1229 return NTB_LNK_STA_ACTIVE(ndev->lnk_sta); 1230 } 1231 1232 enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd) 1233 { 1234 switch (ppd & XEON_PPD_TOPO_MASK) { 1235 case XEON_PPD_TOPO_B2B_USD: 1236 return NTB_TOPO_B2B_USD; 1237 1238 case XEON_PPD_TOPO_B2B_DSD: 1239 return NTB_TOPO_B2B_DSD; 1240 1241 case XEON_PPD_TOPO_PRI_USD: 1242 case XEON_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */ 1243 return NTB_TOPO_PRI; 1244 1245 case XEON_PPD_TOPO_SEC_USD: 1246 case XEON_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */ 1247 return NTB_TOPO_SEC; 1248 } 1249 1250 return NTB_TOPO_NONE; 1251 } 1252 1253 static inline int xeon_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd) 1254 { 1255 if (ppd & XEON_PPD_SPLIT_BAR_MASK) { 1256 dev_dbg(&ndev->ntb.pdev->dev, "PPD %d split bar\n", ppd); 1257 return 1; 1258 } 1259 return 0; 1260 } 1261 1262 static int xeon_init_isr(struct intel_ntb_dev *ndev) 1263 { 1264 return ndev_init_isr(ndev, XEON_DB_MSIX_VECTOR_COUNT, 1265 XEON_DB_MSIX_VECTOR_COUNT, 1266 XEON_DB_MSIX_VECTOR_SHIFT, 1267 XEON_DB_TOTAL_SHIFT); 1268 } 1269 1270 static void xeon_deinit_isr(struct intel_ntb_dev *ndev) 1271 { 1272 ndev_deinit_isr(ndev); 1273 } 1274 1275 static int xeon_setup_b2b_mw(struct intel_ntb_dev *ndev, 1276 const struct intel_b2b_addr *addr, 1277 const struct intel_b2b_addr *peer_addr) 1278 { 1279 struct pci_dev *pdev; 1280 void __iomem *mmio; 1281 resource_size_t bar_size; 1282 phys_addr_t bar_addr; 1283 int b2b_bar; 1284 u8 bar_sz; 1285 1286 pdev = ndev->ntb.pdev; 1287 mmio = ndev->self_mmio; 1288 1289 if (ndev->b2b_idx == UINT_MAX) { 1290 dev_dbg(&pdev->dev, "not using b2b mw\n"); 1291 b2b_bar = 0; 1292 ndev->b2b_off = 0; 1293 } else { 1294 b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx); 1295 if (b2b_bar < 0) 1296 return -EIO; 1297 1298 dev_dbg(&pdev->dev, "using b2b mw bar %d\n", b2b_bar); 1299 1300 bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar); 1301 1302 dev_dbg(&pdev->dev, "b2b bar size %#llx\n", bar_size); 1303 1304 if (b2b_mw_share && XEON_B2B_MIN_SIZE <= bar_size >> 1) { 1305 dev_dbg(&pdev->dev, "b2b using first half of bar\n"); 1306 ndev->b2b_off = bar_size >> 1; 1307 } else if (XEON_B2B_MIN_SIZE <= bar_size) { 1308 dev_dbg(&pdev->dev, "b2b using whole bar\n"); 1309 ndev->b2b_off = 0; 1310 --ndev->mw_count; 1311 } else { 1312 dev_dbg(&pdev->dev, "b2b bar size is too small\n"); 1313 return -EIO; 1314 } 1315 } 1316 1317 /* Reset the secondary bar sizes to match the primary bar sizes, 1318 * except disable or halve the size of the b2b secondary bar. 1319 * 1320 * Note: code for each specific bar size register, because the register 1321 * offsets are not in a consistent order (bar5sz comes after ppd, odd). 1322 */ 1323 pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &bar_sz); 1324 dev_dbg(&pdev->dev, "PBAR23SZ %#x\n", bar_sz); 1325 if (b2b_bar == 2) { 1326 if (ndev->b2b_off) 1327 bar_sz -= 1; 1328 else 1329 bar_sz = 0; 1330 } 1331 pci_write_config_byte(pdev, XEON_SBAR23SZ_OFFSET, bar_sz); 1332 pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &bar_sz); 1333 dev_dbg(&pdev->dev, "SBAR23SZ %#x\n", bar_sz); 1334 1335 if (!ndev->bar4_split) { 1336 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &bar_sz); 1337 dev_dbg(&pdev->dev, "PBAR45SZ %#x\n", bar_sz); 1338 if (b2b_bar == 4) { 1339 if (ndev->b2b_off) 1340 bar_sz -= 1; 1341 else 1342 bar_sz = 0; 1343 } 1344 pci_write_config_byte(pdev, XEON_SBAR45SZ_OFFSET, bar_sz); 1345 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &bar_sz); 1346 dev_dbg(&pdev->dev, "SBAR45SZ %#x\n", bar_sz); 1347 } else { 1348 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &bar_sz); 1349 dev_dbg(&pdev->dev, "PBAR4SZ %#x\n", bar_sz); 1350 if (b2b_bar == 4) { 1351 if (ndev->b2b_off) 1352 bar_sz -= 1; 1353 else 1354 bar_sz = 0; 1355 } 1356 pci_write_config_byte(pdev, XEON_SBAR4SZ_OFFSET, bar_sz); 1357 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &bar_sz); 1358 dev_dbg(&pdev->dev, "SBAR4SZ %#x\n", bar_sz); 1359 1360 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &bar_sz); 1361 dev_dbg(&pdev->dev, "PBAR5SZ %#x\n", bar_sz); 1362 if (b2b_bar == 5) { 1363 if (ndev->b2b_off) 1364 bar_sz -= 1; 1365 else 1366 bar_sz = 0; 1367 } 1368 pci_write_config_byte(pdev, XEON_SBAR5SZ_OFFSET, bar_sz); 1369 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &bar_sz); 1370 dev_dbg(&pdev->dev, "SBAR5SZ %#x\n", bar_sz); 1371 } 1372 1373 /* SBAR01 hit by first part of the b2b bar */ 1374 if (b2b_bar == 0) 1375 bar_addr = addr->bar0_addr; 1376 else if (b2b_bar == 2) 1377 bar_addr = addr->bar2_addr64; 1378 else if (b2b_bar == 4 && !ndev->bar4_split) 1379 bar_addr = addr->bar4_addr64; 1380 else if (b2b_bar == 4) 1381 bar_addr = addr->bar4_addr32; 1382 else if (b2b_bar == 5) 1383 bar_addr = addr->bar5_addr32; 1384 else 1385 return -EIO; 1386 1387 dev_dbg(&pdev->dev, "SBAR01 %#018llx\n", bar_addr); 1388 iowrite64(bar_addr, mmio + XEON_SBAR0BASE_OFFSET); 1389 1390 /* Other SBAR are normally hit by the PBAR xlat, except for b2b bar. 1391 * The b2b bar is either disabled above, or configured half-size, and 1392 * it starts at the PBAR xlat + offset. 1393 */ 1394 1395 bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0); 1396 iowrite64(bar_addr, mmio + XEON_SBAR23BASE_OFFSET); 1397 bar_addr = ioread64(mmio + XEON_SBAR23BASE_OFFSET); 1398 dev_dbg(&pdev->dev, "SBAR23 %#018llx\n", bar_addr); 1399 1400 if (!ndev->bar4_split) { 1401 bar_addr = addr->bar4_addr64 + 1402 (b2b_bar == 4 ? ndev->b2b_off : 0); 1403 iowrite64(bar_addr, mmio + XEON_SBAR45BASE_OFFSET); 1404 bar_addr = ioread64(mmio + XEON_SBAR45BASE_OFFSET); 1405 dev_dbg(&pdev->dev, "SBAR45 %#018llx\n", bar_addr); 1406 } else { 1407 bar_addr = addr->bar4_addr32 + 1408 (b2b_bar == 4 ? ndev->b2b_off : 0); 1409 iowrite32(bar_addr, mmio + XEON_SBAR4BASE_OFFSET); 1410 bar_addr = ioread32(mmio + XEON_SBAR4BASE_OFFSET); 1411 dev_dbg(&pdev->dev, "SBAR4 %#010llx\n", bar_addr); 1412 1413 bar_addr = addr->bar5_addr32 + 1414 (b2b_bar == 5 ? ndev->b2b_off : 0); 1415 iowrite32(bar_addr, mmio + XEON_SBAR5BASE_OFFSET); 1416 bar_addr = ioread32(mmio + XEON_SBAR5BASE_OFFSET); 1417 dev_dbg(&pdev->dev, "SBAR5 %#010llx\n", bar_addr); 1418 } 1419 1420 /* setup incoming bar limits == base addrs (zero length windows) */ 1421 1422 bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0); 1423 iowrite64(bar_addr, mmio + XEON_SBAR23LMT_OFFSET); 1424 bar_addr = ioread64(mmio + XEON_SBAR23LMT_OFFSET); 1425 dev_dbg(&pdev->dev, "SBAR23LMT %#018llx\n", bar_addr); 1426 1427 if (!ndev->bar4_split) { 1428 bar_addr = addr->bar4_addr64 + 1429 (b2b_bar == 4 ? ndev->b2b_off : 0); 1430 iowrite64(bar_addr, mmio + XEON_SBAR45LMT_OFFSET); 1431 bar_addr = ioread64(mmio + XEON_SBAR45LMT_OFFSET); 1432 dev_dbg(&pdev->dev, "SBAR45LMT %#018llx\n", bar_addr); 1433 } else { 1434 bar_addr = addr->bar4_addr32 + 1435 (b2b_bar == 4 ? ndev->b2b_off : 0); 1436 iowrite32(bar_addr, mmio + XEON_SBAR4LMT_OFFSET); 1437 bar_addr = ioread32(mmio + XEON_SBAR4LMT_OFFSET); 1438 dev_dbg(&pdev->dev, "SBAR4LMT %#010llx\n", bar_addr); 1439 1440 bar_addr = addr->bar5_addr32 + 1441 (b2b_bar == 5 ? ndev->b2b_off : 0); 1442 iowrite32(bar_addr, mmio + XEON_SBAR5LMT_OFFSET); 1443 bar_addr = ioread32(mmio + XEON_SBAR5LMT_OFFSET); 1444 dev_dbg(&pdev->dev, "SBAR5LMT %#05llx\n", bar_addr); 1445 } 1446 1447 /* zero incoming translation addrs */ 1448 iowrite64(0, mmio + XEON_SBAR23XLAT_OFFSET); 1449 1450 if (!ndev->bar4_split) { 1451 iowrite64(0, mmio + XEON_SBAR45XLAT_OFFSET); 1452 } else { 1453 iowrite32(0, mmio + XEON_SBAR4XLAT_OFFSET); 1454 iowrite32(0, mmio + XEON_SBAR5XLAT_OFFSET); 1455 } 1456 1457 /* zero outgoing translation limits (whole bar size windows) */ 1458 iowrite64(0, mmio + XEON_PBAR23LMT_OFFSET); 1459 if (!ndev->bar4_split) { 1460 iowrite64(0, mmio + XEON_PBAR45LMT_OFFSET); 1461 } else { 1462 iowrite32(0, mmio + XEON_PBAR4LMT_OFFSET); 1463 iowrite32(0, mmio + XEON_PBAR5LMT_OFFSET); 1464 } 1465 1466 /* set outgoing translation offsets */ 1467 bar_addr = peer_addr->bar2_addr64; 1468 iowrite64(bar_addr, mmio + XEON_PBAR23XLAT_OFFSET); 1469 bar_addr = ioread64(mmio + XEON_PBAR23XLAT_OFFSET); 1470 dev_dbg(&pdev->dev, "PBAR23XLAT %#018llx\n", bar_addr); 1471 1472 if (!ndev->bar4_split) { 1473 bar_addr = peer_addr->bar4_addr64; 1474 iowrite64(bar_addr, mmio + XEON_PBAR45XLAT_OFFSET); 1475 bar_addr = ioread64(mmio + XEON_PBAR45XLAT_OFFSET); 1476 dev_dbg(&pdev->dev, "PBAR45XLAT %#018llx\n", bar_addr); 1477 } else { 1478 bar_addr = peer_addr->bar4_addr32; 1479 iowrite32(bar_addr, mmio + XEON_PBAR4XLAT_OFFSET); 1480 bar_addr = ioread32(mmio + XEON_PBAR4XLAT_OFFSET); 1481 dev_dbg(&pdev->dev, "PBAR4XLAT %#010llx\n", bar_addr); 1482 1483 bar_addr = peer_addr->bar5_addr32; 1484 iowrite32(bar_addr, mmio + XEON_PBAR5XLAT_OFFSET); 1485 bar_addr = ioread32(mmio + XEON_PBAR5XLAT_OFFSET); 1486 dev_dbg(&pdev->dev, "PBAR5XLAT %#010llx\n", bar_addr); 1487 } 1488 1489 /* set the translation offset for b2b registers */ 1490 if (b2b_bar == 0) 1491 bar_addr = peer_addr->bar0_addr; 1492 else if (b2b_bar == 2) 1493 bar_addr = peer_addr->bar2_addr64; 1494 else if (b2b_bar == 4 && !ndev->bar4_split) 1495 bar_addr = peer_addr->bar4_addr64; 1496 else if (b2b_bar == 4) 1497 bar_addr = peer_addr->bar4_addr32; 1498 else if (b2b_bar == 5) 1499 bar_addr = peer_addr->bar5_addr32; 1500 else 1501 return -EIO; 1502 1503 /* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */ 1504 dev_dbg(&pdev->dev, "B2BXLAT %#018llx\n", bar_addr); 1505 iowrite32(bar_addr, mmio + XEON_B2B_XLAT_OFFSETL); 1506 iowrite32(bar_addr >> 32, mmio + XEON_B2B_XLAT_OFFSETU); 1507 1508 if (b2b_bar) { 1509 /* map peer ntb mmio config space registers */ 1510 ndev->peer_mmio = pci_iomap(pdev, b2b_bar, 1511 XEON_B2B_MIN_SIZE); 1512 if (!ndev->peer_mmio) 1513 return -EIO; 1514 1515 ndev->peer_addr = pci_resource_start(pdev, b2b_bar); 1516 } 1517 1518 return 0; 1519 } 1520 1521 static int xeon_init_ntb(struct intel_ntb_dev *ndev) 1522 { 1523 struct device *dev = &ndev->ntb.pdev->dev; 1524 int rc; 1525 u32 ntb_ctl; 1526 1527 if (ndev->bar4_split) 1528 ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT; 1529 else 1530 ndev->mw_count = XEON_MW_COUNT; 1531 1532 ndev->spad_count = XEON_SPAD_COUNT; 1533 ndev->db_count = XEON_DB_COUNT; 1534 ndev->db_link_mask = XEON_DB_LINK_BIT; 1535 1536 switch (ndev->ntb.topo) { 1537 case NTB_TOPO_PRI: 1538 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) { 1539 dev_err(dev, "NTB Primary config disabled\n"); 1540 return -EINVAL; 1541 } 1542 1543 /* enable link to allow secondary side device to appear */ 1544 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl); 1545 ntb_ctl &= ~NTB_CTL_DISABLE; 1546 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl); 1547 1548 /* use half the spads for the peer */ 1549 ndev->spad_count >>= 1; 1550 ndev->self_reg = &xeon_pri_reg; 1551 ndev->peer_reg = &xeon_sec_reg; 1552 ndev->xlat_reg = &xeon_sec_xlat; 1553 break; 1554 1555 case NTB_TOPO_SEC: 1556 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) { 1557 dev_err(dev, "NTB Secondary config disabled\n"); 1558 return -EINVAL; 1559 } 1560 /* use half the spads for the peer */ 1561 ndev->spad_count >>= 1; 1562 ndev->self_reg = &xeon_sec_reg; 1563 ndev->peer_reg = &xeon_pri_reg; 1564 ndev->xlat_reg = &xeon_pri_xlat; 1565 break; 1566 1567 case NTB_TOPO_B2B_USD: 1568 case NTB_TOPO_B2B_DSD: 1569 ndev->self_reg = &xeon_pri_reg; 1570 ndev->peer_reg = &xeon_b2b_reg; 1571 ndev->xlat_reg = &xeon_sec_xlat; 1572 1573 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) { 1574 ndev->peer_reg = &xeon_pri_reg; 1575 1576 if (b2b_mw_idx < 0) 1577 ndev->b2b_idx = b2b_mw_idx + ndev->mw_count; 1578 else 1579 ndev->b2b_idx = b2b_mw_idx; 1580 1581 if (ndev->b2b_idx >= ndev->mw_count) { 1582 dev_dbg(dev, 1583 "b2b_mw_idx %d invalid for mw_count %u\n", 1584 b2b_mw_idx, ndev->mw_count); 1585 return -EINVAL; 1586 } 1587 1588 dev_dbg(dev, "setting up b2b mw idx %d means %d\n", 1589 b2b_mw_idx, ndev->b2b_idx); 1590 1591 } else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) { 1592 dev_warn(dev, "Reduce doorbell count by 1\n"); 1593 ndev->db_count -= 1; 1594 } 1595 1596 if (ndev->ntb.topo == NTB_TOPO_B2B_USD) { 1597 rc = xeon_setup_b2b_mw(ndev, 1598 &xeon_b2b_dsd_addr, 1599 &xeon_b2b_usd_addr); 1600 } else { 1601 rc = xeon_setup_b2b_mw(ndev, 1602 &xeon_b2b_usd_addr, 1603 &xeon_b2b_dsd_addr); 1604 } 1605 if (rc) 1606 return rc; 1607 1608 /* Enable Bus Master and Memory Space on the secondary side */ 1609 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER, 1610 ndev->self_mmio + XEON_SPCICMD_OFFSET); 1611 1612 break; 1613 1614 default: 1615 return -EINVAL; 1616 } 1617 1618 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; 1619 1620 ndev->reg->db_iowrite(ndev->db_valid_mask, 1621 ndev->self_mmio + 1622 ndev->self_reg->db_mask); 1623 1624 return 0; 1625 } 1626 1627 static int xeon_init_dev(struct intel_ntb_dev *ndev) 1628 { 1629 struct pci_dev *pdev; 1630 u8 ppd; 1631 int rc, mem; 1632 1633 pdev = ndev->ntb.pdev; 1634 1635 switch (pdev->device) { 1636 /* There is a Xeon hardware errata related to writes to SDOORBELL or 1637 * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space, 1638 * which may hang the system. To workaround this use the second memory 1639 * window to access the interrupt and scratch pad registers on the 1640 * remote system. 1641 */ 1642 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF: 1643 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF: 1644 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF: 1645 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB: 1646 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB: 1647 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB: 1648 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: 1649 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: 1650 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: 1651 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: 1652 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: 1653 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: 1654 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX: 1655 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX: 1656 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX: 1657 ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP; 1658 break; 1659 } 1660 1661 switch (pdev->device) { 1662 /* There is a hardware errata related to accessing any register in 1663 * SB01BASE in the presence of bidirectional traffic crossing the NTB. 1664 */ 1665 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: 1666 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: 1667 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: 1668 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: 1669 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: 1670 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: 1671 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX: 1672 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX: 1673 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX: 1674 ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP; 1675 break; 1676 } 1677 1678 switch (pdev->device) { 1679 /* HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1680 * mirrored to the remote system. Shrink the number of bits by one, 1681 * since bit 14 is the last bit. 1682 */ 1683 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF: 1684 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF: 1685 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF: 1686 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB: 1687 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB: 1688 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB: 1689 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: 1690 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: 1691 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: 1692 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: 1693 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: 1694 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: 1695 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX: 1696 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX: 1697 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX: 1698 ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14; 1699 break; 1700 } 1701 1702 ndev->reg = &xeon_reg; 1703 1704 rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd); 1705 if (rc) 1706 return -EIO; 1707 1708 ndev->ntb.topo = xeon_ppd_topo(ndev, ppd); 1709 dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd, 1710 ntb_topo_string(ndev->ntb.topo)); 1711 if (ndev->ntb.topo == NTB_TOPO_NONE) 1712 return -EINVAL; 1713 1714 if (ndev->ntb.topo != NTB_TOPO_SEC) { 1715 ndev->bar4_split = xeon_ppd_bar4_split(ndev, ppd); 1716 dev_dbg(&pdev->dev, "ppd %#x bar4_split %d\n", 1717 ppd, ndev->bar4_split); 1718 } else { 1719 /* This is a way for transparent BAR to figure out if we are 1720 * doing split BAR or not. There is no way for the hw on the 1721 * transparent side to know and set the PPD. 1722 */ 1723 mem = pci_select_bars(pdev, IORESOURCE_MEM); 1724 ndev->bar4_split = hweight32(mem) == 1725 HSX_SPLIT_BAR_MW_COUNT + 1; 1726 dev_dbg(&pdev->dev, "mem %#x bar4_split %d\n", 1727 mem, ndev->bar4_split); 1728 } 1729 1730 rc = xeon_init_ntb(ndev); 1731 if (rc) 1732 return rc; 1733 1734 return xeon_init_isr(ndev); 1735 } 1736 1737 static void xeon_deinit_dev(struct intel_ntb_dev *ndev) 1738 { 1739 xeon_deinit_isr(ndev); 1740 } 1741 1742 static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev) 1743 { 1744 int rc; 1745 1746 pci_set_drvdata(pdev, ndev); 1747 1748 rc = pci_enable_device(pdev); 1749 if (rc) 1750 goto err_pci_enable; 1751 1752 rc = pci_request_regions(pdev, NTB_NAME); 1753 if (rc) 1754 goto err_pci_regions; 1755 1756 pci_set_master(pdev); 1757 1758 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 1759 if (rc) { 1760 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 1761 if (rc) 1762 goto err_dma_mask; 1763 dev_warn(&pdev->dev, "Cannot DMA highmem\n"); 1764 } 1765 1766 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 1767 if (rc) { 1768 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 1769 if (rc) 1770 goto err_dma_mask; 1771 dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n"); 1772 } 1773 rc = dma_coerce_mask_and_coherent(&ndev->ntb.dev, 1774 dma_get_mask(&pdev->dev)); 1775 if (rc) 1776 goto err_dma_mask; 1777 1778 ndev->self_mmio = pci_iomap(pdev, 0, 0); 1779 if (!ndev->self_mmio) { 1780 rc = -EIO; 1781 goto err_mmio; 1782 } 1783 ndev->peer_mmio = ndev->self_mmio; 1784 ndev->peer_addr = pci_resource_start(pdev, 0); 1785 1786 return 0; 1787 1788 err_mmio: 1789 err_dma_mask: 1790 pci_clear_master(pdev); 1791 pci_release_regions(pdev); 1792 err_pci_regions: 1793 pci_disable_device(pdev); 1794 err_pci_enable: 1795 pci_set_drvdata(pdev, NULL); 1796 return rc; 1797 } 1798 1799 static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev) 1800 { 1801 struct pci_dev *pdev = ndev->ntb.pdev; 1802 1803 if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio) 1804 pci_iounmap(pdev, ndev->peer_mmio); 1805 pci_iounmap(pdev, ndev->self_mmio); 1806 1807 pci_clear_master(pdev); 1808 pci_release_regions(pdev); 1809 pci_disable_device(pdev); 1810 pci_set_drvdata(pdev, NULL); 1811 } 1812 1813 static inline void ndev_init_struct(struct intel_ntb_dev *ndev, 1814 struct pci_dev *pdev) 1815 { 1816 ndev->ntb.pdev = pdev; 1817 ndev->ntb.topo = NTB_TOPO_NONE; 1818 ndev->ntb.ops = &intel_ntb_ops; 1819 1820 ndev->b2b_off = 0; 1821 ndev->b2b_idx = UINT_MAX; 1822 1823 ndev->bar4_split = 0; 1824 1825 ndev->mw_count = 0; 1826 ndev->spad_count = 0; 1827 ndev->db_count = 0; 1828 ndev->db_vec_count = 0; 1829 ndev->db_vec_shift = 0; 1830 1831 ndev->ntb_ctl = 0; 1832 ndev->lnk_sta = 0; 1833 1834 ndev->db_valid_mask = 0; 1835 ndev->db_link_mask = 0; 1836 ndev->db_mask = 0; 1837 1838 spin_lock_init(&ndev->db_mask_lock); 1839 } 1840 1841 static int intel_ntb_pci_probe(struct pci_dev *pdev, 1842 const struct pci_device_id *id) 1843 { 1844 struct intel_ntb_dev *ndev; 1845 int rc, node; 1846 1847 node = dev_to_node(&pdev->dev); 1848 1849 if (pdev_is_gen1(pdev)) { 1850 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node); 1851 if (!ndev) { 1852 rc = -ENOMEM; 1853 goto err_ndev; 1854 } 1855 1856 ndev_init_struct(ndev, pdev); 1857 1858 rc = intel_ntb_init_pci(ndev, pdev); 1859 if (rc) 1860 goto err_init_pci; 1861 1862 rc = xeon_init_dev(ndev); 1863 if (rc) 1864 goto err_init_dev; 1865 1866 } else if (pdev_is_gen3(pdev)) { 1867 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node); 1868 if (!ndev) { 1869 rc = -ENOMEM; 1870 goto err_ndev; 1871 } 1872 1873 ndev_init_struct(ndev, pdev); 1874 ndev->ntb.ops = &intel_ntb3_ops; 1875 1876 rc = intel_ntb_init_pci(ndev, pdev); 1877 if (rc) 1878 goto err_init_pci; 1879 1880 rc = gen3_init_dev(ndev); 1881 if (rc) 1882 goto err_init_dev; 1883 1884 } else { 1885 rc = -EINVAL; 1886 goto err_ndev; 1887 } 1888 1889 ndev_reset_unsafe_flags(ndev); 1890 1891 ndev->reg->poll_link(ndev); 1892 1893 ndev_init_debugfs(ndev); 1894 1895 rc = ntb_register_device(&ndev->ntb); 1896 if (rc) 1897 goto err_register; 1898 1899 dev_info(&pdev->dev, "NTB device registered.\n"); 1900 1901 return 0; 1902 1903 err_register: 1904 ndev_deinit_debugfs(ndev); 1905 if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev)) 1906 xeon_deinit_dev(ndev); 1907 err_init_dev: 1908 intel_ntb_deinit_pci(ndev); 1909 err_init_pci: 1910 kfree(ndev); 1911 err_ndev: 1912 return rc; 1913 } 1914 1915 static void intel_ntb_pci_remove(struct pci_dev *pdev) 1916 { 1917 struct intel_ntb_dev *ndev = pci_get_drvdata(pdev); 1918 1919 ntb_unregister_device(&ndev->ntb); 1920 ndev_deinit_debugfs(ndev); 1921 if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev)) 1922 xeon_deinit_dev(ndev); 1923 intel_ntb_deinit_pci(ndev); 1924 kfree(ndev); 1925 } 1926 1927 static const struct intel_ntb_reg xeon_reg = { 1928 .poll_link = xeon_poll_link, 1929 .link_is_up = xeon_link_is_up, 1930 .db_ioread = xeon_db_ioread, 1931 .db_iowrite = xeon_db_iowrite, 1932 .db_size = sizeof(u32), 1933 .ntb_ctl = XEON_NTBCNTL_OFFSET, 1934 .mw_bar = {2, 4, 5}, 1935 }; 1936 1937 static const struct intel_ntb_alt_reg xeon_pri_reg = { 1938 .db_bell = XEON_PDOORBELL_OFFSET, 1939 .db_mask = XEON_PDBMSK_OFFSET, 1940 .spad = XEON_SPAD_OFFSET, 1941 }; 1942 1943 static const struct intel_ntb_alt_reg xeon_sec_reg = { 1944 .db_bell = XEON_SDOORBELL_OFFSET, 1945 .db_mask = XEON_SDBMSK_OFFSET, 1946 /* second half of the scratchpads */ 1947 .spad = XEON_SPAD_OFFSET + (XEON_SPAD_COUNT << 1), 1948 }; 1949 1950 static const struct intel_ntb_alt_reg xeon_b2b_reg = { 1951 .db_bell = XEON_B2B_DOORBELL_OFFSET, 1952 .spad = XEON_B2B_SPAD_OFFSET, 1953 }; 1954 1955 static const struct intel_ntb_xlat_reg xeon_pri_xlat = { 1956 /* Note: no primary .bar0_base visible to the secondary side. 1957 * 1958 * The secondary side cannot get the base address stored in primary 1959 * bars. The base address is necessary to set the limit register to 1960 * any value other than zero, or unlimited. 1961 * 1962 * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the 1963 * window by setting the limit equal to base, nor can it limit the size 1964 * of the memory window by setting the limit to base + size. 1965 */ 1966 .bar2_limit = XEON_PBAR23LMT_OFFSET, 1967 .bar2_xlat = XEON_PBAR23XLAT_OFFSET, 1968 }; 1969 1970 static const struct intel_ntb_xlat_reg xeon_sec_xlat = { 1971 .bar0_base = XEON_SBAR0BASE_OFFSET, 1972 .bar2_limit = XEON_SBAR23LMT_OFFSET, 1973 .bar2_xlat = XEON_SBAR23XLAT_OFFSET, 1974 }; 1975 1976 struct intel_b2b_addr xeon_b2b_usd_addr = { 1977 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 1978 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 1979 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 1980 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 1981 }; 1982 1983 struct intel_b2b_addr xeon_b2b_dsd_addr = { 1984 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 1985 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 1986 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 1987 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 1988 }; 1989 1990 /* operations for primary side of local ntb */ 1991 static const struct ntb_dev_ops intel_ntb_ops = { 1992 .mw_count = intel_ntb_mw_count, 1993 .mw_get_align = intel_ntb_mw_get_align, 1994 .mw_set_trans = intel_ntb_mw_set_trans, 1995 .peer_mw_count = intel_ntb_peer_mw_count, 1996 .peer_mw_get_addr = intel_ntb_peer_mw_get_addr, 1997 .link_is_up = intel_ntb_link_is_up, 1998 .link_enable = intel_ntb_link_enable, 1999 .link_disable = intel_ntb_link_disable, 2000 .db_is_unsafe = intel_ntb_db_is_unsafe, 2001 .db_valid_mask = intel_ntb_db_valid_mask, 2002 .db_vector_count = intel_ntb_db_vector_count, 2003 .db_vector_mask = intel_ntb_db_vector_mask, 2004 .db_read = intel_ntb_db_read, 2005 .db_clear = intel_ntb_db_clear, 2006 .db_set_mask = intel_ntb_db_set_mask, 2007 .db_clear_mask = intel_ntb_db_clear_mask, 2008 .peer_db_addr = intel_ntb_peer_db_addr, 2009 .peer_db_set = intel_ntb_peer_db_set, 2010 .spad_is_unsafe = intel_ntb_spad_is_unsafe, 2011 .spad_count = intel_ntb_spad_count, 2012 .spad_read = intel_ntb_spad_read, 2013 .spad_write = intel_ntb_spad_write, 2014 .peer_spad_addr = intel_ntb_peer_spad_addr, 2015 .peer_spad_read = intel_ntb_peer_spad_read, 2016 .peer_spad_write = intel_ntb_peer_spad_write, 2017 }; 2018 2019 static const struct file_operations intel_ntb_debugfs_info = { 2020 .owner = THIS_MODULE, 2021 .open = simple_open, 2022 .read = ndev_debugfs_read, 2023 }; 2024 2025 static const struct pci_device_id intel_ntb_pci_tbl[] = { 2026 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)}, 2027 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)}, 2028 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)}, 2029 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)}, 2030 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BDX)}, 2031 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)}, 2032 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)}, 2033 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)}, 2034 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)}, 2035 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_BDX)}, 2036 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)}, 2037 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)}, 2038 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)}, 2039 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)}, 2040 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_BDX)}, 2041 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)}, 2042 {0} 2043 }; 2044 MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl); 2045 2046 static struct pci_driver intel_ntb_pci_driver = { 2047 .name = KBUILD_MODNAME, 2048 .id_table = intel_ntb_pci_tbl, 2049 .probe = intel_ntb_pci_probe, 2050 .remove = intel_ntb_pci_remove, 2051 }; 2052 2053 static int __init intel_ntb_pci_driver_init(void) 2054 { 2055 pr_info("%s %s\n", NTB_DESC, NTB_VER); 2056 2057 if (debugfs_initialized()) 2058 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 2059 2060 return pci_register_driver(&intel_ntb_pci_driver); 2061 } 2062 module_init(intel_ntb_pci_driver_init); 2063 2064 static void __exit intel_ntb_pci_driver_exit(void) 2065 { 2066 pci_unregister_driver(&intel_ntb_pci_driver); 2067 2068 debugfs_remove_recursive(debugfs_dir); 2069 } 2070 module_exit(intel_ntb_pci_driver_exit); 2071