1f6e51c35SDave Jiang /* 2f6e51c35SDave Jiang * This file is provided under a dual BSD/GPLv2 license. When using or 3f6e51c35SDave Jiang * redistributing this file, you may do so under either license. 4f6e51c35SDave Jiang * 5f6e51c35SDave Jiang * GPL LICENSE SUMMARY 6f6e51c35SDave Jiang * 7f6e51c35SDave Jiang * Copyright(c) 2012 Intel Corporation. All rights reserved. 8f6e51c35SDave Jiang * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 9f6e51c35SDave Jiang * Copyright (C) 2016 T-Platforms. All Rights Reserved. 10f6e51c35SDave Jiang * 11f6e51c35SDave Jiang * This program is free software; you can redistribute it and/or modify 12f6e51c35SDave Jiang * it under the terms of version 2 of the GNU General Public License as 13f6e51c35SDave Jiang * published by the Free Software Foundation. 14f6e51c35SDave Jiang * 15f6e51c35SDave Jiang * BSD LICENSE 16f6e51c35SDave Jiang * 17f6e51c35SDave Jiang * Copyright(c) 2012 Intel Corporation. All rights reserved. 18f6e51c35SDave Jiang * Copyright (C) 2015 EMC Corporation. All Rights Reserved. 19f6e51c35SDave Jiang * Copyright (C) 2016 T-Platforms. All Rights Reserved. 20f6e51c35SDave Jiang * 21f6e51c35SDave Jiang * Redistribution and use in source and binary forms, with or without 22f6e51c35SDave Jiang * modification, are permitted provided that the following conditions 23f6e51c35SDave Jiang * are met: 24f6e51c35SDave Jiang * 25f6e51c35SDave Jiang * * Redistributions of source code must retain the above copyright 26f6e51c35SDave Jiang * notice, this list of conditions and the following disclaimer. 27f6e51c35SDave Jiang * * Redistributions in binary form must reproduce the above copy 28f6e51c35SDave Jiang * notice, this list of conditions and the following disclaimer in 29f6e51c35SDave Jiang * the documentation and/or other materials provided with the 30f6e51c35SDave Jiang * distribution. 31f6e51c35SDave Jiang * * Neither the name of Intel Corporation nor the names of its 32f6e51c35SDave Jiang * contributors may be used to endorse or promote products derived 33f6e51c35SDave Jiang * from this software without specific prior written permission. 34f6e51c35SDave Jiang * 35f6e51c35SDave Jiang * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 36f6e51c35SDave Jiang * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 37f6e51c35SDave Jiang * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 38f6e51c35SDave Jiang * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 39f6e51c35SDave Jiang * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 40f6e51c35SDave Jiang * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 41f6e51c35SDave Jiang * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 42f6e51c35SDave Jiang * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 43f6e51c35SDave Jiang * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 44f6e51c35SDave Jiang * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 45f6e51c35SDave Jiang * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 46f6e51c35SDave Jiang * 47f6e51c35SDave Jiang * Intel PCIe NTB Linux driver 48f6e51c35SDave Jiang * 49f6e51c35SDave Jiang * Contact Information: 50f6e51c35SDave Jiang * Jon Mason <jon.mason@intel.com> 51f6e51c35SDave Jiang */ 52f6e51c35SDave Jiang 53f6e51c35SDave Jiang #include <linux/debugfs.h> 54f6e51c35SDave Jiang #include <linux/delay.h> 55f6e51c35SDave Jiang #include <linux/init.h> 56f6e51c35SDave Jiang #include <linux/interrupt.h> 57f6e51c35SDave Jiang #include <linux/module.h> 58f6e51c35SDave Jiang #include <linux/pci.h> 59f6e51c35SDave Jiang #include <linux/random.h> 60f6e51c35SDave Jiang #include <linux/slab.h> 61f6e51c35SDave Jiang #include <linux/ntb.h> 62f6e51c35SDave Jiang 63f6e51c35SDave Jiang #include "ntb_hw_intel.h" 64f6e51c35SDave Jiang #include "ntb_hw_gen1.h" 65f6e51c35SDave Jiang #include "ntb_hw_gen3.h" 66f6e51c35SDave Jiang 67f6e51c35SDave Jiang #define NTB_NAME "ntb_hw_intel" 68f6e51c35SDave Jiang #define NTB_DESC "Intel(R) PCI-E Non-Transparent Bridge Driver" 69f6e51c35SDave Jiang #define NTB_VER "2.0" 70f6e51c35SDave Jiang 71f6e51c35SDave Jiang MODULE_DESCRIPTION(NTB_DESC); 72f6e51c35SDave Jiang MODULE_VERSION(NTB_VER); 73f6e51c35SDave Jiang MODULE_LICENSE("Dual BSD/GPL"); 74f6e51c35SDave Jiang MODULE_AUTHOR("Intel Corporation"); 75f6e51c35SDave Jiang 76f6e51c35SDave Jiang #define bar0_off(base, bar) ((base) + ((bar) << 2)) 77f6e51c35SDave Jiang #define bar2_off(base, bar) bar0_off(base, (bar) - 2) 78f6e51c35SDave Jiang 79f6e51c35SDave Jiang static const struct intel_ntb_reg xeon_reg; 80f6e51c35SDave Jiang static const struct intel_ntb_alt_reg xeon_pri_reg; 81f6e51c35SDave Jiang static const struct intel_ntb_alt_reg xeon_sec_reg; 82f6e51c35SDave Jiang static const struct intel_ntb_alt_reg xeon_b2b_reg; 83f6e51c35SDave Jiang static const struct intel_ntb_xlat_reg xeon_pri_xlat; 84f6e51c35SDave Jiang static const struct intel_ntb_xlat_reg xeon_sec_xlat; 85f6e51c35SDave Jiang static const struct ntb_dev_ops intel_ntb_ops; 86f6e51c35SDave Jiang 87f6e51c35SDave Jiang static const struct file_operations intel_ntb_debugfs_info; 88f6e51c35SDave Jiang static struct dentry *debugfs_dir; 89f6e51c35SDave Jiang 90f6e51c35SDave Jiang static int b2b_mw_idx = -1; 91f6e51c35SDave Jiang module_param(b2b_mw_idx, int, 0644); 92f6e51c35SDave Jiang MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb. A " 93f6e51c35SDave Jiang "value of zero or positive starts from first mw idx, and a " 94f6e51c35SDave Jiang "negative value starts from last mw idx. Both sides MUST " 95f6e51c35SDave Jiang "set the same value here!"); 96f6e51c35SDave Jiang 97f6e51c35SDave Jiang static unsigned int b2b_mw_share; 98f6e51c35SDave Jiang module_param(b2b_mw_share, uint, 0644); 99f6e51c35SDave Jiang MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the " 100f6e51c35SDave Jiang "ntb so that the peer ntb only occupies the first half of " 101f6e51c35SDave Jiang "the mw, so the second half can still be used as a mw. Both " 102f6e51c35SDave Jiang "sides MUST set the same value here!"); 103f6e51c35SDave Jiang 104f6e51c35SDave Jiang module_param_named(xeon_b2b_usd_bar2_addr64, 105f6e51c35SDave Jiang xeon_b2b_usd_addr.bar2_addr64, ullong, 0644); 106f6e51c35SDave Jiang MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64, 107f6e51c35SDave Jiang "XEON B2B USD BAR 2 64-bit address"); 108f6e51c35SDave Jiang 109f6e51c35SDave Jiang module_param_named(xeon_b2b_usd_bar4_addr64, 110f6e51c35SDave Jiang xeon_b2b_usd_addr.bar4_addr64, ullong, 0644); 111f6e51c35SDave Jiang MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr64, 112f6e51c35SDave Jiang "XEON B2B USD BAR 4 64-bit address"); 113f6e51c35SDave Jiang 114f6e51c35SDave Jiang module_param_named(xeon_b2b_usd_bar4_addr32, 115f6e51c35SDave Jiang xeon_b2b_usd_addr.bar4_addr32, ullong, 0644); 116f6e51c35SDave Jiang MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr32, 117f6e51c35SDave Jiang "XEON B2B USD split-BAR 4 32-bit address"); 118f6e51c35SDave Jiang 119f6e51c35SDave Jiang module_param_named(xeon_b2b_usd_bar5_addr32, 120f6e51c35SDave Jiang xeon_b2b_usd_addr.bar5_addr32, ullong, 0644); 121f6e51c35SDave Jiang MODULE_PARM_DESC(xeon_b2b_usd_bar5_addr32, 122f6e51c35SDave Jiang "XEON B2B USD split-BAR 5 32-bit address"); 123f6e51c35SDave Jiang 124f6e51c35SDave Jiang module_param_named(xeon_b2b_dsd_bar2_addr64, 125f6e51c35SDave Jiang xeon_b2b_dsd_addr.bar2_addr64, ullong, 0644); 126f6e51c35SDave Jiang MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64, 127f6e51c35SDave Jiang "XEON B2B DSD BAR 2 64-bit address"); 128f6e51c35SDave Jiang 129f6e51c35SDave Jiang module_param_named(xeon_b2b_dsd_bar4_addr64, 130f6e51c35SDave Jiang xeon_b2b_dsd_addr.bar4_addr64, ullong, 0644); 131f6e51c35SDave Jiang MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr64, 132f6e51c35SDave Jiang "XEON B2B DSD BAR 4 64-bit address"); 133f6e51c35SDave Jiang 134f6e51c35SDave Jiang module_param_named(xeon_b2b_dsd_bar4_addr32, 135f6e51c35SDave Jiang xeon_b2b_dsd_addr.bar4_addr32, ullong, 0644); 136f6e51c35SDave Jiang MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr32, 137f6e51c35SDave Jiang "XEON B2B DSD split-BAR 4 32-bit address"); 138f6e51c35SDave Jiang 139f6e51c35SDave Jiang module_param_named(xeon_b2b_dsd_bar5_addr32, 140f6e51c35SDave Jiang xeon_b2b_dsd_addr.bar5_addr32, ullong, 0644); 141f6e51c35SDave Jiang MODULE_PARM_DESC(xeon_b2b_dsd_bar5_addr32, 142f6e51c35SDave Jiang "XEON B2B DSD split-BAR 5 32-bit address"); 143f6e51c35SDave Jiang 144f6e51c35SDave Jiang 145f6e51c35SDave Jiang static int xeon_init_isr(struct intel_ntb_dev *ndev); 146f6e51c35SDave Jiang 147f6e51c35SDave Jiang static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev) 148f6e51c35SDave Jiang { 149f6e51c35SDave Jiang ndev->unsafe_flags = 0; 150f6e51c35SDave Jiang ndev->unsafe_flags_ignore = 0; 151f6e51c35SDave Jiang 152f6e51c35SDave Jiang /* Only B2B has a workaround to avoid SDOORBELL */ 153f6e51c35SDave Jiang if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) 154f6e51c35SDave Jiang if (!ntb_topo_is_b2b(ndev->ntb.topo)) 155f6e51c35SDave Jiang ndev->unsafe_flags |= NTB_UNSAFE_DB; 156f6e51c35SDave Jiang 157f6e51c35SDave Jiang /* No low level workaround to avoid SB01BASE */ 158f6e51c35SDave Jiang if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) { 159f6e51c35SDave Jiang ndev->unsafe_flags |= NTB_UNSAFE_DB; 160f6e51c35SDave Jiang ndev->unsafe_flags |= NTB_UNSAFE_SPAD; 161f6e51c35SDave Jiang } 162f6e51c35SDave Jiang } 163f6e51c35SDave Jiang 164f6e51c35SDave Jiang static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev, 165f6e51c35SDave Jiang unsigned long flag) 166f6e51c35SDave Jiang { 167f6e51c35SDave Jiang return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore); 168f6e51c35SDave Jiang } 169f6e51c35SDave Jiang 170f6e51c35SDave Jiang static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev, 171f6e51c35SDave Jiang unsigned long flag) 172f6e51c35SDave Jiang { 173f6e51c35SDave Jiang flag &= ndev->unsafe_flags; 174f6e51c35SDave Jiang ndev->unsafe_flags_ignore |= flag; 175f6e51c35SDave Jiang 176f6e51c35SDave Jiang return !!flag; 177f6e51c35SDave Jiang } 178f6e51c35SDave Jiang 179f6e51c35SDave Jiang int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx) 180f6e51c35SDave Jiang { 181f6e51c35SDave Jiang if (idx < 0 || idx >= ndev->mw_count) 182f6e51c35SDave Jiang return -EINVAL; 183f6e51c35SDave Jiang return ndev->reg->mw_bar[idx]; 184f6e51c35SDave Jiang } 185f6e51c35SDave Jiang 186f6e51c35SDave Jiang static inline int ndev_db_addr(struct intel_ntb_dev *ndev, 187f6e51c35SDave Jiang phys_addr_t *db_addr, resource_size_t *db_size, 188f6e51c35SDave Jiang phys_addr_t reg_addr, unsigned long reg) 189f6e51c35SDave Jiang { 190f6e51c35SDave Jiang if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) 191f6e51c35SDave Jiang pr_warn_once("%s: NTB unsafe doorbell access", __func__); 192f6e51c35SDave Jiang 193f6e51c35SDave Jiang if (db_addr) { 194f6e51c35SDave Jiang *db_addr = reg_addr + reg; 195f6e51c35SDave Jiang dev_dbg(&ndev->ntb.pdev->dev, "Peer db addr %llx\n", *db_addr); 196f6e51c35SDave Jiang } 197f6e51c35SDave Jiang 198f6e51c35SDave Jiang if (db_size) { 199f6e51c35SDave Jiang *db_size = ndev->reg->db_size; 200f6e51c35SDave Jiang dev_dbg(&ndev->ntb.pdev->dev, "Peer db size %llx\n", *db_size); 201f6e51c35SDave Jiang } 202f6e51c35SDave Jiang 203f6e51c35SDave Jiang return 0; 204f6e51c35SDave Jiang } 205f6e51c35SDave Jiang 206f6e51c35SDave Jiang u64 ndev_db_read(struct intel_ntb_dev *ndev, 207f6e51c35SDave Jiang void __iomem *mmio) 208f6e51c35SDave Jiang { 209f6e51c35SDave Jiang if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) 210f6e51c35SDave Jiang pr_warn_once("%s: NTB unsafe doorbell access", __func__); 211f6e51c35SDave Jiang 212f6e51c35SDave Jiang return ndev->reg->db_ioread(mmio); 213f6e51c35SDave Jiang } 214f6e51c35SDave Jiang 215f6e51c35SDave Jiang int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits, 216f6e51c35SDave Jiang void __iomem *mmio) 217f6e51c35SDave Jiang { 218f6e51c35SDave Jiang if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) 219f6e51c35SDave Jiang pr_warn_once("%s: NTB unsafe doorbell access", __func__); 220f6e51c35SDave Jiang 221f6e51c35SDave Jiang if (db_bits & ~ndev->db_valid_mask) 222f6e51c35SDave Jiang return -EINVAL; 223f6e51c35SDave Jiang 224f6e51c35SDave Jiang ndev->reg->db_iowrite(db_bits, mmio); 225f6e51c35SDave Jiang 226f6e51c35SDave Jiang return 0; 227f6e51c35SDave Jiang } 228f6e51c35SDave Jiang 229f6e51c35SDave Jiang static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits, 230f6e51c35SDave Jiang void __iomem *mmio) 231f6e51c35SDave Jiang { 232f6e51c35SDave Jiang unsigned long irqflags; 233f6e51c35SDave Jiang 234f6e51c35SDave Jiang if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) 235f6e51c35SDave Jiang pr_warn_once("%s: NTB unsafe doorbell access", __func__); 236f6e51c35SDave Jiang 237f6e51c35SDave Jiang if (db_bits & ~ndev->db_valid_mask) 238f6e51c35SDave Jiang return -EINVAL; 239f6e51c35SDave Jiang 240f6e51c35SDave Jiang spin_lock_irqsave(&ndev->db_mask_lock, irqflags); 241f6e51c35SDave Jiang { 242f6e51c35SDave Jiang ndev->db_mask |= db_bits; 243f6e51c35SDave Jiang ndev->reg->db_iowrite(ndev->db_mask, mmio); 244f6e51c35SDave Jiang } 245f6e51c35SDave Jiang spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags); 246f6e51c35SDave Jiang 247f6e51c35SDave Jiang return 0; 248f6e51c35SDave Jiang } 249f6e51c35SDave Jiang 250f6e51c35SDave Jiang static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits, 251f6e51c35SDave Jiang void __iomem *mmio) 252f6e51c35SDave Jiang { 253f6e51c35SDave Jiang unsigned long irqflags; 254f6e51c35SDave Jiang 255f6e51c35SDave Jiang if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) 256f6e51c35SDave Jiang pr_warn_once("%s: NTB unsafe doorbell access", __func__); 257f6e51c35SDave Jiang 258f6e51c35SDave Jiang if (db_bits & ~ndev->db_valid_mask) 259f6e51c35SDave Jiang return -EINVAL; 260f6e51c35SDave Jiang 261f6e51c35SDave Jiang spin_lock_irqsave(&ndev->db_mask_lock, irqflags); 262f6e51c35SDave Jiang { 263f6e51c35SDave Jiang ndev->db_mask &= ~db_bits; 264f6e51c35SDave Jiang ndev->reg->db_iowrite(ndev->db_mask, mmio); 265f6e51c35SDave Jiang } 266f6e51c35SDave Jiang spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags); 267f6e51c35SDave Jiang 268f6e51c35SDave Jiang return 0; 269f6e51c35SDave Jiang } 270f6e51c35SDave Jiang 271f6e51c35SDave Jiang static inline int ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector) 272f6e51c35SDave Jiang { 273f6e51c35SDave Jiang u64 shift, mask; 274f6e51c35SDave Jiang 275f6e51c35SDave Jiang shift = ndev->db_vec_shift; 276f6e51c35SDave Jiang mask = BIT_ULL(shift) - 1; 277f6e51c35SDave Jiang 278f6e51c35SDave Jiang return mask << (shift * db_vector); 279f6e51c35SDave Jiang } 280f6e51c35SDave Jiang 281f6e51c35SDave Jiang static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx, 282f6e51c35SDave Jiang phys_addr_t *spad_addr, phys_addr_t reg_addr, 283f6e51c35SDave Jiang unsigned long reg) 284f6e51c35SDave Jiang { 285f6e51c35SDave Jiang if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD)) 286f6e51c35SDave Jiang pr_warn_once("%s: NTB unsafe scratchpad access", __func__); 287f6e51c35SDave Jiang 288f6e51c35SDave Jiang if (idx < 0 || idx >= ndev->spad_count) 289f6e51c35SDave Jiang return -EINVAL; 290f6e51c35SDave Jiang 291f6e51c35SDave Jiang if (spad_addr) { 292f6e51c35SDave Jiang *spad_addr = reg_addr + reg + (idx << 2); 293f6e51c35SDave Jiang dev_dbg(&ndev->ntb.pdev->dev, "Peer spad addr %llx\n", 294f6e51c35SDave Jiang *spad_addr); 295f6e51c35SDave Jiang } 296f6e51c35SDave Jiang 297f6e51c35SDave Jiang return 0; 298f6e51c35SDave Jiang } 299f6e51c35SDave Jiang 300f6e51c35SDave Jiang static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx, 301f6e51c35SDave Jiang void __iomem *mmio) 302f6e51c35SDave Jiang { 303f6e51c35SDave Jiang if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD)) 304f6e51c35SDave Jiang pr_warn_once("%s: NTB unsafe scratchpad access", __func__); 305f6e51c35SDave Jiang 306f6e51c35SDave Jiang if (idx < 0 || idx >= ndev->spad_count) 307f6e51c35SDave Jiang return 0; 308f6e51c35SDave Jiang 309f6e51c35SDave Jiang return ioread32(mmio + (idx << 2)); 310f6e51c35SDave Jiang } 311f6e51c35SDave Jiang 312f6e51c35SDave Jiang static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val, 313f6e51c35SDave Jiang void __iomem *mmio) 314f6e51c35SDave Jiang { 315f6e51c35SDave Jiang if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD)) 316f6e51c35SDave Jiang pr_warn_once("%s: NTB unsafe scratchpad access", __func__); 317f6e51c35SDave Jiang 318f6e51c35SDave Jiang if (idx < 0 || idx >= ndev->spad_count) 319f6e51c35SDave Jiang return -EINVAL; 320f6e51c35SDave Jiang 321f6e51c35SDave Jiang iowrite32(val, mmio + (idx << 2)); 322f6e51c35SDave Jiang 323f6e51c35SDave Jiang return 0; 324f6e51c35SDave Jiang } 325f6e51c35SDave Jiang 326f6e51c35SDave Jiang static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec) 327f6e51c35SDave Jiang { 328f6e51c35SDave Jiang u64 vec_mask; 329f6e51c35SDave Jiang 330f6e51c35SDave Jiang vec_mask = ndev_vec_mask(ndev, vec); 331f6e51c35SDave Jiang 332f6e51c35SDave Jiang if ((ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) && (vec == 31)) 333f6e51c35SDave Jiang vec_mask |= ndev->db_link_mask; 334f6e51c35SDave Jiang 335f6e51c35SDave Jiang dev_dbg(&ndev->ntb.pdev->dev, "vec %d vec_mask %llx\n", vec, vec_mask); 336f6e51c35SDave Jiang 337f6e51c35SDave Jiang ndev->last_ts = jiffies; 338f6e51c35SDave Jiang 339f6e51c35SDave Jiang if (vec_mask & ndev->db_link_mask) { 340f6e51c35SDave Jiang if (ndev->reg->poll_link(ndev)) 341f6e51c35SDave Jiang ntb_link_event(&ndev->ntb); 342f6e51c35SDave Jiang } 343f6e51c35SDave Jiang 344f6e51c35SDave Jiang if (vec_mask & ndev->db_valid_mask) 345f6e51c35SDave Jiang ntb_db_event(&ndev->ntb, vec); 346f6e51c35SDave Jiang 347f6e51c35SDave Jiang return IRQ_HANDLED; 348f6e51c35SDave Jiang } 349f6e51c35SDave Jiang 350f6e51c35SDave Jiang static irqreturn_t ndev_vec_isr(int irq, void *dev) 351f6e51c35SDave Jiang { 352f6e51c35SDave Jiang struct intel_ntb_vec *nvec = dev; 353f6e51c35SDave Jiang 354f6e51c35SDave Jiang dev_dbg(&nvec->ndev->ntb.pdev->dev, "irq: %d nvec->num: %d\n", 355f6e51c35SDave Jiang irq, nvec->num); 356f6e51c35SDave Jiang 357f6e51c35SDave Jiang return ndev_interrupt(nvec->ndev, nvec->num); 358f6e51c35SDave Jiang } 359f6e51c35SDave Jiang 360f6e51c35SDave Jiang static irqreturn_t ndev_irq_isr(int irq, void *dev) 361f6e51c35SDave Jiang { 362f6e51c35SDave Jiang struct intel_ntb_dev *ndev = dev; 363f6e51c35SDave Jiang 364f6e51c35SDave Jiang return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq); 365f6e51c35SDave Jiang } 366f6e51c35SDave Jiang 367f6e51c35SDave Jiang int ndev_init_isr(struct intel_ntb_dev *ndev, 368f6e51c35SDave Jiang int msix_min, int msix_max, 369f6e51c35SDave Jiang int msix_shift, int total_shift) 370f6e51c35SDave Jiang { 371f6e51c35SDave Jiang struct pci_dev *pdev; 372f6e51c35SDave Jiang int rc, i, msix_count, node; 373f6e51c35SDave Jiang 374f6e51c35SDave Jiang pdev = ndev->ntb.pdev; 375f6e51c35SDave Jiang 376f6e51c35SDave Jiang node = dev_to_node(&pdev->dev); 377f6e51c35SDave Jiang 378f6e51c35SDave Jiang /* Mask all doorbell interrupts */ 379f6e51c35SDave Jiang ndev->db_mask = ndev->db_valid_mask; 380f6e51c35SDave Jiang ndev->reg->db_iowrite(ndev->db_mask, 381f6e51c35SDave Jiang ndev->self_mmio + 382f6e51c35SDave Jiang ndev->self_reg->db_mask); 383f6e51c35SDave Jiang 384f6e51c35SDave Jiang /* Try to set up msix irq */ 385f6e51c35SDave Jiang 386f6e51c35SDave Jiang ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec), 387f6e51c35SDave Jiang GFP_KERNEL, node); 388f6e51c35SDave Jiang if (!ndev->vec) 389f6e51c35SDave Jiang goto err_msix_vec_alloc; 390f6e51c35SDave Jiang 391f6e51c35SDave Jiang ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix), 392f6e51c35SDave Jiang GFP_KERNEL, node); 393f6e51c35SDave Jiang if (!ndev->msix) 394f6e51c35SDave Jiang goto err_msix_alloc; 395f6e51c35SDave Jiang 396f6e51c35SDave Jiang for (i = 0; i < msix_max; ++i) 397f6e51c35SDave Jiang ndev->msix[i].entry = i; 398f6e51c35SDave Jiang 399f6e51c35SDave Jiang msix_count = pci_enable_msix_range(pdev, ndev->msix, 400f6e51c35SDave Jiang msix_min, msix_max); 401f6e51c35SDave Jiang if (msix_count < 0) 402f6e51c35SDave Jiang goto err_msix_enable; 403f6e51c35SDave Jiang 404f6e51c35SDave Jiang for (i = 0; i < msix_count; ++i) { 405f6e51c35SDave Jiang ndev->vec[i].ndev = ndev; 406f6e51c35SDave Jiang ndev->vec[i].num = i; 407f6e51c35SDave Jiang rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0, 408f6e51c35SDave Jiang "ndev_vec_isr", &ndev->vec[i]); 409f6e51c35SDave Jiang if (rc) 410f6e51c35SDave Jiang goto err_msix_request; 411f6e51c35SDave Jiang } 412f6e51c35SDave Jiang 413f6e51c35SDave Jiang dev_dbg(&pdev->dev, "Using %d msix interrupts\n", msix_count); 414f6e51c35SDave Jiang ndev->db_vec_count = msix_count; 415f6e51c35SDave Jiang ndev->db_vec_shift = msix_shift; 416f6e51c35SDave Jiang return 0; 417f6e51c35SDave Jiang 418f6e51c35SDave Jiang err_msix_request: 419f6e51c35SDave Jiang while (i-- > 0) 420f6e51c35SDave Jiang free_irq(ndev->msix[i].vector, &ndev->vec[i]); 421f6e51c35SDave Jiang pci_disable_msix(pdev); 422f6e51c35SDave Jiang err_msix_enable: 423f6e51c35SDave Jiang kfree(ndev->msix); 424f6e51c35SDave Jiang err_msix_alloc: 425f6e51c35SDave Jiang kfree(ndev->vec); 426f6e51c35SDave Jiang err_msix_vec_alloc: 427f6e51c35SDave Jiang ndev->msix = NULL; 428f6e51c35SDave Jiang ndev->vec = NULL; 429f6e51c35SDave Jiang 430f6e51c35SDave Jiang /* Try to set up msi irq */ 431f6e51c35SDave Jiang 432f6e51c35SDave Jiang rc = pci_enable_msi(pdev); 433f6e51c35SDave Jiang if (rc) 434f6e51c35SDave Jiang goto err_msi_enable; 435f6e51c35SDave Jiang 436f6e51c35SDave Jiang rc = request_irq(pdev->irq, ndev_irq_isr, 0, 437f6e51c35SDave Jiang "ndev_irq_isr", ndev); 438f6e51c35SDave Jiang if (rc) 439f6e51c35SDave Jiang goto err_msi_request; 440f6e51c35SDave Jiang 441f6e51c35SDave Jiang dev_dbg(&pdev->dev, "Using msi interrupts\n"); 442f6e51c35SDave Jiang ndev->db_vec_count = 1; 443f6e51c35SDave Jiang ndev->db_vec_shift = total_shift; 444f6e51c35SDave Jiang return 0; 445f6e51c35SDave Jiang 446f6e51c35SDave Jiang err_msi_request: 447f6e51c35SDave Jiang pci_disable_msi(pdev); 448f6e51c35SDave Jiang err_msi_enable: 449f6e51c35SDave Jiang 450f6e51c35SDave Jiang /* Try to set up intx irq */ 451f6e51c35SDave Jiang 452f6e51c35SDave Jiang pci_intx(pdev, 1); 453f6e51c35SDave Jiang 454f6e51c35SDave Jiang rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED, 455f6e51c35SDave Jiang "ndev_irq_isr", ndev); 456f6e51c35SDave Jiang if (rc) 457f6e51c35SDave Jiang goto err_intx_request; 458f6e51c35SDave Jiang 459f6e51c35SDave Jiang dev_dbg(&pdev->dev, "Using intx interrupts\n"); 460f6e51c35SDave Jiang ndev->db_vec_count = 1; 461f6e51c35SDave Jiang ndev->db_vec_shift = total_shift; 462f6e51c35SDave Jiang return 0; 463f6e51c35SDave Jiang 464f6e51c35SDave Jiang err_intx_request: 465f6e51c35SDave Jiang return rc; 466f6e51c35SDave Jiang } 467f6e51c35SDave Jiang 468f6e51c35SDave Jiang static void ndev_deinit_isr(struct intel_ntb_dev *ndev) 469f6e51c35SDave Jiang { 470f6e51c35SDave Jiang struct pci_dev *pdev; 471f6e51c35SDave Jiang int i; 472f6e51c35SDave Jiang 473f6e51c35SDave Jiang pdev = ndev->ntb.pdev; 474f6e51c35SDave Jiang 475f6e51c35SDave Jiang /* Mask all doorbell interrupts */ 476f6e51c35SDave Jiang ndev->db_mask = ndev->db_valid_mask; 477f6e51c35SDave Jiang ndev->reg->db_iowrite(ndev->db_mask, 478f6e51c35SDave Jiang ndev->self_mmio + 479f6e51c35SDave Jiang ndev->self_reg->db_mask); 480f6e51c35SDave Jiang 481f6e51c35SDave Jiang if (ndev->msix) { 482f6e51c35SDave Jiang i = ndev->db_vec_count; 483f6e51c35SDave Jiang while (i--) 484f6e51c35SDave Jiang free_irq(ndev->msix[i].vector, &ndev->vec[i]); 485f6e51c35SDave Jiang pci_disable_msix(pdev); 486f6e51c35SDave Jiang kfree(ndev->msix); 487f6e51c35SDave Jiang kfree(ndev->vec); 488f6e51c35SDave Jiang } else { 489f6e51c35SDave Jiang free_irq(pdev->irq, ndev); 490f6e51c35SDave Jiang if (pci_dev_msi_enabled(pdev)) 491f6e51c35SDave Jiang pci_disable_msi(pdev); 492f6e51c35SDave Jiang } 493f6e51c35SDave Jiang } 494f6e51c35SDave Jiang 495f6e51c35SDave Jiang static ssize_t ndev_ntb_debugfs_read(struct file *filp, char __user *ubuf, 496f6e51c35SDave Jiang size_t count, loff_t *offp) 497f6e51c35SDave Jiang { 498f6e51c35SDave Jiang struct intel_ntb_dev *ndev; 499f6e51c35SDave Jiang struct pci_dev *pdev; 500f6e51c35SDave Jiang void __iomem *mmio; 501f6e51c35SDave Jiang char *buf; 502f6e51c35SDave Jiang size_t buf_size; 503f6e51c35SDave Jiang ssize_t ret, off; 504f6e51c35SDave Jiang union { u64 v64; u32 v32; u16 v16; u8 v8; } u; 505f6e51c35SDave Jiang 506f6e51c35SDave Jiang ndev = filp->private_data; 507f6e51c35SDave Jiang pdev = ndev->ntb.pdev; 508f6e51c35SDave Jiang mmio = ndev->self_mmio; 509f6e51c35SDave Jiang 510f6e51c35SDave Jiang buf_size = min(count, 0x800ul); 511f6e51c35SDave Jiang 512f6e51c35SDave Jiang buf = kmalloc(buf_size, GFP_KERNEL); 513f6e51c35SDave Jiang if (!buf) 514f6e51c35SDave Jiang return -ENOMEM; 515f6e51c35SDave Jiang 516f6e51c35SDave Jiang off = 0; 517f6e51c35SDave Jiang 518f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 519f6e51c35SDave Jiang "NTB Device Information:\n"); 520f6e51c35SDave Jiang 521f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 522f6e51c35SDave Jiang "Connection Topology -\t%s\n", 523f6e51c35SDave Jiang ntb_topo_string(ndev->ntb.topo)); 524f6e51c35SDave Jiang 525f6e51c35SDave Jiang if (ndev->b2b_idx != UINT_MAX) { 526f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 527f6e51c35SDave Jiang "B2B MW Idx -\t\t%u\n", ndev->b2b_idx); 528f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 529f6e51c35SDave Jiang "B2B Offset -\t\t%#lx\n", ndev->b2b_off); 530f6e51c35SDave Jiang } 531f6e51c35SDave Jiang 532f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 533f6e51c35SDave Jiang "BAR4 Split -\t\t%s\n", 534f6e51c35SDave Jiang ndev->bar4_split ? "yes" : "no"); 535f6e51c35SDave Jiang 536f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 537f6e51c35SDave Jiang "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl); 538f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 539f6e51c35SDave Jiang "LNK STA -\t\t%#06x\n", ndev->lnk_sta); 540f6e51c35SDave Jiang 541f6e51c35SDave Jiang if (!ndev->reg->link_is_up(ndev)) { 542f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 543f6e51c35SDave Jiang "Link Status -\t\tDown\n"); 544f6e51c35SDave Jiang } else { 545f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 546f6e51c35SDave Jiang "Link Status -\t\tUp\n"); 547f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 548f6e51c35SDave Jiang "Link Speed -\t\tPCI-E Gen %u\n", 549f6e51c35SDave Jiang NTB_LNK_STA_SPEED(ndev->lnk_sta)); 550f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 551f6e51c35SDave Jiang "Link Width -\t\tx%u\n", 552f6e51c35SDave Jiang NTB_LNK_STA_WIDTH(ndev->lnk_sta)); 553f6e51c35SDave Jiang } 554f6e51c35SDave Jiang 555f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 556f6e51c35SDave Jiang "Memory Window Count -\t%u\n", ndev->mw_count); 557f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 558f6e51c35SDave Jiang "Scratchpad Count -\t%u\n", ndev->spad_count); 559f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 560f6e51c35SDave Jiang "Doorbell Count -\t%u\n", ndev->db_count); 561f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 562f6e51c35SDave Jiang "Doorbell Vector Count -\t%u\n", ndev->db_vec_count); 563f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 564f6e51c35SDave Jiang "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift); 565f6e51c35SDave Jiang 566f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 567f6e51c35SDave Jiang "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask); 568f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 569f6e51c35SDave Jiang "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask); 570f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 571f6e51c35SDave Jiang "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask); 572f6e51c35SDave Jiang 573f6e51c35SDave Jiang u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask); 574f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 575f6e51c35SDave Jiang "Doorbell Mask -\t\t%#llx\n", u.v64); 576f6e51c35SDave Jiang 577f6e51c35SDave Jiang u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell); 578f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 579f6e51c35SDave Jiang "Doorbell Bell -\t\t%#llx\n", u.v64); 580f6e51c35SDave Jiang 581f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 582f6e51c35SDave Jiang "\nNTB Window Size:\n"); 583f6e51c35SDave Jiang 584f6e51c35SDave Jiang pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &u.v8); 585f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 586f6e51c35SDave Jiang "PBAR23SZ %hhu\n", u.v8); 587f6e51c35SDave Jiang if (!ndev->bar4_split) { 588f6e51c35SDave Jiang pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &u.v8); 589f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 590f6e51c35SDave Jiang "PBAR45SZ %hhu\n", u.v8); 591f6e51c35SDave Jiang } else { 592f6e51c35SDave Jiang pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &u.v8); 593f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 594f6e51c35SDave Jiang "PBAR4SZ %hhu\n", u.v8); 595f6e51c35SDave Jiang pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &u.v8); 596f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 597f6e51c35SDave Jiang "PBAR5SZ %hhu\n", u.v8); 598f6e51c35SDave Jiang } 599f6e51c35SDave Jiang 600f6e51c35SDave Jiang pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &u.v8); 601f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 602f6e51c35SDave Jiang "SBAR23SZ %hhu\n", u.v8); 603f6e51c35SDave Jiang if (!ndev->bar4_split) { 604f6e51c35SDave Jiang pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &u.v8); 605f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 606f6e51c35SDave Jiang "SBAR45SZ %hhu\n", u.v8); 607f6e51c35SDave Jiang } else { 608f6e51c35SDave Jiang pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &u.v8); 609f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 610f6e51c35SDave Jiang "SBAR4SZ %hhu\n", u.v8); 611f6e51c35SDave Jiang pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &u.v8); 612f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 613f6e51c35SDave Jiang "SBAR5SZ %hhu\n", u.v8); 614f6e51c35SDave Jiang } 615f6e51c35SDave Jiang 616f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 617f6e51c35SDave Jiang "\nNTB Incoming XLAT:\n"); 618f6e51c35SDave Jiang 619f6e51c35SDave Jiang u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2)); 620f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 621f6e51c35SDave Jiang "XLAT23 -\t\t%#018llx\n", u.v64); 622f6e51c35SDave Jiang 623f6e51c35SDave Jiang if (ndev->bar4_split) { 624f6e51c35SDave Jiang u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4)); 625f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 626f6e51c35SDave Jiang "XLAT4 -\t\t\t%#06x\n", u.v32); 627f6e51c35SDave Jiang 628f6e51c35SDave Jiang u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 5)); 629f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 630f6e51c35SDave Jiang "XLAT5 -\t\t\t%#06x\n", u.v32); 631f6e51c35SDave Jiang } else { 632f6e51c35SDave Jiang u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4)); 633f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 634f6e51c35SDave Jiang "XLAT45 -\t\t%#018llx\n", u.v64); 635f6e51c35SDave Jiang } 636f6e51c35SDave Jiang 637f6e51c35SDave Jiang u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2)); 638f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 639f6e51c35SDave Jiang "LMT23 -\t\t\t%#018llx\n", u.v64); 640f6e51c35SDave Jiang 641f6e51c35SDave Jiang if (ndev->bar4_split) { 642f6e51c35SDave Jiang u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4)); 643f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 644f6e51c35SDave Jiang "LMT4 -\t\t\t%#06x\n", u.v32); 645f6e51c35SDave Jiang u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 5)); 646f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 647f6e51c35SDave Jiang "LMT5 -\t\t\t%#06x\n", u.v32); 648f6e51c35SDave Jiang } else { 649f6e51c35SDave Jiang u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4)); 650f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 651f6e51c35SDave Jiang "LMT45 -\t\t\t%#018llx\n", u.v64); 652f6e51c35SDave Jiang } 653f6e51c35SDave Jiang 654f6e51c35SDave Jiang if (pdev_is_xeon(pdev)) { 655f6e51c35SDave Jiang if (ntb_topo_is_b2b(ndev->ntb.topo)) { 656f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 657f6e51c35SDave Jiang "\nNTB Outgoing B2B XLAT:\n"); 658f6e51c35SDave Jiang 659f6e51c35SDave Jiang u.v64 = ioread64(mmio + XEON_PBAR23XLAT_OFFSET); 660f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 661f6e51c35SDave Jiang "B2B XLAT23 -\t\t%#018llx\n", u.v64); 662f6e51c35SDave Jiang 663f6e51c35SDave Jiang if (ndev->bar4_split) { 664f6e51c35SDave Jiang u.v32 = ioread32(mmio + XEON_PBAR4XLAT_OFFSET); 665f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 666f6e51c35SDave Jiang "B2B XLAT4 -\t\t%#06x\n", 667f6e51c35SDave Jiang u.v32); 668f6e51c35SDave Jiang u.v32 = ioread32(mmio + XEON_PBAR5XLAT_OFFSET); 669f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 670f6e51c35SDave Jiang "B2B XLAT5 -\t\t%#06x\n", 671f6e51c35SDave Jiang u.v32); 672f6e51c35SDave Jiang } else { 673f6e51c35SDave Jiang u.v64 = ioread64(mmio + XEON_PBAR45XLAT_OFFSET); 674f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 675f6e51c35SDave Jiang "B2B XLAT45 -\t\t%#018llx\n", 676f6e51c35SDave Jiang u.v64); 677f6e51c35SDave Jiang } 678f6e51c35SDave Jiang 679f6e51c35SDave Jiang u.v64 = ioread64(mmio + XEON_PBAR23LMT_OFFSET); 680f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 681f6e51c35SDave Jiang "B2B LMT23 -\t\t%#018llx\n", u.v64); 682f6e51c35SDave Jiang 683f6e51c35SDave Jiang if (ndev->bar4_split) { 684f6e51c35SDave Jiang u.v32 = ioread32(mmio + XEON_PBAR4LMT_OFFSET); 685f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 686f6e51c35SDave Jiang "B2B LMT4 -\t\t%#06x\n", 687f6e51c35SDave Jiang u.v32); 688f6e51c35SDave Jiang u.v32 = ioread32(mmio + XEON_PBAR5LMT_OFFSET); 689f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 690f6e51c35SDave Jiang "B2B LMT5 -\t\t%#06x\n", 691f6e51c35SDave Jiang u.v32); 692f6e51c35SDave Jiang } else { 693f6e51c35SDave Jiang u.v64 = ioread64(mmio + XEON_PBAR45LMT_OFFSET); 694f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 695f6e51c35SDave Jiang "B2B LMT45 -\t\t%#018llx\n", 696f6e51c35SDave Jiang u.v64); 697f6e51c35SDave Jiang } 698f6e51c35SDave Jiang 699f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 700f6e51c35SDave Jiang "\nNTB Secondary BAR:\n"); 701f6e51c35SDave Jiang 702f6e51c35SDave Jiang u.v64 = ioread64(mmio + XEON_SBAR0BASE_OFFSET); 703f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 704f6e51c35SDave Jiang "SBAR01 -\t\t%#018llx\n", u.v64); 705f6e51c35SDave Jiang 706f6e51c35SDave Jiang u.v64 = ioread64(mmio + XEON_SBAR23BASE_OFFSET); 707f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 708f6e51c35SDave Jiang "SBAR23 -\t\t%#018llx\n", u.v64); 709f6e51c35SDave Jiang 710f6e51c35SDave Jiang if (ndev->bar4_split) { 711f6e51c35SDave Jiang u.v32 = ioread32(mmio + XEON_SBAR4BASE_OFFSET); 712f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 713f6e51c35SDave Jiang "SBAR4 -\t\t\t%#06x\n", u.v32); 714f6e51c35SDave Jiang u.v32 = ioread32(mmio + XEON_SBAR5BASE_OFFSET); 715f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 716f6e51c35SDave Jiang "SBAR5 -\t\t\t%#06x\n", u.v32); 717f6e51c35SDave Jiang } else { 718f6e51c35SDave Jiang u.v64 = ioread64(mmio + XEON_SBAR45BASE_OFFSET); 719f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 720f6e51c35SDave Jiang "SBAR45 -\t\t%#018llx\n", 721f6e51c35SDave Jiang u.v64); 722f6e51c35SDave Jiang } 723f6e51c35SDave Jiang } 724f6e51c35SDave Jiang 725f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 726f6e51c35SDave Jiang "\nXEON NTB Statistics:\n"); 727f6e51c35SDave Jiang 728f6e51c35SDave Jiang u.v16 = ioread16(mmio + XEON_USMEMMISS_OFFSET); 729f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 730f6e51c35SDave Jiang "Upstream Memory Miss -\t%u\n", u.v16); 731f6e51c35SDave Jiang 732f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 733f6e51c35SDave Jiang "\nXEON NTB Hardware Errors:\n"); 734f6e51c35SDave Jiang 735f6e51c35SDave Jiang if (!pci_read_config_word(pdev, 736f6e51c35SDave Jiang XEON_DEVSTS_OFFSET, &u.v16)) 737f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 738f6e51c35SDave Jiang "DEVSTS -\t\t%#06x\n", u.v16); 739f6e51c35SDave Jiang 740f6e51c35SDave Jiang if (!pci_read_config_word(pdev, 741f6e51c35SDave Jiang XEON_LINK_STATUS_OFFSET, &u.v16)) 742f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 743f6e51c35SDave Jiang "LNKSTS -\t\t%#06x\n", u.v16); 744f6e51c35SDave Jiang 745f6e51c35SDave Jiang if (!pci_read_config_dword(pdev, 746f6e51c35SDave Jiang XEON_UNCERRSTS_OFFSET, &u.v32)) 747f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 748f6e51c35SDave Jiang "UNCERRSTS -\t\t%#06x\n", u.v32); 749f6e51c35SDave Jiang 750f6e51c35SDave Jiang if (!pci_read_config_dword(pdev, 751f6e51c35SDave Jiang XEON_CORERRSTS_OFFSET, &u.v32)) 752f6e51c35SDave Jiang off += scnprintf(buf + off, buf_size - off, 753f6e51c35SDave Jiang "CORERRSTS -\t\t%#06x\n", u.v32); 754f6e51c35SDave Jiang } 755f6e51c35SDave Jiang 756f6e51c35SDave Jiang ret = simple_read_from_buffer(ubuf, count, offp, buf, off); 757f6e51c35SDave Jiang kfree(buf); 758f6e51c35SDave Jiang return ret; 759f6e51c35SDave Jiang } 760f6e51c35SDave Jiang 761f6e51c35SDave Jiang static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf, 762f6e51c35SDave Jiang size_t count, loff_t *offp) 763f6e51c35SDave Jiang { 764f6e51c35SDave Jiang struct intel_ntb_dev *ndev = filp->private_data; 765f6e51c35SDave Jiang 766f6e51c35SDave Jiang if (pdev_is_xeon(ndev->ntb.pdev)) 767f6e51c35SDave Jiang return ndev_ntb_debugfs_read(filp, ubuf, count, offp); 768f6e51c35SDave Jiang else if (pdev_is_skx_xeon(ndev->ntb.pdev)) 769f6e51c35SDave Jiang return ndev_ntb3_debugfs_read(filp, ubuf, count, offp); 770f6e51c35SDave Jiang 771f6e51c35SDave Jiang return -ENXIO; 772f6e51c35SDave Jiang } 773f6e51c35SDave Jiang 774f6e51c35SDave Jiang static void ndev_init_debugfs(struct intel_ntb_dev *ndev) 775f6e51c35SDave Jiang { 776f6e51c35SDave Jiang if (!debugfs_dir) { 777f6e51c35SDave Jiang ndev->debugfs_dir = NULL; 778f6e51c35SDave Jiang ndev->debugfs_info = NULL; 779f6e51c35SDave Jiang } else { 780f6e51c35SDave Jiang ndev->debugfs_dir = 781f6e51c35SDave Jiang debugfs_create_dir(pci_name(ndev->ntb.pdev), 782f6e51c35SDave Jiang debugfs_dir); 783f6e51c35SDave Jiang if (!ndev->debugfs_dir) 784f6e51c35SDave Jiang ndev->debugfs_info = NULL; 785f6e51c35SDave Jiang else 786f6e51c35SDave Jiang ndev->debugfs_info = 787f6e51c35SDave Jiang debugfs_create_file("info", S_IRUSR, 788f6e51c35SDave Jiang ndev->debugfs_dir, ndev, 789f6e51c35SDave Jiang &intel_ntb_debugfs_info); 790f6e51c35SDave Jiang } 791f6e51c35SDave Jiang } 792f6e51c35SDave Jiang 793f6e51c35SDave Jiang static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev) 794f6e51c35SDave Jiang { 795f6e51c35SDave Jiang debugfs_remove_recursive(ndev->debugfs_dir); 796f6e51c35SDave Jiang } 797f6e51c35SDave Jiang 798f6e51c35SDave Jiang int intel_ntb_mw_count(struct ntb_dev *ntb, int pidx) 799f6e51c35SDave Jiang { 800f6e51c35SDave Jiang if (pidx != NTB_DEF_PEER_IDX) 801f6e51c35SDave Jiang return -EINVAL; 802f6e51c35SDave Jiang 803f6e51c35SDave Jiang return ntb_ndev(ntb)->mw_count; 804f6e51c35SDave Jiang } 805f6e51c35SDave Jiang 806f6e51c35SDave Jiang int intel_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx, 807f6e51c35SDave Jiang resource_size_t *addr_align, 808f6e51c35SDave Jiang resource_size_t *size_align, 809f6e51c35SDave Jiang resource_size_t *size_max) 810f6e51c35SDave Jiang { 811f6e51c35SDave Jiang struct intel_ntb_dev *ndev = ntb_ndev(ntb); 812f6e51c35SDave Jiang resource_size_t bar_size, mw_size; 813f6e51c35SDave Jiang int bar; 814f6e51c35SDave Jiang 815f6e51c35SDave Jiang if (pidx != NTB_DEF_PEER_IDX) 816f6e51c35SDave Jiang return -EINVAL; 817f6e51c35SDave Jiang 818f6e51c35SDave Jiang if (idx >= ndev->b2b_idx && !ndev->b2b_off) 819f6e51c35SDave Jiang idx += 1; 820f6e51c35SDave Jiang 821f6e51c35SDave Jiang bar = ndev_mw_to_bar(ndev, idx); 822f6e51c35SDave Jiang if (bar < 0) 823f6e51c35SDave Jiang return bar; 824f6e51c35SDave Jiang 825f6e51c35SDave Jiang bar_size = pci_resource_len(ndev->ntb.pdev, bar); 826f6e51c35SDave Jiang 827f6e51c35SDave Jiang if (idx == ndev->b2b_idx) 828f6e51c35SDave Jiang mw_size = bar_size - ndev->b2b_off; 829f6e51c35SDave Jiang else 830f6e51c35SDave Jiang mw_size = bar_size; 831f6e51c35SDave Jiang 832f6e51c35SDave Jiang if (addr_align) 833f6e51c35SDave Jiang *addr_align = pci_resource_len(ndev->ntb.pdev, bar); 834f6e51c35SDave Jiang 835f6e51c35SDave Jiang if (size_align) 836f6e51c35SDave Jiang *size_align = 1; 837f6e51c35SDave Jiang 838f6e51c35SDave Jiang if (size_max) 839f6e51c35SDave Jiang *size_max = mw_size; 840f6e51c35SDave Jiang 841f6e51c35SDave Jiang return 0; 842f6e51c35SDave Jiang } 843f6e51c35SDave Jiang 844f6e51c35SDave Jiang static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, 845f6e51c35SDave Jiang dma_addr_t addr, resource_size_t size) 846f6e51c35SDave Jiang { 847f6e51c35SDave Jiang struct intel_ntb_dev *ndev = ntb_ndev(ntb); 848f6e51c35SDave Jiang unsigned long base_reg, xlat_reg, limit_reg; 849f6e51c35SDave Jiang resource_size_t bar_size, mw_size; 850f6e51c35SDave Jiang void __iomem *mmio; 851f6e51c35SDave Jiang u64 base, limit, reg_val; 852f6e51c35SDave Jiang int bar; 853f6e51c35SDave Jiang 854f6e51c35SDave Jiang if (pidx != NTB_DEF_PEER_IDX) 855f6e51c35SDave Jiang return -EINVAL; 856f6e51c35SDave Jiang 857f6e51c35SDave Jiang if (idx >= ndev->b2b_idx && !ndev->b2b_off) 858f6e51c35SDave Jiang idx += 1; 859f6e51c35SDave Jiang 860f6e51c35SDave Jiang bar = ndev_mw_to_bar(ndev, idx); 861f6e51c35SDave Jiang if (bar < 0) 862f6e51c35SDave Jiang return bar; 863f6e51c35SDave Jiang 864f6e51c35SDave Jiang bar_size = pci_resource_len(ndev->ntb.pdev, bar); 865f6e51c35SDave Jiang 866f6e51c35SDave Jiang if (idx == ndev->b2b_idx) 867f6e51c35SDave Jiang mw_size = bar_size - ndev->b2b_off; 868f6e51c35SDave Jiang else 869f6e51c35SDave Jiang mw_size = bar_size; 870f6e51c35SDave Jiang 871f6e51c35SDave Jiang /* hardware requires that addr is aligned to bar size */ 872f6e51c35SDave Jiang if (addr & (bar_size - 1)) 873f6e51c35SDave Jiang return -EINVAL; 874f6e51c35SDave Jiang 875f6e51c35SDave Jiang /* make sure the range fits in the usable mw size */ 876f6e51c35SDave Jiang if (size > mw_size) 877f6e51c35SDave Jiang return -EINVAL; 878f6e51c35SDave Jiang 879f6e51c35SDave Jiang mmio = ndev->self_mmio; 880f6e51c35SDave Jiang base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar); 881f6e51c35SDave Jiang xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar); 882f6e51c35SDave Jiang limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar); 883f6e51c35SDave Jiang 884f6e51c35SDave Jiang if (bar < 4 || !ndev->bar4_split) { 885f6e51c35SDave Jiang base = ioread64(mmio + base_reg) & NTB_BAR_MASK_64; 886f6e51c35SDave Jiang 887f6e51c35SDave Jiang /* Set the limit if supported, if size is not mw_size */ 888f6e51c35SDave Jiang if (limit_reg && size != mw_size) 889f6e51c35SDave Jiang limit = base + size; 890f6e51c35SDave Jiang else 891f6e51c35SDave Jiang limit = 0; 892f6e51c35SDave Jiang 893f6e51c35SDave Jiang /* set and verify setting the translation address */ 894f6e51c35SDave Jiang iowrite64(addr, mmio + xlat_reg); 895f6e51c35SDave Jiang reg_val = ioread64(mmio + xlat_reg); 896f6e51c35SDave Jiang if (reg_val != addr) { 897f6e51c35SDave Jiang iowrite64(0, mmio + xlat_reg); 898f6e51c35SDave Jiang return -EIO; 899f6e51c35SDave Jiang } 900f6e51c35SDave Jiang 901f6e51c35SDave Jiang /* set and verify setting the limit */ 902f6e51c35SDave Jiang iowrite64(limit, mmio + limit_reg); 903f6e51c35SDave Jiang reg_val = ioread64(mmio + limit_reg); 904f6e51c35SDave Jiang if (reg_val != limit) { 905f6e51c35SDave Jiang iowrite64(base, mmio + limit_reg); 906f6e51c35SDave Jiang iowrite64(0, mmio + xlat_reg); 907f6e51c35SDave Jiang return -EIO; 908f6e51c35SDave Jiang } 909f6e51c35SDave Jiang } else { 910f6e51c35SDave Jiang /* split bar addr range must all be 32 bit */ 911f6e51c35SDave Jiang if (addr & (~0ull << 32)) 912f6e51c35SDave Jiang return -EINVAL; 913f6e51c35SDave Jiang if ((addr + size) & (~0ull << 32)) 914f6e51c35SDave Jiang return -EINVAL; 915f6e51c35SDave Jiang 916f6e51c35SDave Jiang base = ioread32(mmio + base_reg) & NTB_BAR_MASK_32; 917f6e51c35SDave Jiang 918f6e51c35SDave Jiang /* Set the limit if supported, if size is not mw_size */ 919f6e51c35SDave Jiang if (limit_reg && size != mw_size) 920f6e51c35SDave Jiang limit = base + size; 921f6e51c35SDave Jiang else 922f6e51c35SDave Jiang limit = 0; 923f6e51c35SDave Jiang 924f6e51c35SDave Jiang /* set and verify setting the translation address */ 925f6e51c35SDave Jiang iowrite32(addr, mmio + xlat_reg); 926f6e51c35SDave Jiang reg_val = ioread32(mmio + xlat_reg); 927f6e51c35SDave Jiang if (reg_val != addr) { 928f6e51c35SDave Jiang iowrite32(0, mmio + xlat_reg); 929f6e51c35SDave Jiang return -EIO; 930f6e51c35SDave Jiang } 931f6e51c35SDave Jiang 932f6e51c35SDave Jiang /* set and verify setting the limit */ 933f6e51c35SDave Jiang iowrite32(limit, mmio + limit_reg); 934f6e51c35SDave Jiang reg_val = ioread32(mmio + limit_reg); 935f6e51c35SDave Jiang if (reg_val != limit) { 936f6e51c35SDave Jiang iowrite32(base, mmio + limit_reg); 937f6e51c35SDave Jiang iowrite32(0, mmio + xlat_reg); 938f6e51c35SDave Jiang return -EIO; 939f6e51c35SDave Jiang } 940f6e51c35SDave Jiang } 941f6e51c35SDave Jiang 942f6e51c35SDave Jiang return 0; 943f6e51c35SDave Jiang } 944f6e51c35SDave Jiang 945f6e51c35SDave Jiang u64 intel_ntb_link_is_up(struct ntb_dev *ntb, enum ntb_speed *speed, 946f6e51c35SDave Jiang enum ntb_width *width) 947f6e51c35SDave Jiang { 948f6e51c35SDave Jiang struct intel_ntb_dev *ndev = ntb_ndev(ntb); 949f6e51c35SDave Jiang 950f6e51c35SDave Jiang if (ndev->reg->link_is_up(ndev)) { 951f6e51c35SDave Jiang if (speed) 952f6e51c35SDave Jiang *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta); 953f6e51c35SDave Jiang if (width) 954f6e51c35SDave Jiang *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta); 955f6e51c35SDave Jiang return 1; 956f6e51c35SDave Jiang } else { 957f6e51c35SDave Jiang /* TODO MAYBE: is it possible to observe the link speed and 958f6e51c35SDave Jiang * width while link is training? */ 959f6e51c35SDave Jiang if (speed) 960f6e51c35SDave Jiang *speed = NTB_SPEED_NONE; 961f6e51c35SDave Jiang if (width) 962f6e51c35SDave Jiang *width = NTB_WIDTH_NONE; 963f6e51c35SDave Jiang return 0; 964f6e51c35SDave Jiang } 965f6e51c35SDave Jiang } 966f6e51c35SDave Jiang 967f6e51c35SDave Jiang static int intel_ntb_link_enable(struct ntb_dev *ntb, 968f6e51c35SDave Jiang enum ntb_speed max_speed, 969f6e51c35SDave Jiang enum ntb_width max_width) 970f6e51c35SDave Jiang { 971f6e51c35SDave Jiang struct intel_ntb_dev *ndev; 972f6e51c35SDave Jiang u32 ntb_ctl; 973f6e51c35SDave Jiang 974f6e51c35SDave Jiang ndev = container_of(ntb, struct intel_ntb_dev, ntb); 975f6e51c35SDave Jiang 976f6e51c35SDave Jiang if (ndev->ntb.topo == NTB_TOPO_SEC) 977f6e51c35SDave Jiang return -EINVAL; 978f6e51c35SDave Jiang 979f6e51c35SDave Jiang dev_dbg(&ntb->pdev->dev, 980f6e51c35SDave Jiang "Enabling link with max_speed %d max_width %d\n", 981f6e51c35SDave Jiang max_speed, max_width); 982f6e51c35SDave Jiang if (max_speed != NTB_SPEED_AUTO) 983f6e51c35SDave Jiang dev_dbg(&ntb->pdev->dev, "ignoring max_speed %d\n", max_speed); 984f6e51c35SDave Jiang if (max_width != NTB_WIDTH_AUTO) 985f6e51c35SDave Jiang dev_dbg(&ntb->pdev->dev, "ignoring max_width %d\n", max_width); 986f6e51c35SDave Jiang 987f6e51c35SDave Jiang ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl); 988f6e51c35SDave Jiang ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK); 989f6e51c35SDave Jiang ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP; 990f6e51c35SDave Jiang ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP; 991f6e51c35SDave Jiang if (ndev->bar4_split) 992f6e51c35SDave Jiang ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP; 993f6e51c35SDave Jiang iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl); 994f6e51c35SDave Jiang 995f6e51c35SDave Jiang return 0; 996f6e51c35SDave Jiang } 997f6e51c35SDave Jiang 998f6e51c35SDave Jiang int intel_ntb_link_disable(struct ntb_dev *ntb) 999f6e51c35SDave Jiang { 1000f6e51c35SDave Jiang struct intel_ntb_dev *ndev; 1001f6e51c35SDave Jiang u32 ntb_cntl; 1002f6e51c35SDave Jiang 1003f6e51c35SDave Jiang ndev = container_of(ntb, struct intel_ntb_dev, ntb); 1004f6e51c35SDave Jiang 1005f6e51c35SDave Jiang if (ndev->ntb.topo == NTB_TOPO_SEC) 1006f6e51c35SDave Jiang return -EINVAL; 1007f6e51c35SDave Jiang 1008f6e51c35SDave Jiang dev_dbg(&ntb->pdev->dev, "Disabling link\n"); 1009f6e51c35SDave Jiang 1010f6e51c35SDave Jiang /* Bring NTB link down */ 1011f6e51c35SDave Jiang ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl); 1012f6e51c35SDave Jiang ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP); 1013f6e51c35SDave Jiang ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP); 1014f6e51c35SDave Jiang if (ndev->bar4_split) 1015f6e51c35SDave Jiang ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP); 1016f6e51c35SDave Jiang ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK; 1017f6e51c35SDave Jiang iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl); 1018f6e51c35SDave Jiang 1019f6e51c35SDave Jiang return 0; 1020f6e51c35SDave Jiang } 1021f6e51c35SDave Jiang 1022f6e51c35SDave Jiang int intel_ntb_peer_mw_count(struct ntb_dev *ntb) 1023f6e51c35SDave Jiang { 1024f6e51c35SDave Jiang /* Numbers of inbound and outbound memory windows match */ 1025f6e51c35SDave Jiang return ntb_ndev(ntb)->mw_count; 1026f6e51c35SDave Jiang } 1027f6e51c35SDave Jiang 1028f6e51c35SDave Jiang int intel_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx, 1029f6e51c35SDave Jiang phys_addr_t *base, resource_size_t *size) 1030f6e51c35SDave Jiang { 1031f6e51c35SDave Jiang struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1032f6e51c35SDave Jiang int bar; 1033f6e51c35SDave Jiang 1034f6e51c35SDave Jiang if (idx >= ndev->b2b_idx && !ndev->b2b_off) 1035f6e51c35SDave Jiang idx += 1; 1036f6e51c35SDave Jiang 1037f6e51c35SDave Jiang bar = ndev_mw_to_bar(ndev, idx); 1038f6e51c35SDave Jiang if (bar < 0) 1039f6e51c35SDave Jiang return bar; 1040f6e51c35SDave Jiang 1041f6e51c35SDave Jiang if (base) 1042f6e51c35SDave Jiang *base = pci_resource_start(ndev->ntb.pdev, bar) + 1043f6e51c35SDave Jiang (idx == ndev->b2b_idx ? ndev->b2b_off : 0); 1044f6e51c35SDave Jiang 1045f6e51c35SDave Jiang if (size) 1046f6e51c35SDave Jiang *size = pci_resource_len(ndev->ntb.pdev, bar) - 1047f6e51c35SDave Jiang (idx == ndev->b2b_idx ? ndev->b2b_off : 0); 1048f6e51c35SDave Jiang 1049f6e51c35SDave Jiang return 0; 1050f6e51c35SDave Jiang } 1051f6e51c35SDave Jiang 1052f6e51c35SDave Jiang static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb) 1053f6e51c35SDave Jiang { 1054f6e51c35SDave Jiang return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB); 1055f6e51c35SDave Jiang } 1056f6e51c35SDave Jiang 1057f6e51c35SDave Jiang u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb) 1058f6e51c35SDave Jiang { 1059f6e51c35SDave Jiang return ntb_ndev(ntb)->db_valid_mask; 1060f6e51c35SDave Jiang } 1061f6e51c35SDave Jiang 1062f6e51c35SDave Jiang int intel_ntb_db_vector_count(struct ntb_dev *ntb) 1063f6e51c35SDave Jiang { 1064f6e51c35SDave Jiang struct intel_ntb_dev *ndev; 1065f6e51c35SDave Jiang 1066f6e51c35SDave Jiang ndev = container_of(ntb, struct intel_ntb_dev, ntb); 1067f6e51c35SDave Jiang 1068f6e51c35SDave Jiang return ndev->db_vec_count; 1069f6e51c35SDave Jiang } 1070f6e51c35SDave Jiang 1071f6e51c35SDave Jiang u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) 1072f6e51c35SDave Jiang { 1073f6e51c35SDave Jiang struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1074f6e51c35SDave Jiang 1075f6e51c35SDave Jiang if (db_vector < 0 || db_vector > ndev->db_vec_count) 1076f6e51c35SDave Jiang return 0; 1077f6e51c35SDave Jiang 1078f6e51c35SDave Jiang return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector); 1079f6e51c35SDave Jiang } 1080f6e51c35SDave Jiang 1081f6e51c35SDave Jiang static u64 intel_ntb_db_read(struct ntb_dev *ntb) 1082f6e51c35SDave Jiang { 1083f6e51c35SDave Jiang struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1084f6e51c35SDave Jiang 1085f6e51c35SDave Jiang return ndev_db_read(ndev, 1086f6e51c35SDave Jiang ndev->self_mmio + 1087f6e51c35SDave Jiang ndev->self_reg->db_bell); 1088f6e51c35SDave Jiang } 1089f6e51c35SDave Jiang 1090f6e51c35SDave Jiang static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) 1091f6e51c35SDave Jiang { 1092f6e51c35SDave Jiang struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1093f6e51c35SDave Jiang 1094f6e51c35SDave Jiang return ndev_db_write(ndev, db_bits, 1095f6e51c35SDave Jiang ndev->self_mmio + 1096f6e51c35SDave Jiang ndev->self_reg->db_bell); 1097f6e51c35SDave Jiang } 1098f6e51c35SDave Jiang 1099f6e51c35SDave Jiang int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) 1100f6e51c35SDave Jiang { 1101f6e51c35SDave Jiang struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1102f6e51c35SDave Jiang 1103f6e51c35SDave Jiang return ndev_db_set_mask(ndev, db_bits, 1104f6e51c35SDave Jiang ndev->self_mmio + 1105f6e51c35SDave Jiang ndev->self_reg->db_mask); 1106f6e51c35SDave Jiang } 1107f6e51c35SDave Jiang 1108f6e51c35SDave Jiang int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) 1109f6e51c35SDave Jiang { 1110f6e51c35SDave Jiang struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1111f6e51c35SDave Jiang 1112f6e51c35SDave Jiang return ndev_db_clear_mask(ndev, db_bits, 1113f6e51c35SDave Jiang ndev->self_mmio + 1114f6e51c35SDave Jiang ndev->self_reg->db_mask); 1115f6e51c35SDave Jiang } 1116f6e51c35SDave Jiang 1117f6e51c35SDave Jiang int intel_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr, 1118f6e51c35SDave Jiang resource_size_t *db_size) 1119f6e51c35SDave Jiang { 1120f6e51c35SDave Jiang struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1121f6e51c35SDave Jiang 1122f6e51c35SDave Jiang return ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr, 1123f6e51c35SDave Jiang ndev->peer_reg->db_bell); 1124f6e51c35SDave Jiang } 1125f6e51c35SDave Jiang 1126f6e51c35SDave Jiang static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) 1127f6e51c35SDave Jiang { 1128f6e51c35SDave Jiang struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1129f6e51c35SDave Jiang 1130f6e51c35SDave Jiang return ndev_db_write(ndev, db_bits, 1131f6e51c35SDave Jiang ndev->peer_mmio + 1132f6e51c35SDave Jiang ndev->peer_reg->db_bell); 1133f6e51c35SDave Jiang } 1134f6e51c35SDave Jiang 1135f6e51c35SDave Jiang int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb) 1136f6e51c35SDave Jiang { 1137f6e51c35SDave Jiang return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD); 1138f6e51c35SDave Jiang } 1139f6e51c35SDave Jiang 1140f6e51c35SDave Jiang int intel_ntb_spad_count(struct ntb_dev *ntb) 1141f6e51c35SDave Jiang { 1142f6e51c35SDave Jiang struct intel_ntb_dev *ndev; 1143f6e51c35SDave Jiang 1144f6e51c35SDave Jiang ndev = container_of(ntb, struct intel_ntb_dev, ntb); 1145f6e51c35SDave Jiang 1146f6e51c35SDave Jiang return ndev->spad_count; 1147f6e51c35SDave Jiang } 1148f6e51c35SDave Jiang 1149f6e51c35SDave Jiang u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx) 1150f6e51c35SDave Jiang { 1151f6e51c35SDave Jiang struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1152f6e51c35SDave Jiang 1153f6e51c35SDave Jiang return ndev_spad_read(ndev, idx, 1154f6e51c35SDave Jiang ndev->self_mmio + 1155f6e51c35SDave Jiang ndev->self_reg->spad); 1156f6e51c35SDave Jiang } 1157f6e51c35SDave Jiang 1158f6e51c35SDave Jiang int intel_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val) 1159f6e51c35SDave Jiang { 1160f6e51c35SDave Jiang struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1161f6e51c35SDave Jiang 1162f6e51c35SDave Jiang return ndev_spad_write(ndev, idx, val, 1163f6e51c35SDave Jiang ndev->self_mmio + 1164f6e51c35SDave Jiang ndev->self_reg->spad); 1165f6e51c35SDave Jiang } 1166f6e51c35SDave Jiang 1167f6e51c35SDave Jiang int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx, 1168f6e51c35SDave Jiang phys_addr_t *spad_addr) 1169f6e51c35SDave Jiang { 1170f6e51c35SDave Jiang struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1171f6e51c35SDave Jiang 1172f6e51c35SDave Jiang return ndev_spad_addr(ndev, sidx, spad_addr, ndev->peer_addr, 1173f6e51c35SDave Jiang ndev->peer_reg->spad); 1174f6e51c35SDave Jiang } 1175f6e51c35SDave Jiang 1176f6e51c35SDave Jiang u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx) 1177f6e51c35SDave Jiang { 1178f6e51c35SDave Jiang struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1179f6e51c35SDave Jiang 1180f6e51c35SDave Jiang return ndev_spad_read(ndev, sidx, 1181f6e51c35SDave Jiang ndev->peer_mmio + 1182f6e51c35SDave Jiang ndev->peer_reg->spad); 1183f6e51c35SDave Jiang } 1184f6e51c35SDave Jiang 1185f6e51c35SDave Jiang int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, int sidx, 1186f6e51c35SDave Jiang u32 val) 1187f6e51c35SDave Jiang { 1188f6e51c35SDave Jiang struct intel_ntb_dev *ndev = ntb_ndev(ntb); 1189f6e51c35SDave Jiang 1190f6e51c35SDave Jiang return ndev_spad_write(ndev, sidx, val, 1191f6e51c35SDave Jiang ndev->peer_mmio + 1192f6e51c35SDave Jiang ndev->peer_reg->spad); 1193f6e51c35SDave Jiang } 1194f6e51c35SDave Jiang 1195f6e51c35SDave Jiang static u64 xeon_db_ioread(void __iomem *mmio) 1196f6e51c35SDave Jiang { 1197f6e51c35SDave Jiang return (u64)ioread16(mmio); 1198f6e51c35SDave Jiang } 1199f6e51c35SDave Jiang 1200f6e51c35SDave Jiang static void xeon_db_iowrite(u64 bits, void __iomem *mmio) 1201f6e51c35SDave Jiang { 1202f6e51c35SDave Jiang iowrite16((u16)bits, mmio); 1203f6e51c35SDave Jiang } 1204f6e51c35SDave Jiang 1205f6e51c35SDave Jiang static int xeon_poll_link(struct intel_ntb_dev *ndev) 1206f6e51c35SDave Jiang { 1207f6e51c35SDave Jiang u16 reg_val; 1208f6e51c35SDave Jiang int rc; 1209f6e51c35SDave Jiang 1210f6e51c35SDave Jiang ndev->reg->db_iowrite(ndev->db_link_mask, 1211f6e51c35SDave Jiang ndev->self_mmio + 1212f6e51c35SDave Jiang ndev->self_reg->db_bell); 1213f6e51c35SDave Jiang 1214f6e51c35SDave Jiang rc = pci_read_config_word(ndev->ntb.pdev, 1215f6e51c35SDave Jiang XEON_LINK_STATUS_OFFSET, ®_val); 1216f6e51c35SDave Jiang if (rc) 1217f6e51c35SDave Jiang return 0; 1218f6e51c35SDave Jiang 1219f6e51c35SDave Jiang if (reg_val == ndev->lnk_sta) 1220f6e51c35SDave Jiang return 0; 1221f6e51c35SDave Jiang 1222f6e51c35SDave Jiang ndev->lnk_sta = reg_val; 1223f6e51c35SDave Jiang 1224f6e51c35SDave Jiang return 1; 1225f6e51c35SDave Jiang } 1226f6e51c35SDave Jiang 1227f6e51c35SDave Jiang int xeon_link_is_up(struct intel_ntb_dev *ndev) 1228f6e51c35SDave Jiang { 1229f6e51c35SDave Jiang if (ndev->ntb.topo == NTB_TOPO_SEC) 1230f6e51c35SDave Jiang return 1; 1231f6e51c35SDave Jiang 1232f6e51c35SDave Jiang return NTB_LNK_STA_ACTIVE(ndev->lnk_sta); 1233f6e51c35SDave Jiang } 1234f6e51c35SDave Jiang 1235f6e51c35SDave Jiang enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd) 1236f6e51c35SDave Jiang { 1237f6e51c35SDave Jiang switch (ppd & XEON_PPD_TOPO_MASK) { 1238f6e51c35SDave Jiang case XEON_PPD_TOPO_B2B_USD: 1239f6e51c35SDave Jiang return NTB_TOPO_B2B_USD; 1240f6e51c35SDave Jiang 1241f6e51c35SDave Jiang case XEON_PPD_TOPO_B2B_DSD: 1242f6e51c35SDave Jiang return NTB_TOPO_B2B_DSD; 1243f6e51c35SDave Jiang 1244f6e51c35SDave Jiang case XEON_PPD_TOPO_PRI_USD: 1245f6e51c35SDave Jiang case XEON_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */ 1246f6e51c35SDave Jiang return NTB_TOPO_PRI; 1247f6e51c35SDave Jiang 1248f6e51c35SDave Jiang case XEON_PPD_TOPO_SEC_USD: 1249f6e51c35SDave Jiang case XEON_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */ 1250f6e51c35SDave Jiang return NTB_TOPO_SEC; 1251f6e51c35SDave Jiang } 1252f6e51c35SDave Jiang 1253f6e51c35SDave Jiang return NTB_TOPO_NONE; 1254f6e51c35SDave Jiang } 1255f6e51c35SDave Jiang 1256f6e51c35SDave Jiang static inline int xeon_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd) 1257f6e51c35SDave Jiang { 1258f6e51c35SDave Jiang if (ppd & XEON_PPD_SPLIT_BAR_MASK) { 1259f6e51c35SDave Jiang dev_dbg(&ndev->ntb.pdev->dev, "PPD %d split bar\n", ppd); 1260f6e51c35SDave Jiang return 1; 1261f6e51c35SDave Jiang } 1262f6e51c35SDave Jiang return 0; 1263f6e51c35SDave Jiang } 1264f6e51c35SDave Jiang 1265f6e51c35SDave Jiang static int xeon_init_isr(struct intel_ntb_dev *ndev) 1266f6e51c35SDave Jiang { 1267f6e51c35SDave Jiang return ndev_init_isr(ndev, XEON_DB_MSIX_VECTOR_COUNT, 1268f6e51c35SDave Jiang XEON_DB_MSIX_VECTOR_COUNT, 1269f6e51c35SDave Jiang XEON_DB_MSIX_VECTOR_SHIFT, 1270f6e51c35SDave Jiang XEON_DB_TOTAL_SHIFT); 1271f6e51c35SDave Jiang } 1272f6e51c35SDave Jiang 1273f6e51c35SDave Jiang static void xeon_deinit_isr(struct intel_ntb_dev *ndev) 1274f6e51c35SDave Jiang { 1275f6e51c35SDave Jiang ndev_deinit_isr(ndev); 1276f6e51c35SDave Jiang } 1277f6e51c35SDave Jiang 1278f6e51c35SDave Jiang static int xeon_setup_b2b_mw(struct intel_ntb_dev *ndev, 1279f6e51c35SDave Jiang const struct intel_b2b_addr *addr, 1280f6e51c35SDave Jiang const struct intel_b2b_addr *peer_addr) 1281f6e51c35SDave Jiang { 1282f6e51c35SDave Jiang struct pci_dev *pdev; 1283f6e51c35SDave Jiang void __iomem *mmio; 1284f6e51c35SDave Jiang resource_size_t bar_size; 1285f6e51c35SDave Jiang phys_addr_t bar_addr; 1286f6e51c35SDave Jiang int b2b_bar; 1287f6e51c35SDave Jiang u8 bar_sz; 1288f6e51c35SDave Jiang 1289f6e51c35SDave Jiang pdev = ndev->ntb.pdev; 1290f6e51c35SDave Jiang mmio = ndev->self_mmio; 1291f6e51c35SDave Jiang 1292f6e51c35SDave Jiang if (ndev->b2b_idx == UINT_MAX) { 1293f6e51c35SDave Jiang dev_dbg(&pdev->dev, "not using b2b mw\n"); 1294f6e51c35SDave Jiang b2b_bar = 0; 1295f6e51c35SDave Jiang ndev->b2b_off = 0; 1296f6e51c35SDave Jiang } else { 1297f6e51c35SDave Jiang b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx); 1298f6e51c35SDave Jiang if (b2b_bar < 0) 1299f6e51c35SDave Jiang return -EIO; 1300f6e51c35SDave Jiang 1301f6e51c35SDave Jiang dev_dbg(&pdev->dev, "using b2b mw bar %d\n", b2b_bar); 1302f6e51c35SDave Jiang 1303f6e51c35SDave Jiang bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar); 1304f6e51c35SDave Jiang 1305f6e51c35SDave Jiang dev_dbg(&pdev->dev, "b2b bar size %#llx\n", bar_size); 1306f6e51c35SDave Jiang 1307f6e51c35SDave Jiang if (b2b_mw_share && XEON_B2B_MIN_SIZE <= bar_size >> 1) { 1308f6e51c35SDave Jiang dev_dbg(&pdev->dev, "b2b using first half of bar\n"); 1309f6e51c35SDave Jiang ndev->b2b_off = bar_size >> 1; 1310f6e51c35SDave Jiang } else if (XEON_B2B_MIN_SIZE <= bar_size) { 1311f6e51c35SDave Jiang dev_dbg(&pdev->dev, "b2b using whole bar\n"); 1312f6e51c35SDave Jiang ndev->b2b_off = 0; 1313f6e51c35SDave Jiang --ndev->mw_count; 1314f6e51c35SDave Jiang } else { 1315f6e51c35SDave Jiang dev_dbg(&pdev->dev, "b2b bar size is too small\n"); 1316f6e51c35SDave Jiang return -EIO; 1317f6e51c35SDave Jiang } 1318f6e51c35SDave Jiang } 1319f6e51c35SDave Jiang 1320f6e51c35SDave Jiang /* Reset the secondary bar sizes to match the primary bar sizes, 1321f6e51c35SDave Jiang * except disable or halve the size of the b2b secondary bar. 1322f6e51c35SDave Jiang * 1323f6e51c35SDave Jiang * Note: code for each specific bar size register, because the register 1324f6e51c35SDave Jiang * offsets are not in a consistent order (bar5sz comes after ppd, odd). 1325f6e51c35SDave Jiang */ 1326f6e51c35SDave Jiang pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &bar_sz); 1327f6e51c35SDave Jiang dev_dbg(&pdev->dev, "PBAR23SZ %#x\n", bar_sz); 1328f6e51c35SDave Jiang if (b2b_bar == 2) { 1329f6e51c35SDave Jiang if (ndev->b2b_off) 1330f6e51c35SDave Jiang bar_sz -= 1; 1331f6e51c35SDave Jiang else 1332f6e51c35SDave Jiang bar_sz = 0; 1333f6e51c35SDave Jiang } 1334f6e51c35SDave Jiang pci_write_config_byte(pdev, XEON_SBAR23SZ_OFFSET, bar_sz); 1335f6e51c35SDave Jiang pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &bar_sz); 1336f6e51c35SDave Jiang dev_dbg(&pdev->dev, "SBAR23SZ %#x\n", bar_sz); 1337f6e51c35SDave Jiang 1338f6e51c35SDave Jiang if (!ndev->bar4_split) { 1339f6e51c35SDave Jiang pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &bar_sz); 1340f6e51c35SDave Jiang dev_dbg(&pdev->dev, "PBAR45SZ %#x\n", bar_sz); 1341f6e51c35SDave Jiang if (b2b_bar == 4) { 1342f6e51c35SDave Jiang if (ndev->b2b_off) 1343f6e51c35SDave Jiang bar_sz -= 1; 1344f6e51c35SDave Jiang else 1345f6e51c35SDave Jiang bar_sz = 0; 1346f6e51c35SDave Jiang } 1347f6e51c35SDave Jiang pci_write_config_byte(pdev, XEON_SBAR45SZ_OFFSET, bar_sz); 1348f6e51c35SDave Jiang pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &bar_sz); 1349f6e51c35SDave Jiang dev_dbg(&pdev->dev, "SBAR45SZ %#x\n", bar_sz); 1350f6e51c35SDave Jiang } else { 1351f6e51c35SDave Jiang pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &bar_sz); 1352f6e51c35SDave Jiang dev_dbg(&pdev->dev, "PBAR4SZ %#x\n", bar_sz); 1353f6e51c35SDave Jiang if (b2b_bar == 4) { 1354f6e51c35SDave Jiang if (ndev->b2b_off) 1355f6e51c35SDave Jiang bar_sz -= 1; 1356f6e51c35SDave Jiang else 1357f6e51c35SDave Jiang bar_sz = 0; 1358f6e51c35SDave Jiang } 1359f6e51c35SDave Jiang pci_write_config_byte(pdev, XEON_SBAR4SZ_OFFSET, bar_sz); 1360f6e51c35SDave Jiang pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &bar_sz); 1361f6e51c35SDave Jiang dev_dbg(&pdev->dev, "SBAR4SZ %#x\n", bar_sz); 1362f6e51c35SDave Jiang 1363f6e51c35SDave Jiang pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &bar_sz); 1364f6e51c35SDave Jiang dev_dbg(&pdev->dev, "PBAR5SZ %#x\n", bar_sz); 1365f6e51c35SDave Jiang if (b2b_bar == 5) { 1366f6e51c35SDave Jiang if (ndev->b2b_off) 1367f6e51c35SDave Jiang bar_sz -= 1; 1368f6e51c35SDave Jiang else 1369f6e51c35SDave Jiang bar_sz = 0; 1370f6e51c35SDave Jiang } 1371f6e51c35SDave Jiang pci_write_config_byte(pdev, XEON_SBAR5SZ_OFFSET, bar_sz); 1372f6e51c35SDave Jiang pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &bar_sz); 1373f6e51c35SDave Jiang dev_dbg(&pdev->dev, "SBAR5SZ %#x\n", bar_sz); 1374f6e51c35SDave Jiang } 1375f6e51c35SDave Jiang 1376f6e51c35SDave Jiang /* SBAR01 hit by first part of the b2b bar */ 1377f6e51c35SDave Jiang if (b2b_bar == 0) 1378f6e51c35SDave Jiang bar_addr = addr->bar0_addr; 1379f6e51c35SDave Jiang else if (b2b_bar == 2) 1380f6e51c35SDave Jiang bar_addr = addr->bar2_addr64; 1381f6e51c35SDave Jiang else if (b2b_bar == 4 && !ndev->bar4_split) 1382f6e51c35SDave Jiang bar_addr = addr->bar4_addr64; 1383f6e51c35SDave Jiang else if (b2b_bar == 4) 1384f6e51c35SDave Jiang bar_addr = addr->bar4_addr32; 1385f6e51c35SDave Jiang else if (b2b_bar == 5) 1386f6e51c35SDave Jiang bar_addr = addr->bar5_addr32; 1387f6e51c35SDave Jiang else 1388f6e51c35SDave Jiang return -EIO; 1389f6e51c35SDave Jiang 1390f6e51c35SDave Jiang dev_dbg(&pdev->dev, "SBAR01 %#018llx\n", bar_addr); 1391f6e51c35SDave Jiang iowrite64(bar_addr, mmio + XEON_SBAR0BASE_OFFSET); 1392f6e51c35SDave Jiang 1393f6e51c35SDave Jiang /* Other SBAR are normally hit by the PBAR xlat, except for b2b bar. 1394f6e51c35SDave Jiang * The b2b bar is either disabled above, or configured half-size, and 1395f6e51c35SDave Jiang * it starts at the PBAR xlat + offset. 1396f6e51c35SDave Jiang */ 1397f6e51c35SDave Jiang 1398f6e51c35SDave Jiang bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0); 1399f6e51c35SDave Jiang iowrite64(bar_addr, mmio + XEON_SBAR23BASE_OFFSET); 1400f6e51c35SDave Jiang bar_addr = ioread64(mmio + XEON_SBAR23BASE_OFFSET); 1401f6e51c35SDave Jiang dev_dbg(&pdev->dev, "SBAR23 %#018llx\n", bar_addr); 1402f6e51c35SDave Jiang 1403f6e51c35SDave Jiang if (!ndev->bar4_split) { 1404f6e51c35SDave Jiang bar_addr = addr->bar4_addr64 + 1405f6e51c35SDave Jiang (b2b_bar == 4 ? ndev->b2b_off : 0); 1406f6e51c35SDave Jiang iowrite64(bar_addr, mmio + XEON_SBAR45BASE_OFFSET); 1407f6e51c35SDave Jiang bar_addr = ioread64(mmio + XEON_SBAR45BASE_OFFSET); 1408f6e51c35SDave Jiang dev_dbg(&pdev->dev, "SBAR45 %#018llx\n", bar_addr); 1409f6e51c35SDave Jiang } else { 1410f6e51c35SDave Jiang bar_addr = addr->bar4_addr32 + 1411f6e51c35SDave Jiang (b2b_bar == 4 ? ndev->b2b_off : 0); 1412f6e51c35SDave Jiang iowrite32(bar_addr, mmio + XEON_SBAR4BASE_OFFSET); 1413f6e51c35SDave Jiang bar_addr = ioread32(mmio + XEON_SBAR4BASE_OFFSET); 1414f6e51c35SDave Jiang dev_dbg(&pdev->dev, "SBAR4 %#010llx\n", bar_addr); 1415f6e51c35SDave Jiang 1416f6e51c35SDave Jiang bar_addr = addr->bar5_addr32 + 1417f6e51c35SDave Jiang (b2b_bar == 5 ? ndev->b2b_off : 0); 1418f6e51c35SDave Jiang iowrite32(bar_addr, mmio + XEON_SBAR5BASE_OFFSET); 1419f6e51c35SDave Jiang bar_addr = ioread32(mmio + XEON_SBAR5BASE_OFFSET); 1420f6e51c35SDave Jiang dev_dbg(&pdev->dev, "SBAR5 %#010llx\n", bar_addr); 1421f6e51c35SDave Jiang } 1422f6e51c35SDave Jiang 1423f6e51c35SDave Jiang /* setup incoming bar limits == base addrs (zero length windows) */ 1424f6e51c35SDave Jiang 1425f6e51c35SDave Jiang bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0); 1426f6e51c35SDave Jiang iowrite64(bar_addr, mmio + XEON_SBAR23LMT_OFFSET); 1427f6e51c35SDave Jiang bar_addr = ioread64(mmio + XEON_SBAR23LMT_OFFSET); 1428f6e51c35SDave Jiang dev_dbg(&pdev->dev, "SBAR23LMT %#018llx\n", bar_addr); 1429f6e51c35SDave Jiang 1430f6e51c35SDave Jiang if (!ndev->bar4_split) { 1431f6e51c35SDave Jiang bar_addr = addr->bar4_addr64 + 1432f6e51c35SDave Jiang (b2b_bar == 4 ? ndev->b2b_off : 0); 1433f6e51c35SDave Jiang iowrite64(bar_addr, mmio + XEON_SBAR45LMT_OFFSET); 1434f6e51c35SDave Jiang bar_addr = ioread64(mmio + XEON_SBAR45LMT_OFFSET); 1435f6e51c35SDave Jiang dev_dbg(&pdev->dev, "SBAR45LMT %#018llx\n", bar_addr); 1436f6e51c35SDave Jiang } else { 1437f6e51c35SDave Jiang bar_addr = addr->bar4_addr32 + 1438f6e51c35SDave Jiang (b2b_bar == 4 ? ndev->b2b_off : 0); 1439f6e51c35SDave Jiang iowrite32(bar_addr, mmio + XEON_SBAR4LMT_OFFSET); 1440f6e51c35SDave Jiang bar_addr = ioread32(mmio + XEON_SBAR4LMT_OFFSET); 1441f6e51c35SDave Jiang dev_dbg(&pdev->dev, "SBAR4LMT %#010llx\n", bar_addr); 1442f6e51c35SDave Jiang 1443f6e51c35SDave Jiang bar_addr = addr->bar5_addr32 + 1444f6e51c35SDave Jiang (b2b_bar == 5 ? ndev->b2b_off : 0); 1445f6e51c35SDave Jiang iowrite32(bar_addr, mmio + XEON_SBAR5LMT_OFFSET); 1446f6e51c35SDave Jiang bar_addr = ioread32(mmio + XEON_SBAR5LMT_OFFSET); 1447f6e51c35SDave Jiang dev_dbg(&pdev->dev, "SBAR5LMT %#05llx\n", bar_addr); 1448f6e51c35SDave Jiang } 1449f6e51c35SDave Jiang 1450f6e51c35SDave Jiang /* zero incoming translation addrs */ 1451f6e51c35SDave Jiang iowrite64(0, mmio + XEON_SBAR23XLAT_OFFSET); 1452f6e51c35SDave Jiang 1453f6e51c35SDave Jiang if (!ndev->bar4_split) { 1454f6e51c35SDave Jiang iowrite64(0, mmio + XEON_SBAR45XLAT_OFFSET); 1455f6e51c35SDave Jiang } else { 1456f6e51c35SDave Jiang iowrite32(0, mmio + XEON_SBAR4XLAT_OFFSET); 1457f6e51c35SDave Jiang iowrite32(0, mmio + XEON_SBAR5XLAT_OFFSET); 1458f6e51c35SDave Jiang } 1459f6e51c35SDave Jiang 1460f6e51c35SDave Jiang /* zero outgoing translation limits (whole bar size windows) */ 1461f6e51c35SDave Jiang iowrite64(0, mmio + XEON_PBAR23LMT_OFFSET); 1462f6e51c35SDave Jiang if (!ndev->bar4_split) { 1463f6e51c35SDave Jiang iowrite64(0, mmio + XEON_PBAR45LMT_OFFSET); 1464f6e51c35SDave Jiang } else { 1465f6e51c35SDave Jiang iowrite32(0, mmio + XEON_PBAR4LMT_OFFSET); 1466f6e51c35SDave Jiang iowrite32(0, mmio + XEON_PBAR5LMT_OFFSET); 1467f6e51c35SDave Jiang } 1468f6e51c35SDave Jiang 1469f6e51c35SDave Jiang /* set outgoing translation offsets */ 1470f6e51c35SDave Jiang bar_addr = peer_addr->bar2_addr64; 1471f6e51c35SDave Jiang iowrite64(bar_addr, mmio + XEON_PBAR23XLAT_OFFSET); 1472f6e51c35SDave Jiang bar_addr = ioread64(mmio + XEON_PBAR23XLAT_OFFSET); 1473f6e51c35SDave Jiang dev_dbg(&pdev->dev, "PBAR23XLAT %#018llx\n", bar_addr); 1474f6e51c35SDave Jiang 1475f6e51c35SDave Jiang if (!ndev->bar4_split) { 1476f6e51c35SDave Jiang bar_addr = peer_addr->bar4_addr64; 1477f6e51c35SDave Jiang iowrite64(bar_addr, mmio + XEON_PBAR45XLAT_OFFSET); 1478f6e51c35SDave Jiang bar_addr = ioread64(mmio + XEON_PBAR45XLAT_OFFSET); 1479f6e51c35SDave Jiang dev_dbg(&pdev->dev, "PBAR45XLAT %#018llx\n", bar_addr); 1480f6e51c35SDave Jiang } else { 1481f6e51c35SDave Jiang bar_addr = peer_addr->bar4_addr32; 1482f6e51c35SDave Jiang iowrite32(bar_addr, mmio + XEON_PBAR4XLAT_OFFSET); 1483f6e51c35SDave Jiang bar_addr = ioread32(mmio + XEON_PBAR4XLAT_OFFSET); 1484f6e51c35SDave Jiang dev_dbg(&pdev->dev, "PBAR4XLAT %#010llx\n", bar_addr); 1485f6e51c35SDave Jiang 1486f6e51c35SDave Jiang bar_addr = peer_addr->bar5_addr32; 1487f6e51c35SDave Jiang iowrite32(bar_addr, mmio + XEON_PBAR5XLAT_OFFSET); 1488f6e51c35SDave Jiang bar_addr = ioread32(mmio + XEON_PBAR5XLAT_OFFSET); 1489f6e51c35SDave Jiang dev_dbg(&pdev->dev, "PBAR5XLAT %#010llx\n", bar_addr); 1490f6e51c35SDave Jiang } 1491f6e51c35SDave Jiang 1492f6e51c35SDave Jiang /* set the translation offset for b2b registers */ 1493f6e51c35SDave Jiang if (b2b_bar == 0) 1494f6e51c35SDave Jiang bar_addr = peer_addr->bar0_addr; 1495f6e51c35SDave Jiang else if (b2b_bar == 2) 1496f6e51c35SDave Jiang bar_addr = peer_addr->bar2_addr64; 1497f6e51c35SDave Jiang else if (b2b_bar == 4 && !ndev->bar4_split) 1498f6e51c35SDave Jiang bar_addr = peer_addr->bar4_addr64; 1499f6e51c35SDave Jiang else if (b2b_bar == 4) 1500f6e51c35SDave Jiang bar_addr = peer_addr->bar4_addr32; 1501f6e51c35SDave Jiang else if (b2b_bar == 5) 1502f6e51c35SDave Jiang bar_addr = peer_addr->bar5_addr32; 1503f6e51c35SDave Jiang else 1504f6e51c35SDave Jiang return -EIO; 1505f6e51c35SDave Jiang 1506f6e51c35SDave Jiang /* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */ 1507f6e51c35SDave Jiang dev_dbg(&pdev->dev, "B2BXLAT %#018llx\n", bar_addr); 1508f6e51c35SDave Jiang iowrite32(bar_addr, mmio + XEON_B2B_XLAT_OFFSETL); 1509f6e51c35SDave Jiang iowrite32(bar_addr >> 32, mmio + XEON_B2B_XLAT_OFFSETU); 1510f6e51c35SDave Jiang 1511f6e51c35SDave Jiang if (b2b_bar) { 1512f6e51c35SDave Jiang /* map peer ntb mmio config space registers */ 1513f6e51c35SDave Jiang ndev->peer_mmio = pci_iomap(pdev, b2b_bar, 1514f6e51c35SDave Jiang XEON_B2B_MIN_SIZE); 1515f6e51c35SDave Jiang if (!ndev->peer_mmio) 1516f6e51c35SDave Jiang return -EIO; 1517f6e51c35SDave Jiang 1518f6e51c35SDave Jiang ndev->peer_addr = pci_resource_start(pdev, b2b_bar); 1519f6e51c35SDave Jiang } 1520f6e51c35SDave Jiang 1521f6e51c35SDave Jiang return 0; 1522f6e51c35SDave Jiang } 1523f6e51c35SDave Jiang 1524f6e51c35SDave Jiang static int xeon_init_ntb(struct intel_ntb_dev *ndev) 1525f6e51c35SDave Jiang { 1526f6e51c35SDave Jiang struct device *dev = &ndev->ntb.pdev->dev; 1527f6e51c35SDave Jiang int rc; 1528f6e51c35SDave Jiang u32 ntb_ctl; 1529f6e51c35SDave Jiang 1530f6e51c35SDave Jiang if (ndev->bar4_split) 1531f6e51c35SDave Jiang ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT; 1532f6e51c35SDave Jiang else 1533f6e51c35SDave Jiang ndev->mw_count = XEON_MW_COUNT; 1534f6e51c35SDave Jiang 1535f6e51c35SDave Jiang ndev->spad_count = XEON_SPAD_COUNT; 1536f6e51c35SDave Jiang ndev->db_count = XEON_DB_COUNT; 1537f6e51c35SDave Jiang ndev->db_link_mask = XEON_DB_LINK_BIT; 1538f6e51c35SDave Jiang 1539f6e51c35SDave Jiang switch (ndev->ntb.topo) { 1540f6e51c35SDave Jiang case NTB_TOPO_PRI: 1541f6e51c35SDave Jiang if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) { 1542f6e51c35SDave Jiang dev_err(dev, "NTB Primary config disabled\n"); 1543f6e51c35SDave Jiang return -EINVAL; 1544f6e51c35SDave Jiang } 1545f6e51c35SDave Jiang 1546f6e51c35SDave Jiang /* enable link to allow secondary side device to appear */ 1547f6e51c35SDave Jiang ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl); 1548f6e51c35SDave Jiang ntb_ctl &= ~NTB_CTL_DISABLE; 1549f6e51c35SDave Jiang iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl); 1550f6e51c35SDave Jiang 1551f6e51c35SDave Jiang /* use half the spads for the peer */ 1552f6e51c35SDave Jiang ndev->spad_count >>= 1; 1553f6e51c35SDave Jiang ndev->self_reg = &xeon_pri_reg; 1554f6e51c35SDave Jiang ndev->peer_reg = &xeon_sec_reg; 1555f6e51c35SDave Jiang ndev->xlat_reg = &xeon_sec_xlat; 1556f6e51c35SDave Jiang break; 1557f6e51c35SDave Jiang 1558f6e51c35SDave Jiang case NTB_TOPO_SEC: 1559f6e51c35SDave Jiang if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) { 1560f6e51c35SDave Jiang dev_err(dev, "NTB Secondary config disabled\n"); 1561f6e51c35SDave Jiang return -EINVAL; 1562f6e51c35SDave Jiang } 1563f6e51c35SDave Jiang /* use half the spads for the peer */ 1564f6e51c35SDave Jiang ndev->spad_count >>= 1; 1565f6e51c35SDave Jiang ndev->self_reg = &xeon_sec_reg; 1566f6e51c35SDave Jiang ndev->peer_reg = &xeon_pri_reg; 1567f6e51c35SDave Jiang ndev->xlat_reg = &xeon_pri_xlat; 1568f6e51c35SDave Jiang break; 1569f6e51c35SDave Jiang 1570f6e51c35SDave Jiang case NTB_TOPO_B2B_USD: 1571f6e51c35SDave Jiang case NTB_TOPO_B2B_DSD: 1572f6e51c35SDave Jiang ndev->self_reg = &xeon_pri_reg; 1573f6e51c35SDave Jiang ndev->peer_reg = &xeon_b2b_reg; 1574f6e51c35SDave Jiang ndev->xlat_reg = &xeon_sec_xlat; 1575f6e51c35SDave Jiang 1576f6e51c35SDave Jiang if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) { 1577f6e51c35SDave Jiang ndev->peer_reg = &xeon_pri_reg; 1578f6e51c35SDave Jiang 1579f6e51c35SDave Jiang if (b2b_mw_idx < 0) 1580f6e51c35SDave Jiang ndev->b2b_idx = b2b_mw_idx + ndev->mw_count; 1581f6e51c35SDave Jiang else 1582f6e51c35SDave Jiang ndev->b2b_idx = b2b_mw_idx; 1583f6e51c35SDave Jiang 1584f6e51c35SDave Jiang if (ndev->b2b_idx >= ndev->mw_count) { 1585f6e51c35SDave Jiang dev_dbg(dev, 1586f6e51c35SDave Jiang "b2b_mw_idx %d invalid for mw_count %u\n", 1587f6e51c35SDave Jiang b2b_mw_idx, ndev->mw_count); 1588f6e51c35SDave Jiang return -EINVAL; 1589f6e51c35SDave Jiang } 1590f6e51c35SDave Jiang 1591f6e51c35SDave Jiang dev_dbg(dev, "setting up b2b mw idx %d means %d\n", 1592f6e51c35SDave Jiang b2b_mw_idx, ndev->b2b_idx); 1593f6e51c35SDave Jiang 1594f6e51c35SDave Jiang } else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) { 1595f6e51c35SDave Jiang dev_warn(dev, "Reduce doorbell count by 1\n"); 1596f6e51c35SDave Jiang ndev->db_count -= 1; 1597f6e51c35SDave Jiang } 1598f6e51c35SDave Jiang 1599f6e51c35SDave Jiang if (ndev->ntb.topo == NTB_TOPO_B2B_USD) { 1600f6e51c35SDave Jiang rc = xeon_setup_b2b_mw(ndev, 1601f6e51c35SDave Jiang &xeon_b2b_dsd_addr, 1602f6e51c35SDave Jiang &xeon_b2b_usd_addr); 1603f6e51c35SDave Jiang } else { 1604f6e51c35SDave Jiang rc = xeon_setup_b2b_mw(ndev, 1605f6e51c35SDave Jiang &xeon_b2b_usd_addr, 1606f6e51c35SDave Jiang &xeon_b2b_dsd_addr); 1607f6e51c35SDave Jiang } 1608f6e51c35SDave Jiang if (rc) 1609f6e51c35SDave Jiang return rc; 1610f6e51c35SDave Jiang 1611f6e51c35SDave Jiang /* Enable Bus Master and Memory Space on the secondary side */ 1612f6e51c35SDave Jiang iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER, 1613f6e51c35SDave Jiang ndev->self_mmio + XEON_SPCICMD_OFFSET); 1614f6e51c35SDave Jiang 1615f6e51c35SDave Jiang break; 1616f6e51c35SDave Jiang 1617f6e51c35SDave Jiang default: 1618f6e51c35SDave Jiang return -EINVAL; 1619f6e51c35SDave Jiang } 1620f6e51c35SDave Jiang 1621f6e51c35SDave Jiang ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; 1622f6e51c35SDave Jiang 1623f6e51c35SDave Jiang ndev->reg->db_iowrite(ndev->db_valid_mask, 1624f6e51c35SDave Jiang ndev->self_mmio + 1625f6e51c35SDave Jiang ndev->self_reg->db_mask); 1626f6e51c35SDave Jiang 1627f6e51c35SDave Jiang return 0; 1628f6e51c35SDave Jiang } 1629f6e51c35SDave Jiang 1630f6e51c35SDave Jiang static int xeon_init_dev(struct intel_ntb_dev *ndev) 1631f6e51c35SDave Jiang { 1632f6e51c35SDave Jiang struct pci_dev *pdev; 1633f6e51c35SDave Jiang u8 ppd; 1634f6e51c35SDave Jiang int rc, mem; 1635f6e51c35SDave Jiang 1636f6e51c35SDave Jiang pdev = ndev->ntb.pdev; 1637f6e51c35SDave Jiang 1638f6e51c35SDave Jiang switch (pdev->device) { 1639f6e51c35SDave Jiang /* There is a Xeon hardware errata related to writes to SDOORBELL or 1640f6e51c35SDave Jiang * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space, 1641f6e51c35SDave Jiang * which may hang the system. To workaround this use the second memory 1642f6e51c35SDave Jiang * window to access the interrupt and scratch pad registers on the 1643f6e51c35SDave Jiang * remote system. 1644f6e51c35SDave Jiang */ 1645f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_SS_JSF: 1646f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_PS_JSF: 1647f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF: 1648f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_SS_SNB: 1649f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_PS_SNB: 1650f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB: 1651f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: 1652f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: 1653f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: 1654f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: 1655f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: 1656f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: 1657f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_SS_BDX: 1658f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_PS_BDX: 1659f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX: 1660f6e51c35SDave Jiang ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP; 1661f6e51c35SDave Jiang break; 1662f6e51c35SDave Jiang } 1663f6e51c35SDave Jiang 1664f6e51c35SDave Jiang switch (pdev->device) { 1665f6e51c35SDave Jiang /* There is a hardware errata related to accessing any register in 1666f6e51c35SDave Jiang * SB01BASE in the presence of bidirectional traffic crossing the NTB. 1667f6e51c35SDave Jiang */ 1668f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: 1669f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: 1670f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: 1671f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: 1672f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: 1673f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: 1674f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_SS_BDX: 1675f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_PS_BDX: 1676f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX: 1677f6e51c35SDave Jiang ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP; 1678f6e51c35SDave Jiang break; 1679f6e51c35SDave Jiang } 1680f6e51c35SDave Jiang 1681f6e51c35SDave Jiang switch (pdev->device) { 1682f6e51c35SDave Jiang /* HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1683f6e51c35SDave Jiang * mirrored to the remote system. Shrink the number of bits by one, 1684f6e51c35SDave Jiang * since bit 14 is the last bit. 1685f6e51c35SDave Jiang */ 1686f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_SS_JSF: 1687f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_PS_JSF: 1688f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF: 1689f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_SS_SNB: 1690f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_PS_SNB: 1691f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB: 1692f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: 1693f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: 1694f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: 1695f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: 1696f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: 1697f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: 1698f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_SS_BDX: 1699f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_PS_BDX: 1700f6e51c35SDave Jiang case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX: 1701f6e51c35SDave Jiang ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14; 1702f6e51c35SDave Jiang break; 1703f6e51c35SDave Jiang } 1704f6e51c35SDave Jiang 1705f6e51c35SDave Jiang ndev->reg = &xeon_reg; 1706f6e51c35SDave Jiang 1707f6e51c35SDave Jiang rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd); 1708f6e51c35SDave Jiang if (rc) 1709f6e51c35SDave Jiang return -EIO; 1710f6e51c35SDave Jiang 1711f6e51c35SDave Jiang ndev->ntb.topo = xeon_ppd_topo(ndev, ppd); 1712f6e51c35SDave Jiang dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd, 1713f6e51c35SDave Jiang ntb_topo_string(ndev->ntb.topo)); 1714f6e51c35SDave Jiang if (ndev->ntb.topo == NTB_TOPO_NONE) 1715f6e51c35SDave Jiang return -EINVAL; 1716f6e51c35SDave Jiang 1717f6e51c35SDave Jiang if (ndev->ntb.topo != NTB_TOPO_SEC) { 1718f6e51c35SDave Jiang ndev->bar4_split = xeon_ppd_bar4_split(ndev, ppd); 1719f6e51c35SDave Jiang dev_dbg(&pdev->dev, "ppd %#x bar4_split %d\n", 1720f6e51c35SDave Jiang ppd, ndev->bar4_split); 1721f6e51c35SDave Jiang } else { 1722f6e51c35SDave Jiang /* This is a way for transparent BAR to figure out if we are 1723f6e51c35SDave Jiang * doing split BAR or not. There is no way for the hw on the 1724f6e51c35SDave Jiang * transparent side to know and set the PPD. 1725f6e51c35SDave Jiang */ 1726f6e51c35SDave Jiang mem = pci_select_bars(pdev, IORESOURCE_MEM); 1727f6e51c35SDave Jiang ndev->bar4_split = hweight32(mem) == 1728f6e51c35SDave Jiang HSX_SPLIT_BAR_MW_COUNT + 1; 1729f6e51c35SDave Jiang dev_dbg(&pdev->dev, "mem %#x bar4_split %d\n", 1730f6e51c35SDave Jiang mem, ndev->bar4_split); 1731f6e51c35SDave Jiang } 1732f6e51c35SDave Jiang 1733f6e51c35SDave Jiang rc = xeon_init_ntb(ndev); 1734f6e51c35SDave Jiang if (rc) 1735f6e51c35SDave Jiang return rc; 1736f6e51c35SDave Jiang 1737f6e51c35SDave Jiang return xeon_init_isr(ndev); 1738f6e51c35SDave Jiang } 1739f6e51c35SDave Jiang 1740f6e51c35SDave Jiang static void xeon_deinit_dev(struct intel_ntb_dev *ndev) 1741f6e51c35SDave Jiang { 1742f6e51c35SDave Jiang xeon_deinit_isr(ndev); 1743f6e51c35SDave Jiang } 1744f6e51c35SDave Jiang 1745f6e51c35SDave Jiang static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev) 1746f6e51c35SDave Jiang { 1747f6e51c35SDave Jiang int rc; 1748f6e51c35SDave Jiang 1749f6e51c35SDave Jiang pci_set_drvdata(pdev, ndev); 1750f6e51c35SDave Jiang 1751f6e51c35SDave Jiang rc = pci_enable_device(pdev); 1752f6e51c35SDave Jiang if (rc) 1753f6e51c35SDave Jiang goto err_pci_enable; 1754f6e51c35SDave Jiang 1755f6e51c35SDave Jiang rc = pci_request_regions(pdev, NTB_NAME); 1756f6e51c35SDave Jiang if (rc) 1757f6e51c35SDave Jiang goto err_pci_regions; 1758f6e51c35SDave Jiang 1759f6e51c35SDave Jiang pci_set_master(pdev); 1760f6e51c35SDave Jiang 1761f6e51c35SDave Jiang rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 1762f6e51c35SDave Jiang if (rc) { 1763f6e51c35SDave Jiang rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 1764f6e51c35SDave Jiang if (rc) 1765f6e51c35SDave Jiang goto err_dma_mask; 1766f6e51c35SDave Jiang dev_warn(&pdev->dev, "Cannot DMA highmem\n"); 1767f6e51c35SDave Jiang } 1768f6e51c35SDave Jiang 1769f6e51c35SDave Jiang rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 1770f6e51c35SDave Jiang if (rc) { 1771f6e51c35SDave Jiang rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 1772f6e51c35SDave Jiang if (rc) 1773f6e51c35SDave Jiang goto err_dma_mask; 1774f6e51c35SDave Jiang dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n"); 1775f6e51c35SDave Jiang } 1776f6e51c35SDave Jiang rc = dma_coerce_mask_and_coherent(&ndev->ntb.dev, 1777f6e51c35SDave Jiang dma_get_mask(&pdev->dev)); 1778f6e51c35SDave Jiang if (rc) 1779f6e51c35SDave Jiang goto err_dma_mask; 1780f6e51c35SDave Jiang 1781f6e51c35SDave Jiang ndev->self_mmio = pci_iomap(pdev, 0, 0); 1782f6e51c35SDave Jiang if (!ndev->self_mmio) { 1783f6e51c35SDave Jiang rc = -EIO; 1784f6e51c35SDave Jiang goto err_mmio; 1785f6e51c35SDave Jiang } 1786f6e51c35SDave Jiang ndev->peer_mmio = ndev->self_mmio; 1787f6e51c35SDave Jiang ndev->peer_addr = pci_resource_start(pdev, 0); 1788f6e51c35SDave Jiang 1789f6e51c35SDave Jiang return 0; 1790f6e51c35SDave Jiang 1791f6e51c35SDave Jiang err_mmio: 1792f6e51c35SDave Jiang err_dma_mask: 1793f6e51c35SDave Jiang pci_clear_master(pdev); 1794f6e51c35SDave Jiang pci_release_regions(pdev); 1795f6e51c35SDave Jiang err_pci_regions: 1796f6e51c35SDave Jiang pci_disable_device(pdev); 1797f6e51c35SDave Jiang err_pci_enable: 1798f6e51c35SDave Jiang pci_set_drvdata(pdev, NULL); 1799f6e51c35SDave Jiang return rc; 1800f6e51c35SDave Jiang } 1801f6e51c35SDave Jiang 1802f6e51c35SDave Jiang static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev) 1803f6e51c35SDave Jiang { 1804f6e51c35SDave Jiang struct pci_dev *pdev = ndev->ntb.pdev; 1805f6e51c35SDave Jiang 1806f6e51c35SDave Jiang if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio) 1807f6e51c35SDave Jiang pci_iounmap(pdev, ndev->peer_mmio); 1808f6e51c35SDave Jiang pci_iounmap(pdev, ndev->self_mmio); 1809f6e51c35SDave Jiang 1810f6e51c35SDave Jiang pci_clear_master(pdev); 1811f6e51c35SDave Jiang pci_release_regions(pdev); 1812f6e51c35SDave Jiang pci_disable_device(pdev); 1813f6e51c35SDave Jiang pci_set_drvdata(pdev, NULL); 1814f6e51c35SDave Jiang } 1815f6e51c35SDave Jiang 1816f6e51c35SDave Jiang static inline void ndev_init_struct(struct intel_ntb_dev *ndev, 1817f6e51c35SDave Jiang struct pci_dev *pdev) 1818f6e51c35SDave Jiang { 1819f6e51c35SDave Jiang ndev->ntb.pdev = pdev; 1820f6e51c35SDave Jiang ndev->ntb.topo = NTB_TOPO_NONE; 1821f6e51c35SDave Jiang ndev->ntb.ops = &intel_ntb_ops; 1822f6e51c35SDave Jiang 1823f6e51c35SDave Jiang ndev->b2b_off = 0; 1824f6e51c35SDave Jiang ndev->b2b_idx = UINT_MAX; 1825f6e51c35SDave Jiang 1826f6e51c35SDave Jiang ndev->bar4_split = 0; 1827f6e51c35SDave Jiang 1828f6e51c35SDave Jiang ndev->mw_count = 0; 1829f6e51c35SDave Jiang ndev->spad_count = 0; 1830f6e51c35SDave Jiang ndev->db_count = 0; 1831f6e51c35SDave Jiang ndev->db_vec_count = 0; 1832f6e51c35SDave Jiang ndev->db_vec_shift = 0; 1833f6e51c35SDave Jiang 1834f6e51c35SDave Jiang ndev->ntb_ctl = 0; 1835f6e51c35SDave Jiang ndev->lnk_sta = 0; 1836f6e51c35SDave Jiang 1837f6e51c35SDave Jiang ndev->db_valid_mask = 0; 1838f6e51c35SDave Jiang ndev->db_link_mask = 0; 1839f6e51c35SDave Jiang ndev->db_mask = 0; 1840f6e51c35SDave Jiang 1841f6e51c35SDave Jiang spin_lock_init(&ndev->db_mask_lock); 1842f6e51c35SDave Jiang } 1843f6e51c35SDave Jiang 1844f6e51c35SDave Jiang static int intel_ntb_pci_probe(struct pci_dev *pdev, 1845f6e51c35SDave Jiang const struct pci_device_id *id) 1846f6e51c35SDave Jiang { 1847f6e51c35SDave Jiang struct intel_ntb_dev *ndev; 1848f6e51c35SDave Jiang int rc, node; 1849f6e51c35SDave Jiang 1850f6e51c35SDave Jiang node = dev_to_node(&pdev->dev); 1851f6e51c35SDave Jiang 1852f6e51c35SDave Jiang if (pdev_is_xeon(pdev)) { 1853f6e51c35SDave Jiang ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node); 1854f6e51c35SDave Jiang if (!ndev) { 1855f6e51c35SDave Jiang rc = -ENOMEM; 1856f6e51c35SDave Jiang goto err_ndev; 1857f6e51c35SDave Jiang } 1858f6e51c35SDave Jiang 1859f6e51c35SDave Jiang ndev_init_struct(ndev, pdev); 1860f6e51c35SDave Jiang 1861f6e51c35SDave Jiang rc = intel_ntb_init_pci(ndev, pdev); 1862f6e51c35SDave Jiang if (rc) 1863f6e51c35SDave Jiang goto err_init_pci; 1864f6e51c35SDave Jiang 1865f6e51c35SDave Jiang rc = xeon_init_dev(ndev); 1866f6e51c35SDave Jiang if (rc) 1867f6e51c35SDave Jiang goto err_init_dev; 1868f6e51c35SDave Jiang 1869f6e51c35SDave Jiang } else if (pdev_is_skx_xeon(pdev)) { 1870f6e51c35SDave Jiang ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node); 1871f6e51c35SDave Jiang if (!ndev) { 1872f6e51c35SDave Jiang rc = -ENOMEM; 1873f6e51c35SDave Jiang goto err_ndev; 1874f6e51c35SDave Jiang } 1875f6e51c35SDave Jiang 1876f6e51c35SDave Jiang ndev_init_struct(ndev, pdev); 1877f6e51c35SDave Jiang ndev->ntb.ops = &intel_ntb3_ops; 1878f6e51c35SDave Jiang 1879f6e51c35SDave Jiang rc = intel_ntb_init_pci(ndev, pdev); 1880f6e51c35SDave Jiang if (rc) 1881f6e51c35SDave Jiang goto err_init_pci; 1882f6e51c35SDave Jiang 1883f6e51c35SDave Jiang rc = skx_init_dev(ndev); 1884f6e51c35SDave Jiang if (rc) 1885f6e51c35SDave Jiang goto err_init_dev; 1886f6e51c35SDave Jiang 1887f6e51c35SDave Jiang } else { 1888f6e51c35SDave Jiang rc = -EINVAL; 1889f6e51c35SDave Jiang goto err_ndev; 1890f6e51c35SDave Jiang } 1891f6e51c35SDave Jiang 1892f6e51c35SDave Jiang ndev_reset_unsafe_flags(ndev); 1893f6e51c35SDave Jiang 1894f6e51c35SDave Jiang ndev->reg->poll_link(ndev); 1895f6e51c35SDave Jiang 1896f6e51c35SDave Jiang ndev_init_debugfs(ndev); 1897f6e51c35SDave Jiang 1898f6e51c35SDave Jiang rc = ntb_register_device(&ndev->ntb); 1899f6e51c35SDave Jiang if (rc) 1900f6e51c35SDave Jiang goto err_register; 1901f6e51c35SDave Jiang 1902f6e51c35SDave Jiang dev_info(&pdev->dev, "NTB device registered.\n"); 1903f6e51c35SDave Jiang 1904f6e51c35SDave Jiang return 0; 1905f6e51c35SDave Jiang 1906f6e51c35SDave Jiang err_register: 1907f6e51c35SDave Jiang ndev_deinit_debugfs(ndev); 1908f6e51c35SDave Jiang if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev)) 1909f6e51c35SDave Jiang xeon_deinit_dev(ndev); 1910f6e51c35SDave Jiang err_init_dev: 1911f6e51c35SDave Jiang intel_ntb_deinit_pci(ndev); 1912f6e51c35SDave Jiang err_init_pci: 1913f6e51c35SDave Jiang kfree(ndev); 1914f6e51c35SDave Jiang err_ndev: 1915f6e51c35SDave Jiang return rc; 1916f6e51c35SDave Jiang } 1917f6e51c35SDave Jiang 1918f6e51c35SDave Jiang static void intel_ntb_pci_remove(struct pci_dev *pdev) 1919f6e51c35SDave Jiang { 1920f6e51c35SDave Jiang struct intel_ntb_dev *ndev = pci_get_drvdata(pdev); 1921f6e51c35SDave Jiang 1922f6e51c35SDave Jiang ntb_unregister_device(&ndev->ntb); 1923f6e51c35SDave Jiang ndev_deinit_debugfs(ndev); 1924f6e51c35SDave Jiang if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev)) 1925f6e51c35SDave Jiang xeon_deinit_dev(ndev); 1926f6e51c35SDave Jiang intel_ntb_deinit_pci(ndev); 1927f6e51c35SDave Jiang kfree(ndev); 1928f6e51c35SDave Jiang } 1929f6e51c35SDave Jiang 1930f6e51c35SDave Jiang static const struct intel_ntb_reg xeon_reg = { 1931f6e51c35SDave Jiang .poll_link = xeon_poll_link, 1932f6e51c35SDave Jiang .link_is_up = xeon_link_is_up, 1933f6e51c35SDave Jiang .db_ioread = xeon_db_ioread, 1934f6e51c35SDave Jiang .db_iowrite = xeon_db_iowrite, 1935f6e51c35SDave Jiang .db_size = sizeof(u32), 1936f6e51c35SDave Jiang .ntb_ctl = XEON_NTBCNTL_OFFSET, 1937f6e51c35SDave Jiang .mw_bar = {2, 4, 5}, 1938f6e51c35SDave Jiang }; 1939f6e51c35SDave Jiang 1940f6e51c35SDave Jiang static const struct intel_ntb_alt_reg xeon_pri_reg = { 1941f6e51c35SDave Jiang .db_bell = XEON_PDOORBELL_OFFSET, 1942f6e51c35SDave Jiang .db_mask = XEON_PDBMSK_OFFSET, 1943f6e51c35SDave Jiang .spad = XEON_SPAD_OFFSET, 1944f6e51c35SDave Jiang }; 1945f6e51c35SDave Jiang 1946f6e51c35SDave Jiang static const struct intel_ntb_alt_reg xeon_sec_reg = { 1947f6e51c35SDave Jiang .db_bell = XEON_SDOORBELL_OFFSET, 1948f6e51c35SDave Jiang .db_mask = XEON_SDBMSK_OFFSET, 1949f6e51c35SDave Jiang /* second half of the scratchpads */ 1950f6e51c35SDave Jiang .spad = XEON_SPAD_OFFSET + (XEON_SPAD_COUNT << 1), 1951f6e51c35SDave Jiang }; 1952f6e51c35SDave Jiang 1953f6e51c35SDave Jiang static const struct intel_ntb_alt_reg xeon_b2b_reg = { 1954f6e51c35SDave Jiang .db_bell = XEON_B2B_DOORBELL_OFFSET, 1955f6e51c35SDave Jiang .spad = XEON_B2B_SPAD_OFFSET, 1956f6e51c35SDave Jiang }; 1957f6e51c35SDave Jiang 1958f6e51c35SDave Jiang static const struct intel_ntb_xlat_reg xeon_pri_xlat = { 1959f6e51c35SDave Jiang /* Note: no primary .bar0_base visible to the secondary side. 1960f6e51c35SDave Jiang * 1961f6e51c35SDave Jiang * The secondary side cannot get the base address stored in primary 1962f6e51c35SDave Jiang * bars. The base address is necessary to set the limit register to 1963f6e51c35SDave Jiang * any value other than zero, or unlimited. 1964f6e51c35SDave Jiang * 1965f6e51c35SDave Jiang * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the 1966f6e51c35SDave Jiang * window by setting the limit equal to base, nor can it limit the size 1967f6e51c35SDave Jiang * of the memory window by setting the limit to base + size. 1968f6e51c35SDave Jiang */ 1969f6e51c35SDave Jiang .bar2_limit = XEON_PBAR23LMT_OFFSET, 1970f6e51c35SDave Jiang .bar2_xlat = XEON_PBAR23XLAT_OFFSET, 1971f6e51c35SDave Jiang }; 1972f6e51c35SDave Jiang 1973f6e51c35SDave Jiang static const struct intel_ntb_xlat_reg xeon_sec_xlat = { 1974f6e51c35SDave Jiang .bar0_base = XEON_SBAR0BASE_OFFSET, 1975f6e51c35SDave Jiang .bar2_limit = XEON_SBAR23LMT_OFFSET, 1976f6e51c35SDave Jiang .bar2_xlat = XEON_SBAR23XLAT_OFFSET, 1977f6e51c35SDave Jiang }; 1978f6e51c35SDave Jiang 1979f6e51c35SDave Jiang struct intel_b2b_addr xeon_b2b_usd_addr = { 1980f6e51c35SDave Jiang .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 1981f6e51c35SDave Jiang .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 1982f6e51c35SDave Jiang .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 1983f6e51c35SDave Jiang .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 1984f6e51c35SDave Jiang }; 1985f6e51c35SDave Jiang 1986f6e51c35SDave Jiang struct intel_b2b_addr xeon_b2b_dsd_addr = { 1987f6e51c35SDave Jiang .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 1988f6e51c35SDave Jiang .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 1989f6e51c35SDave Jiang .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 1990f6e51c35SDave Jiang .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 1991f6e51c35SDave Jiang }; 1992f6e51c35SDave Jiang 1993f6e51c35SDave Jiang /* operations for primary side of local ntb */ 1994f6e51c35SDave Jiang static const struct ntb_dev_ops intel_ntb_ops = { 1995f6e51c35SDave Jiang .mw_count = intel_ntb_mw_count, 1996f6e51c35SDave Jiang .mw_get_align = intel_ntb_mw_get_align, 1997f6e51c35SDave Jiang .mw_set_trans = intel_ntb_mw_set_trans, 1998f6e51c35SDave Jiang .peer_mw_count = intel_ntb_peer_mw_count, 1999f6e51c35SDave Jiang .peer_mw_get_addr = intel_ntb_peer_mw_get_addr, 2000f6e51c35SDave Jiang .link_is_up = intel_ntb_link_is_up, 2001f6e51c35SDave Jiang .link_enable = intel_ntb_link_enable, 2002f6e51c35SDave Jiang .link_disable = intel_ntb_link_disable, 2003f6e51c35SDave Jiang .db_is_unsafe = intel_ntb_db_is_unsafe, 2004f6e51c35SDave Jiang .db_valid_mask = intel_ntb_db_valid_mask, 2005f6e51c35SDave Jiang .db_vector_count = intel_ntb_db_vector_count, 2006f6e51c35SDave Jiang .db_vector_mask = intel_ntb_db_vector_mask, 2007f6e51c35SDave Jiang .db_read = intel_ntb_db_read, 2008f6e51c35SDave Jiang .db_clear = intel_ntb_db_clear, 2009f6e51c35SDave Jiang .db_set_mask = intel_ntb_db_set_mask, 2010f6e51c35SDave Jiang .db_clear_mask = intel_ntb_db_clear_mask, 2011f6e51c35SDave Jiang .peer_db_addr = intel_ntb_peer_db_addr, 2012f6e51c35SDave Jiang .peer_db_set = intel_ntb_peer_db_set, 2013f6e51c35SDave Jiang .spad_is_unsafe = intel_ntb_spad_is_unsafe, 2014f6e51c35SDave Jiang .spad_count = intel_ntb_spad_count, 2015f6e51c35SDave Jiang .spad_read = intel_ntb_spad_read, 2016f6e51c35SDave Jiang .spad_write = intel_ntb_spad_write, 2017f6e51c35SDave Jiang .peer_spad_addr = intel_ntb_peer_spad_addr, 2018f6e51c35SDave Jiang .peer_spad_read = intel_ntb_peer_spad_read, 2019f6e51c35SDave Jiang .peer_spad_write = intel_ntb_peer_spad_write, 2020f6e51c35SDave Jiang }; 2021f6e51c35SDave Jiang 2022f6e51c35SDave Jiang static const struct file_operations intel_ntb_debugfs_info = { 2023f6e51c35SDave Jiang .owner = THIS_MODULE, 2024f6e51c35SDave Jiang .open = simple_open, 2025f6e51c35SDave Jiang .read = ndev_debugfs_read, 2026f6e51c35SDave Jiang }; 2027f6e51c35SDave Jiang 2028f6e51c35SDave Jiang static const struct pci_device_id intel_ntb_pci_tbl[] = { 2029f6e51c35SDave Jiang {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)}, 2030f6e51c35SDave Jiang {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)}, 2031f6e51c35SDave Jiang {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)}, 2032f6e51c35SDave Jiang {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)}, 2033f6e51c35SDave Jiang {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BDX)}, 2034f6e51c35SDave Jiang {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)}, 2035f6e51c35SDave Jiang {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)}, 2036f6e51c35SDave Jiang {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)}, 2037f6e51c35SDave Jiang {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)}, 2038f6e51c35SDave Jiang {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_BDX)}, 2039f6e51c35SDave Jiang {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)}, 2040f6e51c35SDave Jiang {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)}, 2041f6e51c35SDave Jiang {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)}, 2042f6e51c35SDave Jiang {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)}, 2043f6e51c35SDave Jiang {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_BDX)}, 2044f6e51c35SDave Jiang {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)}, 2045f6e51c35SDave Jiang {0} 2046f6e51c35SDave Jiang }; 2047f6e51c35SDave Jiang MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl); 2048f6e51c35SDave Jiang 2049f6e51c35SDave Jiang static struct pci_driver intel_ntb_pci_driver = { 2050f6e51c35SDave Jiang .name = KBUILD_MODNAME, 2051f6e51c35SDave Jiang .id_table = intel_ntb_pci_tbl, 2052f6e51c35SDave Jiang .probe = intel_ntb_pci_probe, 2053f6e51c35SDave Jiang .remove = intel_ntb_pci_remove, 2054f6e51c35SDave Jiang }; 2055f6e51c35SDave Jiang 2056f6e51c35SDave Jiang static int __init intel_ntb_pci_driver_init(void) 2057f6e51c35SDave Jiang { 2058f6e51c35SDave Jiang pr_info("%s %s\n", NTB_DESC, NTB_VER); 2059f6e51c35SDave Jiang 2060f6e51c35SDave Jiang if (debugfs_initialized()) 2061f6e51c35SDave Jiang debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); 2062f6e51c35SDave Jiang 2063f6e51c35SDave Jiang return pci_register_driver(&intel_ntb_pci_driver); 2064f6e51c35SDave Jiang } 2065f6e51c35SDave Jiang module_init(intel_ntb_pci_driver_init); 2066f6e51c35SDave Jiang 2067f6e51c35SDave Jiang static void __exit intel_ntb_pci_driver_exit(void) 2068f6e51c35SDave Jiang { 2069f6e51c35SDave Jiang pci_unregister_driver(&intel_ntb_pci_driver); 2070f6e51c35SDave Jiang 2071f6e51c35SDave Jiang debugfs_remove_recursive(debugfs_dir); 2072f6e51c35SDave Jiang } 2073f6e51c35SDave Jiang module_exit(intel_ntb_pci_driver_exit); 2074