1 /* 2 * This file is provided under a GPLv2 license. When using or 3 * redistributing this file, you may do so under that license. 4 * 5 * GPL LICENSE SUMMARY 6 * 7 * Copyright (C) 2016 T-Platforms All Rights Reserved. 8 * 9 * This program is free software; you can redistribute it and/or modify it 10 * under the terms and conditions of the GNU General Public License, 11 * version 2, as published by the Free Software Foundation. 12 * 13 * This program is distributed in the hope that it will be useful, but 14 * WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General 16 * Public License for more details. 17 * 18 * You should have received a copy of the GNU General Public License along 19 * with this program; if not, one can be found http://www.gnu.org/licenses/. 20 * 21 * The full GNU General Public License is included in this distribution in 22 * the file called "COPYING". 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 27 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 28 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 29 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 30 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 31 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 32 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 34 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 * 36 * IDT PCIe-switch NTB Linux driver 37 * 38 * Contact Information: 39 * Serge Semin <fancer.lancer@gmail.com>, <Sergey.Semin@t-platforms.ru> 40 */ 41 42 #include <linux/stddef.h> 43 #include <linux/types.h> 44 #include <linux/kernel.h> 45 #include <linux/bitops.h> 46 #include <linux/sizes.h> 47 #include <linux/module.h> 48 #include <linux/moduleparam.h> 49 #include <linux/init.h> 50 #include <linux/interrupt.h> 51 #include <linux/spinlock.h> 52 #include <linux/pci.h> 53 #include <linux/aer.h> 54 #include <linux/slab.h> 55 #include <linux/list.h> 56 #include <linux/debugfs.h> 57 #include <linux/ntb.h> 58 59 #include "ntb_hw_idt.h" 60 61 #define NTB_NAME "ntb_hw_idt" 62 #define NTB_DESC "IDT PCI-E Non-Transparent Bridge Driver" 63 #define NTB_VER "2.0" 64 #define NTB_IRQNAME "ntb_irq_idt" 65 66 MODULE_DESCRIPTION(NTB_DESC); 67 MODULE_VERSION(NTB_VER); 68 MODULE_LICENSE("GPL v2"); 69 MODULE_AUTHOR("T-platforms"); 70 71 /* 72 * NT Endpoint registers table simplifying a loop access to the functionally 73 * related registers 74 */ 75 static const struct idt_ntb_regs ntdata_tbl = { 76 { {IDT_NT_BARSETUP0, IDT_NT_BARLIMIT0, 77 IDT_NT_BARLTBASE0, IDT_NT_BARUTBASE0}, 78 {IDT_NT_BARSETUP1, IDT_NT_BARLIMIT1, 79 IDT_NT_BARLTBASE1, IDT_NT_BARUTBASE1}, 80 {IDT_NT_BARSETUP2, IDT_NT_BARLIMIT2, 81 IDT_NT_BARLTBASE2, IDT_NT_BARUTBASE2}, 82 {IDT_NT_BARSETUP3, IDT_NT_BARLIMIT3, 83 IDT_NT_BARLTBASE3, IDT_NT_BARUTBASE3}, 84 {IDT_NT_BARSETUP4, IDT_NT_BARLIMIT4, 85 IDT_NT_BARLTBASE4, IDT_NT_BARUTBASE4}, 86 {IDT_NT_BARSETUP5, IDT_NT_BARLIMIT5, 87 IDT_NT_BARLTBASE5, IDT_NT_BARUTBASE5} }, 88 { {IDT_NT_INMSG0, IDT_NT_OUTMSG0, IDT_NT_INMSGSRC0}, 89 {IDT_NT_INMSG1, IDT_NT_OUTMSG1, IDT_NT_INMSGSRC1}, 90 {IDT_NT_INMSG2, IDT_NT_OUTMSG2, IDT_NT_INMSGSRC2}, 91 {IDT_NT_INMSG3, IDT_NT_OUTMSG3, IDT_NT_INMSGSRC3} } 92 }; 93 94 /* 95 * NT Endpoint ports data table with the corresponding pcie command, link 96 * status, control and BAR-related registers 97 */ 98 static const struct idt_ntb_port portdata_tbl[IDT_MAX_NR_PORTS] = { 99 /*0*/ { IDT_SW_NTP0_PCIECMDSTS, IDT_SW_NTP0_PCIELCTLSTS, 100 IDT_SW_NTP0_NTCTL, 101 IDT_SW_SWPORT0CTL, IDT_SW_SWPORT0STS, 102 { {IDT_SW_NTP0_BARSETUP0, IDT_SW_NTP0_BARLIMIT0, 103 IDT_SW_NTP0_BARLTBASE0, IDT_SW_NTP0_BARUTBASE0}, 104 {IDT_SW_NTP0_BARSETUP1, IDT_SW_NTP0_BARLIMIT1, 105 IDT_SW_NTP0_BARLTBASE1, IDT_SW_NTP0_BARUTBASE1}, 106 {IDT_SW_NTP0_BARSETUP2, IDT_SW_NTP0_BARLIMIT2, 107 IDT_SW_NTP0_BARLTBASE2, IDT_SW_NTP0_BARUTBASE2}, 108 {IDT_SW_NTP0_BARSETUP3, IDT_SW_NTP0_BARLIMIT3, 109 IDT_SW_NTP0_BARLTBASE3, IDT_SW_NTP0_BARUTBASE3}, 110 {IDT_SW_NTP0_BARSETUP4, IDT_SW_NTP0_BARLIMIT4, 111 IDT_SW_NTP0_BARLTBASE4, IDT_SW_NTP0_BARUTBASE4}, 112 {IDT_SW_NTP0_BARSETUP5, IDT_SW_NTP0_BARLIMIT5, 113 IDT_SW_NTP0_BARLTBASE5, IDT_SW_NTP0_BARUTBASE5} } }, 114 /*1*/ {0}, 115 /*2*/ { IDT_SW_NTP2_PCIECMDSTS, IDT_SW_NTP2_PCIELCTLSTS, 116 IDT_SW_NTP2_NTCTL, 117 IDT_SW_SWPORT2CTL, IDT_SW_SWPORT2STS, 118 { {IDT_SW_NTP2_BARSETUP0, IDT_SW_NTP2_BARLIMIT0, 119 IDT_SW_NTP2_BARLTBASE0, IDT_SW_NTP2_BARUTBASE0}, 120 {IDT_SW_NTP2_BARSETUP1, IDT_SW_NTP2_BARLIMIT1, 121 IDT_SW_NTP2_BARLTBASE1, IDT_SW_NTP2_BARUTBASE1}, 122 {IDT_SW_NTP2_BARSETUP2, IDT_SW_NTP2_BARLIMIT2, 123 IDT_SW_NTP2_BARLTBASE2, IDT_SW_NTP2_BARUTBASE2}, 124 {IDT_SW_NTP2_BARSETUP3, IDT_SW_NTP2_BARLIMIT3, 125 IDT_SW_NTP2_BARLTBASE3, IDT_SW_NTP2_BARUTBASE3}, 126 {IDT_SW_NTP2_BARSETUP4, IDT_SW_NTP2_BARLIMIT4, 127 IDT_SW_NTP2_BARLTBASE4, IDT_SW_NTP2_BARUTBASE4}, 128 {IDT_SW_NTP2_BARSETUP5, IDT_SW_NTP2_BARLIMIT5, 129 IDT_SW_NTP2_BARLTBASE5, IDT_SW_NTP2_BARUTBASE5} } }, 130 /*3*/ {0}, 131 /*4*/ { IDT_SW_NTP4_PCIECMDSTS, IDT_SW_NTP4_PCIELCTLSTS, 132 IDT_SW_NTP4_NTCTL, 133 IDT_SW_SWPORT4CTL, IDT_SW_SWPORT4STS, 134 { {IDT_SW_NTP4_BARSETUP0, IDT_SW_NTP4_BARLIMIT0, 135 IDT_SW_NTP4_BARLTBASE0, IDT_SW_NTP4_BARUTBASE0}, 136 {IDT_SW_NTP4_BARSETUP1, IDT_SW_NTP4_BARLIMIT1, 137 IDT_SW_NTP4_BARLTBASE1, IDT_SW_NTP4_BARUTBASE1}, 138 {IDT_SW_NTP4_BARSETUP2, IDT_SW_NTP4_BARLIMIT2, 139 IDT_SW_NTP4_BARLTBASE2, IDT_SW_NTP4_BARUTBASE2}, 140 {IDT_SW_NTP4_BARSETUP3, IDT_SW_NTP4_BARLIMIT3, 141 IDT_SW_NTP4_BARLTBASE3, IDT_SW_NTP4_BARUTBASE3}, 142 {IDT_SW_NTP4_BARSETUP4, IDT_SW_NTP4_BARLIMIT4, 143 IDT_SW_NTP4_BARLTBASE4, IDT_SW_NTP4_BARUTBASE4}, 144 {IDT_SW_NTP4_BARSETUP5, IDT_SW_NTP4_BARLIMIT5, 145 IDT_SW_NTP4_BARLTBASE5, IDT_SW_NTP4_BARUTBASE5} } }, 146 /*5*/ {0}, 147 /*6*/ { IDT_SW_NTP6_PCIECMDSTS, IDT_SW_NTP6_PCIELCTLSTS, 148 IDT_SW_NTP6_NTCTL, 149 IDT_SW_SWPORT6CTL, IDT_SW_SWPORT6STS, 150 { {IDT_SW_NTP6_BARSETUP0, IDT_SW_NTP6_BARLIMIT0, 151 IDT_SW_NTP6_BARLTBASE0, IDT_SW_NTP6_BARUTBASE0}, 152 {IDT_SW_NTP6_BARSETUP1, IDT_SW_NTP6_BARLIMIT1, 153 IDT_SW_NTP6_BARLTBASE1, IDT_SW_NTP6_BARUTBASE1}, 154 {IDT_SW_NTP6_BARSETUP2, IDT_SW_NTP6_BARLIMIT2, 155 IDT_SW_NTP6_BARLTBASE2, IDT_SW_NTP6_BARUTBASE2}, 156 {IDT_SW_NTP6_BARSETUP3, IDT_SW_NTP6_BARLIMIT3, 157 IDT_SW_NTP6_BARLTBASE3, IDT_SW_NTP6_BARUTBASE3}, 158 {IDT_SW_NTP6_BARSETUP4, IDT_SW_NTP6_BARLIMIT4, 159 IDT_SW_NTP6_BARLTBASE4, IDT_SW_NTP6_BARUTBASE4}, 160 {IDT_SW_NTP6_BARSETUP5, IDT_SW_NTP6_BARLIMIT5, 161 IDT_SW_NTP6_BARLTBASE5, IDT_SW_NTP6_BARUTBASE5} } }, 162 /*7*/ {0}, 163 /*8*/ { IDT_SW_NTP8_PCIECMDSTS, IDT_SW_NTP8_PCIELCTLSTS, 164 IDT_SW_NTP8_NTCTL, 165 IDT_SW_SWPORT8CTL, IDT_SW_SWPORT8STS, 166 { {IDT_SW_NTP8_BARSETUP0, IDT_SW_NTP8_BARLIMIT0, 167 IDT_SW_NTP8_BARLTBASE0, IDT_SW_NTP8_BARUTBASE0}, 168 {IDT_SW_NTP8_BARSETUP1, IDT_SW_NTP8_BARLIMIT1, 169 IDT_SW_NTP8_BARLTBASE1, IDT_SW_NTP8_BARUTBASE1}, 170 {IDT_SW_NTP8_BARSETUP2, IDT_SW_NTP8_BARLIMIT2, 171 IDT_SW_NTP8_BARLTBASE2, IDT_SW_NTP8_BARUTBASE2}, 172 {IDT_SW_NTP8_BARSETUP3, IDT_SW_NTP8_BARLIMIT3, 173 IDT_SW_NTP8_BARLTBASE3, IDT_SW_NTP8_BARUTBASE3}, 174 {IDT_SW_NTP8_BARSETUP4, IDT_SW_NTP8_BARLIMIT4, 175 IDT_SW_NTP8_BARLTBASE4, IDT_SW_NTP8_BARUTBASE4}, 176 {IDT_SW_NTP8_BARSETUP5, IDT_SW_NTP8_BARLIMIT5, 177 IDT_SW_NTP8_BARLTBASE5, IDT_SW_NTP8_BARUTBASE5} } }, 178 /*9*/ {0}, 179 /*10*/ {0}, 180 /*11*/ {0}, 181 /*12*/ { IDT_SW_NTP12_PCIECMDSTS, IDT_SW_NTP12_PCIELCTLSTS, 182 IDT_SW_NTP12_NTCTL, 183 IDT_SW_SWPORT12CTL, IDT_SW_SWPORT12STS, 184 { {IDT_SW_NTP12_BARSETUP0, IDT_SW_NTP12_BARLIMIT0, 185 IDT_SW_NTP12_BARLTBASE0, IDT_SW_NTP12_BARUTBASE0}, 186 {IDT_SW_NTP12_BARSETUP1, IDT_SW_NTP12_BARLIMIT1, 187 IDT_SW_NTP12_BARLTBASE1, IDT_SW_NTP12_BARUTBASE1}, 188 {IDT_SW_NTP12_BARSETUP2, IDT_SW_NTP12_BARLIMIT2, 189 IDT_SW_NTP12_BARLTBASE2, IDT_SW_NTP12_BARUTBASE2}, 190 {IDT_SW_NTP12_BARSETUP3, IDT_SW_NTP12_BARLIMIT3, 191 IDT_SW_NTP12_BARLTBASE3, IDT_SW_NTP12_BARUTBASE3}, 192 {IDT_SW_NTP12_BARSETUP4, IDT_SW_NTP12_BARLIMIT4, 193 IDT_SW_NTP12_BARLTBASE4, IDT_SW_NTP12_BARUTBASE4}, 194 {IDT_SW_NTP12_BARSETUP5, IDT_SW_NTP12_BARLIMIT5, 195 IDT_SW_NTP12_BARLTBASE5, IDT_SW_NTP12_BARUTBASE5} } }, 196 /*13*/ {0}, 197 /*14*/ {0}, 198 /*15*/ {0}, 199 /*16*/ { IDT_SW_NTP16_PCIECMDSTS, IDT_SW_NTP16_PCIELCTLSTS, 200 IDT_SW_NTP16_NTCTL, 201 IDT_SW_SWPORT16CTL, IDT_SW_SWPORT16STS, 202 { {IDT_SW_NTP16_BARSETUP0, IDT_SW_NTP16_BARLIMIT0, 203 IDT_SW_NTP16_BARLTBASE0, IDT_SW_NTP16_BARUTBASE0}, 204 {IDT_SW_NTP16_BARSETUP1, IDT_SW_NTP16_BARLIMIT1, 205 IDT_SW_NTP16_BARLTBASE1, IDT_SW_NTP16_BARUTBASE1}, 206 {IDT_SW_NTP16_BARSETUP2, IDT_SW_NTP16_BARLIMIT2, 207 IDT_SW_NTP16_BARLTBASE2, IDT_SW_NTP16_BARUTBASE2}, 208 {IDT_SW_NTP16_BARSETUP3, IDT_SW_NTP16_BARLIMIT3, 209 IDT_SW_NTP16_BARLTBASE3, IDT_SW_NTP16_BARUTBASE3}, 210 {IDT_SW_NTP16_BARSETUP4, IDT_SW_NTP16_BARLIMIT4, 211 IDT_SW_NTP16_BARLTBASE4, IDT_SW_NTP16_BARUTBASE4}, 212 {IDT_SW_NTP16_BARSETUP5, IDT_SW_NTP16_BARLIMIT5, 213 IDT_SW_NTP16_BARLTBASE5, IDT_SW_NTP16_BARUTBASE5} } }, 214 /*17*/ {0}, 215 /*18*/ {0}, 216 /*19*/ {0}, 217 /*20*/ { IDT_SW_NTP20_PCIECMDSTS, IDT_SW_NTP20_PCIELCTLSTS, 218 IDT_SW_NTP20_NTCTL, 219 IDT_SW_SWPORT20CTL, IDT_SW_SWPORT20STS, 220 { {IDT_SW_NTP20_BARSETUP0, IDT_SW_NTP20_BARLIMIT0, 221 IDT_SW_NTP20_BARLTBASE0, IDT_SW_NTP20_BARUTBASE0}, 222 {IDT_SW_NTP20_BARSETUP1, IDT_SW_NTP20_BARLIMIT1, 223 IDT_SW_NTP20_BARLTBASE1, IDT_SW_NTP20_BARUTBASE1}, 224 {IDT_SW_NTP20_BARSETUP2, IDT_SW_NTP20_BARLIMIT2, 225 IDT_SW_NTP20_BARLTBASE2, IDT_SW_NTP20_BARUTBASE2}, 226 {IDT_SW_NTP20_BARSETUP3, IDT_SW_NTP20_BARLIMIT3, 227 IDT_SW_NTP20_BARLTBASE3, IDT_SW_NTP20_BARUTBASE3}, 228 {IDT_SW_NTP20_BARSETUP4, IDT_SW_NTP20_BARLIMIT4, 229 IDT_SW_NTP20_BARLTBASE4, IDT_SW_NTP20_BARUTBASE4}, 230 {IDT_SW_NTP20_BARSETUP5, IDT_SW_NTP20_BARLIMIT5, 231 IDT_SW_NTP20_BARLTBASE5, IDT_SW_NTP20_BARUTBASE5} } }, 232 /*21*/ {0}, 233 /*22*/ {0}, 234 /*23*/ {0} 235 }; 236 237 /* 238 * IDT PCIe-switch partitions table with the corresponding control, status 239 * and messages control registers 240 */ 241 static const struct idt_ntb_part partdata_tbl[IDT_MAX_NR_PARTS] = { 242 /*0*/ { IDT_SW_SWPART0CTL, IDT_SW_SWPART0STS, 243 {IDT_SW_SWP0MSGCTL0, IDT_SW_SWP0MSGCTL1, 244 IDT_SW_SWP0MSGCTL2, IDT_SW_SWP0MSGCTL3} }, 245 /*1*/ { IDT_SW_SWPART1CTL, IDT_SW_SWPART1STS, 246 {IDT_SW_SWP1MSGCTL0, IDT_SW_SWP1MSGCTL1, 247 IDT_SW_SWP1MSGCTL2, IDT_SW_SWP1MSGCTL3} }, 248 /*2*/ { IDT_SW_SWPART2CTL, IDT_SW_SWPART2STS, 249 {IDT_SW_SWP2MSGCTL0, IDT_SW_SWP2MSGCTL1, 250 IDT_SW_SWP2MSGCTL2, IDT_SW_SWP2MSGCTL3} }, 251 /*3*/ { IDT_SW_SWPART3CTL, IDT_SW_SWPART3STS, 252 {IDT_SW_SWP3MSGCTL0, IDT_SW_SWP3MSGCTL1, 253 IDT_SW_SWP3MSGCTL2, IDT_SW_SWP3MSGCTL3} }, 254 /*4*/ { IDT_SW_SWPART4CTL, IDT_SW_SWPART4STS, 255 {IDT_SW_SWP4MSGCTL0, IDT_SW_SWP4MSGCTL1, 256 IDT_SW_SWP4MSGCTL2, IDT_SW_SWP4MSGCTL3} }, 257 /*5*/ { IDT_SW_SWPART5CTL, IDT_SW_SWPART5STS, 258 {IDT_SW_SWP5MSGCTL0, IDT_SW_SWP5MSGCTL1, 259 IDT_SW_SWP5MSGCTL2, IDT_SW_SWP5MSGCTL3} }, 260 /*6*/ { IDT_SW_SWPART6CTL, IDT_SW_SWPART6STS, 261 {IDT_SW_SWP6MSGCTL0, IDT_SW_SWP6MSGCTL1, 262 IDT_SW_SWP6MSGCTL2, IDT_SW_SWP6MSGCTL3} }, 263 /*7*/ { IDT_SW_SWPART7CTL, IDT_SW_SWPART7STS, 264 {IDT_SW_SWP7MSGCTL0, IDT_SW_SWP7MSGCTL1, 265 IDT_SW_SWP7MSGCTL2, IDT_SW_SWP7MSGCTL3} } 266 }; 267 268 /* 269 * DebugFS directory to place the driver debug file 270 */ 271 static struct dentry *dbgfs_topdir; 272 273 /*============================================================================= 274 * 1. IDT PCIe-switch registers IO-functions 275 * 276 * Beside ordinary configuration space registers IDT PCIe-switch expose 277 * global configuration registers, which are used to determine state of other 278 * device ports as well as being notified of some switch-related events. 279 * Additionally all the configuration space registers of all the IDT 280 * PCIe-switch functions are mapped to the Global Address space, so each 281 * function can determine a configuration of any other PCI-function. 282 * Functions declared in this chapter are created to encapsulate access 283 * to configuration and global registers, so the driver code just need to 284 * provide IDT NTB hardware descriptor and a register address. 285 *============================================================================= 286 */ 287 288 /* 289 * idt_nt_write() - PCI configuration space registers write method 290 * @ndev: IDT NTB hardware driver descriptor 291 * @reg: Register to write data to 292 * @data: Value to write to the register 293 * 294 * IDT PCIe-switch registers are all Little endian. 295 */ 296 static void idt_nt_write(struct idt_ntb_dev *ndev, 297 const unsigned int reg, const u32 data) 298 { 299 /* 300 * It's obvious bug to request a register exceeding the maximum possible 301 * value as well as to have it unaligned. 302 */ 303 if (WARN_ON(reg > IDT_REG_PCI_MAX || !IS_ALIGNED(reg, IDT_REG_ALIGN))) 304 return; 305 306 /* Just write the value to the specified register */ 307 iowrite32(data, ndev->cfgspc + (ptrdiff_t)reg); 308 } 309 310 /* 311 * idt_nt_read() - PCI configuration space registers read method 312 * @ndev: IDT NTB hardware driver descriptor 313 * @reg: Register to write data to 314 * 315 * IDT PCIe-switch Global configuration registers are all Little endian. 316 * 317 * Return: register value 318 */ 319 static u32 idt_nt_read(struct idt_ntb_dev *ndev, const unsigned int reg) 320 { 321 /* 322 * It's obvious bug to request a register exceeding the maximum possible 323 * value as well as to have it unaligned. 324 */ 325 if (WARN_ON(reg > IDT_REG_PCI_MAX || !IS_ALIGNED(reg, IDT_REG_ALIGN))) 326 return ~0; 327 328 /* Just read the value from the specified register */ 329 return ioread32(ndev->cfgspc + (ptrdiff_t)reg); 330 } 331 332 /* 333 * idt_sw_write() - Global registers write method 334 * @ndev: IDT NTB hardware driver descriptor 335 * @reg: Register to write data to 336 * @data: Value to write to the register 337 * 338 * IDT PCIe-switch Global configuration registers are all Little endian. 339 */ 340 static void idt_sw_write(struct idt_ntb_dev *ndev, 341 const unsigned int reg, const u32 data) 342 { 343 unsigned long irqflags; 344 345 /* 346 * It's obvious bug to request a register exceeding the maximum possible 347 * value as well as to have it unaligned. 348 */ 349 if (WARN_ON(reg > IDT_REG_SW_MAX || !IS_ALIGNED(reg, IDT_REG_ALIGN))) 350 return; 351 352 /* Lock GASA registers operations */ 353 spin_lock_irqsave(&ndev->gasa_lock, irqflags); 354 /* Set the global register address */ 355 iowrite32((u32)reg, ndev->cfgspc + (ptrdiff_t)IDT_NT_GASAADDR); 356 /* Put the new value of the register */ 357 iowrite32(data, ndev->cfgspc + (ptrdiff_t)IDT_NT_GASADATA); 358 /* Make sure the PCIe transactions are executed */ 359 mmiowb(); 360 /* Unlock GASA registers operations */ 361 spin_unlock_irqrestore(&ndev->gasa_lock, irqflags); 362 } 363 364 /* 365 * idt_sw_read() - Global registers read method 366 * @ndev: IDT NTB hardware driver descriptor 367 * @reg: Register to write data to 368 * 369 * IDT PCIe-switch Global configuration registers are all Little endian. 370 * 371 * Return: register value 372 */ 373 static u32 idt_sw_read(struct idt_ntb_dev *ndev, const unsigned int reg) 374 { 375 unsigned long irqflags; 376 u32 data; 377 378 /* 379 * It's obvious bug to request a register exceeding the maximum possible 380 * value as well as to have it unaligned. 381 */ 382 if (WARN_ON(reg > IDT_REG_SW_MAX || !IS_ALIGNED(reg, IDT_REG_ALIGN))) 383 return ~0; 384 385 /* Lock GASA registers operations */ 386 spin_lock_irqsave(&ndev->gasa_lock, irqflags); 387 /* Set the global register address */ 388 iowrite32((u32)reg, ndev->cfgspc + (ptrdiff_t)IDT_NT_GASAADDR); 389 /* Get the data of the register (read ops acts as MMIO barrier) */ 390 data = ioread32(ndev->cfgspc + (ptrdiff_t)IDT_NT_GASADATA); 391 /* Unlock GASA registers operations */ 392 spin_unlock_irqrestore(&ndev->gasa_lock, irqflags); 393 394 return data; 395 } 396 397 /* 398 * idt_reg_set_bits() - set bits of a passed register 399 * @ndev: IDT NTB hardware driver descriptor 400 * @reg: Register to change bits of 401 * @reg_lock: Register access spin lock 402 * @valid_mask: Mask of valid bits 403 * @set_bits: Bitmask to set 404 * 405 * Helper method to check whether a passed bitfield is valid and set 406 * corresponding bits of a register. 407 * 408 * WARNING! Make sure the passed register isn't accessed over plane 409 * idt_nt_write() method (read method is ok to be used concurrently). 410 * 411 * Return: zero on success, negative error on invalid bitmask. 412 */ 413 static inline int idt_reg_set_bits(struct idt_ntb_dev *ndev, unsigned int reg, 414 spinlock_t *reg_lock, 415 u64 valid_mask, u64 set_bits) 416 { 417 unsigned long irqflags; 418 u32 data; 419 420 if (set_bits & ~(u64)valid_mask) 421 return -EINVAL; 422 423 /* Lock access to the register unless the change is written back */ 424 spin_lock_irqsave(reg_lock, irqflags); 425 data = idt_nt_read(ndev, reg) | (u32)set_bits; 426 idt_nt_write(ndev, reg, data); 427 /* Unlock the register */ 428 spin_unlock_irqrestore(reg_lock, irqflags); 429 430 return 0; 431 } 432 433 /* 434 * idt_reg_clear_bits() - clear bits of a passed register 435 * @ndev: IDT NTB hardware driver descriptor 436 * @reg: Register to change bits of 437 * @reg_lock: Register access spin lock 438 * @set_bits: Bitmask to clear 439 * 440 * Helper method to check whether a passed bitfield is valid and clear 441 * corresponding bits of a register. 442 * 443 * NOTE! Invalid bits are always considered cleared so it's not an error 444 * to clear them over. 445 * 446 * WARNING! Make sure the passed register isn't accessed over plane 447 * idt_nt_write() method (read method is ok to use concurrently). 448 */ 449 static inline void idt_reg_clear_bits(struct idt_ntb_dev *ndev, 450 unsigned int reg, spinlock_t *reg_lock, 451 u64 clear_bits) 452 { 453 unsigned long irqflags; 454 u32 data; 455 456 /* Lock access to the register unless the change is written back */ 457 spin_lock_irqsave(reg_lock, irqflags); 458 data = idt_nt_read(ndev, reg) & ~(u32)clear_bits; 459 idt_nt_write(ndev, reg, data); 460 /* Unlock the register */ 461 spin_unlock_irqrestore(reg_lock, irqflags); 462 } 463 464 /*=========================================================================== 465 * 2. Ports operations 466 * 467 * IDT PCIe-switches can have from 3 up to 8 ports with possible 468 * NT-functions enabled. So all the possible ports need to be scanned looking 469 * for NTB activated. NTB API will have enumerated only the ports with NTB. 470 *=========================================================================== 471 */ 472 473 /* 474 * idt_scan_ports() - scan IDT PCIe-switch ports collecting info in the tables 475 * @ndev: Pointer to the PCI device descriptor 476 * 477 * Return: zero on success, otherwise a negative error number. 478 */ 479 static int idt_scan_ports(struct idt_ntb_dev *ndev) 480 { 481 unsigned char pidx, port, part; 482 u32 data, portsts, partsts; 483 484 /* Retrieve the local port number */ 485 data = idt_nt_read(ndev, IDT_NT_PCIELCAP); 486 ndev->port = GET_FIELD(PCIELCAP_PORTNUM, data); 487 488 /* Retrieve the local partition number */ 489 portsts = idt_sw_read(ndev, portdata_tbl[ndev->port].sts); 490 ndev->part = GET_FIELD(SWPORTxSTS_SWPART, portsts); 491 492 /* Initialize port/partition -> index tables with invalid values */ 493 memset(ndev->port_idx_map, -EINVAL, sizeof(ndev->port_idx_map)); 494 memset(ndev->part_idx_map, -EINVAL, sizeof(ndev->part_idx_map)); 495 496 /* 497 * Walk over all the possible ports checking whether any of them has 498 * NT-function activated 499 */ 500 ndev->peer_cnt = 0; 501 for (pidx = 0; pidx < ndev->swcfg->port_cnt; pidx++) { 502 port = ndev->swcfg->ports[pidx]; 503 /* Skip local port */ 504 if (port == ndev->port) 505 continue; 506 507 /* Read the port status register to get it partition */ 508 portsts = idt_sw_read(ndev, portdata_tbl[port].sts); 509 part = GET_FIELD(SWPORTxSTS_SWPART, portsts); 510 511 /* Retrieve the partition status */ 512 partsts = idt_sw_read(ndev, partdata_tbl[part].sts); 513 /* Check if partition state is active and port has NTB */ 514 if (IS_FLD_SET(SWPARTxSTS_STATE, partsts, ACT) && 515 (IS_FLD_SET(SWPORTxSTS_MODE, portsts, NT) || 516 IS_FLD_SET(SWPORTxSTS_MODE, portsts, USNT) || 517 IS_FLD_SET(SWPORTxSTS_MODE, portsts, USNTDMA) || 518 IS_FLD_SET(SWPORTxSTS_MODE, portsts, NTDMA))) { 519 /* Save the port and partition numbers */ 520 ndev->peers[ndev->peer_cnt].port = port; 521 ndev->peers[ndev->peer_cnt].part = part; 522 /* Fill in the port/partition -> index tables */ 523 ndev->port_idx_map[port] = ndev->peer_cnt; 524 ndev->part_idx_map[part] = ndev->peer_cnt; 525 ndev->peer_cnt++; 526 } 527 } 528 529 dev_dbg(&ndev->ntb.pdev->dev, "Local port: %hhu, num of peers: %hhu\n", 530 ndev->port, ndev->peer_cnt); 531 532 /* It's useless to have this driver loaded if there is no any peer */ 533 if (ndev->peer_cnt == 0) { 534 dev_warn(&ndev->ntb.pdev->dev, "No active peer found\n"); 535 return -ENODEV; 536 } 537 538 return 0; 539 } 540 541 /* 542 * idt_ntb_port_number() - get the local port number 543 * @ntb: NTB device context. 544 * 545 * Return: the local port number 546 */ 547 static int idt_ntb_port_number(struct ntb_dev *ntb) 548 { 549 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 550 551 return ndev->port; 552 } 553 554 /* 555 * idt_ntb_peer_port_count() - get the number of peer ports 556 * @ntb: NTB device context. 557 * 558 * Return the count of detected peer NT-functions. 559 * 560 * Return: number of peer ports 561 */ 562 static int idt_ntb_peer_port_count(struct ntb_dev *ntb) 563 { 564 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 565 566 return ndev->peer_cnt; 567 } 568 569 /* 570 * idt_ntb_peer_port_number() - get peer port by given index 571 * @ntb: NTB device context. 572 * @pidx: Peer port index. 573 * 574 * Return: peer port or negative error 575 */ 576 static int idt_ntb_peer_port_number(struct ntb_dev *ntb, int pidx) 577 { 578 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 579 580 if (pidx < 0 || ndev->peer_cnt <= pidx) 581 return -EINVAL; 582 583 /* Return the detected NT-function port number */ 584 return ndev->peers[pidx].port; 585 } 586 587 /* 588 * idt_ntb_peer_port_idx() - get peer port index by given port number 589 * @ntb: NTB device context. 590 * @port: Peer port number. 591 * 592 * Internal port -> index table is pre-initialized with -EINVAL values, 593 * so we just need to return it value 594 * 595 * Return: peer NT-function port index or negative error 596 */ 597 static int idt_ntb_peer_port_idx(struct ntb_dev *ntb, int port) 598 { 599 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 600 601 if (port < 0 || IDT_MAX_NR_PORTS <= port) 602 return -EINVAL; 603 604 return ndev->port_idx_map[port]; 605 } 606 607 /*=========================================================================== 608 * 3. Link status operations 609 * There is no any ready-to-use method to have peer ports notified if NTB 610 * link is set up or got down. Instead global signal can be used instead. 611 * In case if any one of ports changes local NTB link state, it sends 612 * global signal and clears corresponding global state bit. Then all the ports 613 * receive a notification of that, so to make client driver being aware of 614 * possible NTB link change. 615 * Additionally each of active NT-functions is subscribed to PCIe-link 616 * state changes of peer ports. 617 *=========================================================================== 618 */ 619 620 static void idt_ntb_local_link_disable(struct idt_ntb_dev *ndev); 621 622 /* 623 * idt_init_link() - Initialize NTB link state notification subsystem 624 * @ndev: IDT NTB hardware driver descriptor 625 * 626 * Function performs the basic initialization of some global registers 627 * needed to enable IRQ-based notifications of PCIe Link Up/Down and 628 * Global Signal events. 629 * NOTE Since it's not possible to determine when all the NTB peer drivers are 630 * unloaded as well as have those registers accessed concurrently, we must 631 * preinitialize them with the same value and leave it uncleared on local 632 * driver unload. 633 */ 634 static void idt_init_link(struct idt_ntb_dev *ndev) 635 { 636 u32 part_mask, port_mask, se_mask; 637 unsigned char pidx; 638 639 /* Initialize spin locker of Mapping Table access registers */ 640 spin_lock_init(&ndev->mtbl_lock); 641 642 /* Walk over all detected peers collecting port and partition masks */ 643 port_mask = ~BIT(ndev->port); 644 part_mask = ~BIT(ndev->part); 645 for (pidx = 0; pidx < ndev->peer_cnt; pidx++) { 646 port_mask &= ~BIT(ndev->peers[pidx].port); 647 part_mask &= ~BIT(ndev->peers[pidx].part); 648 } 649 650 /* Clean the Link Up/Down and GLobal Signal status registers */ 651 idt_sw_write(ndev, IDT_SW_SELINKUPSTS, (u32)-1); 652 idt_sw_write(ndev, IDT_SW_SELINKDNSTS, (u32)-1); 653 idt_sw_write(ndev, IDT_SW_SEGSIGSTS, (u32)-1); 654 655 /* Unmask NT-activated partitions to receive Global Switch events */ 656 idt_sw_write(ndev, IDT_SW_SEPMSK, part_mask); 657 658 /* Enable PCIe Link Up events of NT-activated ports */ 659 idt_sw_write(ndev, IDT_SW_SELINKUPMSK, port_mask); 660 661 /* Enable PCIe Link Down events of NT-activated ports */ 662 idt_sw_write(ndev, IDT_SW_SELINKDNMSK, port_mask); 663 664 /* Unmask NT-activated partitions to receive Global Signal events */ 665 idt_sw_write(ndev, IDT_SW_SEGSIGMSK, part_mask); 666 667 /* Unmask Link Up/Down and Global Switch Events */ 668 se_mask = ~(IDT_SEMSK_LINKUP | IDT_SEMSK_LINKDN | IDT_SEMSK_GSIGNAL); 669 idt_sw_write(ndev, IDT_SW_SEMSK, se_mask); 670 671 dev_dbg(&ndev->ntb.pdev->dev, "NTB link status events initialized"); 672 } 673 674 /* 675 * idt_deinit_link() - deinitialize link subsystem 676 * @ndev: IDT NTB hardware driver descriptor 677 * 678 * Just disable the link back. 679 */ 680 static void idt_deinit_link(struct idt_ntb_dev *ndev) 681 { 682 /* Disable the link */ 683 idt_ntb_local_link_disable(ndev); 684 685 dev_dbg(&ndev->ntb.pdev->dev, "NTB link status events deinitialized"); 686 } 687 688 /* 689 * idt_se_isr() - switch events ISR 690 * @ndev: IDT NTB hardware driver descriptor 691 * @ntint_sts: NT-function interrupt status 692 * 693 * This driver doesn't support IDT PCIe-switch dynamic reconfigurations, 694 * Failover capability, etc, so switch events are utilized to notify of 695 * PCIe and NTB link events. 696 * The method is called from PCIe ISR bottom-half routine. 697 */ 698 static void idt_se_isr(struct idt_ntb_dev *ndev, u32 ntint_sts) 699 { 700 u32 sests; 701 702 /* Read Switch Events status */ 703 sests = idt_sw_read(ndev, IDT_SW_SESTS); 704 705 /* Clean the Link Up/Down and Global Signal status registers */ 706 idt_sw_write(ndev, IDT_SW_SELINKUPSTS, (u32)-1); 707 idt_sw_write(ndev, IDT_SW_SELINKDNSTS, (u32)-1); 708 idt_sw_write(ndev, IDT_SW_SEGSIGSTS, (u32)-1); 709 710 /* Clean the corresponding interrupt bit */ 711 idt_nt_write(ndev, IDT_NT_NTINTSTS, IDT_NTINTSTS_SEVENT); 712 713 dev_dbg(&ndev->ntb.pdev->dev, "SE IRQ detected %#08x (SESTS %#08x)", 714 ntint_sts, sests); 715 716 /* Notify the client driver of possible link state change */ 717 ntb_link_event(&ndev->ntb); 718 } 719 720 /* 721 * idt_ntb_local_link_enable() - enable the local NTB link. 722 * @ndev: IDT NTB hardware driver descriptor 723 * 724 * In order to enable the NTB link we need: 725 * - enable Completion TLPs translation 726 * - initialize mapping table to enable the Request ID translation 727 * - notify peers of NTB link state change 728 */ 729 static void idt_ntb_local_link_enable(struct idt_ntb_dev *ndev) 730 { 731 u32 reqid, mtbldata = 0; 732 unsigned long irqflags; 733 734 /* Enable the ID protection and Completion TLPs translation */ 735 idt_nt_write(ndev, IDT_NT_NTCTL, IDT_NTCTL_CPEN); 736 737 /* Retrieve the current Requester ID (Bus:Device:Function) */ 738 reqid = idt_nt_read(ndev, IDT_NT_REQIDCAP); 739 740 /* 741 * Set the corresponding NT Mapping table entry of port partition index 742 * with the data to perform the Request ID translation 743 */ 744 mtbldata = SET_FIELD(NTMTBLDATA_REQID, 0, reqid) | 745 SET_FIELD(NTMTBLDATA_PART, 0, ndev->part) | 746 IDT_NTMTBLDATA_VALID; 747 spin_lock_irqsave(&ndev->mtbl_lock, irqflags); 748 idt_nt_write(ndev, IDT_NT_NTMTBLADDR, ndev->part); 749 idt_nt_write(ndev, IDT_NT_NTMTBLDATA, mtbldata); 750 mmiowb(); 751 spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags); 752 753 /* Notify the peers by setting and clearing the global signal bit */ 754 idt_nt_write(ndev, IDT_NT_NTGSIGNAL, IDT_NTGSIGNAL_SET); 755 idt_sw_write(ndev, IDT_SW_SEGSIGSTS, (u32)1 << ndev->part); 756 } 757 758 /* 759 * idt_ntb_local_link_disable() - disable the local NTB link. 760 * @ndev: IDT NTB hardware driver descriptor 761 * 762 * In order to enable the NTB link we need: 763 * - disable Completion TLPs translation 764 * - clear corresponding mapping table entry 765 * - notify peers of NTB link state change 766 */ 767 static void idt_ntb_local_link_disable(struct idt_ntb_dev *ndev) 768 { 769 unsigned long irqflags; 770 771 /* Disable Completion TLPs translation */ 772 idt_nt_write(ndev, IDT_NT_NTCTL, 0); 773 774 /* Clear the corresponding NT Mapping table entry */ 775 spin_lock_irqsave(&ndev->mtbl_lock, irqflags); 776 idt_nt_write(ndev, IDT_NT_NTMTBLADDR, ndev->part); 777 idt_nt_write(ndev, IDT_NT_NTMTBLDATA, 0); 778 mmiowb(); 779 spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags); 780 781 /* Notify the peers by setting and clearing the global signal bit */ 782 idt_nt_write(ndev, IDT_NT_NTGSIGNAL, IDT_NTGSIGNAL_SET); 783 idt_sw_write(ndev, IDT_SW_SEGSIGSTS, (u32)1 << ndev->part); 784 } 785 786 /* 787 * idt_ntb_local_link_is_up() - test wethter local NTB link is up 788 * @ndev: IDT NTB hardware driver descriptor 789 * 790 * Local link is up under the following conditions: 791 * - Bus mastering is enabled 792 * - NTCTL has Completion TLPs translation enabled 793 * - Mapping table permits Request TLPs translation 794 * NOTE: We don't need to check PCIe link state since it's obviously 795 * up while we are able to communicate with IDT PCIe-switch 796 * 797 * Return: true if link is up, otherwise false 798 */ 799 static bool idt_ntb_local_link_is_up(struct idt_ntb_dev *ndev) 800 { 801 unsigned long irqflags; 802 u32 data; 803 804 /* Read the local Bus Master Enable status */ 805 data = idt_nt_read(ndev, IDT_NT_PCICMDSTS); 806 if (!(data & IDT_PCICMDSTS_BME)) 807 return false; 808 809 /* Read the local Completion TLPs translation enable status */ 810 data = idt_nt_read(ndev, IDT_NT_NTCTL); 811 if (!(data & IDT_NTCTL_CPEN)) 812 return false; 813 814 /* Read Mapping table entry corresponding to the local partition */ 815 spin_lock_irqsave(&ndev->mtbl_lock, irqflags); 816 idt_nt_write(ndev, IDT_NT_NTMTBLADDR, ndev->part); 817 data = idt_nt_read(ndev, IDT_NT_NTMTBLDATA); 818 spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags); 819 820 return !!(data & IDT_NTMTBLDATA_VALID); 821 } 822 823 /* 824 * idt_ntb_peer_link_is_up() - test whether peer NTB link is up 825 * @ndev: IDT NTB hardware driver descriptor 826 * @pidx: Peer port index 827 * 828 * Peer link is up under the following conditions: 829 * - PCIe link is up 830 * - Bus mastering is enabled 831 * - NTCTL has Completion TLPs translation enabled 832 * - Mapping table permits Request TLPs translation 833 * 834 * Return: true if link is up, otherwise false 835 */ 836 static bool idt_ntb_peer_link_is_up(struct idt_ntb_dev *ndev, int pidx) 837 { 838 unsigned long irqflags; 839 unsigned char port; 840 u32 data; 841 842 /* Retrieve the device port number */ 843 port = ndev->peers[pidx].port; 844 845 /* Check whether PCIe link is up */ 846 data = idt_sw_read(ndev, portdata_tbl[port].sts); 847 if (!(data & IDT_SWPORTxSTS_LINKUP)) 848 return false; 849 850 /* Check whether bus mastering is enabled on the peer port */ 851 data = idt_sw_read(ndev, portdata_tbl[port].pcicmdsts); 852 if (!(data & IDT_PCICMDSTS_BME)) 853 return false; 854 855 /* Check if Completion TLPs translation is enabled on the peer port */ 856 data = idt_sw_read(ndev, portdata_tbl[port].ntctl); 857 if (!(data & IDT_NTCTL_CPEN)) 858 return false; 859 860 /* Read Mapping table entry corresponding to the peer partition */ 861 spin_lock_irqsave(&ndev->mtbl_lock, irqflags); 862 idt_nt_write(ndev, IDT_NT_NTMTBLADDR, ndev->peers[pidx].part); 863 data = idt_nt_read(ndev, IDT_NT_NTMTBLDATA); 864 spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags); 865 866 return !!(data & IDT_NTMTBLDATA_VALID); 867 } 868 869 /* 870 * idt_ntb_link_is_up() - get the current ntb link state (NTB API callback) 871 * @ntb: NTB device context. 872 * @speed: OUT - The link speed expressed as PCIe generation number. 873 * @width: OUT - The link width expressed as the number of PCIe lanes. 874 * 875 * Get the bitfield of NTB link states for all peer ports 876 * 877 * Return: bitfield of indexed ports link state: bit is set/cleared if the 878 * link is up/down respectively. 879 */ 880 static u64 idt_ntb_link_is_up(struct ntb_dev *ntb, 881 enum ntb_speed *speed, enum ntb_width *width) 882 { 883 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 884 unsigned char pidx; 885 u64 status; 886 u32 data; 887 888 /* Retrieve the local link speed and width */ 889 if (speed != NULL || width != NULL) { 890 data = idt_nt_read(ndev, IDT_NT_PCIELCTLSTS); 891 if (speed != NULL) 892 *speed = GET_FIELD(PCIELCTLSTS_CLS, data); 893 if (width != NULL) 894 *width = GET_FIELD(PCIELCTLSTS_NLW, data); 895 } 896 897 /* If local NTB link isn't up then all the links are considered down */ 898 if (!idt_ntb_local_link_is_up(ndev)) 899 return 0; 900 901 /* Collect all the peer ports link states into the bitfield */ 902 status = 0; 903 for (pidx = 0; pidx < ndev->peer_cnt; pidx++) { 904 if (idt_ntb_peer_link_is_up(ndev, pidx)) 905 status |= ((u64)1 << pidx); 906 } 907 908 return status; 909 } 910 911 /* 912 * idt_ntb_link_enable() - enable local port ntb link (NTB API callback) 913 * @ntb: NTB device context. 914 * @max_speed: The maximum link speed expressed as PCIe generation number. 915 * @max_width: The maximum link width expressed as the number of PCIe lanes. 916 * 917 * Enable just local NTB link. PCIe link parameters are ignored. 918 * 919 * Return: always zero. 920 */ 921 static int idt_ntb_link_enable(struct ntb_dev *ntb, enum ntb_speed speed, 922 enum ntb_width width) 923 { 924 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 925 926 /* Just enable the local NTB link */ 927 idt_ntb_local_link_enable(ndev); 928 929 dev_dbg(&ndev->ntb.pdev->dev, "Local NTB link enabled"); 930 931 return 0; 932 } 933 934 /* 935 * idt_ntb_link_disable() - disable local port ntb link (NTB API callback) 936 * @ntb: NTB device context. 937 * 938 * Disable just local NTB link. 939 * 940 * Return: always zero. 941 */ 942 static int idt_ntb_link_disable(struct ntb_dev *ntb) 943 { 944 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 945 946 /* Just disable the local NTB link */ 947 idt_ntb_local_link_disable(ndev); 948 949 dev_dbg(&ndev->ntb.pdev->dev, "Local NTB link disabled"); 950 951 return 0; 952 } 953 954 /*============================================================================= 955 * 4. Memory Window operations 956 * 957 * IDT PCIe-switches have two types of memory windows: MWs with direct 958 * address translation and MWs with LUT based translation. The first type of 959 * MWs is simple map of corresponding BAR address space to a memory space 960 * of specified target port. So it implemets just ont-to-one mapping. Lookup 961 * table in its turn can map one BAR address space to up to 24 different 962 * memory spaces of different ports. 963 * NT-functions BARs can be turned on to implement either direct or lookup 964 * table based address translations, so: 965 * BAR0 - NT configuration registers space/direct address translation 966 * BAR1 - direct address translation/upper address of BAR0x64 967 * BAR2 - direct address translation/Lookup table with either 12 or 24 entries 968 * BAR3 - direct address translation/upper address of BAR2x64 969 * BAR4 - direct address translation/Lookup table with either 12 or 24 entries 970 * BAR5 - direct address translation/upper address of BAR4x64 971 * Additionally BAR2 and BAR4 can't have 24-entries LUT enabled at the same 972 * time. Since the BARs setup can be rather complicated this driver implements 973 * a scanning algorithm to have all the possible memory windows configuration 974 * covered. 975 * 976 * NOTE 1 BAR setup must be done before Linux kernel enumerated NT-function 977 * of any port, so this driver would have memory windows configurations fixed. 978 * In this way all initializations must be performed either by platform BIOS 979 * or using EEPROM connected to IDT PCIe-switch master SMBus. 980 * 981 * NOTE 2 This driver expects BAR0 mapping NT-function configuration space. 982 * Easy calculation can give us an upper boundary of 29 possible memory windows 983 * per each NT-function if all the BARs are of 32bit type. 984 *============================================================================= 985 */ 986 987 /* 988 * idt_get_mw_count() - get memory window count 989 * @mw_type: Memory window type 990 * 991 * Return: number of memory windows with respect to the BAR type 992 */ 993 static inline unsigned char idt_get_mw_count(enum idt_mw_type mw_type) 994 { 995 switch (mw_type) { 996 case IDT_MW_DIR: 997 return 1; 998 case IDT_MW_LUT12: 999 return 12; 1000 case IDT_MW_LUT24: 1001 return 24; 1002 default: 1003 break; 1004 } 1005 1006 return 0; 1007 } 1008 1009 /* 1010 * idt_get_mw_name() - get memory window name 1011 * @mw_type: Memory window type 1012 * 1013 * Return: pointer to a string with name 1014 */ 1015 static inline char *idt_get_mw_name(enum idt_mw_type mw_type) 1016 { 1017 switch (mw_type) { 1018 case IDT_MW_DIR: 1019 return "DIR "; 1020 case IDT_MW_LUT12: 1021 return "LUT12"; 1022 case IDT_MW_LUT24: 1023 return "LUT24"; 1024 default: 1025 break; 1026 } 1027 1028 return "unknown"; 1029 } 1030 1031 /* 1032 * idt_scan_mws() - scan memory windows of the port 1033 * @ndev: IDT NTB hardware driver descriptor 1034 * @port: Port to get number of memory windows for 1035 * @mw_cnt: Out - number of memory windows 1036 * 1037 * It walks over BAR setup registers of the specified port and determines 1038 * the memory windows parameters if any activated. 1039 * 1040 * Return: array of memory windows 1041 */ 1042 static struct idt_mw_cfg *idt_scan_mws(struct idt_ntb_dev *ndev, int port, 1043 unsigned char *mw_cnt) 1044 { 1045 struct idt_mw_cfg mws[IDT_MAX_NR_MWS], *ret_mws; 1046 const struct idt_ntb_bar *bars; 1047 enum idt_mw_type mw_type; 1048 unsigned char widx, bidx, en_cnt; 1049 bool bar_64bit = false; 1050 int aprt_size; 1051 u32 data; 1052 1053 /* Retrieve the array of the BARs registers */ 1054 bars = portdata_tbl[port].bars; 1055 1056 /* Scan all the BARs belonging to the port */ 1057 *mw_cnt = 0; 1058 for (bidx = 0; bidx < IDT_BAR_CNT; bidx += 1 + bar_64bit) { 1059 /* Read BARSETUP register value */ 1060 data = idt_sw_read(ndev, bars[bidx].setup); 1061 1062 /* Skip disabled BARs */ 1063 if (!(data & IDT_BARSETUP_EN)) { 1064 bar_64bit = false; 1065 continue; 1066 } 1067 1068 /* Skip next BARSETUP if current one has 64bit addressing */ 1069 bar_64bit = IS_FLD_SET(BARSETUP_TYPE, data, 64); 1070 1071 /* Skip configuration space mapping BARs */ 1072 if (data & IDT_BARSETUP_MODE_CFG) 1073 continue; 1074 1075 /* Retrieve MW type/entries count and aperture size */ 1076 mw_type = GET_FIELD(BARSETUP_ATRAN, data); 1077 en_cnt = idt_get_mw_count(mw_type); 1078 aprt_size = (u64)1 << GET_FIELD(BARSETUP_SIZE, data); 1079 1080 /* Save configurations of all available memory windows */ 1081 for (widx = 0; widx < en_cnt; widx++, (*mw_cnt)++) { 1082 /* 1083 * IDT can expose a limited number of MWs, so it's bug 1084 * to have more than the driver expects 1085 */ 1086 if (*mw_cnt >= IDT_MAX_NR_MWS) 1087 return ERR_PTR(-EINVAL); 1088 1089 /* Save basic MW info */ 1090 mws[*mw_cnt].type = mw_type; 1091 mws[*mw_cnt].bar = bidx; 1092 mws[*mw_cnt].idx = widx; 1093 /* It's always DWORD aligned */ 1094 mws[*mw_cnt].addr_align = IDT_TRANS_ALIGN; 1095 /* DIR and LUT approachs differently configure MWs */ 1096 if (mw_type == IDT_MW_DIR) 1097 mws[*mw_cnt].size_max = aprt_size; 1098 else if (mw_type == IDT_MW_LUT12) 1099 mws[*mw_cnt].size_max = aprt_size / 16; 1100 else 1101 mws[*mw_cnt].size_max = aprt_size / 32; 1102 mws[*mw_cnt].size_align = (mw_type == IDT_MW_DIR) ? 1103 IDT_DIR_SIZE_ALIGN : mws[*mw_cnt].size_max; 1104 } 1105 } 1106 1107 /* Allocate memory for memory window descriptors */ 1108 ret_mws = devm_kcalloc(&ndev->ntb.pdev->dev, *mw_cnt, 1109 sizeof(*ret_mws), GFP_KERNEL); 1110 if (IS_ERR_OR_NULL(ret_mws)) 1111 return ERR_PTR(-ENOMEM); 1112 1113 /* Copy the info of detected memory windows */ 1114 memcpy(ret_mws, mws, (*mw_cnt)*sizeof(*ret_mws)); 1115 1116 return ret_mws; 1117 } 1118 1119 /* 1120 * idt_init_mws() - initialize memory windows subsystem 1121 * @ndev: IDT NTB hardware driver descriptor 1122 * 1123 * Scan BAR setup registers of local and peer ports to determine the 1124 * outbound and inbound memory windows parameters 1125 * 1126 * Return: zero on success, otherwise a negative error number 1127 */ 1128 static int idt_init_mws(struct idt_ntb_dev *ndev) 1129 { 1130 struct idt_ntb_peer *peer; 1131 unsigned char pidx; 1132 1133 /* Scan memory windows of the local port */ 1134 ndev->mws = idt_scan_mws(ndev, ndev->port, &ndev->mw_cnt); 1135 if (IS_ERR(ndev->mws)) { 1136 dev_err(&ndev->ntb.pdev->dev, 1137 "Failed to scan mws of local port %hhu", ndev->port); 1138 return PTR_ERR(ndev->mws); 1139 } 1140 1141 /* Scan memory windows of the peer ports */ 1142 for (pidx = 0; pidx < ndev->peer_cnt; pidx++) { 1143 peer = &ndev->peers[pidx]; 1144 peer->mws = idt_scan_mws(ndev, peer->port, &peer->mw_cnt); 1145 if (IS_ERR(peer->mws)) { 1146 dev_err(&ndev->ntb.pdev->dev, 1147 "Failed to scan mws of port %hhu", peer->port); 1148 return PTR_ERR(peer->mws); 1149 } 1150 } 1151 1152 /* Initialize spin locker of the LUT registers */ 1153 spin_lock_init(&ndev->lut_lock); 1154 1155 dev_dbg(&ndev->ntb.pdev->dev, "Outbound and inbound MWs initialized"); 1156 1157 return 0; 1158 } 1159 1160 /* 1161 * idt_ntb_mw_count() - number of inbound memory windows (NTB API callback) 1162 * @ntb: NTB device context. 1163 * @pidx: Port index of peer device. 1164 * 1165 * The value is returned for the specified peer, so generally speaking it can 1166 * be different for different port depending on the IDT PCIe-switch 1167 * initialization. 1168 * 1169 * Return: the number of memory windows. 1170 */ 1171 static int idt_ntb_mw_count(struct ntb_dev *ntb, int pidx) 1172 { 1173 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1174 1175 if (pidx < 0 || ndev->peer_cnt <= pidx) 1176 return -EINVAL; 1177 1178 return ndev->peers[pidx].mw_cnt; 1179 } 1180 1181 /* 1182 * idt_ntb_mw_get_align() - inbound memory window parameters (NTB API callback) 1183 * @ntb: NTB device context. 1184 * @pidx: Port index of peer device. 1185 * @widx: Memory window index. 1186 * @addr_align: OUT - the base alignment for translating the memory window 1187 * @size_align: OUT - the size alignment for translating the memory window 1188 * @size_max: OUT - the maximum size of the memory window 1189 * 1190 * The peer memory window parameters have already been determined, so just 1191 * return the corresponding values, which mustn't change within session. 1192 * 1193 * Return: Zero on success, otherwise a negative error number. 1194 */ 1195 static int idt_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int widx, 1196 resource_size_t *addr_align, 1197 resource_size_t *size_align, 1198 resource_size_t *size_max) 1199 { 1200 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1201 struct idt_ntb_peer *peer; 1202 1203 if (pidx < 0 || ndev->peer_cnt <= pidx) 1204 return -EINVAL; 1205 1206 peer = &ndev->peers[pidx]; 1207 1208 if (widx < 0 || peer->mw_cnt <= widx) 1209 return -EINVAL; 1210 1211 if (addr_align != NULL) 1212 *addr_align = peer->mws[widx].addr_align; 1213 1214 if (size_align != NULL) 1215 *size_align = peer->mws[widx].size_align; 1216 1217 if (size_max != NULL) 1218 *size_max = peer->mws[widx].size_max; 1219 1220 return 0; 1221 } 1222 1223 /* 1224 * idt_ntb_peer_mw_count() - number of outbound memory windows 1225 * (NTB API callback) 1226 * @ntb: NTB device context. 1227 * 1228 * Outbound memory windows parameters have been determined based on the 1229 * BAR setup registers value, which are mostly constants within one session. 1230 * 1231 * Return: the number of memory windows. 1232 */ 1233 static int idt_ntb_peer_mw_count(struct ntb_dev *ntb) 1234 { 1235 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1236 1237 return ndev->mw_cnt; 1238 } 1239 1240 /* 1241 * idt_ntb_peer_mw_get_addr() - get map address of an outbound memory window 1242 * (NTB API callback) 1243 * @ntb: NTB device context. 1244 * @widx: Memory window index (within ntb_peer_mw_count() return value). 1245 * @base: OUT - the base address of mapping region. 1246 * @size: OUT - the size of mapping region. 1247 * 1248 * Return just parameters of BAR resources mapping. Size reflects just the size 1249 * of the resource 1250 * 1251 * Return: Zero on success, otherwise a negative error number. 1252 */ 1253 static int idt_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int widx, 1254 phys_addr_t *base, resource_size_t *size) 1255 { 1256 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1257 1258 if (widx < 0 || ndev->mw_cnt <= widx) 1259 return -EINVAL; 1260 1261 /* Mapping address is just properly shifted BAR resource start */ 1262 if (base != NULL) 1263 *base = pci_resource_start(ntb->pdev, ndev->mws[widx].bar) + 1264 ndev->mws[widx].idx * ndev->mws[widx].size_max; 1265 1266 /* Mapping size has already been calculated at MWs scanning */ 1267 if (size != NULL) 1268 *size = ndev->mws[widx].size_max; 1269 1270 return 0; 1271 } 1272 1273 /* 1274 * idt_ntb_peer_mw_set_trans() - set a translation address of a memory window 1275 * (NTB API callback) 1276 * @ntb: NTB device context. 1277 * @pidx: Port index of peer device the translation address received from. 1278 * @widx: Memory window index. 1279 * @addr: The dma address of the shared memory to access. 1280 * @size: The size of the shared memory to access. 1281 * 1282 * The Direct address translation and LUT base translation is initialized a 1283 * bit differenet. Although the parameters restriction are now determined by 1284 * the same code. 1285 * 1286 * Return: Zero on success, otherwise an error number. 1287 */ 1288 static int idt_ntb_peer_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx, 1289 u64 addr, resource_size_t size) 1290 { 1291 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1292 struct idt_mw_cfg *mw_cfg; 1293 u32 data = 0, lutoff = 0; 1294 1295 if (pidx < 0 || ndev->peer_cnt <= pidx) 1296 return -EINVAL; 1297 1298 if (widx < 0 || ndev->mw_cnt <= widx) 1299 return -EINVAL; 1300 1301 /* 1302 * Retrieve the memory window config to make sure the passed arguments 1303 * fit it restrictions 1304 */ 1305 mw_cfg = &ndev->mws[widx]; 1306 if (!IS_ALIGNED(addr, mw_cfg->addr_align)) 1307 return -EINVAL; 1308 if (!IS_ALIGNED(size, mw_cfg->size_align) || size > mw_cfg->size_max) 1309 return -EINVAL; 1310 1311 /* DIR and LUT based translations are initialized differently */ 1312 if (mw_cfg->type == IDT_MW_DIR) { 1313 const struct idt_ntb_bar *bar = &ntdata_tbl.bars[mw_cfg->bar]; 1314 u64 limit; 1315 /* Set destination partition of translation */ 1316 data = idt_nt_read(ndev, bar->setup); 1317 data = SET_FIELD(BARSETUP_TPART, data, ndev->peers[pidx].part); 1318 idt_nt_write(ndev, bar->setup, data); 1319 /* Set translation base address */ 1320 idt_nt_write(ndev, bar->ltbase, (u32)addr); 1321 idt_nt_write(ndev, bar->utbase, (u32)(addr >> 32)); 1322 /* Set the custom BAR aperture limit */ 1323 limit = pci_resource_start(ntb->pdev, mw_cfg->bar) + size; 1324 idt_nt_write(ndev, bar->limit, (u32)limit); 1325 if (IS_FLD_SET(BARSETUP_TYPE, data, 64)) 1326 idt_nt_write(ndev, (bar + 1)->limit, (limit >> 32)); 1327 } else { 1328 unsigned long irqflags; 1329 /* Initialize corresponding LUT entry */ 1330 lutoff = SET_FIELD(LUTOFFSET_INDEX, 0, mw_cfg->idx) | 1331 SET_FIELD(LUTOFFSET_BAR, 0, mw_cfg->bar); 1332 data = SET_FIELD(LUTUDATA_PART, 0, ndev->peers[pidx].part) | 1333 IDT_LUTUDATA_VALID; 1334 spin_lock_irqsave(&ndev->lut_lock, irqflags); 1335 idt_nt_write(ndev, IDT_NT_LUTOFFSET, lutoff); 1336 idt_nt_write(ndev, IDT_NT_LUTLDATA, (u32)addr); 1337 idt_nt_write(ndev, IDT_NT_LUTMDATA, (u32)(addr >> 32)); 1338 idt_nt_write(ndev, IDT_NT_LUTUDATA, data); 1339 mmiowb(); 1340 spin_unlock_irqrestore(&ndev->lut_lock, irqflags); 1341 /* Limit address isn't specified since size is fixed for LUT */ 1342 } 1343 1344 return 0; 1345 } 1346 1347 /* 1348 * idt_ntb_peer_mw_clear_trans() - clear the outbound MW translation address 1349 * (NTB API callback) 1350 * @ntb: NTB device context. 1351 * @pidx: Port index of peer device. 1352 * @widx: Memory window index. 1353 * 1354 * It effectively disables the translation over the specified outbound MW. 1355 * 1356 * Return: Zero on success, otherwise an error number. 1357 */ 1358 static int idt_ntb_peer_mw_clear_trans(struct ntb_dev *ntb, int pidx, 1359 int widx) 1360 { 1361 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1362 struct idt_mw_cfg *mw_cfg; 1363 1364 if (pidx < 0 || ndev->peer_cnt <= pidx) 1365 return -EINVAL; 1366 1367 if (widx < 0 || ndev->mw_cnt <= widx) 1368 return -EINVAL; 1369 1370 mw_cfg = &ndev->mws[widx]; 1371 1372 /* DIR and LUT based translations are initialized differently */ 1373 if (mw_cfg->type == IDT_MW_DIR) { 1374 const struct idt_ntb_bar *bar = &ntdata_tbl.bars[mw_cfg->bar]; 1375 u32 data; 1376 /* Read BARSETUP to check BAR type */ 1377 data = idt_nt_read(ndev, bar->setup); 1378 /* Disable translation by specifying zero BAR limit */ 1379 idt_nt_write(ndev, bar->limit, 0); 1380 if (IS_FLD_SET(BARSETUP_TYPE, data, 64)) 1381 idt_nt_write(ndev, (bar + 1)->limit, 0); 1382 } else { 1383 unsigned long irqflags; 1384 u32 lutoff; 1385 /* Clear the corresponding LUT entry up */ 1386 lutoff = SET_FIELD(LUTOFFSET_INDEX, 0, mw_cfg->idx) | 1387 SET_FIELD(LUTOFFSET_BAR, 0, mw_cfg->bar); 1388 spin_lock_irqsave(&ndev->lut_lock, irqflags); 1389 idt_nt_write(ndev, IDT_NT_LUTOFFSET, lutoff); 1390 idt_nt_write(ndev, IDT_NT_LUTLDATA, 0); 1391 idt_nt_write(ndev, IDT_NT_LUTMDATA, 0); 1392 idt_nt_write(ndev, IDT_NT_LUTUDATA, 0); 1393 mmiowb(); 1394 spin_unlock_irqrestore(&ndev->lut_lock, irqflags); 1395 } 1396 1397 return 0; 1398 } 1399 1400 /*============================================================================= 1401 * 5. Doorbell operations 1402 * 1403 * Doorbell functionality of IDT PCIe-switches is pretty unusual. First of 1404 * all there is global doorbell register which state can by changed by any 1405 * NT-function of the IDT device in accordance with global permissions. These 1406 * permissions configs are not supported by NTB API, so it must be done by 1407 * either BIOS or EEPROM settings. In the same way the state of the global 1408 * doorbell is reflected to the NT-functions local inbound doorbell registers. 1409 * It can lead to situations when client driver sets some peer doorbell bits 1410 * and get them bounced back to local inbound doorbell if permissions are 1411 * granted. 1412 * Secondly there is just one IRQ vector for Doorbell, Message, Temperature 1413 * and Switch events, so if client driver left any of Doorbell bits set and 1414 * some other event occurred, the driver will be notified of Doorbell event 1415 * again. 1416 *============================================================================= 1417 */ 1418 1419 /* 1420 * idt_db_isr() - doorbell event ISR 1421 * @ndev: IDT NTB hardware driver descriptor 1422 * @ntint_sts: NT-function interrupt status 1423 * 1424 * Doorbell event happans when DBELL bit of NTINTSTS switches from 0 to 1. 1425 * It happens only when unmasked doorbell bits are set to ones on completely 1426 * zeroed doorbell register. 1427 * The method is called from PCIe ISR bottom-half routine. 1428 */ 1429 static void idt_db_isr(struct idt_ntb_dev *ndev, u32 ntint_sts) 1430 { 1431 /* 1432 * Doorbell IRQ status will be cleaned only when client 1433 * driver unsets all the doorbell bits. 1434 */ 1435 dev_dbg(&ndev->ntb.pdev->dev, "DB IRQ detected %#08x", ntint_sts); 1436 1437 /* Notify the client driver of possible doorbell state change */ 1438 ntb_db_event(&ndev->ntb, 0); 1439 } 1440 1441 /* 1442 * idt_ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb 1443 * (NTB API callback) 1444 * @ntb: NTB device context. 1445 * 1446 * IDT PCIe-switches expose just one Doorbell register of DWORD size. 1447 * 1448 * Return: A mask of doorbell bits supported by the ntb. 1449 */ 1450 static u64 idt_ntb_db_valid_mask(struct ntb_dev *ntb) 1451 { 1452 return IDT_DBELL_MASK; 1453 } 1454 1455 /* 1456 * idt_ntb_db_read() - read the local doorbell register (NTB API callback) 1457 * @ntb: NTB device context. 1458 * 1459 * There is just on inbound doorbell register of each NT-function, so 1460 * this method return it value. 1461 * 1462 * Return: The bits currently set in the local doorbell register. 1463 */ 1464 static u64 idt_ntb_db_read(struct ntb_dev *ntb) 1465 { 1466 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1467 1468 return idt_nt_read(ndev, IDT_NT_INDBELLSTS); 1469 } 1470 1471 /* 1472 * idt_ntb_db_clear() - clear bits in the local doorbell register 1473 * (NTB API callback) 1474 * @ntb: NTB device context. 1475 * @db_bits: Doorbell bits to clear. 1476 * 1477 * Clear bits of inbound doorbell register by writing ones to it. 1478 * 1479 * NOTE! Invalid bits are always considered cleared so it's not an error 1480 * to clear them over. 1481 * 1482 * Return: always zero as success. 1483 */ 1484 static int idt_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) 1485 { 1486 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1487 1488 idt_nt_write(ndev, IDT_NT_INDBELLSTS, (u32)db_bits); 1489 1490 return 0; 1491 } 1492 1493 /* 1494 * idt_ntb_db_read_mask() - read the local doorbell mask (NTB API callback) 1495 * @ntb: NTB device context. 1496 * 1497 * Each inbound doorbell bit can be masked from generating IRQ by setting 1498 * the corresponding bit in inbound doorbell mask. So this method returns 1499 * the value of the register. 1500 * 1501 * Return: The bits currently set in the local doorbell mask register. 1502 */ 1503 static u64 idt_ntb_db_read_mask(struct ntb_dev *ntb) 1504 { 1505 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1506 1507 return idt_nt_read(ndev, IDT_NT_INDBELLMSK); 1508 } 1509 1510 /* 1511 * idt_ntb_db_set_mask() - set bits in the local doorbell mask 1512 * (NTB API callback) 1513 * @ntb: NTB device context. 1514 * @db_bits: Doorbell mask bits to set. 1515 * 1516 * The inbound doorbell register mask value must be read, then OR'ed with 1517 * passed field and only then set back. 1518 * 1519 * Return: zero on success, negative error if invalid argument passed. 1520 */ 1521 static int idt_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) 1522 { 1523 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1524 1525 return idt_reg_set_bits(ndev, IDT_NT_INDBELLMSK, &ndev->db_mask_lock, 1526 IDT_DBELL_MASK, db_bits); 1527 } 1528 1529 /* 1530 * idt_ntb_db_clear_mask() - clear bits in the local doorbell mask 1531 * (NTB API callback) 1532 * @ntb: NTB device context. 1533 * @db_bits: Doorbell bits to clear. 1534 * 1535 * The method just clears the set bits up in accordance with the passed 1536 * bitfield. IDT PCIe-switch shall generate an interrupt if there hasn't 1537 * been any unmasked bit set before current unmasking. Otherwise IRQ won't 1538 * be generated since there is only one IRQ vector for all doorbells. 1539 * 1540 * Return: always zero as success 1541 */ 1542 static int idt_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) 1543 { 1544 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1545 1546 idt_reg_clear_bits(ndev, IDT_NT_INDBELLMSK, &ndev->db_mask_lock, 1547 db_bits); 1548 1549 return 0; 1550 } 1551 1552 /* 1553 * idt_ntb_peer_db_set() - set bits in the peer doorbell register 1554 * (NTB API callback) 1555 * @ntb: NTB device context. 1556 * @db_bits: Doorbell bits to set. 1557 * 1558 * IDT PCIe-switches exposes local outbound doorbell register to change peer 1559 * inbound doorbell register state. 1560 * 1561 * Return: zero on success, negative error if invalid argument passed. 1562 */ 1563 static int idt_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) 1564 { 1565 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1566 1567 if (db_bits & ~(u64)IDT_DBELL_MASK) 1568 return -EINVAL; 1569 1570 idt_nt_write(ndev, IDT_NT_OUTDBELLSET, (u32)db_bits); 1571 return 0; 1572 } 1573 1574 /*============================================================================= 1575 * 6. Messaging operations 1576 * 1577 * Each NT-function of IDT PCIe-switch has four inbound and four outbound 1578 * message registers. Each outbound message register can be connected to one or 1579 * even more than one peer inbound message registers by setting global 1580 * configurations. Since NTB API permits one-on-one message registers mapping 1581 * only, the driver acts in according with that restriction. 1582 *============================================================================= 1583 */ 1584 1585 /* 1586 * idt_init_msg() - initialize messaging interface 1587 * @ndev: IDT NTB hardware driver descriptor 1588 * 1589 * Just initialize the message registers routing tables locker. 1590 */ 1591 static void idt_init_msg(struct idt_ntb_dev *ndev) 1592 { 1593 unsigned char midx; 1594 1595 /* Init the messages routing table lockers */ 1596 for (midx = 0; midx < IDT_MSG_CNT; midx++) 1597 spin_lock_init(&ndev->msg_locks[midx]); 1598 1599 dev_dbg(&ndev->ntb.pdev->dev, "NTB Messaging initialized"); 1600 } 1601 1602 /* 1603 * idt_msg_isr() - message event ISR 1604 * @ndev: IDT NTB hardware driver descriptor 1605 * @ntint_sts: NT-function interrupt status 1606 * 1607 * Message event happens when MSG bit of NTINTSTS switches from 0 to 1. 1608 * It happens only when unmasked message status bits are set to ones on 1609 * completely zeroed message status register. 1610 * The method is called from PCIe ISR bottom-half routine. 1611 */ 1612 static void idt_msg_isr(struct idt_ntb_dev *ndev, u32 ntint_sts) 1613 { 1614 /* 1615 * Message IRQ status will be cleaned only when client 1616 * driver unsets all the message status bits. 1617 */ 1618 dev_dbg(&ndev->ntb.pdev->dev, "Message IRQ detected %#08x", ntint_sts); 1619 1620 /* Notify the client driver of possible message status change */ 1621 ntb_msg_event(&ndev->ntb); 1622 } 1623 1624 /* 1625 * idt_ntb_msg_count() - get the number of message registers (NTB API callback) 1626 * @ntb: NTB device context. 1627 * 1628 * IDT PCIe-switches support four message registers. 1629 * 1630 * Return: the number of message registers. 1631 */ 1632 static int idt_ntb_msg_count(struct ntb_dev *ntb) 1633 { 1634 return IDT_MSG_CNT; 1635 } 1636 1637 /* 1638 * idt_ntb_msg_inbits() - get a bitfield of inbound message registers status 1639 * (NTB API callback) 1640 * @ntb: NTB device context. 1641 * 1642 * NT message status register is shared between inbound and outbound message 1643 * registers status 1644 * 1645 * Return: bitfield of inbound message registers. 1646 */ 1647 static u64 idt_ntb_msg_inbits(struct ntb_dev *ntb) 1648 { 1649 return (u64)IDT_INMSG_MASK; 1650 } 1651 1652 /* 1653 * idt_ntb_msg_outbits() - get a bitfield of outbound message registers status 1654 * (NTB API callback) 1655 * @ntb: NTB device context. 1656 * 1657 * NT message status register is shared between inbound and outbound message 1658 * registers status 1659 * 1660 * Return: bitfield of outbound message registers. 1661 */ 1662 static u64 idt_ntb_msg_outbits(struct ntb_dev *ntb) 1663 { 1664 return (u64)IDT_OUTMSG_MASK; 1665 } 1666 1667 /* 1668 * idt_ntb_msg_read_sts() - read the message registers status (NTB API callback) 1669 * @ntb: NTB device context. 1670 * 1671 * IDT PCIe-switches expose message status registers to notify drivers of 1672 * incoming data and failures in case if peer message register isn't freed. 1673 * 1674 * Return: status bits of message registers 1675 */ 1676 static u64 idt_ntb_msg_read_sts(struct ntb_dev *ntb) 1677 { 1678 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1679 1680 return idt_nt_read(ndev, IDT_NT_MSGSTS); 1681 } 1682 1683 /* 1684 * idt_ntb_msg_clear_sts() - clear status bits of message registers 1685 * (NTB API callback) 1686 * @ntb: NTB device context. 1687 * @sts_bits: Status bits to clear. 1688 * 1689 * Clear bits in the status register by writing ones. 1690 * 1691 * NOTE! Invalid bits are always considered cleared so it's not an error 1692 * to clear them over. 1693 * 1694 * Return: always zero as success. 1695 */ 1696 static int idt_ntb_msg_clear_sts(struct ntb_dev *ntb, u64 sts_bits) 1697 { 1698 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1699 1700 idt_nt_write(ndev, IDT_NT_MSGSTS, sts_bits); 1701 1702 return 0; 1703 } 1704 1705 /* 1706 * idt_ntb_msg_set_mask() - set mask of message register status bits 1707 * (NTB API callback) 1708 * @ntb: NTB device context. 1709 * @mask_bits: Mask bits. 1710 * 1711 * Mask the message status bits from raising an IRQ. 1712 * 1713 * Return: zero on success, negative error if invalid argument passed. 1714 */ 1715 static int idt_ntb_msg_set_mask(struct ntb_dev *ntb, u64 mask_bits) 1716 { 1717 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1718 1719 return idt_reg_set_bits(ndev, IDT_NT_MSGSTSMSK, &ndev->msg_mask_lock, 1720 IDT_MSG_MASK, mask_bits); 1721 } 1722 1723 /* 1724 * idt_ntb_msg_clear_mask() - clear message registers mask 1725 * (NTB API callback) 1726 * @ntb: NTB device context. 1727 * @mask_bits: Mask bits. 1728 * 1729 * Clear mask of message status bits IRQs. 1730 * 1731 * Return: always zero as success. 1732 */ 1733 static int idt_ntb_msg_clear_mask(struct ntb_dev *ntb, u64 mask_bits) 1734 { 1735 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1736 1737 idt_reg_clear_bits(ndev, IDT_NT_MSGSTSMSK, &ndev->msg_mask_lock, 1738 mask_bits); 1739 1740 return 0; 1741 } 1742 1743 /* 1744 * idt_ntb_msg_read() - read message register with specified index 1745 * (NTB API callback) 1746 * @ntb: NTB device context. 1747 * @pidx: OUT - Port index of peer device a message retrieved from 1748 * @midx: Message register index 1749 * 1750 * Read data from the specified message register and source register. 1751 * 1752 * Return: inbound message register value. 1753 */ 1754 static u32 idt_ntb_msg_read(struct ntb_dev *ntb, int *pidx, int midx) 1755 { 1756 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1757 1758 if (midx < 0 || IDT_MSG_CNT <= midx) 1759 return ~(u32)0; 1760 1761 /* Retrieve source port index of the message */ 1762 if (pidx != NULL) { 1763 u32 srcpart; 1764 1765 srcpart = idt_nt_read(ndev, ntdata_tbl.msgs[midx].src); 1766 *pidx = ndev->part_idx_map[srcpart]; 1767 1768 /* Sanity check partition index (for initial case) */ 1769 if (*pidx == -EINVAL) 1770 *pidx = 0; 1771 } 1772 1773 /* Retrieve data of the corresponding message register */ 1774 return idt_nt_read(ndev, ntdata_tbl.msgs[midx].in); 1775 } 1776 1777 /* 1778 * idt_ntb_peer_msg_write() - write data to the specified message register 1779 * (NTB API callback) 1780 * @ntb: NTB device context. 1781 * @pidx: Port index of peer device a message being sent to 1782 * @midx: Message register index 1783 * @msg: Data to send 1784 * 1785 * Just try to send data to a peer. Message status register should be 1786 * checked by client driver. 1787 * 1788 * Return: zero on success, negative error if invalid argument passed. 1789 */ 1790 static int idt_ntb_peer_msg_write(struct ntb_dev *ntb, int pidx, int midx, 1791 u32 msg) 1792 { 1793 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb); 1794 unsigned long irqflags; 1795 u32 swpmsgctl = 0; 1796 1797 if (midx < 0 || IDT_MSG_CNT <= midx) 1798 return -EINVAL; 1799 1800 if (pidx < 0 || ndev->peer_cnt <= pidx) 1801 return -EINVAL; 1802 1803 /* Collect the routing information */ 1804 swpmsgctl = SET_FIELD(SWPxMSGCTL_REG, 0, midx) | 1805 SET_FIELD(SWPxMSGCTL_PART, 0, ndev->peers[pidx].part); 1806 1807 /* Lock the messages routing table of the specified register */ 1808 spin_lock_irqsave(&ndev->msg_locks[midx], irqflags); 1809 /* Set the route and send the data */ 1810 idt_sw_write(ndev, partdata_tbl[ndev->part].msgctl[midx], swpmsgctl); 1811 idt_nt_write(ndev, ntdata_tbl.msgs[midx].out, msg); 1812 mmiowb(); 1813 /* Unlock the messages routing table */ 1814 spin_unlock_irqrestore(&ndev->msg_locks[midx], irqflags); 1815 1816 /* Client driver shall check the status register */ 1817 return 0; 1818 } 1819 1820 /*============================================================================= 1821 * 7. Temperature sensor operations 1822 * 1823 * IDT PCIe-switch has an embedded temperature sensor, which can be used to 1824 * warn a user-space of possible chip overheating. Since workload temperature 1825 * can be different on different platforms, temperature thresholds as well as 1826 * general sensor settings must be setup in the framework of BIOS/EEPROM 1827 * initializations. It includes the actual sensor enabling as well. 1828 *============================================================================= 1829 */ 1830 1831 /* 1832 * idt_read_temp() - read temperature from chip sensor 1833 * @ntb: NTB device context. 1834 * @val: OUT - integer value of temperature 1835 * @frac: OUT - fraction 1836 */ 1837 static void idt_read_temp(struct idt_ntb_dev *ndev, unsigned char *val, 1838 unsigned char *frac) 1839 { 1840 u32 data; 1841 1842 /* Read the data from TEMP field of the TMPSTS register */ 1843 data = idt_sw_read(ndev, IDT_SW_TMPSTS); 1844 data = GET_FIELD(TMPSTS_TEMP, data); 1845 /* TEMP field has one fractional bit and seven integer bits */ 1846 *val = data >> 1; 1847 *frac = ((data & 0x1) ? 5 : 0); 1848 } 1849 1850 /* 1851 * idt_temp_isr() - temperature sensor alarm events ISR 1852 * @ndev: IDT NTB hardware driver descriptor 1853 * @ntint_sts: NT-function interrupt status 1854 * 1855 * It handles events of temperature crossing alarm thresholds. Since reading 1856 * of TMPALARM register clears it up, the function doesn't analyze the 1857 * read value, instead the current temperature value just warningly printed to 1858 * log. 1859 * The method is called from PCIe ISR bottom-half routine. 1860 */ 1861 static void idt_temp_isr(struct idt_ntb_dev *ndev, u32 ntint_sts) 1862 { 1863 unsigned char val, frac; 1864 1865 /* Read the current temperature value */ 1866 idt_read_temp(ndev, &val, &frac); 1867 1868 /* Read the temperature alarm to clean the alarm status out */ 1869 /*(void)idt_sw_read(ndev, IDT_SW_TMPALARM);*/ 1870 1871 /* Clean the corresponding interrupt bit */ 1872 idt_nt_write(ndev, IDT_NT_NTINTSTS, IDT_NTINTSTS_TMPSENSOR); 1873 1874 dev_dbg(&ndev->ntb.pdev->dev, 1875 "Temp sensor IRQ detected %#08x", ntint_sts); 1876 1877 /* Print temperature value to log */ 1878 dev_warn(&ndev->ntb.pdev->dev, "Temperature %hhu.%hhu", val, frac); 1879 } 1880 1881 /*============================================================================= 1882 * 8. ISRs related operations 1883 * 1884 * IDT PCIe-switch has strangely developed IRQ system. There is just one 1885 * interrupt vector for doorbell and message registers. So the hardware driver 1886 * can't determine actual source of IRQ if, for example, message event happened 1887 * while any of unmasked doorbell is still set. The similar situation may be if 1888 * switch or temperature sensor events pop up. The difference is that SEVENT 1889 * and TMPSENSOR bits of NT interrupt status register can be cleaned by 1890 * IRQ handler so a next interrupt request won't have false handling of 1891 * corresponding events. 1892 * The hardware driver has only bottom-half handler of the IRQ, since if any 1893 * of events happened the device won't raise it again before the last one is 1894 * handled by clearing of corresponding NTINTSTS bit. 1895 *============================================================================= 1896 */ 1897 1898 static irqreturn_t idt_thread_isr(int irq, void *devid); 1899 1900 /* 1901 * idt_init_isr() - initialize PCIe interrupt handler 1902 * @ndev: IDT NTB hardware driver descriptor 1903 * 1904 * Return: zero on success, otherwise a negative error number. 1905 */ 1906 static int idt_init_isr(struct idt_ntb_dev *ndev) 1907 { 1908 struct pci_dev *pdev = ndev->ntb.pdev; 1909 u32 ntint_mask; 1910 int ret; 1911 1912 /* Allocate just one interrupt vector for the ISR */ 1913 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI | PCI_IRQ_LEGACY); 1914 if (ret != 1) { 1915 dev_err(&pdev->dev, "Failed to allocate IRQ vector"); 1916 return ret; 1917 } 1918 1919 /* Retrieve the IRQ vector */ 1920 ret = pci_irq_vector(pdev, 0); 1921 if (ret < 0) { 1922 dev_err(&pdev->dev, "Failed to get IRQ vector"); 1923 goto err_free_vectors; 1924 } 1925 1926 /* Set the IRQ handler */ 1927 ret = devm_request_threaded_irq(&pdev->dev, ret, NULL, idt_thread_isr, 1928 IRQF_ONESHOT, NTB_IRQNAME, ndev); 1929 if (ret != 0) { 1930 dev_err(&pdev->dev, "Failed to set MSI IRQ handler, %d", ret); 1931 goto err_free_vectors; 1932 } 1933 1934 /* Unmask Message/Doorbell/SE/Temperature interrupts */ 1935 ntint_mask = idt_nt_read(ndev, IDT_NT_NTINTMSK) & ~IDT_NTINTMSK_ALL; 1936 idt_nt_write(ndev, IDT_NT_NTINTMSK, ntint_mask); 1937 1938 /* From now on the interrupts are enabled */ 1939 dev_dbg(&pdev->dev, "NTB interrupts initialized"); 1940 1941 return 0; 1942 1943 err_free_vectors: 1944 pci_free_irq_vectors(pdev); 1945 1946 return ret; 1947 } 1948 1949 1950 /* 1951 * idt_deinit_ist() - deinitialize PCIe interrupt handler 1952 * @ndev: IDT NTB hardware driver descriptor 1953 * 1954 * Disable corresponding interrupts and free allocated IRQ vectors. 1955 */ 1956 static void idt_deinit_isr(struct idt_ntb_dev *ndev) 1957 { 1958 struct pci_dev *pdev = ndev->ntb.pdev; 1959 u32 ntint_mask; 1960 1961 /* Mask interrupts back */ 1962 ntint_mask = idt_nt_read(ndev, IDT_NT_NTINTMSK) | IDT_NTINTMSK_ALL; 1963 idt_nt_write(ndev, IDT_NT_NTINTMSK, ntint_mask); 1964 1965 /* Manually free IRQ otherwise PCI free irq vectors will fail */ 1966 devm_free_irq(&pdev->dev, pci_irq_vector(pdev, 0), ndev); 1967 1968 /* Free allocated IRQ vectors */ 1969 pci_free_irq_vectors(pdev); 1970 1971 dev_dbg(&pdev->dev, "NTB interrupts deinitialized"); 1972 } 1973 1974 /* 1975 * idt_thread_isr() - NT function interrupts handler 1976 * @irq: IRQ number 1977 * @devid: Custom buffer 1978 * 1979 * It reads current NT interrupts state register and handles all the event 1980 * it declares. 1981 * The method is bottom-half routine of actual default PCIe IRQ handler. 1982 */ 1983 static irqreturn_t idt_thread_isr(int irq, void *devid) 1984 { 1985 struct idt_ntb_dev *ndev = devid; 1986 bool handled = false; 1987 u32 ntint_sts; 1988 1989 /* Read the NT interrupts status register */ 1990 ntint_sts = idt_nt_read(ndev, IDT_NT_NTINTSTS); 1991 1992 /* Handle messaging interrupts */ 1993 if (ntint_sts & IDT_NTINTSTS_MSG) { 1994 idt_msg_isr(ndev, ntint_sts); 1995 handled = true; 1996 } 1997 1998 /* Handle doorbell interrupts */ 1999 if (ntint_sts & IDT_NTINTSTS_DBELL) { 2000 idt_db_isr(ndev, ntint_sts); 2001 handled = true; 2002 } 2003 2004 /* Handle switch event interrupts */ 2005 if (ntint_sts & IDT_NTINTSTS_SEVENT) { 2006 idt_se_isr(ndev, ntint_sts); 2007 handled = true; 2008 } 2009 2010 /* Handle temperature sensor interrupt */ 2011 if (ntint_sts & IDT_NTINTSTS_TMPSENSOR) { 2012 idt_temp_isr(ndev, ntint_sts); 2013 handled = true; 2014 } 2015 2016 dev_dbg(&ndev->ntb.pdev->dev, "IDT IRQs 0x%08x handled", ntint_sts); 2017 2018 return handled ? IRQ_HANDLED : IRQ_NONE; 2019 } 2020 2021 /*=========================================================================== 2022 * 9. NTB hardware driver initialization 2023 *=========================================================================== 2024 */ 2025 2026 /* 2027 * NTB API operations 2028 */ 2029 static const struct ntb_dev_ops idt_ntb_ops = { 2030 .port_number = idt_ntb_port_number, 2031 .peer_port_count = idt_ntb_peer_port_count, 2032 .peer_port_number = idt_ntb_peer_port_number, 2033 .peer_port_idx = idt_ntb_peer_port_idx, 2034 .link_is_up = idt_ntb_link_is_up, 2035 .link_enable = idt_ntb_link_enable, 2036 .link_disable = idt_ntb_link_disable, 2037 .mw_count = idt_ntb_mw_count, 2038 .mw_get_align = idt_ntb_mw_get_align, 2039 .peer_mw_count = idt_ntb_peer_mw_count, 2040 .peer_mw_get_addr = idt_ntb_peer_mw_get_addr, 2041 .peer_mw_set_trans = idt_ntb_peer_mw_set_trans, 2042 .peer_mw_clear_trans = idt_ntb_peer_mw_clear_trans, 2043 .db_valid_mask = idt_ntb_db_valid_mask, 2044 .db_read = idt_ntb_db_read, 2045 .db_clear = idt_ntb_db_clear, 2046 .db_read_mask = idt_ntb_db_read_mask, 2047 .db_set_mask = idt_ntb_db_set_mask, 2048 .db_clear_mask = idt_ntb_db_clear_mask, 2049 .peer_db_set = idt_ntb_peer_db_set, 2050 .msg_count = idt_ntb_msg_count, 2051 .msg_inbits = idt_ntb_msg_inbits, 2052 .msg_outbits = idt_ntb_msg_outbits, 2053 .msg_read_sts = idt_ntb_msg_read_sts, 2054 .msg_clear_sts = idt_ntb_msg_clear_sts, 2055 .msg_set_mask = idt_ntb_msg_set_mask, 2056 .msg_clear_mask = idt_ntb_msg_clear_mask, 2057 .msg_read = idt_ntb_msg_read, 2058 .peer_msg_write = idt_ntb_peer_msg_write 2059 }; 2060 2061 /* 2062 * idt_register_device() - register IDT NTB device 2063 * @ndev: IDT NTB hardware driver descriptor 2064 * 2065 * Return: zero on success, otherwise a negative error number. 2066 */ 2067 static int idt_register_device(struct idt_ntb_dev *ndev) 2068 { 2069 int ret; 2070 2071 /* Initialize the rest of NTB device structure and register it */ 2072 ndev->ntb.ops = &idt_ntb_ops; 2073 ndev->ntb.topo = NTB_TOPO_SWITCH; 2074 2075 ret = ntb_register_device(&ndev->ntb); 2076 if (ret != 0) { 2077 dev_err(&ndev->ntb.pdev->dev, "Failed to register NTB device"); 2078 return ret; 2079 } 2080 2081 dev_dbg(&ndev->ntb.pdev->dev, "NTB device successfully registered"); 2082 2083 return 0; 2084 } 2085 2086 /* 2087 * idt_unregister_device() - unregister IDT NTB device 2088 * @ndev: IDT NTB hardware driver descriptor 2089 */ 2090 static void idt_unregister_device(struct idt_ntb_dev *ndev) 2091 { 2092 /* Just unregister the NTB device */ 2093 ntb_unregister_device(&ndev->ntb); 2094 2095 dev_dbg(&ndev->ntb.pdev->dev, "NTB device unregistered"); 2096 } 2097 2098 /*============================================================================= 2099 * 10. DebugFS node initialization 2100 *============================================================================= 2101 */ 2102 2103 static ssize_t idt_dbgfs_info_read(struct file *filp, char __user *ubuf, 2104 size_t count, loff_t *offp); 2105 2106 /* 2107 * Driver DebugFS info file operations 2108 */ 2109 static const struct file_operations idt_dbgfs_info_ops = { 2110 .owner = THIS_MODULE, 2111 .open = simple_open, 2112 .read = idt_dbgfs_info_read 2113 }; 2114 2115 /* 2116 * idt_dbgfs_info_read() - DebugFS read info node callback 2117 * @file: File node descriptor. 2118 * @ubuf: User-space buffer to put data to 2119 * @count: Size of the buffer 2120 * @offp: Offset within the buffer 2121 */ 2122 static ssize_t idt_dbgfs_info_read(struct file *filp, char __user *ubuf, 2123 size_t count, loff_t *offp) 2124 { 2125 struct idt_ntb_dev *ndev = filp->private_data; 2126 unsigned char temp, frac, idx, pidx, cnt; 2127 ssize_t ret = 0, off = 0; 2128 unsigned long irqflags; 2129 enum ntb_speed speed; 2130 enum ntb_width width; 2131 char *strbuf; 2132 size_t size; 2133 u32 data; 2134 2135 /* Lets limit the buffer size the way the Intel/AMD drivers do */ 2136 size = min_t(size_t, count, 0x1000U); 2137 2138 /* Allocate the memory for the buffer */ 2139 strbuf = kmalloc(size, GFP_KERNEL); 2140 if (strbuf == NULL) 2141 return -ENOMEM; 2142 2143 /* Put the data into the string buffer */ 2144 off += scnprintf(strbuf + off, size - off, 2145 "\n\t\tIDT NTB device Information:\n\n"); 2146 2147 /* General local device configurations */ 2148 off += scnprintf(strbuf + off, size - off, 2149 "Local Port %hhu, Partition %hhu\n", ndev->port, ndev->part); 2150 2151 /* Peer ports information */ 2152 off += scnprintf(strbuf + off, size - off, "Peers:\n"); 2153 for (idx = 0; idx < ndev->peer_cnt; idx++) { 2154 off += scnprintf(strbuf + off, size - off, 2155 "\t%hhu. Port %hhu, Partition %hhu\n", 2156 idx, ndev->peers[idx].port, ndev->peers[idx].part); 2157 } 2158 2159 /* Links status */ 2160 data = idt_ntb_link_is_up(&ndev->ntb, &speed, &width); 2161 off += scnprintf(strbuf + off, size - off, 2162 "NTB link status\t- 0x%08x, ", data); 2163 off += scnprintf(strbuf + off, size - off, "PCIe Gen %d x%d lanes\n", 2164 speed, width); 2165 2166 /* Mapping table entries */ 2167 off += scnprintf(strbuf + off, size - off, "NTB Mapping Table:\n"); 2168 for (idx = 0; idx < IDT_MTBL_ENTRY_CNT; idx++) { 2169 spin_lock_irqsave(&ndev->mtbl_lock, irqflags); 2170 idt_nt_write(ndev, IDT_NT_NTMTBLADDR, idx); 2171 data = idt_nt_read(ndev, IDT_NT_NTMTBLDATA); 2172 spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags); 2173 2174 /* Print valid entries only */ 2175 if (data & IDT_NTMTBLDATA_VALID) { 2176 off += scnprintf(strbuf + off, size - off, 2177 "\t%hhu. Partition %d, Requester ID 0x%04x\n", 2178 idx, GET_FIELD(NTMTBLDATA_PART, data), 2179 GET_FIELD(NTMTBLDATA_REQID, data)); 2180 } 2181 } 2182 off += scnprintf(strbuf + off, size - off, "\n"); 2183 2184 /* Outbound memory windows information */ 2185 off += scnprintf(strbuf + off, size - off, 2186 "Outbound Memory Windows:\n"); 2187 for (idx = 0; idx < ndev->mw_cnt; idx += cnt) { 2188 data = ndev->mws[idx].type; 2189 cnt = idt_get_mw_count(data); 2190 2191 /* Print Memory Window information */ 2192 if (data == IDT_MW_DIR) 2193 off += scnprintf(strbuf + off, size - off, 2194 "\t%hhu.\t", idx); 2195 else 2196 off += scnprintf(strbuf + off, size - off, 2197 "\t%hhu-%hhu.\t", idx, idx + cnt - 1); 2198 2199 off += scnprintf(strbuf + off, size - off, "%s BAR%hhu, ", 2200 idt_get_mw_name(data), ndev->mws[idx].bar); 2201 2202 off += scnprintf(strbuf + off, size - off, 2203 "Address align 0x%08llx, ", ndev->mws[idx].addr_align); 2204 2205 off += scnprintf(strbuf + off, size - off, 2206 "Size align 0x%08llx, Size max %llu\n", 2207 ndev->mws[idx].size_align, ndev->mws[idx].size_max); 2208 } 2209 2210 /* Inbound memory windows information */ 2211 for (pidx = 0; pidx < ndev->peer_cnt; pidx++) { 2212 off += scnprintf(strbuf + off, size - off, 2213 "Inbound Memory Windows for peer %hhu (Port %hhu):\n", 2214 pidx, ndev->peers[pidx].port); 2215 2216 /* Print Memory Windows information */ 2217 for (idx = 0; idx < ndev->peers[pidx].mw_cnt; idx += cnt) { 2218 data = ndev->peers[pidx].mws[idx].type; 2219 cnt = idt_get_mw_count(data); 2220 2221 if (data == IDT_MW_DIR) 2222 off += scnprintf(strbuf + off, size - off, 2223 "\t%hhu.\t", idx); 2224 else 2225 off += scnprintf(strbuf + off, size - off, 2226 "\t%hhu-%hhu.\t", idx, idx + cnt - 1); 2227 2228 off += scnprintf(strbuf + off, size - off, 2229 "%s BAR%hhu, ", idt_get_mw_name(data), 2230 ndev->peers[pidx].mws[idx].bar); 2231 2232 off += scnprintf(strbuf + off, size - off, 2233 "Address align 0x%08llx, ", 2234 ndev->peers[pidx].mws[idx].addr_align); 2235 2236 off += scnprintf(strbuf + off, size - off, 2237 "Size align 0x%08llx, Size max %llu\n", 2238 ndev->peers[pidx].mws[idx].size_align, 2239 ndev->peers[pidx].mws[idx].size_max); 2240 } 2241 } 2242 off += scnprintf(strbuf + off, size - off, "\n"); 2243 2244 /* Doorbell information */ 2245 data = idt_sw_read(ndev, IDT_SW_GDBELLSTS); 2246 off += scnprintf(strbuf + off, size - off, 2247 "Global Doorbell state\t- 0x%08x\n", data); 2248 data = idt_ntb_db_read(&ndev->ntb); 2249 off += scnprintf(strbuf + off, size - off, 2250 "Local Doorbell state\t- 0x%08x\n", data); 2251 data = idt_nt_read(ndev, IDT_NT_INDBELLMSK); 2252 off += scnprintf(strbuf + off, size - off, 2253 "Local Doorbell mask\t- 0x%08x\n", data); 2254 off += scnprintf(strbuf + off, size - off, "\n"); 2255 2256 /* Messaging information */ 2257 off += scnprintf(strbuf + off, size - off, 2258 "Message event valid\t- 0x%08x\n", IDT_MSG_MASK); 2259 data = idt_ntb_msg_read_sts(&ndev->ntb); 2260 off += scnprintf(strbuf + off, size - off, 2261 "Message event status\t- 0x%08x\n", data); 2262 data = idt_nt_read(ndev, IDT_NT_MSGSTSMSK); 2263 off += scnprintf(strbuf + off, size - off, 2264 "Message event mask\t- 0x%08x\n", data); 2265 off += scnprintf(strbuf + off, size - off, 2266 "Message data:\n"); 2267 for (idx = 0; idx < IDT_MSG_CNT; idx++) { 2268 int src; 2269 data = idt_ntb_msg_read(&ndev->ntb, &src, idx); 2270 off += scnprintf(strbuf + off, size - off, 2271 "\t%hhu. 0x%08x from peer %hhu (Port %hhu)\n", 2272 idx, data, src, ndev->peers[src].port); 2273 } 2274 off += scnprintf(strbuf + off, size - off, "\n"); 2275 2276 /* Current temperature */ 2277 idt_read_temp(ndev, &temp, &frac); 2278 off += scnprintf(strbuf + off, size - off, 2279 "Switch temperature\t\t- %hhu.%hhuC\n", temp, frac); 2280 2281 /* Copy the buffer to the User Space */ 2282 ret = simple_read_from_buffer(ubuf, count, offp, strbuf, off); 2283 kfree(strbuf); 2284 2285 return ret; 2286 } 2287 2288 /* 2289 * idt_init_dbgfs() - initialize DebugFS node 2290 * @ndev: IDT NTB hardware driver descriptor 2291 * 2292 * Return: zero on success, otherwise a negative error number. 2293 */ 2294 static int idt_init_dbgfs(struct idt_ntb_dev *ndev) 2295 { 2296 char devname[64]; 2297 2298 /* If the top directory is not created then do nothing */ 2299 if (IS_ERR_OR_NULL(dbgfs_topdir)) { 2300 dev_info(&ndev->ntb.pdev->dev, "Top DebugFS directory absent"); 2301 return PTR_ERR(dbgfs_topdir); 2302 } 2303 2304 /* Create the info file node */ 2305 snprintf(devname, 64, "info:%s", pci_name(ndev->ntb.pdev)); 2306 ndev->dbgfs_info = debugfs_create_file(devname, 0400, dbgfs_topdir, 2307 ndev, &idt_dbgfs_info_ops); 2308 if (IS_ERR(ndev->dbgfs_info)) { 2309 dev_dbg(&ndev->ntb.pdev->dev, "Failed to create DebugFS node"); 2310 return PTR_ERR(ndev->dbgfs_info); 2311 } 2312 2313 dev_dbg(&ndev->ntb.pdev->dev, "NTB device DebugFS node created"); 2314 2315 return 0; 2316 } 2317 2318 /* 2319 * idt_deinit_dbgfs() - deinitialize DebugFS node 2320 * @ndev: IDT NTB hardware driver descriptor 2321 * 2322 * Just discard the info node from DebugFS 2323 */ 2324 static void idt_deinit_dbgfs(struct idt_ntb_dev *ndev) 2325 { 2326 debugfs_remove(ndev->dbgfs_info); 2327 2328 dev_dbg(&ndev->ntb.pdev->dev, "NTB device DebugFS node discarded"); 2329 } 2330 2331 /*============================================================================= 2332 * 11. Basic PCIe device initialization 2333 *============================================================================= 2334 */ 2335 2336 /* 2337 * idt_check_setup() - Check whether the IDT PCIe-swtich is properly 2338 * pre-initialized 2339 * @pdev: Pointer to the PCI device descriptor 2340 * 2341 * Return: zero on success, otherwise a negative error number. 2342 */ 2343 static int idt_check_setup(struct pci_dev *pdev) 2344 { 2345 u32 data; 2346 int ret; 2347 2348 /* Read the BARSETUP0 */ 2349 ret = pci_read_config_dword(pdev, IDT_NT_BARSETUP0, &data); 2350 if (ret != 0) { 2351 dev_err(&pdev->dev, 2352 "Failed to read BARSETUP0 config register"); 2353 return ret; 2354 } 2355 2356 /* Check whether the BAR0 register is enabled to be of config space */ 2357 if (!(data & IDT_BARSETUP_EN) || !(data & IDT_BARSETUP_MODE_CFG)) { 2358 dev_err(&pdev->dev, "BAR0 doesn't map config space"); 2359 return -EINVAL; 2360 } 2361 2362 /* Configuration space BAR0 must have certain size */ 2363 if ((data & IDT_BARSETUP_SIZE_MASK) != IDT_BARSETUP_SIZE_CFG) { 2364 dev_err(&pdev->dev, "Invalid size of config space"); 2365 return -EINVAL; 2366 } 2367 2368 dev_dbg(&pdev->dev, "NTB device pre-initialized correctly"); 2369 2370 return 0; 2371 } 2372 2373 /* 2374 * Create the IDT PCIe-switch driver descriptor 2375 * @pdev: Pointer to the PCI device descriptor 2376 * @id: IDT PCIe-device configuration 2377 * 2378 * It just allocates a memory for IDT PCIe-switch device structure and 2379 * initializes some commonly used fields. 2380 * 2381 * No need of release method, since managed device resource is used for 2382 * memory allocation. 2383 * 2384 * Return: pointer to the descriptor, otherwise a negative error number. 2385 */ 2386 static struct idt_ntb_dev *idt_create_dev(struct pci_dev *pdev, 2387 const struct pci_device_id *id) 2388 { 2389 struct idt_ntb_dev *ndev; 2390 2391 /* Allocate memory for the IDT PCIe-device descriptor */ 2392 ndev = devm_kzalloc(&pdev->dev, sizeof(*ndev), GFP_KERNEL); 2393 if (IS_ERR_OR_NULL(ndev)) { 2394 dev_err(&pdev->dev, "Memory allocation failed for descriptor"); 2395 return ERR_PTR(-ENOMEM); 2396 } 2397 2398 /* Save the IDT PCIe-switch ports configuration */ 2399 ndev->swcfg = (struct idt_89hpes_cfg *)id->driver_data; 2400 /* Save the PCI-device pointer inside the NTB device structure */ 2401 ndev->ntb.pdev = pdev; 2402 2403 /* Initialize spin locker of Doorbell, Message and GASA registers */ 2404 spin_lock_init(&ndev->db_mask_lock); 2405 spin_lock_init(&ndev->msg_mask_lock); 2406 spin_lock_init(&ndev->gasa_lock); 2407 2408 dev_info(&pdev->dev, "IDT %s discovered", ndev->swcfg->name); 2409 2410 dev_dbg(&pdev->dev, "NTB device descriptor created"); 2411 2412 return ndev; 2413 } 2414 2415 /* 2416 * idt_init_pci() - initialize the basic PCI-related subsystem 2417 * @ndev: Pointer to the IDT PCIe-switch driver descriptor 2418 * 2419 * Managed device resources will be freed automatically in case of failure or 2420 * driver detachment. 2421 * 2422 * Return: zero on success, otherwise negative error number. 2423 */ 2424 static int idt_init_pci(struct idt_ntb_dev *ndev) 2425 { 2426 struct pci_dev *pdev = ndev->ntb.pdev; 2427 int ret; 2428 2429 /* Initialize the bit mask of PCI/NTB DMA */ 2430 ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 2431 if (ret != 0) { 2432 ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 2433 if (ret != 0) { 2434 dev_err(&pdev->dev, "Failed to set DMA bit mask\n"); 2435 return ret; 2436 } 2437 dev_warn(&pdev->dev, "Cannot set DMA highmem bit mask\n"); 2438 } 2439 ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 2440 if (ret != 0) { 2441 ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 2442 if (ret != 0) { 2443 dev_err(&pdev->dev, 2444 "Failed to set consistent DMA bit mask\n"); 2445 return ret; 2446 } 2447 dev_warn(&pdev->dev, 2448 "Cannot set consistent DMA highmem bit mask\n"); 2449 } 2450 ret = dma_coerce_mask_and_coherent(&ndev->ntb.dev, 2451 dma_get_mask(&pdev->dev)); 2452 if (ret != 0) { 2453 dev_err(&pdev->dev, "Failed to set NTB device DMA bit mask\n"); 2454 return ret; 2455 } 2456 2457 /* 2458 * Enable the device advanced error reporting. It's not critical to 2459 * have AER disabled in the kernel. 2460 */ 2461 ret = pci_enable_pcie_error_reporting(pdev); 2462 if (ret != 0) 2463 dev_warn(&pdev->dev, "PCIe AER capability disabled\n"); 2464 else /* Cleanup uncorrectable error status before getting to init */ 2465 pci_cleanup_aer_uncorrect_error_status(pdev); 2466 2467 /* First enable the PCI device */ 2468 ret = pcim_enable_device(pdev); 2469 if (ret != 0) { 2470 dev_err(&pdev->dev, "Failed to enable PCIe device\n"); 2471 goto err_disable_aer; 2472 } 2473 2474 /* 2475 * Enable the bus mastering, which effectively enables MSI IRQs and 2476 * Request TLPs translation 2477 */ 2478 pci_set_master(pdev); 2479 2480 /* Request all BARs resources and map BAR0 only */ 2481 ret = pcim_iomap_regions_request_all(pdev, 1, NTB_NAME); 2482 if (ret != 0) { 2483 dev_err(&pdev->dev, "Failed to request resources\n"); 2484 goto err_clear_master; 2485 } 2486 2487 /* Retrieve virtual address of BAR0 - PCI configuration space */ 2488 ndev->cfgspc = pcim_iomap_table(pdev)[0]; 2489 2490 /* Put the IDT driver data pointer to the PCI-device private pointer */ 2491 pci_set_drvdata(pdev, ndev); 2492 2493 dev_dbg(&pdev->dev, "NT-function PCIe interface initialized"); 2494 2495 return 0; 2496 2497 err_clear_master: 2498 pci_clear_master(pdev); 2499 err_disable_aer: 2500 (void)pci_disable_pcie_error_reporting(pdev); 2501 2502 return ret; 2503 } 2504 2505 /* 2506 * idt_deinit_pci() - deinitialize the basic PCI-related subsystem 2507 * @ndev: Pointer to the IDT PCIe-switch driver descriptor 2508 * 2509 * Managed resources will be freed on the driver detachment 2510 */ 2511 static void idt_deinit_pci(struct idt_ntb_dev *ndev) 2512 { 2513 struct pci_dev *pdev = ndev->ntb.pdev; 2514 2515 /* Clean up the PCI-device private data pointer */ 2516 pci_set_drvdata(pdev, NULL); 2517 2518 /* Clear the bus master disabling the Request TLPs translation */ 2519 pci_clear_master(pdev); 2520 2521 /* Disable the AER capability */ 2522 (void)pci_disable_pcie_error_reporting(pdev); 2523 2524 dev_dbg(&pdev->dev, "NT-function PCIe interface cleared"); 2525 } 2526 2527 /*=========================================================================== 2528 * 12. PCI bus callback functions 2529 *=========================================================================== 2530 */ 2531 2532 /* 2533 * idt_pci_probe() - PCI device probe callback 2534 * @pdev: Pointer to PCI device structure 2535 * @id: PCIe device custom descriptor 2536 * 2537 * Return: zero on success, otherwise negative error number 2538 */ 2539 static int idt_pci_probe(struct pci_dev *pdev, 2540 const struct pci_device_id *id) 2541 { 2542 struct idt_ntb_dev *ndev; 2543 int ret; 2544 2545 /* Check whether IDT PCIe-switch is properly pre-initialized */ 2546 ret = idt_check_setup(pdev); 2547 if (ret != 0) 2548 return ret; 2549 2550 /* Allocate the memory for IDT NTB device data */ 2551 ndev = idt_create_dev(pdev, id); 2552 if (IS_ERR_OR_NULL(ndev)) 2553 return PTR_ERR(ndev); 2554 2555 /* Initialize the basic PCI subsystem of the device */ 2556 ret = idt_init_pci(ndev); 2557 if (ret != 0) 2558 return ret; 2559 2560 /* Scan ports of the IDT PCIe-switch */ 2561 (void)idt_scan_ports(ndev); 2562 2563 /* Initialize NTB link events subsystem */ 2564 idt_init_link(ndev); 2565 2566 /* Initialize MWs subsystem */ 2567 ret = idt_init_mws(ndev); 2568 if (ret != 0) 2569 goto err_deinit_link; 2570 2571 /* Initialize Messaging subsystem */ 2572 idt_init_msg(ndev); 2573 2574 /* Initialize IDT interrupts handler */ 2575 ret = idt_init_isr(ndev); 2576 if (ret != 0) 2577 goto err_deinit_link; 2578 2579 /* Register IDT NTB devices on the NTB bus */ 2580 ret = idt_register_device(ndev); 2581 if (ret != 0) 2582 goto err_deinit_isr; 2583 2584 /* Initialize DebugFS info node */ 2585 (void)idt_init_dbgfs(ndev); 2586 2587 /* IDT PCIe-switch NTB driver is finally initialized */ 2588 dev_info(&pdev->dev, "IDT NTB device is ready"); 2589 2590 /* May the force be with us... */ 2591 return 0; 2592 2593 err_deinit_isr: 2594 idt_deinit_isr(ndev); 2595 err_deinit_link: 2596 idt_deinit_link(ndev); 2597 idt_deinit_pci(ndev); 2598 2599 return ret; 2600 } 2601 2602 /* 2603 * idt_pci_probe() - PCI device remove callback 2604 * @pdev: Pointer to PCI device structure 2605 */ 2606 static void idt_pci_remove(struct pci_dev *pdev) 2607 { 2608 struct idt_ntb_dev *ndev = pci_get_drvdata(pdev); 2609 2610 /* Deinit the DebugFS node */ 2611 idt_deinit_dbgfs(ndev); 2612 2613 /* Unregister NTB device */ 2614 idt_unregister_device(ndev); 2615 2616 /* Stop the interrupts handling */ 2617 idt_deinit_isr(ndev); 2618 2619 /* Deinitialize link event subsystem */ 2620 idt_deinit_link(ndev); 2621 2622 /* Deinit basic PCI subsystem */ 2623 idt_deinit_pci(ndev); 2624 2625 /* IDT PCIe-switch NTB driver is finally initialized */ 2626 dev_info(&pdev->dev, "IDT NTB device is removed"); 2627 2628 /* Sayonara... */ 2629 } 2630 2631 /* 2632 * IDT PCIe-switch models ports configuration structures 2633 */ 2634 static const struct idt_89hpes_cfg idt_89hpes24nt6ag2_config = { 2635 .name = "89HPES24NT6AG2", 2636 .port_cnt = 6, .ports = {0, 2, 4, 6, 8, 12} 2637 }; 2638 static const struct idt_89hpes_cfg idt_89hpes32nt8ag2_config = { 2639 .name = "89HPES32NT8AG2", 2640 .port_cnt = 8, .ports = {0, 2, 4, 6, 8, 12, 16, 20} 2641 }; 2642 static const struct idt_89hpes_cfg idt_89hpes32nt8bg2_config = { 2643 .name = "89HPES32NT8BG2", 2644 .port_cnt = 8, .ports = {0, 2, 4, 6, 8, 12, 16, 20} 2645 }; 2646 static const struct idt_89hpes_cfg idt_89hpes12nt12g2_config = { 2647 .name = "89HPES12NT12G2", 2648 .port_cnt = 3, .ports = {0, 8, 16} 2649 }; 2650 static const struct idt_89hpes_cfg idt_89hpes16nt16g2_config = { 2651 .name = "89HPES16NT16G2", 2652 .port_cnt = 4, .ports = {0, 8, 12, 16} 2653 }; 2654 static const struct idt_89hpes_cfg idt_89hpes24nt24g2_config = { 2655 .name = "89HPES24NT24G2", 2656 .port_cnt = 8, .ports = {0, 2, 4, 6, 8, 12, 16, 20} 2657 }; 2658 static const struct idt_89hpes_cfg idt_89hpes32nt24ag2_config = { 2659 .name = "89HPES32NT24AG2", 2660 .port_cnt = 8, .ports = {0, 2, 4, 6, 8, 12, 16, 20} 2661 }; 2662 static const struct idt_89hpes_cfg idt_89hpes32nt24bg2_config = { 2663 .name = "89HPES32NT24BG2", 2664 .port_cnt = 8, .ports = {0, 2, 4, 6, 8, 12, 16, 20} 2665 }; 2666 2667 /* 2668 * PCI-ids table of the supported IDT PCIe-switch devices 2669 */ 2670 static const struct pci_device_id idt_pci_tbl[] = { 2671 {IDT_PCI_DEVICE_IDS(89HPES24NT6AG2, idt_89hpes24nt6ag2_config)}, 2672 {IDT_PCI_DEVICE_IDS(89HPES32NT8AG2, idt_89hpes32nt8ag2_config)}, 2673 {IDT_PCI_DEVICE_IDS(89HPES32NT8BG2, idt_89hpes32nt8bg2_config)}, 2674 {IDT_PCI_DEVICE_IDS(89HPES12NT12G2, idt_89hpes12nt12g2_config)}, 2675 {IDT_PCI_DEVICE_IDS(89HPES16NT16G2, idt_89hpes16nt16g2_config)}, 2676 {IDT_PCI_DEVICE_IDS(89HPES24NT24G2, idt_89hpes24nt24g2_config)}, 2677 {IDT_PCI_DEVICE_IDS(89HPES32NT24AG2, idt_89hpes32nt24ag2_config)}, 2678 {IDT_PCI_DEVICE_IDS(89HPES32NT24BG2, idt_89hpes32nt24bg2_config)}, 2679 {0} 2680 }; 2681 MODULE_DEVICE_TABLE(pci, idt_pci_tbl); 2682 2683 /* 2684 * IDT PCIe-switch NT-function device driver structure definition 2685 */ 2686 static struct pci_driver idt_pci_driver = { 2687 .name = KBUILD_MODNAME, 2688 .probe = idt_pci_probe, 2689 .remove = idt_pci_remove, 2690 .id_table = idt_pci_tbl, 2691 }; 2692 2693 static int __init idt_pci_driver_init(void) 2694 { 2695 pr_info("%s %s\n", NTB_DESC, NTB_VER); 2696 2697 /* Create the top DebugFS directory if the FS is initialized */ 2698 if (debugfs_initialized()) 2699 dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, NULL); 2700 2701 /* Register the NTB hardware driver to handle the PCI device */ 2702 return pci_register_driver(&idt_pci_driver); 2703 } 2704 module_init(idt_pci_driver_init); 2705 2706 static void __exit idt_pci_driver_exit(void) 2707 { 2708 /* Unregister the NTB hardware driver */ 2709 pci_unregister_driver(&idt_pci_driver); 2710 2711 /* Discard the top DebugFS directory */ 2712 debugfs_remove_recursive(dbgfs_topdir); 2713 } 2714 module_exit(idt_pci_driver_exit); 2715 2716