1*6e173d3bSMartin Habets /* SPDX-License-Identifier: GPL-2.0-only */ 2*6e173d3bSMartin Habets /**************************************************************************** 3*6e173d3bSMartin Habets * Driver for Solarflare network controllers and boards 4*6e173d3bSMartin Habets * Copyright 2005-2006 Fen Systems Ltd. 5*6e173d3bSMartin Habets * Copyright 2006-2013 Solarflare Communications Inc. 6*6e173d3bSMartin Habets */ 7*6e173d3bSMartin Habets 8*6e173d3bSMartin Habets #ifndef EFX_IO_H 9*6e173d3bSMartin Habets #define EFX_IO_H 10*6e173d3bSMartin Habets 11*6e173d3bSMartin Habets #include <linux/io.h> 12*6e173d3bSMartin Habets #include <linux/spinlock.h> 13*6e173d3bSMartin Habets 14*6e173d3bSMartin Habets /************************************************************************** 15*6e173d3bSMartin Habets * 16*6e173d3bSMartin Habets * NIC register I/O 17*6e173d3bSMartin Habets * 18*6e173d3bSMartin Habets ************************************************************************** 19*6e173d3bSMartin Habets * 20*6e173d3bSMartin Habets * Notes on locking strategy for the Falcon architecture: 21*6e173d3bSMartin Habets * 22*6e173d3bSMartin Habets * Many CSRs are very wide and cannot be read or written atomically. 23*6e173d3bSMartin Habets * Writes from the host are buffered by the Bus Interface Unit (BIU) 24*6e173d3bSMartin Habets * up to 128 bits. Whenever the host writes part of such a register, 25*6e173d3bSMartin Habets * the BIU collects the written value and does not write to the 26*6e173d3bSMartin Habets * underlying register until all 4 dwords have been written. A 27*6e173d3bSMartin Habets * similar buffering scheme applies to host access to the NIC's 64-bit 28*6e173d3bSMartin Habets * SRAM. 29*6e173d3bSMartin Habets * 30*6e173d3bSMartin Habets * Writes to different CSRs and 64-bit SRAM words must be serialised, 31*6e173d3bSMartin Habets * since interleaved access can result in lost writes. We use 32*6e173d3bSMartin Habets * efx_nic::biu_lock for this. 33*6e173d3bSMartin Habets * 34*6e173d3bSMartin Habets * We also serialise reads from 128-bit CSRs and SRAM with the same 35*6e173d3bSMartin Habets * spinlock. This may not be necessary, but it doesn't really matter 36*6e173d3bSMartin Habets * as there are no such reads on the fast path. 37*6e173d3bSMartin Habets * 38*6e173d3bSMartin Habets * The DMA descriptor pointers (RX_DESC_UPD and TX_DESC_UPD) are 39*6e173d3bSMartin Habets * 128-bit but are special-cased in the BIU to avoid the need for 40*6e173d3bSMartin Habets * locking in the host: 41*6e173d3bSMartin Habets * 42*6e173d3bSMartin Habets * - They are write-only. 43*6e173d3bSMartin Habets * - The semantics of writing to these registers are such that 44*6e173d3bSMartin Habets * replacing the low 96 bits with zero does not affect functionality. 45*6e173d3bSMartin Habets * - If the host writes to the last dword address of such a register 46*6e173d3bSMartin Habets * (i.e. the high 32 bits) the underlying register will always be 47*6e173d3bSMartin Habets * written. If the collector and the current write together do not 48*6e173d3bSMartin Habets * provide values for all 128 bits of the register, the low 96 bits 49*6e173d3bSMartin Habets * will be written as zero. 50*6e173d3bSMartin Habets * - If the host writes to the address of any other part of such a 51*6e173d3bSMartin Habets * register while the collector already holds values for some other 52*6e173d3bSMartin Habets * register, the write is discarded and the collector maintains its 53*6e173d3bSMartin Habets * current state. 54*6e173d3bSMartin Habets * 55*6e173d3bSMartin Habets * The EF10 architecture exposes very few registers to the host and 56*6e173d3bSMartin Habets * most of them are only 32 bits wide. The only exceptions are the MC 57*6e173d3bSMartin Habets * doorbell register pair, which has its own latching, and 58*6e173d3bSMartin Habets * TX_DESC_UPD, which works in a similar way to the Falcon 59*6e173d3bSMartin Habets * architecture. 60*6e173d3bSMartin Habets */ 61*6e173d3bSMartin Habets 62*6e173d3bSMartin Habets #if BITS_PER_LONG == 64 63*6e173d3bSMartin Habets #define EFX_USE_QWORD_IO 1 64*6e173d3bSMartin Habets #endif 65*6e173d3bSMartin Habets 66*6e173d3bSMartin Habets /* Hardware issue requires that only 64-bit naturally aligned writes 67*6e173d3bSMartin Habets * are seen by hardware. Its not strictly necessary to restrict to 68*6e173d3bSMartin Habets * x86_64 arch, but done for safety since unusual write combining behaviour 69*6e173d3bSMartin Habets * can break PIO. 70*6e173d3bSMartin Habets */ 71*6e173d3bSMartin Habets #ifdef CONFIG_X86_64 72*6e173d3bSMartin Habets /* PIO is a win only if write-combining is possible */ 73*6e173d3bSMartin Habets #ifdef ARCH_HAS_IOREMAP_WC 74*6e173d3bSMartin Habets #define EFX_USE_PIO 1 75*6e173d3bSMartin Habets #endif 76*6e173d3bSMartin Habets #endif 77*6e173d3bSMartin Habets 78*6e173d3bSMartin Habets static inline u32 efx_reg(struct efx_nic *efx, unsigned int reg) 79*6e173d3bSMartin Habets { 80*6e173d3bSMartin Habets return efx->reg_base + reg; 81*6e173d3bSMartin Habets } 82*6e173d3bSMartin Habets 83*6e173d3bSMartin Habets #ifdef EFX_USE_QWORD_IO 84*6e173d3bSMartin Habets static inline void _efx_writeq(struct efx_nic *efx, __le64 value, 85*6e173d3bSMartin Habets unsigned int reg) 86*6e173d3bSMartin Habets { 87*6e173d3bSMartin Habets __raw_writeq((__force u64)value, efx->membase + reg); 88*6e173d3bSMartin Habets } 89*6e173d3bSMartin Habets static inline __le64 _efx_readq(struct efx_nic *efx, unsigned int reg) 90*6e173d3bSMartin Habets { 91*6e173d3bSMartin Habets return (__force __le64)__raw_readq(efx->membase + reg); 92*6e173d3bSMartin Habets } 93*6e173d3bSMartin Habets #endif 94*6e173d3bSMartin Habets 95*6e173d3bSMartin Habets static inline void _efx_writed(struct efx_nic *efx, __le32 value, 96*6e173d3bSMartin Habets unsigned int reg) 97*6e173d3bSMartin Habets { 98*6e173d3bSMartin Habets __raw_writel((__force u32)value, efx->membase + reg); 99*6e173d3bSMartin Habets } 100*6e173d3bSMartin Habets static inline __le32 _efx_readd(struct efx_nic *efx, unsigned int reg) 101*6e173d3bSMartin Habets { 102*6e173d3bSMartin Habets return (__force __le32)__raw_readl(efx->membase + reg); 103*6e173d3bSMartin Habets } 104*6e173d3bSMartin Habets 105*6e173d3bSMartin Habets /* Write a normal 128-bit CSR, locking as appropriate. */ 106*6e173d3bSMartin Habets static inline void efx_writeo(struct efx_nic *efx, const efx_oword_t *value, 107*6e173d3bSMartin Habets unsigned int reg) 108*6e173d3bSMartin Habets { 109*6e173d3bSMartin Habets unsigned long flags __attribute__ ((unused)); 110*6e173d3bSMartin Habets 111*6e173d3bSMartin Habets netif_vdbg(efx, hw, efx->net_dev, 112*6e173d3bSMartin Habets "writing register %x with " EFX_OWORD_FMT "\n", reg, 113*6e173d3bSMartin Habets EFX_OWORD_VAL(*value)); 114*6e173d3bSMartin Habets 115*6e173d3bSMartin Habets spin_lock_irqsave(&efx->biu_lock, flags); 116*6e173d3bSMartin Habets #ifdef EFX_USE_QWORD_IO 117*6e173d3bSMartin Habets _efx_writeq(efx, value->u64[0], reg + 0); 118*6e173d3bSMartin Habets _efx_writeq(efx, value->u64[1], reg + 8); 119*6e173d3bSMartin Habets #else 120*6e173d3bSMartin Habets _efx_writed(efx, value->u32[0], reg + 0); 121*6e173d3bSMartin Habets _efx_writed(efx, value->u32[1], reg + 4); 122*6e173d3bSMartin Habets _efx_writed(efx, value->u32[2], reg + 8); 123*6e173d3bSMartin Habets _efx_writed(efx, value->u32[3], reg + 12); 124*6e173d3bSMartin Habets #endif 125*6e173d3bSMartin Habets spin_unlock_irqrestore(&efx->biu_lock, flags); 126*6e173d3bSMartin Habets } 127*6e173d3bSMartin Habets 128*6e173d3bSMartin Habets /* Write 64-bit SRAM through the supplied mapping, locking as appropriate. */ 129*6e173d3bSMartin Habets static inline void efx_sram_writeq(struct efx_nic *efx, void __iomem *membase, 130*6e173d3bSMartin Habets const efx_qword_t *value, unsigned int index) 131*6e173d3bSMartin Habets { 132*6e173d3bSMartin Habets unsigned int addr = index * sizeof(*value); 133*6e173d3bSMartin Habets unsigned long flags __attribute__ ((unused)); 134*6e173d3bSMartin Habets 135*6e173d3bSMartin Habets netif_vdbg(efx, hw, efx->net_dev, 136*6e173d3bSMartin Habets "writing SRAM address %x with " EFX_QWORD_FMT "\n", 137*6e173d3bSMartin Habets addr, EFX_QWORD_VAL(*value)); 138*6e173d3bSMartin Habets 139*6e173d3bSMartin Habets spin_lock_irqsave(&efx->biu_lock, flags); 140*6e173d3bSMartin Habets #ifdef EFX_USE_QWORD_IO 141*6e173d3bSMartin Habets __raw_writeq((__force u64)value->u64[0], membase + addr); 142*6e173d3bSMartin Habets #else 143*6e173d3bSMartin Habets __raw_writel((__force u32)value->u32[0], membase + addr); 144*6e173d3bSMartin Habets __raw_writel((__force u32)value->u32[1], membase + addr + 4); 145*6e173d3bSMartin Habets #endif 146*6e173d3bSMartin Habets spin_unlock_irqrestore(&efx->biu_lock, flags); 147*6e173d3bSMartin Habets } 148*6e173d3bSMartin Habets 149*6e173d3bSMartin Habets /* Write a 32-bit CSR or the last dword of a special 128-bit CSR */ 150*6e173d3bSMartin Habets static inline void efx_writed(struct efx_nic *efx, const efx_dword_t *value, 151*6e173d3bSMartin Habets unsigned int reg) 152*6e173d3bSMartin Habets { 153*6e173d3bSMartin Habets netif_vdbg(efx, hw, efx->net_dev, 154*6e173d3bSMartin Habets "writing register %x with "EFX_DWORD_FMT"\n", 155*6e173d3bSMartin Habets reg, EFX_DWORD_VAL(*value)); 156*6e173d3bSMartin Habets 157*6e173d3bSMartin Habets /* No lock required */ 158*6e173d3bSMartin Habets _efx_writed(efx, value->u32[0], reg); 159*6e173d3bSMartin Habets } 160*6e173d3bSMartin Habets 161*6e173d3bSMartin Habets /* Read a 128-bit CSR, locking as appropriate. */ 162*6e173d3bSMartin Habets static inline void efx_reado(struct efx_nic *efx, efx_oword_t *value, 163*6e173d3bSMartin Habets unsigned int reg) 164*6e173d3bSMartin Habets { 165*6e173d3bSMartin Habets unsigned long flags __attribute__ ((unused)); 166*6e173d3bSMartin Habets 167*6e173d3bSMartin Habets spin_lock_irqsave(&efx->biu_lock, flags); 168*6e173d3bSMartin Habets value->u32[0] = _efx_readd(efx, reg + 0); 169*6e173d3bSMartin Habets value->u32[1] = _efx_readd(efx, reg + 4); 170*6e173d3bSMartin Habets value->u32[2] = _efx_readd(efx, reg + 8); 171*6e173d3bSMartin Habets value->u32[3] = _efx_readd(efx, reg + 12); 172*6e173d3bSMartin Habets spin_unlock_irqrestore(&efx->biu_lock, flags); 173*6e173d3bSMartin Habets 174*6e173d3bSMartin Habets netif_vdbg(efx, hw, efx->net_dev, 175*6e173d3bSMartin Habets "read from register %x, got " EFX_OWORD_FMT "\n", reg, 176*6e173d3bSMartin Habets EFX_OWORD_VAL(*value)); 177*6e173d3bSMartin Habets } 178*6e173d3bSMartin Habets 179*6e173d3bSMartin Habets /* Read 64-bit SRAM through the supplied mapping, locking as appropriate. */ 180*6e173d3bSMartin Habets static inline void efx_sram_readq(struct efx_nic *efx, void __iomem *membase, 181*6e173d3bSMartin Habets efx_qword_t *value, unsigned int index) 182*6e173d3bSMartin Habets { 183*6e173d3bSMartin Habets unsigned int addr = index * sizeof(*value); 184*6e173d3bSMartin Habets unsigned long flags __attribute__ ((unused)); 185*6e173d3bSMartin Habets 186*6e173d3bSMartin Habets spin_lock_irqsave(&efx->biu_lock, flags); 187*6e173d3bSMartin Habets #ifdef EFX_USE_QWORD_IO 188*6e173d3bSMartin Habets value->u64[0] = (__force __le64)__raw_readq(membase + addr); 189*6e173d3bSMartin Habets #else 190*6e173d3bSMartin Habets value->u32[0] = (__force __le32)__raw_readl(membase + addr); 191*6e173d3bSMartin Habets value->u32[1] = (__force __le32)__raw_readl(membase + addr + 4); 192*6e173d3bSMartin Habets #endif 193*6e173d3bSMartin Habets spin_unlock_irqrestore(&efx->biu_lock, flags); 194*6e173d3bSMartin Habets 195*6e173d3bSMartin Habets netif_vdbg(efx, hw, efx->net_dev, 196*6e173d3bSMartin Habets "read from SRAM address %x, got "EFX_QWORD_FMT"\n", 197*6e173d3bSMartin Habets addr, EFX_QWORD_VAL(*value)); 198*6e173d3bSMartin Habets } 199*6e173d3bSMartin Habets 200*6e173d3bSMartin Habets /* Read a 32-bit CSR or SRAM */ 201*6e173d3bSMartin Habets static inline void efx_readd(struct efx_nic *efx, efx_dword_t *value, 202*6e173d3bSMartin Habets unsigned int reg) 203*6e173d3bSMartin Habets { 204*6e173d3bSMartin Habets value->u32[0] = _efx_readd(efx, reg); 205*6e173d3bSMartin Habets netif_vdbg(efx, hw, efx->net_dev, 206*6e173d3bSMartin Habets "read from register %x, got "EFX_DWORD_FMT"\n", 207*6e173d3bSMartin Habets reg, EFX_DWORD_VAL(*value)); 208*6e173d3bSMartin Habets } 209*6e173d3bSMartin Habets 210*6e173d3bSMartin Habets /* Write a 128-bit CSR forming part of a table */ 211*6e173d3bSMartin Habets static inline void 212*6e173d3bSMartin Habets efx_writeo_table(struct efx_nic *efx, const efx_oword_t *value, 213*6e173d3bSMartin Habets unsigned int reg, unsigned int index) 214*6e173d3bSMartin Habets { 215*6e173d3bSMartin Habets efx_writeo(efx, value, reg + index * sizeof(efx_oword_t)); 216*6e173d3bSMartin Habets } 217*6e173d3bSMartin Habets 218*6e173d3bSMartin Habets /* Read a 128-bit CSR forming part of a table */ 219*6e173d3bSMartin Habets static inline void efx_reado_table(struct efx_nic *efx, efx_oword_t *value, 220*6e173d3bSMartin Habets unsigned int reg, unsigned int index) 221*6e173d3bSMartin Habets { 222*6e173d3bSMartin Habets efx_reado(efx, value, reg + index * sizeof(efx_oword_t)); 223*6e173d3bSMartin Habets } 224*6e173d3bSMartin Habets 225*6e173d3bSMartin Habets /* default VI stride (step between per-VI registers) is 8K on EF10 and 226*6e173d3bSMartin Habets * 64K on EF100 227*6e173d3bSMartin Habets */ 228*6e173d3bSMartin Habets #define EFX_DEFAULT_VI_STRIDE 0x2000 229*6e173d3bSMartin Habets #define EF100_DEFAULT_VI_STRIDE 0x10000 230*6e173d3bSMartin Habets 231*6e173d3bSMartin Habets /* Calculate offset to page-mapped register */ 232*6e173d3bSMartin Habets static inline unsigned int efx_paged_reg(struct efx_nic *efx, unsigned int page, 233*6e173d3bSMartin Habets unsigned int reg) 234*6e173d3bSMartin Habets { 235*6e173d3bSMartin Habets return page * efx->vi_stride + reg; 236*6e173d3bSMartin Habets } 237*6e173d3bSMartin Habets 238*6e173d3bSMartin Habets /* Write the whole of RX_DESC_UPD or TX_DESC_UPD */ 239*6e173d3bSMartin Habets static inline void _efx_writeo_page(struct efx_nic *efx, efx_oword_t *value, 240*6e173d3bSMartin Habets unsigned int reg, unsigned int page) 241*6e173d3bSMartin Habets { 242*6e173d3bSMartin Habets reg = efx_paged_reg(efx, page, reg); 243*6e173d3bSMartin Habets 244*6e173d3bSMartin Habets netif_vdbg(efx, hw, efx->net_dev, 245*6e173d3bSMartin Habets "writing register %x with " EFX_OWORD_FMT "\n", reg, 246*6e173d3bSMartin Habets EFX_OWORD_VAL(*value)); 247*6e173d3bSMartin Habets 248*6e173d3bSMartin Habets #ifdef EFX_USE_QWORD_IO 249*6e173d3bSMartin Habets _efx_writeq(efx, value->u64[0], reg + 0); 250*6e173d3bSMartin Habets _efx_writeq(efx, value->u64[1], reg + 8); 251*6e173d3bSMartin Habets #else 252*6e173d3bSMartin Habets _efx_writed(efx, value->u32[0], reg + 0); 253*6e173d3bSMartin Habets _efx_writed(efx, value->u32[1], reg + 4); 254*6e173d3bSMartin Habets _efx_writed(efx, value->u32[2], reg + 8); 255*6e173d3bSMartin Habets _efx_writed(efx, value->u32[3], reg + 12); 256*6e173d3bSMartin Habets #endif 257*6e173d3bSMartin Habets } 258*6e173d3bSMartin Habets #define efx_writeo_page(efx, value, reg, page) \ 259*6e173d3bSMartin Habets _efx_writeo_page(efx, value, \ 260*6e173d3bSMartin Habets reg + \ 261*6e173d3bSMartin Habets BUILD_BUG_ON_ZERO((reg) != 0x830 && (reg) != 0xa10), \ 262*6e173d3bSMartin Habets page) 263*6e173d3bSMartin Habets 264*6e173d3bSMartin Habets /* Write a page-mapped 32-bit CSR (EVQ_RPTR, EVQ_TMR (EF10), or the 265*6e173d3bSMartin Habets * high bits of RX_DESC_UPD or TX_DESC_UPD) 266*6e173d3bSMartin Habets */ 267*6e173d3bSMartin Habets static inline void 268*6e173d3bSMartin Habets _efx_writed_page(struct efx_nic *efx, const efx_dword_t *value, 269*6e173d3bSMartin Habets unsigned int reg, unsigned int page) 270*6e173d3bSMartin Habets { 271*6e173d3bSMartin Habets efx_writed(efx, value, efx_paged_reg(efx, page, reg)); 272*6e173d3bSMartin Habets } 273*6e173d3bSMartin Habets #define efx_writed_page(efx, value, reg, page) \ 274*6e173d3bSMartin Habets _efx_writed_page(efx, value, \ 275*6e173d3bSMartin Habets reg + \ 276*6e173d3bSMartin Habets BUILD_BUG_ON_ZERO((reg) != 0x180 && \ 277*6e173d3bSMartin Habets (reg) != 0x200 && \ 278*6e173d3bSMartin Habets (reg) != 0x400 && \ 279*6e173d3bSMartin Habets (reg) != 0x420 && \ 280*6e173d3bSMartin Habets (reg) != 0x830 && \ 281*6e173d3bSMartin Habets (reg) != 0x83c && \ 282*6e173d3bSMartin Habets (reg) != 0xa18 && \ 283*6e173d3bSMartin Habets (reg) != 0xa1c), \ 284*6e173d3bSMartin Habets page) 285*6e173d3bSMartin Habets 286*6e173d3bSMartin Habets /* Write TIMER_COMMAND. This is a page-mapped 32-bit CSR, but a bug 287*6e173d3bSMartin Habets * in the BIU means that writes to TIMER_COMMAND[0] invalidate the 288*6e173d3bSMartin Habets * collector register. 289*6e173d3bSMartin Habets */ 290*6e173d3bSMartin Habets static inline void _efx_writed_page_locked(struct efx_nic *efx, 291*6e173d3bSMartin Habets const efx_dword_t *value, 292*6e173d3bSMartin Habets unsigned int reg, 293*6e173d3bSMartin Habets unsigned int page) 294*6e173d3bSMartin Habets { 295*6e173d3bSMartin Habets unsigned long flags __attribute__ ((unused)); 296*6e173d3bSMartin Habets 297*6e173d3bSMartin Habets if (page == 0) { 298*6e173d3bSMartin Habets spin_lock_irqsave(&efx->biu_lock, flags); 299*6e173d3bSMartin Habets efx_writed(efx, value, efx_paged_reg(efx, page, reg)); 300*6e173d3bSMartin Habets spin_unlock_irqrestore(&efx->biu_lock, flags); 301*6e173d3bSMartin Habets } else { 302*6e173d3bSMartin Habets efx_writed(efx, value, efx_paged_reg(efx, page, reg)); 303*6e173d3bSMartin Habets } 304*6e173d3bSMartin Habets } 305*6e173d3bSMartin Habets #define efx_writed_page_locked(efx, value, reg, page) \ 306*6e173d3bSMartin Habets _efx_writed_page_locked(efx, value, \ 307*6e173d3bSMartin Habets reg + BUILD_BUG_ON_ZERO((reg) != 0x420), \ 308*6e173d3bSMartin Habets page) 309*6e173d3bSMartin Habets 310*6e173d3bSMartin Habets #endif /* EFX_IO_H */ 311