1*ca9c54d2SDexuan Cui // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2*ca9c54d2SDexuan Cui /* Copyright (c) 2021, Microsoft Corporation. */ 3*ca9c54d2SDexuan Cui 4*ca9c54d2SDexuan Cui #include <linux/delay.h> 5*ca9c54d2SDexuan Cui #include <linux/device.h> 6*ca9c54d2SDexuan Cui #include <linux/io.h> 7*ca9c54d2SDexuan Cui #include <linux/mm.h> 8*ca9c54d2SDexuan Cui 9*ca9c54d2SDexuan Cui #include "shm_channel.h" 10*ca9c54d2SDexuan Cui 11*ca9c54d2SDexuan Cui #define PAGE_FRAME_L48_WIDTH_BYTES 6 12*ca9c54d2SDexuan Cui #define PAGE_FRAME_L48_WIDTH_BITS (PAGE_FRAME_L48_WIDTH_BYTES * 8) 13*ca9c54d2SDexuan Cui #define PAGE_FRAME_L48_MASK 0x0000FFFFFFFFFFFF 14*ca9c54d2SDexuan Cui #define PAGE_FRAME_H4_WIDTH_BITS 4 15*ca9c54d2SDexuan Cui #define VECTOR_MASK 0xFFFF 16*ca9c54d2SDexuan Cui #define SHMEM_VF_RESET_STATE ((u32)-1) 17*ca9c54d2SDexuan Cui 18*ca9c54d2SDexuan Cui #define SMC_MSG_TYPE_ESTABLISH_HWC 1 19*ca9c54d2SDexuan Cui #define SMC_MSG_TYPE_ESTABLISH_HWC_VERSION 0 20*ca9c54d2SDexuan Cui 21*ca9c54d2SDexuan Cui #define SMC_MSG_TYPE_DESTROY_HWC 2 22*ca9c54d2SDexuan Cui #define SMC_MSG_TYPE_DESTROY_HWC_VERSION 0 23*ca9c54d2SDexuan Cui 24*ca9c54d2SDexuan Cui #define SMC_MSG_DIRECTION_REQUEST 0 25*ca9c54d2SDexuan Cui #define SMC_MSG_DIRECTION_RESPONSE 1 26*ca9c54d2SDexuan Cui 27*ca9c54d2SDexuan Cui /* Structures labeled with "HW DATA" are exchanged with the hardware. All of 28*ca9c54d2SDexuan Cui * them are naturally aligned and hence don't need __packed. 29*ca9c54d2SDexuan Cui */ 30*ca9c54d2SDexuan Cui 31*ca9c54d2SDexuan Cui /* Shared memory channel protocol header 32*ca9c54d2SDexuan Cui * 33*ca9c54d2SDexuan Cui * msg_type: set on request and response; response matches request. 34*ca9c54d2SDexuan Cui * msg_version: newer PF writes back older response (matching request) 35*ca9c54d2SDexuan Cui * older PF acts on latest version known and sets that version in result 36*ca9c54d2SDexuan Cui * (less than request). 37*ca9c54d2SDexuan Cui * direction: 0 for request, VF->PF; 1 for response, PF->VF. 38*ca9c54d2SDexuan Cui * status: 0 on request, 39*ca9c54d2SDexuan Cui * operation result on response (success = 0, failure = 1 or greater). 40*ca9c54d2SDexuan Cui * reset_vf: If set on either establish or destroy request, indicates perform 41*ca9c54d2SDexuan Cui * FLR before/after the operation. 42*ca9c54d2SDexuan Cui * owner_is_pf: 1 indicates PF owned, 0 indicates VF owned. 43*ca9c54d2SDexuan Cui */ 44*ca9c54d2SDexuan Cui union smc_proto_hdr { 45*ca9c54d2SDexuan Cui u32 as_uint32; 46*ca9c54d2SDexuan Cui 47*ca9c54d2SDexuan Cui struct { 48*ca9c54d2SDexuan Cui u8 msg_type : 3; 49*ca9c54d2SDexuan Cui u8 msg_version : 3; 50*ca9c54d2SDexuan Cui u8 reserved_1 : 1; 51*ca9c54d2SDexuan Cui u8 direction : 1; 52*ca9c54d2SDexuan Cui 53*ca9c54d2SDexuan Cui u8 status; 54*ca9c54d2SDexuan Cui 55*ca9c54d2SDexuan Cui u8 reserved_2; 56*ca9c54d2SDexuan Cui 57*ca9c54d2SDexuan Cui u8 reset_vf : 1; 58*ca9c54d2SDexuan Cui u8 reserved_3 : 6; 59*ca9c54d2SDexuan Cui u8 owner_is_pf : 1; 60*ca9c54d2SDexuan Cui }; 61*ca9c54d2SDexuan Cui }; /* HW DATA */ 62*ca9c54d2SDexuan Cui 63*ca9c54d2SDexuan Cui #define SMC_APERTURE_BITS 256 64*ca9c54d2SDexuan Cui #define SMC_BASIC_UNIT (sizeof(u32)) 65*ca9c54d2SDexuan Cui #define SMC_APERTURE_DWORDS (SMC_APERTURE_BITS / (SMC_BASIC_UNIT * 8)) 66*ca9c54d2SDexuan Cui #define SMC_LAST_DWORD (SMC_APERTURE_DWORDS - 1) 67*ca9c54d2SDexuan Cui 68*ca9c54d2SDexuan Cui static int mana_smc_poll_register(void __iomem *base, bool reset) 69*ca9c54d2SDexuan Cui { 70*ca9c54d2SDexuan Cui void __iomem *ptr = base + SMC_LAST_DWORD * SMC_BASIC_UNIT; 71*ca9c54d2SDexuan Cui u32 last_dword; 72*ca9c54d2SDexuan Cui int i; 73*ca9c54d2SDexuan Cui 74*ca9c54d2SDexuan Cui /* Poll the hardware for the ownership bit. This should be pretty fast, 75*ca9c54d2SDexuan Cui * but let's do it in a loop just in case the hardware or the PF 76*ca9c54d2SDexuan Cui * driver are temporarily busy. 77*ca9c54d2SDexuan Cui */ 78*ca9c54d2SDexuan Cui for (i = 0; i < 20 * 1000; i++) { 79*ca9c54d2SDexuan Cui last_dword = readl(ptr); 80*ca9c54d2SDexuan Cui 81*ca9c54d2SDexuan Cui /* shmem reads as 0xFFFFFFFF in the reset case */ 82*ca9c54d2SDexuan Cui if (reset && last_dword == SHMEM_VF_RESET_STATE) 83*ca9c54d2SDexuan Cui return 0; 84*ca9c54d2SDexuan Cui 85*ca9c54d2SDexuan Cui /* If bit_31 is set, the PF currently owns the SMC. */ 86*ca9c54d2SDexuan Cui if (!(last_dword & BIT(31))) 87*ca9c54d2SDexuan Cui return 0; 88*ca9c54d2SDexuan Cui 89*ca9c54d2SDexuan Cui usleep_range(1000, 2000); 90*ca9c54d2SDexuan Cui } 91*ca9c54d2SDexuan Cui 92*ca9c54d2SDexuan Cui return -ETIMEDOUT; 93*ca9c54d2SDexuan Cui } 94*ca9c54d2SDexuan Cui 95*ca9c54d2SDexuan Cui static int mana_smc_read_response(struct shm_channel *sc, u32 msg_type, 96*ca9c54d2SDexuan Cui u32 msg_version, bool reset_vf) 97*ca9c54d2SDexuan Cui { 98*ca9c54d2SDexuan Cui void __iomem *base = sc->base; 99*ca9c54d2SDexuan Cui union smc_proto_hdr hdr; 100*ca9c54d2SDexuan Cui int err; 101*ca9c54d2SDexuan Cui 102*ca9c54d2SDexuan Cui /* Wait for PF to respond. */ 103*ca9c54d2SDexuan Cui err = mana_smc_poll_register(base, reset_vf); 104*ca9c54d2SDexuan Cui if (err) 105*ca9c54d2SDexuan Cui return err; 106*ca9c54d2SDexuan Cui 107*ca9c54d2SDexuan Cui hdr.as_uint32 = readl(base + SMC_LAST_DWORD * SMC_BASIC_UNIT); 108*ca9c54d2SDexuan Cui 109*ca9c54d2SDexuan Cui if (reset_vf && hdr.as_uint32 == SHMEM_VF_RESET_STATE) 110*ca9c54d2SDexuan Cui return 0; 111*ca9c54d2SDexuan Cui 112*ca9c54d2SDexuan Cui /* Validate protocol fields from the PF driver */ 113*ca9c54d2SDexuan Cui if (hdr.msg_type != msg_type || hdr.msg_version > msg_version || 114*ca9c54d2SDexuan Cui hdr.direction != SMC_MSG_DIRECTION_RESPONSE) { 115*ca9c54d2SDexuan Cui dev_err(sc->dev, "Wrong SMC response 0x%x, type=%d, ver=%d\n", 116*ca9c54d2SDexuan Cui hdr.as_uint32, msg_type, msg_version); 117*ca9c54d2SDexuan Cui return -EPROTO; 118*ca9c54d2SDexuan Cui } 119*ca9c54d2SDexuan Cui 120*ca9c54d2SDexuan Cui /* Validate the operation result */ 121*ca9c54d2SDexuan Cui if (hdr.status != 0) { 122*ca9c54d2SDexuan Cui dev_err(sc->dev, "SMC operation failed: 0x%x\n", hdr.status); 123*ca9c54d2SDexuan Cui return -EPROTO; 124*ca9c54d2SDexuan Cui } 125*ca9c54d2SDexuan Cui 126*ca9c54d2SDexuan Cui return 0; 127*ca9c54d2SDexuan Cui } 128*ca9c54d2SDexuan Cui 129*ca9c54d2SDexuan Cui void mana_smc_init(struct shm_channel *sc, struct device *dev, 130*ca9c54d2SDexuan Cui void __iomem *base) 131*ca9c54d2SDexuan Cui { 132*ca9c54d2SDexuan Cui sc->dev = dev; 133*ca9c54d2SDexuan Cui sc->base = base; 134*ca9c54d2SDexuan Cui } 135*ca9c54d2SDexuan Cui 136*ca9c54d2SDexuan Cui int mana_smc_setup_hwc(struct shm_channel *sc, bool reset_vf, u64 eq_addr, 137*ca9c54d2SDexuan Cui u64 cq_addr, u64 rq_addr, u64 sq_addr, 138*ca9c54d2SDexuan Cui u32 eq_msix_index) 139*ca9c54d2SDexuan Cui { 140*ca9c54d2SDexuan Cui union smc_proto_hdr *hdr; 141*ca9c54d2SDexuan Cui u16 all_addr_h4bits = 0; 142*ca9c54d2SDexuan Cui u16 frame_addr_seq = 0; 143*ca9c54d2SDexuan Cui u64 frame_addr = 0; 144*ca9c54d2SDexuan Cui u8 shm_buf[32]; 145*ca9c54d2SDexuan Cui u64 *shmem; 146*ca9c54d2SDexuan Cui u32 *dword; 147*ca9c54d2SDexuan Cui u8 *ptr; 148*ca9c54d2SDexuan Cui int err; 149*ca9c54d2SDexuan Cui int i; 150*ca9c54d2SDexuan Cui 151*ca9c54d2SDexuan Cui /* Ensure VF already has possession of shared memory */ 152*ca9c54d2SDexuan Cui err = mana_smc_poll_register(sc->base, false); 153*ca9c54d2SDexuan Cui if (err) { 154*ca9c54d2SDexuan Cui dev_err(sc->dev, "Timeout when setting up HWC: %d\n", err); 155*ca9c54d2SDexuan Cui return err; 156*ca9c54d2SDexuan Cui } 157*ca9c54d2SDexuan Cui 158*ca9c54d2SDexuan Cui if (!PAGE_ALIGNED(eq_addr) || !PAGE_ALIGNED(cq_addr) || 159*ca9c54d2SDexuan Cui !PAGE_ALIGNED(rq_addr) || !PAGE_ALIGNED(sq_addr)) 160*ca9c54d2SDexuan Cui return -EINVAL; 161*ca9c54d2SDexuan Cui 162*ca9c54d2SDexuan Cui if ((eq_msix_index & VECTOR_MASK) != eq_msix_index) 163*ca9c54d2SDexuan Cui return -EINVAL; 164*ca9c54d2SDexuan Cui 165*ca9c54d2SDexuan Cui /* Scheme for packing four addresses and extra info into 256 bits. 166*ca9c54d2SDexuan Cui * 167*ca9c54d2SDexuan Cui * Addresses must be page frame aligned, so only frame address bits 168*ca9c54d2SDexuan Cui * are transferred. 169*ca9c54d2SDexuan Cui * 170*ca9c54d2SDexuan Cui * 52-bit frame addresses are split into the lower 48 bits and upper 171*ca9c54d2SDexuan Cui * 4 bits. Lower 48 bits of 4 address are written sequentially from 172*ca9c54d2SDexuan Cui * the start of the 256-bit shared memory region followed by 16 bits 173*ca9c54d2SDexuan Cui * containing the upper 4 bits of the 4 addresses in sequence. 174*ca9c54d2SDexuan Cui * 175*ca9c54d2SDexuan Cui * A 16 bit EQ vector number fills out the next-to-last 32-bit dword. 176*ca9c54d2SDexuan Cui * 177*ca9c54d2SDexuan Cui * The final 32-bit dword is used for protocol control information as 178*ca9c54d2SDexuan Cui * defined in smc_proto_hdr. 179*ca9c54d2SDexuan Cui */ 180*ca9c54d2SDexuan Cui 181*ca9c54d2SDexuan Cui memset(shm_buf, 0, sizeof(shm_buf)); 182*ca9c54d2SDexuan Cui ptr = shm_buf; 183*ca9c54d2SDexuan Cui 184*ca9c54d2SDexuan Cui /* EQ addr: low 48 bits of frame address */ 185*ca9c54d2SDexuan Cui shmem = (u64 *)ptr; 186*ca9c54d2SDexuan Cui frame_addr = PHYS_PFN(eq_addr); 187*ca9c54d2SDexuan Cui *shmem = frame_addr & PAGE_FRAME_L48_MASK; 188*ca9c54d2SDexuan Cui all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) << 189*ca9c54d2SDexuan Cui (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS); 190*ca9c54d2SDexuan Cui ptr += PAGE_FRAME_L48_WIDTH_BYTES; 191*ca9c54d2SDexuan Cui 192*ca9c54d2SDexuan Cui /* CQ addr: low 48 bits of frame address */ 193*ca9c54d2SDexuan Cui shmem = (u64 *)ptr; 194*ca9c54d2SDexuan Cui frame_addr = PHYS_PFN(cq_addr); 195*ca9c54d2SDexuan Cui *shmem = frame_addr & PAGE_FRAME_L48_MASK; 196*ca9c54d2SDexuan Cui all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) << 197*ca9c54d2SDexuan Cui (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS); 198*ca9c54d2SDexuan Cui ptr += PAGE_FRAME_L48_WIDTH_BYTES; 199*ca9c54d2SDexuan Cui 200*ca9c54d2SDexuan Cui /* RQ addr: low 48 bits of frame address */ 201*ca9c54d2SDexuan Cui shmem = (u64 *)ptr; 202*ca9c54d2SDexuan Cui frame_addr = PHYS_PFN(rq_addr); 203*ca9c54d2SDexuan Cui *shmem = frame_addr & PAGE_FRAME_L48_MASK; 204*ca9c54d2SDexuan Cui all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) << 205*ca9c54d2SDexuan Cui (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS); 206*ca9c54d2SDexuan Cui ptr += PAGE_FRAME_L48_WIDTH_BYTES; 207*ca9c54d2SDexuan Cui 208*ca9c54d2SDexuan Cui /* SQ addr: low 48 bits of frame address */ 209*ca9c54d2SDexuan Cui shmem = (u64 *)ptr; 210*ca9c54d2SDexuan Cui frame_addr = PHYS_PFN(sq_addr); 211*ca9c54d2SDexuan Cui *shmem = frame_addr & PAGE_FRAME_L48_MASK; 212*ca9c54d2SDexuan Cui all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) << 213*ca9c54d2SDexuan Cui (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS); 214*ca9c54d2SDexuan Cui ptr += PAGE_FRAME_L48_WIDTH_BYTES; 215*ca9c54d2SDexuan Cui 216*ca9c54d2SDexuan Cui /* High 4 bits of the four frame addresses */ 217*ca9c54d2SDexuan Cui *((u16 *)ptr) = all_addr_h4bits; 218*ca9c54d2SDexuan Cui ptr += sizeof(u16); 219*ca9c54d2SDexuan Cui 220*ca9c54d2SDexuan Cui /* EQ MSIX vector number */ 221*ca9c54d2SDexuan Cui *((u16 *)ptr) = (u16)eq_msix_index; 222*ca9c54d2SDexuan Cui ptr += sizeof(u16); 223*ca9c54d2SDexuan Cui 224*ca9c54d2SDexuan Cui /* 32-bit protocol header in final dword */ 225*ca9c54d2SDexuan Cui *((u32 *)ptr) = 0; 226*ca9c54d2SDexuan Cui 227*ca9c54d2SDexuan Cui hdr = (union smc_proto_hdr *)ptr; 228*ca9c54d2SDexuan Cui hdr->msg_type = SMC_MSG_TYPE_ESTABLISH_HWC; 229*ca9c54d2SDexuan Cui hdr->msg_version = SMC_MSG_TYPE_ESTABLISH_HWC_VERSION; 230*ca9c54d2SDexuan Cui hdr->direction = SMC_MSG_DIRECTION_REQUEST; 231*ca9c54d2SDexuan Cui hdr->reset_vf = reset_vf; 232*ca9c54d2SDexuan Cui 233*ca9c54d2SDexuan Cui /* Write 256-message buffer to shared memory (final 32-bit write 234*ca9c54d2SDexuan Cui * triggers HW to set possession bit to PF). 235*ca9c54d2SDexuan Cui */ 236*ca9c54d2SDexuan Cui dword = (u32 *)shm_buf; 237*ca9c54d2SDexuan Cui for (i = 0; i < SMC_APERTURE_DWORDS; i++) 238*ca9c54d2SDexuan Cui writel(*dword++, sc->base + i * SMC_BASIC_UNIT); 239*ca9c54d2SDexuan Cui 240*ca9c54d2SDexuan Cui /* Read shmem response (polling for VF possession) and validate. 241*ca9c54d2SDexuan Cui * For setup, waiting for response on shared memory is not strictly 242*ca9c54d2SDexuan Cui * necessary, since wait occurs later for results to appear in EQE's. 243*ca9c54d2SDexuan Cui */ 244*ca9c54d2SDexuan Cui err = mana_smc_read_response(sc, SMC_MSG_TYPE_ESTABLISH_HWC, 245*ca9c54d2SDexuan Cui SMC_MSG_TYPE_ESTABLISH_HWC_VERSION, 246*ca9c54d2SDexuan Cui reset_vf); 247*ca9c54d2SDexuan Cui if (err) { 248*ca9c54d2SDexuan Cui dev_err(sc->dev, "Error when setting up HWC: %d\n", err); 249*ca9c54d2SDexuan Cui return err; 250*ca9c54d2SDexuan Cui } 251*ca9c54d2SDexuan Cui 252*ca9c54d2SDexuan Cui return 0; 253*ca9c54d2SDexuan Cui } 254*ca9c54d2SDexuan Cui 255*ca9c54d2SDexuan Cui int mana_smc_teardown_hwc(struct shm_channel *sc, bool reset_vf) 256*ca9c54d2SDexuan Cui { 257*ca9c54d2SDexuan Cui union smc_proto_hdr hdr = {}; 258*ca9c54d2SDexuan Cui int err; 259*ca9c54d2SDexuan Cui 260*ca9c54d2SDexuan Cui /* Ensure already has possession of shared memory */ 261*ca9c54d2SDexuan Cui err = mana_smc_poll_register(sc->base, false); 262*ca9c54d2SDexuan Cui if (err) { 263*ca9c54d2SDexuan Cui dev_err(sc->dev, "Timeout when tearing down HWC\n"); 264*ca9c54d2SDexuan Cui return err; 265*ca9c54d2SDexuan Cui } 266*ca9c54d2SDexuan Cui 267*ca9c54d2SDexuan Cui /* Set up protocol header for HWC destroy message */ 268*ca9c54d2SDexuan Cui hdr.msg_type = SMC_MSG_TYPE_DESTROY_HWC; 269*ca9c54d2SDexuan Cui hdr.msg_version = SMC_MSG_TYPE_DESTROY_HWC_VERSION; 270*ca9c54d2SDexuan Cui hdr.direction = SMC_MSG_DIRECTION_REQUEST; 271*ca9c54d2SDexuan Cui hdr.reset_vf = reset_vf; 272*ca9c54d2SDexuan Cui 273*ca9c54d2SDexuan Cui /* Write message in high 32 bits of 256-bit shared memory, causing HW 274*ca9c54d2SDexuan Cui * to set possession bit to PF. 275*ca9c54d2SDexuan Cui */ 276*ca9c54d2SDexuan Cui writel(hdr.as_uint32, sc->base + SMC_LAST_DWORD * SMC_BASIC_UNIT); 277*ca9c54d2SDexuan Cui 278*ca9c54d2SDexuan Cui /* Read shmem response (polling for VF possession) and validate. 279*ca9c54d2SDexuan Cui * For teardown, waiting for response is required to ensure hardware 280*ca9c54d2SDexuan Cui * invalidates MST entries before software frees memory. 281*ca9c54d2SDexuan Cui */ 282*ca9c54d2SDexuan Cui err = mana_smc_read_response(sc, SMC_MSG_TYPE_DESTROY_HWC, 283*ca9c54d2SDexuan Cui SMC_MSG_TYPE_DESTROY_HWC_VERSION, 284*ca9c54d2SDexuan Cui reset_vf); 285*ca9c54d2SDexuan Cui if (err) { 286*ca9c54d2SDexuan Cui dev_err(sc->dev, "Error when tearing down HWC: %d\n", err); 287*ca9c54d2SDexuan Cui return err; 288*ca9c54d2SDexuan Cui } 289*ca9c54d2SDexuan Cui 290*ca9c54d2SDexuan Cui return 0; 291*ca9c54d2SDexuan Cui } 292