1ca9c54d2SDexuan Cui // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2ca9c54d2SDexuan Cui /* Copyright (c) 2021, Microsoft Corporation. */
3ca9c54d2SDexuan Cui
4ca9c54d2SDexuan Cui #include <linux/delay.h>
5ca9c54d2SDexuan Cui #include <linux/device.h>
6ca9c54d2SDexuan Cui #include <linux/io.h>
7ca9c54d2SDexuan Cui #include <linux/mm.h>
8ca9c54d2SDexuan Cui
9*fd325cd6SLong Li #include <net/mana/shm_channel.h>
10ca9c54d2SDexuan Cui
11ca9c54d2SDexuan Cui #define PAGE_FRAME_L48_WIDTH_BYTES 6
12ca9c54d2SDexuan Cui #define PAGE_FRAME_L48_WIDTH_BITS (PAGE_FRAME_L48_WIDTH_BYTES * 8)
13ca9c54d2SDexuan Cui #define PAGE_FRAME_L48_MASK 0x0000FFFFFFFFFFFF
14ca9c54d2SDexuan Cui #define PAGE_FRAME_H4_WIDTH_BITS 4
15ca9c54d2SDexuan Cui #define VECTOR_MASK 0xFFFF
16ca9c54d2SDexuan Cui #define SHMEM_VF_RESET_STATE ((u32)-1)
17ca9c54d2SDexuan Cui
18ca9c54d2SDexuan Cui #define SMC_MSG_TYPE_ESTABLISH_HWC 1
19ca9c54d2SDexuan Cui #define SMC_MSG_TYPE_ESTABLISH_HWC_VERSION 0
20ca9c54d2SDexuan Cui
21ca9c54d2SDexuan Cui #define SMC_MSG_TYPE_DESTROY_HWC 2
22ca9c54d2SDexuan Cui #define SMC_MSG_TYPE_DESTROY_HWC_VERSION 0
23ca9c54d2SDexuan Cui
24ca9c54d2SDexuan Cui #define SMC_MSG_DIRECTION_REQUEST 0
25ca9c54d2SDexuan Cui #define SMC_MSG_DIRECTION_RESPONSE 1
26ca9c54d2SDexuan Cui
27ca9c54d2SDexuan Cui /* Structures labeled with "HW DATA" are exchanged with the hardware. All of
28ca9c54d2SDexuan Cui * them are naturally aligned and hence don't need __packed.
29ca9c54d2SDexuan Cui */
30ca9c54d2SDexuan Cui
31ca9c54d2SDexuan Cui /* Shared memory channel protocol header
32ca9c54d2SDexuan Cui *
33ca9c54d2SDexuan Cui * msg_type: set on request and response; response matches request.
34ca9c54d2SDexuan Cui * msg_version: newer PF writes back older response (matching request)
35ca9c54d2SDexuan Cui * older PF acts on latest version known and sets that version in result
36ca9c54d2SDexuan Cui * (less than request).
37ca9c54d2SDexuan Cui * direction: 0 for request, VF->PF; 1 for response, PF->VF.
38ca9c54d2SDexuan Cui * status: 0 on request,
39ca9c54d2SDexuan Cui * operation result on response (success = 0, failure = 1 or greater).
40ca9c54d2SDexuan Cui * reset_vf: If set on either establish or destroy request, indicates perform
41ca9c54d2SDexuan Cui * FLR before/after the operation.
42ca9c54d2SDexuan Cui * owner_is_pf: 1 indicates PF owned, 0 indicates VF owned.
43ca9c54d2SDexuan Cui */
44ca9c54d2SDexuan Cui union smc_proto_hdr {
45ca9c54d2SDexuan Cui u32 as_uint32;
46ca9c54d2SDexuan Cui
47ca9c54d2SDexuan Cui struct {
48ca9c54d2SDexuan Cui u8 msg_type : 3;
49ca9c54d2SDexuan Cui u8 msg_version : 3;
50ca9c54d2SDexuan Cui u8 reserved_1 : 1;
51ca9c54d2SDexuan Cui u8 direction : 1;
52ca9c54d2SDexuan Cui
53ca9c54d2SDexuan Cui u8 status;
54ca9c54d2SDexuan Cui
55ca9c54d2SDexuan Cui u8 reserved_2;
56ca9c54d2SDexuan Cui
57ca9c54d2SDexuan Cui u8 reset_vf : 1;
58ca9c54d2SDexuan Cui u8 reserved_3 : 6;
59ca9c54d2SDexuan Cui u8 owner_is_pf : 1;
60ca9c54d2SDexuan Cui };
61ca9c54d2SDexuan Cui }; /* HW DATA */
62ca9c54d2SDexuan Cui
63ca9c54d2SDexuan Cui #define SMC_APERTURE_BITS 256
64ca9c54d2SDexuan Cui #define SMC_BASIC_UNIT (sizeof(u32))
65ca9c54d2SDexuan Cui #define SMC_APERTURE_DWORDS (SMC_APERTURE_BITS / (SMC_BASIC_UNIT * 8))
66ca9c54d2SDexuan Cui #define SMC_LAST_DWORD (SMC_APERTURE_DWORDS - 1)
67ca9c54d2SDexuan Cui
mana_smc_poll_register(void __iomem * base,bool reset)68ca9c54d2SDexuan Cui static int mana_smc_poll_register(void __iomem *base, bool reset)
69ca9c54d2SDexuan Cui {
70ca9c54d2SDexuan Cui void __iomem *ptr = base + SMC_LAST_DWORD * SMC_BASIC_UNIT;
71ca9c54d2SDexuan Cui u32 last_dword;
72ca9c54d2SDexuan Cui int i;
73ca9c54d2SDexuan Cui
74ca9c54d2SDexuan Cui /* Poll the hardware for the ownership bit. This should be pretty fast,
75ca9c54d2SDexuan Cui * but let's do it in a loop just in case the hardware or the PF
76ca9c54d2SDexuan Cui * driver are temporarily busy.
77ca9c54d2SDexuan Cui */
78ca9c54d2SDexuan Cui for (i = 0; i < 20 * 1000; i++) {
79ca9c54d2SDexuan Cui last_dword = readl(ptr);
80ca9c54d2SDexuan Cui
81ca9c54d2SDexuan Cui /* shmem reads as 0xFFFFFFFF in the reset case */
82ca9c54d2SDexuan Cui if (reset && last_dword == SHMEM_VF_RESET_STATE)
83ca9c54d2SDexuan Cui return 0;
84ca9c54d2SDexuan Cui
85ca9c54d2SDexuan Cui /* If bit_31 is set, the PF currently owns the SMC. */
86ca9c54d2SDexuan Cui if (!(last_dword & BIT(31)))
87ca9c54d2SDexuan Cui return 0;
88ca9c54d2SDexuan Cui
89ca9c54d2SDexuan Cui usleep_range(1000, 2000);
90ca9c54d2SDexuan Cui }
91ca9c54d2SDexuan Cui
92ca9c54d2SDexuan Cui return -ETIMEDOUT;
93ca9c54d2SDexuan Cui }
94ca9c54d2SDexuan Cui
mana_smc_read_response(struct shm_channel * sc,u32 msg_type,u32 msg_version,bool reset_vf)95ca9c54d2SDexuan Cui static int mana_smc_read_response(struct shm_channel *sc, u32 msg_type,
96ca9c54d2SDexuan Cui u32 msg_version, bool reset_vf)
97ca9c54d2SDexuan Cui {
98ca9c54d2SDexuan Cui void __iomem *base = sc->base;
99ca9c54d2SDexuan Cui union smc_proto_hdr hdr;
100ca9c54d2SDexuan Cui int err;
101ca9c54d2SDexuan Cui
102ca9c54d2SDexuan Cui /* Wait for PF to respond. */
103ca9c54d2SDexuan Cui err = mana_smc_poll_register(base, reset_vf);
104ca9c54d2SDexuan Cui if (err)
105ca9c54d2SDexuan Cui return err;
106ca9c54d2SDexuan Cui
107ca9c54d2SDexuan Cui hdr.as_uint32 = readl(base + SMC_LAST_DWORD * SMC_BASIC_UNIT);
108ca9c54d2SDexuan Cui
109ca9c54d2SDexuan Cui if (reset_vf && hdr.as_uint32 == SHMEM_VF_RESET_STATE)
110ca9c54d2SDexuan Cui return 0;
111ca9c54d2SDexuan Cui
112ca9c54d2SDexuan Cui /* Validate protocol fields from the PF driver */
113ca9c54d2SDexuan Cui if (hdr.msg_type != msg_type || hdr.msg_version > msg_version ||
114ca9c54d2SDexuan Cui hdr.direction != SMC_MSG_DIRECTION_RESPONSE) {
115ca9c54d2SDexuan Cui dev_err(sc->dev, "Wrong SMC response 0x%x, type=%d, ver=%d\n",
116ca9c54d2SDexuan Cui hdr.as_uint32, msg_type, msg_version);
117ca9c54d2SDexuan Cui return -EPROTO;
118ca9c54d2SDexuan Cui }
119ca9c54d2SDexuan Cui
120ca9c54d2SDexuan Cui /* Validate the operation result */
121ca9c54d2SDexuan Cui if (hdr.status != 0) {
122ca9c54d2SDexuan Cui dev_err(sc->dev, "SMC operation failed: 0x%x\n", hdr.status);
123ca9c54d2SDexuan Cui return -EPROTO;
124ca9c54d2SDexuan Cui }
125ca9c54d2SDexuan Cui
126ca9c54d2SDexuan Cui return 0;
127ca9c54d2SDexuan Cui }
128ca9c54d2SDexuan Cui
mana_smc_init(struct shm_channel * sc,struct device * dev,void __iomem * base)129ca9c54d2SDexuan Cui void mana_smc_init(struct shm_channel *sc, struct device *dev,
130ca9c54d2SDexuan Cui void __iomem *base)
131ca9c54d2SDexuan Cui {
132ca9c54d2SDexuan Cui sc->dev = dev;
133ca9c54d2SDexuan Cui sc->base = base;
134ca9c54d2SDexuan Cui }
135ca9c54d2SDexuan Cui
mana_smc_setup_hwc(struct shm_channel * sc,bool reset_vf,u64 eq_addr,u64 cq_addr,u64 rq_addr,u64 sq_addr,u32 eq_msix_index)136ca9c54d2SDexuan Cui int mana_smc_setup_hwc(struct shm_channel *sc, bool reset_vf, u64 eq_addr,
137ca9c54d2SDexuan Cui u64 cq_addr, u64 rq_addr, u64 sq_addr,
138ca9c54d2SDexuan Cui u32 eq_msix_index)
139ca9c54d2SDexuan Cui {
140ca9c54d2SDexuan Cui union smc_proto_hdr *hdr;
141ca9c54d2SDexuan Cui u16 all_addr_h4bits = 0;
142ca9c54d2SDexuan Cui u16 frame_addr_seq = 0;
143ca9c54d2SDexuan Cui u64 frame_addr = 0;
144ca9c54d2SDexuan Cui u8 shm_buf[32];
145ca9c54d2SDexuan Cui u64 *shmem;
146ca9c54d2SDexuan Cui u32 *dword;
147ca9c54d2SDexuan Cui u8 *ptr;
148ca9c54d2SDexuan Cui int err;
149ca9c54d2SDexuan Cui int i;
150ca9c54d2SDexuan Cui
151ca9c54d2SDexuan Cui /* Ensure VF already has possession of shared memory */
152ca9c54d2SDexuan Cui err = mana_smc_poll_register(sc->base, false);
153ca9c54d2SDexuan Cui if (err) {
154ca9c54d2SDexuan Cui dev_err(sc->dev, "Timeout when setting up HWC: %d\n", err);
155ca9c54d2SDexuan Cui return err;
156ca9c54d2SDexuan Cui }
157ca9c54d2SDexuan Cui
158ca9c54d2SDexuan Cui if (!PAGE_ALIGNED(eq_addr) || !PAGE_ALIGNED(cq_addr) ||
159ca9c54d2SDexuan Cui !PAGE_ALIGNED(rq_addr) || !PAGE_ALIGNED(sq_addr))
160ca9c54d2SDexuan Cui return -EINVAL;
161ca9c54d2SDexuan Cui
162ca9c54d2SDexuan Cui if ((eq_msix_index & VECTOR_MASK) != eq_msix_index)
163ca9c54d2SDexuan Cui return -EINVAL;
164ca9c54d2SDexuan Cui
165ca9c54d2SDexuan Cui /* Scheme for packing four addresses and extra info into 256 bits.
166ca9c54d2SDexuan Cui *
167ca9c54d2SDexuan Cui * Addresses must be page frame aligned, so only frame address bits
168ca9c54d2SDexuan Cui * are transferred.
169ca9c54d2SDexuan Cui *
170ca9c54d2SDexuan Cui * 52-bit frame addresses are split into the lower 48 bits and upper
171ca9c54d2SDexuan Cui * 4 bits. Lower 48 bits of 4 address are written sequentially from
172ca9c54d2SDexuan Cui * the start of the 256-bit shared memory region followed by 16 bits
173ca9c54d2SDexuan Cui * containing the upper 4 bits of the 4 addresses in sequence.
174ca9c54d2SDexuan Cui *
175ca9c54d2SDexuan Cui * A 16 bit EQ vector number fills out the next-to-last 32-bit dword.
176ca9c54d2SDexuan Cui *
177ca9c54d2SDexuan Cui * The final 32-bit dword is used for protocol control information as
178ca9c54d2SDexuan Cui * defined in smc_proto_hdr.
179ca9c54d2SDexuan Cui */
180ca9c54d2SDexuan Cui
181ca9c54d2SDexuan Cui memset(shm_buf, 0, sizeof(shm_buf));
182ca9c54d2SDexuan Cui ptr = shm_buf;
183ca9c54d2SDexuan Cui
184ca9c54d2SDexuan Cui /* EQ addr: low 48 bits of frame address */
185ca9c54d2SDexuan Cui shmem = (u64 *)ptr;
186ca9c54d2SDexuan Cui frame_addr = PHYS_PFN(eq_addr);
187ca9c54d2SDexuan Cui *shmem = frame_addr & PAGE_FRAME_L48_MASK;
188ca9c54d2SDexuan Cui all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
189ca9c54d2SDexuan Cui (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
190ca9c54d2SDexuan Cui ptr += PAGE_FRAME_L48_WIDTH_BYTES;
191ca9c54d2SDexuan Cui
192ca9c54d2SDexuan Cui /* CQ addr: low 48 bits of frame address */
193ca9c54d2SDexuan Cui shmem = (u64 *)ptr;
194ca9c54d2SDexuan Cui frame_addr = PHYS_PFN(cq_addr);
195ca9c54d2SDexuan Cui *shmem = frame_addr & PAGE_FRAME_L48_MASK;
196ca9c54d2SDexuan Cui all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
197ca9c54d2SDexuan Cui (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
198ca9c54d2SDexuan Cui ptr += PAGE_FRAME_L48_WIDTH_BYTES;
199ca9c54d2SDexuan Cui
200ca9c54d2SDexuan Cui /* RQ addr: low 48 bits of frame address */
201ca9c54d2SDexuan Cui shmem = (u64 *)ptr;
202ca9c54d2SDexuan Cui frame_addr = PHYS_PFN(rq_addr);
203ca9c54d2SDexuan Cui *shmem = frame_addr & PAGE_FRAME_L48_MASK;
204ca9c54d2SDexuan Cui all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
205ca9c54d2SDexuan Cui (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
206ca9c54d2SDexuan Cui ptr += PAGE_FRAME_L48_WIDTH_BYTES;
207ca9c54d2SDexuan Cui
208ca9c54d2SDexuan Cui /* SQ addr: low 48 bits of frame address */
209ca9c54d2SDexuan Cui shmem = (u64 *)ptr;
210ca9c54d2SDexuan Cui frame_addr = PHYS_PFN(sq_addr);
211ca9c54d2SDexuan Cui *shmem = frame_addr & PAGE_FRAME_L48_MASK;
212ca9c54d2SDexuan Cui all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
213ca9c54d2SDexuan Cui (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
214ca9c54d2SDexuan Cui ptr += PAGE_FRAME_L48_WIDTH_BYTES;
215ca9c54d2SDexuan Cui
216ca9c54d2SDexuan Cui /* High 4 bits of the four frame addresses */
217ca9c54d2SDexuan Cui *((u16 *)ptr) = all_addr_h4bits;
218ca9c54d2SDexuan Cui ptr += sizeof(u16);
219ca9c54d2SDexuan Cui
220ca9c54d2SDexuan Cui /* EQ MSIX vector number */
221ca9c54d2SDexuan Cui *((u16 *)ptr) = (u16)eq_msix_index;
222ca9c54d2SDexuan Cui ptr += sizeof(u16);
223ca9c54d2SDexuan Cui
224ca9c54d2SDexuan Cui /* 32-bit protocol header in final dword */
225ca9c54d2SDexuan Cui *((u32 *)ptr) = 0;
226ca9c54d2SDexuan Cui
227ca9c54d2SDexuan Cui hdr = (union smc_proto_hdr *)ptr;
228ca9c54d2SDexuan Cui hdr->msg_type = SMC_MSG_TYPE_ESTABLISH_HWC;
229ca9c54d2SDexuan Cui hdr->msg_version = SMC_MSG_TYPE_ESTABLISH_HWC_VERSION;
230ca9c54d2SDexuan Cui hdr->direction = SMC_MSG_DIRECTION_REQUEST;
231ca9c54d2SDexuan Cui hdr->reset_vf = reset_vf;
232ca9c54d2SDexuan Cui
233ca9c54d2SDexuan Cui /* Write 256-message buffer to shared memory (final 32-bit write
234ca9c54d2SDexuan Cui * triggers HW to set possession bit to PF).
235ca9c54d2SDexuan Cui */
236ca9c54d2SDexuan Cui dword = (u32 *)shm_buf;
237ca9c54d2SDexuan Cui for (i = 0; i < SMC_APERTURE_DWORDS; i++)
238ca9c54d2SDexuan Cui writel(*dword++, sc->base + i * SMC_BASIC_UNIT);
239ca9c54d2SDexuan Cui
240ca9c54d2SDexuan Cui /* Read shmem response (polling for VF possession) and validate.
241ca9c54d2SDexuan Cui * For setup, waiting for response on shared memory is not strictly
242ca9c54d2SDexuan Cui * necessary, since wait occurs later for results to appear in EQE's.
243ca9c54d2SDexuan Cui */
244ca9c54d2SDexuan Cui err = mana_smc_read_response(sc, SMC_MSG_TYPE_ESTABLISH_HWC,
245ca9c54d2SDexuan Cui SMC_MSG_TYPE_ESTABLISH_HWC_VERSION,
246ca9c54d2SDexuan Cui reset_vf);
247ca9c54d2SDexuan Cui if (err) {
248ca9c54d2SDexuan Cui dev_err(sc->dev, "Error when setting up HWC: %d\n", err);
249ca9c54d2SDexuan Cui return err;
250ca9c54d2SDexuan Cui }
251ca9c54d2SDexuan Cui
252ca9c54d2SDexuan Cui return 0;
253ca9c54d2SDexuan Cui }
254ca9c54d2SDexuan Cui
mana_smc_teardown_hwc(struct shm_channel * sc,bool reset_vf)255ca9c54d2SDexuan Cui int mana_smc_teardown_hwc(struct shm_channel *sc, bool reset_vf)
256ca9c54d2SDexuan Cui {
257ca9c54d2SDexuan Cui union smc_proto_hdr hdr = {};
258ca9c54d2SDexuan Cui int err;
259ca9c54d2SDexuan Cui
260ca9c54d2SDexuan Cui /* Ensure already has possession of shared memory */
261ca9c54d2SDexuan Cui err = mana_smc_poll_register(sc->base, false);
262ca9c54d2SDexuan Cui if (err) {
263ca9c54d2SDexuan Cui dev_err(sc->dev, "Timeout when tearing down HWC\n");
264ca9c54d2SDexuan Cui return err;
265ca9c54d2SDexuan Cui }
266ca9c54d2SDexuan Cui
267ca9c54d2SDexuan Cui /* Set up protocol header for HWC destroy message */
268ca9c54d2SDexuan Cui hdr.msg_type = SMC_MSG_TYPE_DESTROY_HWC;
269ca9c54d2SDexuan Cui hdr.msg_version = SMC_MSG_TYPE_DESTROY_HWC_VERSION;
270ca9c54d2SDexuan Cui hdr.direction = SMC_MSG_DIRECTION_REQUEST;
271ca9c54d2SDexuan Cui hdr.reset_vf = reset_vf;
272ca9c54d2SDexuan Cui
273ca9c54d2SDexuan Cui /* Write message in high 32 bits of 256-bit shared memory, causing HW
274ca9c54d2SDexuan Cui * to set possession bit to PF.
275ca9c54d2SDexuan Cui */
276ca9c54d2SDexuan Cui writel(hdr.as_uint32, sc->base + SMC_LAST_DWORD * SMC_BASIC_UNIT);
277ca9c54d2SDexuan Cui
278ca9c54d2SDexuan Cui /* Read shmem response (polling for VF possession) and validate.
279ca9c54d2SDexuan Cui * For teardown, waiting for response is required to ensure hardware
280ca9c54d2SDexuan Cui * invalidates MST entries before software frees memory.
281ca9c54d2SDexuan Cui */
282ca9c54d2SDexuan Cui err = mana_smc_read_response(sc, SMC_MSG_TYPE_DESTROY_HWC,
283ca9c54d2SDexuan Cui SMC_MSG_TYPE_DESTROY_HWC_VERSION,
284ca9c54d2SDexuan Cui reset_vf);
285ca9c54d2SDexuan Cui if (err) {
286ca9c54d2SDexuan Cui dev_err(sc->dev, "Error when tearing down HWC: %d\n", err);
287ca9c54d2SDexuan Cui return err;
288ca9c54d2SDexuan Cui }
289ca9c54d2SDexuan Cui
290ca9c54d2SDexuan Cui return 0;
291ca9c54d2SDexuan Cui }
292