1ca9c54d2SDexuan Cui // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2ca9c54d2SDexuan Cui /* Copyright (c) 2021, Microsoft Corporation. */
3ca9c54d2SDexuan Cui
4ca9c54d2SDexuan Cui #include <linux/delay.h>
5ca9c54d2SDexuan Cui #include <linux/device.h>
6ca9c54d2SDexuan Cui #include <linux/io.h>
7ca9c54d2SDexuan Cui #include <linux/mm.h>
8ca9c54d2SDexuan Cui
9*6c95c700SHaiyang Zhang #include <net/mana/gdma.h>
10fd325cd6SLong Li #include <net/mana/shm_channel.h>
11ca9c54d2SDexuan Cui
12ca9c54d2SDexuan Cui #define PAGE_FRAME_L48_WIDTH_BYTES 6
13ca9c54d2SDexuan Cui #define PAGE_FRAME_L48_WIDTH_BITS (PAGE_FRAME_L48_WIDTH_BYTES * 8)
14ca9c54d2SDexuan Cui #define PAGE_FRAME_L48_MASK 0x0000FFFFFFFFFFFF
15ca9c54d2SDexuan Cui #define PAGE_FRAME_H4_WIDTH_BITS 4
16ca9c54d2SDexuan Cui #define VECTOR_MASK 0xFFFF
17ca9c54d2SDexuan Cui #define SHMEM_VF_RESET_STATE ((u32)-1)
18ca9c54d2SDexuan Cui
19ca9c54d2SDexuan Cui #define SMC_MSG_TYPE_ESTABLISH_HWC 1
20ca9c54d2SDexuan Cui #define SMC_MSG_TYPE_ESTABLISH_HWC_VERSION 0
21ca9c54d2SDexuan Cui
22ca9c54d2SDexuan Cui #define SMC_MSG_TYPE_DESTROY_HWC 2
23ca9c54d2SDexuan Cui #define SMC_MSG_TYPE_DESTROY_HWC_VERSION 0
24ca9c54d2SDexuan Cui
25ca9c54d2SDexuan Cui #define SMC_MSG_DIRECTION_REQUEST 0
26ca9c54d2SDexuan Cui #define SMC_MSG_DIRECTION_RESPONSE 1
27ca9c54d2SDexuan Cui
28ca9c54d2SDexuan Cui /* Structures labeled with "HW DATA" are exchanged with the hardware. All of
29ca9c54d2SDexuan Cui * them are naturally aligned and hence don't need __packed.
30ca9c54d2SDexuan Cui */
31ca9c54d2SDexuan Cui
32ca9c54d2SDexuan Cui /* Shared memory channel protocol header
33ca9c54d2SDexuan Cui *
34ca9c54d2SDexuan Cui * msg_type: set on request and response; response matches request.
35ca9c54d2SDexuan Cui * msg_version: newer PF writes back older response (matching request)
36ca9c54d2SDexuan Cui * older PF acts on latest version known and sets that version in result
37ca9c54d2SDexuan Cui * (less than request).
38ca9c54d2SDexuan Cui * direction: 0 for request, VF->PF; 1 for response, PF->VF.
39ca9c54d2SDexuan Cui * status: 0 on request,
40ca9c54d2SDexuan Cui * operation result on response (success = 0, failure = 1 or greater).
41ca9c54d2SDexuan Cui * reset_vf: If set on either establish or destroy request, indicates perform
42ca9c54d2SDexuan Cui * FLR before/after the operation.
43ca9c54d2SDexuan Cui * owner_is_pf: 1 indicates PF owned, 0 indicates VF owned.
44ca9c54d2SDexuan Cui */
45ca9c54d2SDexuan Cui union smc_proto_hdr {
46ca9c54d2SDexuan Cui u32 as_uint32;
47ca9c54d2SDexuan Cui
48ca9c54d2SDexuan Cui struct {
49ca9c54d2SDexuan Cui u8 msg_type : 3;
50ca9c54d2SDexuan Cui u8 msg_version : 3;
51ca9c54d2SDexuan Cui u8 reserved_1 : 1;
52ca9c54d2SDexuan Cui u8 direction : 1;
53ca9c54d2SDexuan Cui
54ca9c54d2SDexuan Cui u8 status;
55ca9c54d2SDexuan Cui
56ca9c54d2SDexuan Cui u8 reserved_2;
57ca9c54d2SDexuan Cui
58ca9c54d2SDexuan Cui u8 reset_vf : 1;
59ca9c54d2SDexuan Cui u8 reserved_3 : 6;
60ca9c54d2SDexuan Cui u8 owner_is_pf : 1;
61ca9c54d2SDexuan Cui };
62ca9c54d2SDexuan Cui }; /* HW DATA */
63ca9c54d2SDexuan Cui
64ca9c54d2SDexuan Cui #define SMC_APERTURE_BITS 256
65ca9c54d2SDexuan Cui #define SMC_BASIC_UNIT (sizeof(u32))
66ca9c54d2SDexuan Cui #define SMC_APERTURE_DWORDS (SMC_APERTURE_BITS / (SMC_BASIC_UNIT * 8))
67ca9c54d2SDexuan Cui #define SMC_LAST_DWORD (SMC_APERTURE_DWORDS - 1)
68ca9c54d2SDexuan Cui
mana_smc_poll_register(void __iomem * base,bool reset)69ca9c54d2SDexuan Cui static int mana_smc_poll_register(void __iomem *base, bool reset)
70ca9c54d2SDexuan Cui {
71ca9c54d2SDexuan Cui void __iomem *ptr = base + SMC_LAST_DWORD * SMC_BASIC_UNIT;
72ca9c54d2SDexuan Cui u32 last_dword;
73ca9c54d2SDexuan Cui int i;
74ca9c54d2SDexuan Cui
75ca9c54d2SDexuan Cui /* Poll the hardware for the ownership bit. This should be pretty fast,
76ca9c54d2SDexuan Cui * but let's do it in a loop just in case the hardware or the PF
77ca9c54d2SDexuan Cui * driver are temporarily busy.
78ca9c54d2SDexuan Cui */
79ca9c54d2SDexuan Cui for (i = 0; i < 20 * 1000; i++) {
80ca9c54d2SDexuan Cui last_dword = readl(ptr);
81ca9c54d2SDexuan Cui
82ca9c54d2SDexuan Cui /* shmem reads as 0xFFFFFFFF in the reset case */
83ca9c54d2SDexuan Cui if (reset && last_dword == SHMEM_VF_RESET_STATE)
84ca9c54d2SDexuan Cui return 0;
85ca9c54d2SDexuan Cui
86ca9c54d2SDexuan Cui /* If bit_31 is set, the PF currently owns the SMC. */
87ca9c54d2SDexuan Cui if (!(last_dword & BIT(31)))
88ca9c54d2SDexuan Cui return 0;
89ca9c54d2SDexuan Cui
90ca9c54d2SDexuan Cui usleep_range(1000, 2000);
91ca9c54d2SDexuan Cui }
92ca9c54d2SDexuan Cui
93ca9c54d2SDexuan Cui return -ETIMEDOUT;
94ca9c54d2SDexuan Cui }
95ca9c54d2SDexuan Cui
mana_smc_read_response(struct shm_channel * sc,u32 msg_type,u32 msg_version,bool reset_vf)96ca9c54d2SDexuan Cui static int mana_smc_read_response(struct shm_channel *sc, u32 msg_type,
97ca9c54d2SDexuan Cui u32 msg_version, bool reset_vf)
98ca9c54d2SDexuan Cui {
99ca9c54d2SDexuan Cui void __iomem *base = sc->base;
100ca9c54d2SDexuan Cui union smc_proto_hdr hdr;
101ca9c54d2SDexuan Cui int err;
102ca9c54d2SDexuan Cui
103ca9c54d2SDexuan Cui /* Wait for PF to respond. */
104ca9c54d2SDexuan Cui err = mana_smc_poll_register(base, reset_vf);
105ca9c54d2SDexuan Cui if (err)
106ca9c54d2SDexuan Cui return err;
107ca9c54d2SDexuan Cui
108ca9c54d2SDexuan Cui hdr.as_uint32 = readl(base + SMC_LAST_DWORD * SMC_BASIC_UNIT);
109ca9c54d2SDexuan Cui
110ca9c54d2SDexuan Cui if (reset_vf && hdr.as_uint32 == SHMEM_VF_RESET_STATE)
111ca9c54d2SDexuan Cui return 0;
112ca9c54d2SDexuan Cui
113ca9c54d2SDexuan Cui /* Validate protocol fields from the PF driver */
114ca9c54d2SDexuan Cui if (hdr.msg_type != msg_type || hdr.msg_version > msg_version ||
115ca9c54d2SDexuan Cui hdr.direction != SMC_MSG_DIRECTION_RESPONSE) {
116ca9c54d2SDexuan Cui dev_err(sc->dev, "Wrong SMC response 0x%x, type=%d, ver=%d\n",
117ca9c54d2SDexuan Cui hdr.as_uint32, msg_type, msg_version);
118ca9c54d2SDexuan Cui return -EPROTO;
119ca9c54d2SDexuan Cui }
120ca9c54d2SDexuan Cui
121ca9c54d2SDexuan Cui /* Validate the operation result */
122ca9c54d2SDexuan Cui if (hdr.status != 0) {
123ca9c54d2SDexuan Cui dev_err(sc->dev, "SMC operation failed: 0x%x\n", hdr.status);
124ca9c54d2SDexuan Cui return -EPROTO;
125ca9c54d2SDexuan Cui }
126ca9c54d2SDexuan Cui
127ca9c54d2SDexuan Cui return 0;
128ca9c54d2SDexuan Cui }
129ca9c54d2SDexuan Cui
mana_smc_init(struct shm_channel * sc,struct device * dev,void __iomem * base)130ca9c54d2SDexuan Cui void mana_smc_init(struct shm_channel *sc, struct device *dev,
131ca9c54d2SDexuan Cui void __iomem *base)
132ca9c54d2SDexuan Cui {
133ca9c54d2SDexuan Cui sc->dev = dev;
134ca9c54d2SDexuan Cui sc->base = base;
135ca9c54d2SDexuan Cui }
136ca9c54d2SDexuan Cui
mana_smc_setup_hwc(struct shm_channel * sc,bool reset_vf,u64 eq_addr,u64 cq_addr,u64 rq_addr,u64 sq_addr,u32 eq_msix_index)137ca9c54d2SDexuan Cui int mana_smc_setup_hwc(struct shm_channel *sc, bool reset_vf, u64 eq_addr,
138ca9c54d2SDexuan Cui u64 cq_addr, u64 rq_addr, u64 sq_addr,
139ca9c54d2SDexuan Cui u32 eq_msix_index)
140ca9c54d2SDexuan Cui {
141ca9c54d2SDexuan Cui union smc_proto_hdr *hdr;
142ca9c54d2SDexuan Cui u16 all_addr_h4bits = 0;
143ca9c54d2SDexuan Cui u16 frame_addr_seq = 0;
144ca9c54d2SDexuan Cui u64 frame_addr = 0;
145ca9c54d2SDexuan Cui u8 shm_buf[32];
146ca9c54d2SDexuan Cui u64 *shmem;
147ca9c54d2SDexuan Cui u32 *dword;
148ca9c54d2SDexuan Cui u8 *ptr;
149ca9c54d2SDexuan Cui int err;
150ca9c54d2SDexuan Cui int i;
151ca9c54d2SDexuan Cui
152ca9c54d2SDexuan Cui /* Ensure VF already has possession of shared memory */
153ca9c54d2SDexuan Cui err = mana_smc_poll_register(sc->base, false);
154ca9c54d2SDexuan Cui if (err) {
155ca9c54d2SDexuan Cui dev_err(sc->dev, "Timeout when setting up HWC: %d\n", err);
156ca9c54d2SDexuan Cui return err;
157ca9c54d2SDexuan Cui }
158ca9c54d2SDexuan Cui
159*6c95c700SHaiyang Zhang if (!MANA_PAGE_ALIGNED(eq_addr) || !MANA_PAGE_ALIGNED(cq_addr) ||
160*6c95c700SHaiyang Zhang !MANA_PAGE_ALIGNED(rq_addr) || !MANA_PAGE_ALIGNED(sq_addr))
161ca9c54d2SDexuan Cui return -EINVAL;
162ca9c54d2SDexuan Cui
163ca9c54d2SDexuan Cui if ((eq_msix_index & VECTOR_MASK) != eq_msix_index)
164ca9c54d2SDexuan Cui return -EINVAL;
165ca9c54d2SDexuan Cui
166ca9c54d2SDexuan Cui /* Scheme for packing four addresses and extra info into 256 bits.
167ca9c54d2SDexuan Cui *
168ca9c54d2SDexuan Cui * Addresses must be page frame aligned, so only frame address bits
169ca9c54d2SDexuan Cui * are transferred.
170ca9c54d2SDexuan Cui *
171ca9c54d2SDexuan Cui * 52-bit frame addresses are split into the lower 48 bits and upper
172ca9c54d2SDexuan Cui * 4 bits. Lower 48 bits of 4 address are written sequentially from
173ca9c54d2SDexuan Cui * the start of the 256-bit shared memory region followed by 16 bits
174ca9c54d2SDexuan Cui * containing the upper 4 bits of the 4 addresses in sequence.
175ca9c54d2SDexuan Cui *
176ca9c54d2SDexuan Cui * A 16 bit EQ vector number fills out the next-to-last 32-bit dword.
177ca9c54d2SDexuan Cui *
178ca9c54d2SDexuan Cui * The final 32-bit dword is used for protocol control information as
179ca9c54d2SDexuan Cui * defined in smc_proto_hdr.
180ca9c54d2SDexuan Cui */
181ca9c54d2SDexuan Cui
182ca9c54d2SDexuan Cui memset(shm_buf, 0, sizeof(shm_buf));
183ca9c54d2SDexuan Cui ptr = shm_buf;
184ca9c54d2SDexuan Cui
185ca9c54d2SDexuan Cui /* EQ addr: low 48 bits of frame address */
186ca9c54d2SDexuan Cui shmem = (u64 *)ptr;
187*6c95c700SHaiyang Zhang frame_addr = MANA_PFN(eq_addr);
188ca9c54d2SDexuan Cui *shmem = frame_addr & PAGE_FRAME_L48_MASK;
189ca9c54d2SDexuan Cui all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
190ca9c54d2SDexuan Cui (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
191ca9c54d2SDexuan Cui ptr += PAGE_FRAME_L48_WIDTH_BYTES;
192ca9c54d2SDexuan Cui
193ca9c54d2SDexuan Cui /* CQ addr: low 48 bits of frame address */
194ca9c54d2SDexuan Cui shmem = (u64 *)ptr;
195*6c95c700SHaiyang Zhang frame_addr = MANA_PFN(cq_addr);
196ca9c54d2SDexuan Cui *shmem = frame_addr & PAGE_FRAME_L48_MASK;
197ca9c54d2SDexuan Cui all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
198ca9c54d2SDexuan Cui (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
199ca9c54d2SDexuan Cui ptr += PAGE_FRAME_L48_WIDTH_BYTES;
200ca9c54d2SDexuan Cui
201ca9c54d2SDexuan Cui /* RQ addr: low 48 bits of frame address */
202ca9c54d2SDexuan Cui shmem = (u64 *)ptr;
203*6c95c700SHaiyang Zhang frame_addr = MANA_PFN(rq_addr);
204ca9c54d2SDexuan Cui *shmem = frame_addr & PAGE_FRAME_L48_MASK;
205ca9c54d2SDexuan Cui all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
206ca9c54d2SDexuan Cui (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
207ca9c54d2SDexuan Cui ptr += PAGE_FRAME_L48_WIDTH_BYTES;
208ca9c54d2SDexuan Cui
209ca9c54d2SDexuan Cui /* SQ addr: low 48 bits of frame address */
210ca9c54d2SDexuan Cui shmem = (u64 *)ptr;
211*6c95c700SHaiyang Zhang frame_addr = MANA_PFN(sq_addr);
212ca9c54d2SDexuan Cui *shmem = frame_addr & PAGE_FRAME_L48_MASK;
213ca9c54d2SDexuan Cui all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) <<
214ca9c54d2SDexuan Cui (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS);
215ca9c54d2SDexuan Cui ptr += PAGE_FRAME_L48_WIDTH_BYTES;
216ca9c54d2SDexuan Cui
217ca9c54d2SDexuan Cui /* High 4 bits of the four frame addresses */
218ca9c54d2SDexuan Cui *((u16 *)ptr) = all_addr_h4bits;
219ca9c54d2SDexuan Cui ptr += sizeof(u16);
220ca9c54d2SDexuan Cui
221ca9c54d2SDexuan Cui /* EQ MSIX vector number */
222ca9c54d2SDexuan Cui *((u16 *)ptr) = (u16)eq_msix_index;
223ca9c54d2SDexuan Cui ptr += sizeof(u16);
224ca9c54d2SDexuan Cui
225ca9c54d2SDexuan Cui /* 32-bit protocol header in final dword */
226ca9c54d2SDexuan Cui *((u32 *)ptr) = 0;
227ca9c54d2SDexuan Cui
228ca9c54d2SDexuan Cui hdr = (union smc_proto_hdr *)ptr;
229ca9c54d2SDexuan Cui hdr->msg_type = SMC_MSG_TYPE_ESTABLISH_HWC;
230ca9c54d2SDexuan Cui hdr->msg_version = SMC_MSG_TYPE_ESTABLISH_HWC_VERSION;
231ca9c54d2SDexuan Cui hdr->direction = SMC_MSG_DIRECTION_REQUEST;
232ca9c54d2SDexuan Cui hdr->reset_vf = reset_vf;
233ca9c54d2SDexuan Cui
234ca9c54d2SDexuan Cui /* Write 256-message buffer to shared memory (final 32-bit write
235ca9c54d2SDexuan Cui * triggers HW to set possession bit to PF).
236ca9c54d2SDexuan Cui */
237ca9c54d2SDexuan Cui dword = (u32 *)shm_buf;
238ca9c54d2SDexuan Cui for (i = 0; i < SMC_APERTURE_DWORDS; i++)
239ca9c54d2SDexuan Cui writel(*dword++, sc->base + i * SMC_BASIC_UNIT);
240ca9c54d2SDexuan Cui
241ca9c54d2SDexuan Cui /* Read shmem response (polling for VF possession) and validate.
242ca9c54d2SDexuan Cui * For setup, waiting for response on shared memory is not strictly
243ca9c54d2SDexuan Cui * necessary, since wait occurs later for results to appear in EQE's.
244ca9c54d2SDexuan Cui */
245ca9c54d2SDexuan Cui err = mana_smc_read_response(sc, SMC_MSG_TYPE_ESTABLISH_HWC,
246ca9c54d2SDexuan Cui SMC_MSG_TYPE_ESTABLISH_HWC_VERSION,
247ca9c54d2SDexuan Cui reset_vf);
248ca9c54d2SDexuan Cui if (err) {
249ca9c54d2SDexuan Cui dev_err(sc->dev, "Error when setting up HWC: %d\n", err);
250ca9c54d2SDexuan Cui return err;
251ca9c54d2SDexuan Cui }
252ca9c54d2SDexuan Cui
253ca9c54d2SDexuan Cui return 0;
254ca9c54d2SDexuan Cui }
255ca9c54d2SDexuan Cui
mana_smc_teardown_hwc(struct shm_channel * sc,bool reset_vf)256ca9c54d2SDexuan Cui int mana_smc_teardown_hwc(struct shm_channel *sc, bool reset_vf)
257ca9c54d2SDexuan Cui {
258ca9c54d2SDexuan Cui union smc_proto_hdr hdr = {};
259ca9c54d2SDexuan Cui int err;
260ca9c54d2SDexuan Cui
261ca9c54d2SDexuan Cui /* Ensure already has possession of shared memory */
262ca9c54d2SDexuan Cui err = mana_smc_poll_register(sc->base, false);
263ca9c54d2SDexuan Cui if (err) {
264ca9c54d2SDexuan Cui dev_err(sc->dev, "Timeout when tearing down HWC\n");
265ca9c54d2SDexuan Cui return err;
266ca9c54d2SDexuan Cui }
267ca9c54d2SDexuan Cui
268ca9c54d2SDexuan Cui /* Set up protocol header for HWC destroy message */
269ca9c54d2SDexuan Cui hdr.msg_type = SMC_MSG_TYPE_DESTROY_HWC;
270ca9c54d2SDexuan Cui hdr.msg_version = SMC_MSG_TYPE_DESTROY_HWC_VERSION;
271ca9c54d2SDexuan Cui hdr.direction = SMC_MSG_DIRECTION_REQUEST;
272ca9c54d2SDexuan Cui hdr.reset_vf = reset_vf;
273ca9c54d2SDexuan Cui
274ca9c54d2SDexuan Cui /* Write message in high 32 bits of 256-bit shared memory, causing HW
275ca9c54d2SDexuan Cui * to set possession bit to PF.
276ca9c54d2SDexuan Cui */
277ca9c54d2SDexuan Cui writel(hdr.as_uint32, sc->base + SMC_LAST_DWORD * SMC_BASIC_UNIT);
278ca9c54d2SDexuan Cui
279ca9c54d2SDexuan Cui /* Read shmem response (polling for VF possession) and validate.
280ca9c54d2SDexuan Cui * For teardown, waiting for response is required to ensure hardware
281ca9c54d2SDexuan Cui * invalidates MST entries before software frees memory.
282ca9c54d2SDexuan Cui */
283ca9c54d2SDexuan Cui err = mana_smc_read_response(sc, SMC_MSG_TYPE_DESTROY_HWC,
284ca9c54d2SDexuan Cui SMC_MSG_TYPE_DESTROY_HWC_VERSION,
285ca9c54d2SDexuan Cui reset_vf);
286ca9c54d2SDexuan Cui if (err) {
287ca9c54d2SDexuan Cui dev_err(sc->dev, "Error when tearing down HWC: %d\n", err);
288ca9c54d2SDexuan Cui return err;
289ca9c54d2SDexuan Cui }
290ca9c54d2SDexuan Cui
291ca9c54d2SDexuan Cui return 0;
292ca9c54d2SDexuan Cui }
293